test(rebalance): hold the first run until stop cancels admission

* fix(s3): treat duplicate CreateBucket as owned by the requester

The duplicate-bucket response was decided from the requester's root-owner
flag, so every IAM user recreating a bucket received 409
BucketAlreadyExists, including a user recreating a bucket it created.
BucketAlreadyExists means the bucket belongs to another account; a
deployment owns a single namespace, so the answer is purely region-based:
the default region returns the legacy 200 OK, any other region returns
BucketAlreadyOwnedByYou.

Both the existing-bucket fast path and the BucketExists path now share one
helper. Authorization still runs first, so a requester without
s3:CreateBucket gets AccessDenied and cannot probe bucket existence.

Refs #8493

* test(ci): refresh e2e selections for duplicate CreateBucket

The two new create_bucket_region cases join the e2e-smoke and e2e-full
selections, which main also extended with multipart_upload_id tests.

smoke is measured from a local nextest listing: removing the two new cases
reproduces main's committed digest exactly. e2e-full is derived the same way
from main's listing artifact (run 41933823399, babfeaed0) plus the two
multipart_upload_id cases, which reproduces main's committed linux digest
before the new cases are added; CI re-verifies both platforms.

test_bucket_create_exists_nonowner asserts BucketAlreadyExists for a second
identity, which a single-owner deployment cannot produce. Move it to
excluded_tests.txt with the deployment-ownership rationale; the IAM-user
duplicate paths stay covered by crates/e2e_test/src/create_bucket_region_test.rs.

.config/e2e-distributed-selection.txt

@@ -1,2 +1,2 @@
-sha256-linux=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
-sha256-darwin=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
+sha256-linux=348fa5eed9ae58962c06b33283ff4f15122cff97bce9b4eb1e3ffcb227264b0b
+sha256-darwin=348fa5eed9ae58962c06b33283ff4f15122cff97bce9b4eb1e3ffcb227264b0b

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=67251cb7d9978d38e899268cac4d7898297ff7e5b5c48c9c1546dc760b93568d
-sha256-linux=673da76c2e5710c4622fe2fa32d2b297730895769487fcf1152ee1c2366f32f4
+sha256-darwin=219ac11e720b3f54dfee4d5d5e95171785ba159b2fe01db3037a8f5061c1ad32
+sha256-linux=2d34698cc8ec52d5b6c37ac612919a1d21f36cd56a1af539a7522a4eea518e26

.config/e2e-nightly-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=3814cdde8cacf4187e18188d342ed70931cc32e268e42e5bb438d565bb4d0f81
-sha256-linux=663025a80273579f95ae3f49af02c0afcbfce94df09701c360a13e607b7637db
+sha256-darwin=e84e04208f19dc0596b10d4d1b799ce43d6cc431415902f9c5458aa8232bc2d8
+sha256-linux=1f9c7add72c44d1c12ef4ac050bb7917431be657c0c90b29f70c3df56f8bbae3

.config/e2e-smoke-selection.txt

@@ -1 +1 @@
-sha256=e9dc896774122a304e2b8b90f28c82d4d4e0af8ded36a82b147b30945ec994e3
+sha256=5bdd4c2eadbc41625ae3184fff34062e000d328108436259b2b7c4fe2531ecaf

.config/nextest.toml

@@ -234,7 +234,7 @@ test-group = 'embedded-test-ports'
 # processes cannot exhaust those budgets. Keep deadlines and assertions intact.
 # The short HTTP fixtures are tracked in rustfs/backlog#2714.
 [[profile.default.overrides]]
-filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
+filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
 threads-required = "num-test-threads"
 
 # This cancellation probe hashes the serving executable before acquiring its
@@ -548,7 +548,7 @@ filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnost
 test-group = 'embedded-test-ports'
 
 [[profile.ci.overrides]]
-filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
+filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
 threads-required = "num-test-threads"
 
 # Preserve the cancellation probe's startup budget and assertions in CI too.
@@ -701,7 +701,7 @@ test-group = 'ecstore-serial-flaky'
 [profile.e2e-smoke]
 default-filter = """
     package(e2e_test) & (
-        test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
+        test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|multipart_upload_id|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
         | test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
         | test(/^reliant::lifecycle::/)
         | test(/^conditional_delete_objects_test::/)
@@ -827,7 +827,7 @@ slow-timeout = { period = "120s", terminate-after = 12, grace-period = "10s" }
 # ---------------------------------------------------------------------------
 # e2e-distributed profile — 4-node 4-disk Actions suite
 # ---------------------------------------------------------------------------
-# Storage-sensitive PR / nightly / dispatch lane owned by
+# Nightly and manual-dispatch lane owned by
 # .github/workflows/e2e-distributed.yml.
 # Each case starts four rustfs processes (and for site replication, two
 # clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
@@ -976,6 +976,12 @@ test-group = 'e2e-vault'
 filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
 threads-required = "num-test-threads"
 
+# The mc fixture uploads hundreds of objects concurrently before measuring listing.
+# Reserve the runner so other server fixtures do not starve its disk operations.
+[[profile.e2e-full.overrides]]
+filter = 'package(e2e_test) & test(=mc_mirror_small_bucket_test::test_mc_mirror_small_bucket_completes_without_list_timeout)'
+threads-required = "num-test-threads"
+
 # Fixed, small contract for every executable server change. These existing
 # scenarios exercise write/read, anonymous denial and retained bytes on restart.
 [profile.core-contract]

.config/release-validation-plan.json

@@ -0,0 +1,77 @@
+{
+  "schema_version": 1,
+  "plan_id": "rustfs-release-validation",
+  "description": "Versioned certification plan for every public RustFS release exit (rustfs/backlog#2662, R01). preview, prerelease and stable all require this explicit plan. Scripts/release/release_gate.py fail-closes publication on any missing, duplicate, stale, substituted or non-PASS required item.",
+  "release_types": ["preview", "prerelease", "stable"],
+  "targets": [
+    {"os": "linux", "arch": "x86_64", "abi": "musl"},
+    {"os": "linux", "arch": "aarch64", "abi": "musl"},
+    {"os": "linux", "arch": "x86_64", "abi": "gnu"},
+    {"os": "linux", "arch": "aarch64", "abi": "gnu"},
+    {"os": "macos", "arch": "aarch64", "abi": "none"},
+    {"os": "windows", "arch": "x86_64", "abi": "msvc"}
+  ],
+  "targets_basis": "Platform matrix of .github/workflows/build.yml (prepare-platform-matrix). gnu and macOS targets build with the pyroscope feature; musl and Windows targets build with no extra features.",
+  "features": ["pyroscope"],
+  "trusted_workflows": [
+    {"repository": "rustfs/s3chaos", "workflow_id": "release-gate.yml"}
+  ],
+  "trusted_workflows_basis": "The shared evidence envelope producer lives in rustfs/s3chaos (G01, fault/release_gate.rs) and its plan/live split workflow is .github/workflows/release-gate.yml there. Only results from these identities may enter a release certification; PR-uploaded JSON never gains release authority. R02 must extend this list when additional verifier workflows land, never widen it to untrusted sources.",
+  "lanes": [
+    {
+      "lane": "identity",
+      "harness_repository": "rustfs/s3chaos",
+      "harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
+      "harness_basis": "Fixed G01 evidence-envelope producer revision (rustfs/s3chaos PR #102 head recorded in rustfs/backlog#2658). R02 re-pins this when the live verifier ships.",
+      "timeout_seconds": 1800,
+      "environment_capabilities": ["process-exec"],
+      "cases": ["artifact-identity", "startup"],
+      "applies_to": {
+        "release_types": ["preview", "prerelease", "stable"],
+        "platforms": "all"
+      },
+      "applies_basis": "Artifact identity and startup checks are mandatory for every release type, preview included, on every shipped target (rustfs/backlog#2662 frozen design item 5)."
+    },
+    {
+      "lane": "upgrade",
+      "harness_repository": "rustfs/s3chaos",
+      "harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
+      "harness_basis": "Same fixed envelope producer as the identity lane; the workflow SHA and the product build SHA are bound separately and need not be equal.",
+      "timeout_seconds": 5400,
+      "environment_capabilities": ["process-exec", "previous-release-artifact"],
+      "cases": [
+        "direct-upgrade",
+        "mixed-version-upgrade",
+        "bucket-config-upgrade",
+        "bucket-config-rollback",
+        "multipart-layout-upgrade",
+        "multipart-layout-baseline"
+      ],
+      "cases_basis": "Case IDs of the pinned previous-release matrix in .github/workflows/e2e-upgrade.yml (source.version/asset/sha256 and the case list mapped there).",
+      "applies_to": {
+        "release_types": ["prerelease", "stable"],
+        "platforms": [{"os": "linux", "arch": "x86_64", "abi": "gnu"}]
+      },
+      "applies_basis": "preview releases use the pre-declared smaller set (identity lane only). The upgrade lane is only exercised on linux-x86_64-gnu, the platform of the pinned source assets."
+    }
+  ],
+  "upgrade_sources": {
+    "basis": "Exactly the two previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
+    "sources": [
+      {
+        "version": "1.0.1",
+        "role": "N-1",
+        "artifact_name": "rustfs-linux-x86_64-gnu-v1.0.1.zip",
+        "artifact_sha256": "9742e183cb0dfb0d48fa909797af572fe47b7f827569036baa4156e0d3958ee5"
+      },
+      {
+        "version": "1.0.0-rc.5",
+        "role": "N-2",
+        "artifact_name": "rustfs-linux-x86_64-gnu-v1.0.0-rc.5.zip",
+        "artifact_sha256": "3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d"
+      }
+    ],
+    "rollback_supported": false,
+    "rollback_basis": "Only the bucket-config-rollback case (rollback_to_previous_release_reads_current_bucket_metadata) is exercised. Full data-plane rollback is not certified and is never a default claim."
+  }
+}

.docker/README.md

@@ -22,9 +22,9 @@ The following files in the project root are essential for Docker operations:
 | File | Description | Usage |
 | :--- | :--- | :--- |
 | **`docker-buildx.sh`** | **Multi-Arch Build Script**<br>Automates building and pushing Docker images for `amd64` and `arm64`. Supports release and dev channels. | `./docker-buildx.sh --push` |
-| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads pre-built binaries from GitHub Releases. | `docker build -t rustfs:latest .` |
-| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Useful if you need specific dynamic libraries. | `docker build -f Dockerfile.glibc .` |
-| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code. Includes build tools. Ideal for local development and CI. | `docker build -f Dockerfile.source .` |
+| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads verified RustFS and `rc` binaries from GitHub Releases and bundles `/etc/rustfs/bootstrap`. | `docker build -t rustfs:latest .` |
+| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Includes the bundled bootstrap configuration. | `docker build -f Dockerfile.glibc .` |
+| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code and bundles `rc` and the default bootstrap configuration. | `docker build -f Dockerfile.source .` |
 
 ### Docker Compose Configurations
 

.github/dependabot.yml

@@ -32,6 +32,8 @@ updates:
       - "overtrue"
       - "majinghe"
     ignore:
+      - dependency-name: "pulsar"
+        versions: [ "6.9.x" ]
       - dependency-name: "object_store"
         versions: [ "0.13.x" ]
       - dependency-name: "libunftp"

.github/scheduled-validations.json

@@ -296,6 +296,32 @@
       "origin": "baseline",
       "alert_contract": "scheduled-validation"
     },
+    {
+      "workflow": ".github/workflows/rustfs-validation-sweep.yml",
+      "name": "RustFS Validation Sweep",
+      "roles": [
+        "periodic",
+        "manual"
+      ],
+      "events": [
+        "schedule",
+        "workflow_dispatch"
+      ],
+      "cadence": "nightly",
+      "timezone": "UTC",
+      "schedule": [
+        "30 19 * * *"
+      ],
+      "owner": "@overtrue",
+      "policy": "active",
+      "monitor": "workflow",
+      "max_age_hours": 60,
+      "result_owner": "rustfs-validation-sweep.yml",
+      "disposition": "canonical validation cases executed in rotating batches",
+      "failure_owner": "workflow",
+      "origin": "baseline",
+      "alert_contract": "scheduled-validation"
+    },
     {
       "workflow": ".github/workflows/btf-prototype-reference.yml",
       "name": "BTF prototype reference",
@@ -743,12 +769,10 @@
       "workflow": ".github/workflows/e2e-distributed.yml",
       "name": "e2e-distributed",
       "roles": [
-        "pr",
         "periodic",
         "manual"
       ],
       "events": [
-        "pull_request",
         "workflow_dispatch",
         "schedule"
       ],
@@ -823,12 +847,11 @@
       "workflow": ".github/workflows/e2e-upgrade.yml",
       "name": "Upgrade Compatibility",
       "roles": [
-        "pr",
         "periodic",
-        "manual"
+        "manual",
+        "release"
       ],
       "events": [
-        "pull_request",
         "push",
         "schedule",
         "workflow_dispatch"

.github/workflows/build.yml

@@ -814,13 +814,18 @@ jobs:
           export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
           export AWS_DEFAULT_REGION="auto"
 
-          # Determine upload path based on build type
+          # Determine upload path based on build type. Release and prerelease
+          # builds only stage their bytes here: the official
+          # artifacts/rustfs/release/ path is written solely by the
+          # certification-gated promote-release-artifacts job, from these same
+          # staged objects. Development builds keep the uncertified dev channel
+          # and never touch official or latest channels.
           if [[ "$BUILD_TYPE" == "development" ]]; then
             R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/dev/"
             echo "📤 Uploading development build to R2 dev directory"
           else
-            R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
-            echo "📤 Uploading release build to R2 release directory"
+            R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
+            echo "📤 Staging release build for certification-gated promotion"
           fi
 
           # Upload all rustfs zip files to R2 using S3-compatible API
@@ -869,6 +874,221 @@ jobs:
             echo "This run does not trigger Docker publishing."
           fi
 
+  # Aggregate the parent candidate batch. This job always collects results and
+  # retains its evidence, including on failure, and every publication job
+  # below requires its success. Missing, stale, substituted or non-PASS
+  # evidence fails closed here: only the trusted verification sources
+  # (rustfs/s3chaos release-gate envelopes, rustfs/backlog#2658) can satisfy
+  # it, so a PR-uploaded JSON set never gains release authority.
+  release-certification:
+    name: Release Certification
+    needs: [ build-check, build-rustfs ]
+    if: ${{ !cancelled() && needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') }}
+    runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      actions: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Download all build artifacts
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          path: ./artifacts
+          pattern: rustfs-*
+          merge-multiple: true
+
+      - name: Write candidate and collect trusted validation results
+        env:
+          # Cross-repo read of the trusted verification run. Falls back to the
+          # workflow token, which cannot read rustfs/s3chaos; a missing token
+          # therefore fail-closes instead of certifying.
+          GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
+          PRODUCT_COMMIT: ${{ github.sha }}
+          VERSION: ${{ needs.build-check.outputs.version }}
+          BUILD_TYPE: ${{ needs.build-check.outputs.build_type }}
+          FEATURES: ${{ needs.build-check.outputs.build_type == 'development' && '' || 'pyroscope' }}
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          mkdir -p results release-gate-output
+          PLAN_SHA256=$(python3 -c "import hashlib,json,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('.config/release-validation-plan.json'))))")
+
+          python3 - "$PLAN_SHA256" > candidate.json <<'PY'
+          import hashlib, json, os, sys, zipfile
+
+          plan_sha256 = sys.argv[1]
+          profile = {"release": "stable", "prerelease": "prerelease", "preview": "preview"}.get(os.environ["BUILD_TYPE"])
+          if profile is None:
+              raise SystemExit("release certification only covers preview, prerelease and stable builds")
+
+          def sha256(path):
+              digest = hashlib.sha256()
+              with open(path, "rb") as handle:
+                  for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+                      digest.update(chunk)
+              return digest.hexdigest()
+
+          deliverables = []
+          for name in sorted(os.listdir("artifacts")):
+              if not name.startswith("rustfs-") or not name.endswith(".zip"):
+                  continue
+              path = os.path.join("artifacts", name)
+              members = zipfile.ZipFile(path).namelist()
+              binary_name = "rustfs.exe" if "rustfs.exe" in members else "rustfs"
+              with zipfile.ZipFile(path) as archive:
+                  binary_bytes = archive.read(binary_name)
+              import tempfile
+              with tempfile.NamedTemporaryFile() as handle:
+                  handle.write(binary_bytes)
+                  handle.flush()
+                  binary_sha = sha256(handle.name)
+              deliverables.append({
+                  "kind": "zip",
+                  "name": name,
+                  "location": "https://github.com/rustfs/rustfs/releases/download/{}/{}".format(os.environ["VERSION"], name),
+                  "sha256": sha256(path),
+                  "size": os.path.getsize(path),
+                  "binary_sha256": binary_sha,
+              })
+          if not deliverables:
+              raise SystemExit("no built deliverables found to certify")
+          candidate = {
+              "schema_version": 1,
+              "product_commit": os.environ["PRODUCT_COMMIT"],
+              "version": os.environ["VERSION"],
+              "release_profile": profile,
+              "features": [f for f in os.environ.get("FEATURES", "").split(",") if f],
+              "plan_sha256": plan_sha256,
+              "validation_batch": {"run_id": os.environ["GITHUB_RUN_ID"], "run_attempt": int(os.environ["GITHUB_RUN_ATTEMPT"])},
+              "deliverables": deliverables,
+          }
+          json.dump(candidate, sys.stdout, indent=2, sort_keys=True)
+          print()
+          PY
+
+          # Trusted verification evidence only: the release-gate envelopes of
+          # rustfs/s3chaos for this exact product commit. Absence blocks below.
+          TRUSTED_REPO="rustfs/s3chaos"
+          TRUSTED_WORKFLOW="release-gate.yml"
+          TRUSTED_RUN=$(gh api "repos/${TRUSTED_REPO}/actions/workflows/${TRUSTED_WORKFLOW}/runs?head_sha=${PRODUCT_COMMIT}&status=success&per_page=1" --jq '.workflow_runs[0].id // empty' || true)
+          if [[ -z "$TRUSTED_RUN" ]]; then
+            echo "❌ No trusted verification run of ${TRUSTED_REPO}/${TRUSTED_WORKFLOW} for ${PRODUCT_COMMIT}; refusing to certify" >&2
+          else
+            gh run download "$TRUSTED_RUN" -R "$TRUSTED_REPO" -n release-validation-evidence -D results || \
+              echo "❌ Trusted verification run ${TRUSTED_RUN} published no release-validation-evidence artifact" >&2
+          fi
+
+          python3 scripts/release/release_gate.py validate \
+            --plan .config/release-validation-plan.json \
+            --candidate candidate.json \
+            --results results \
+            --output release-gate-output
+
+      - name: Upload certification evidence
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: release-certification
+          path: |
+            candidate.json
+            results/
+            release-gate-output/
+          if-no-files-found: warn
+          retention-days: 30
+
+  # The only writer of the official R2 release path. Promotion re-checks the
+  # exact bytes about to be published against the passed certification list
+  # and copies the staged objects byte-for-byte: no rebuild-after-verify.
+  promote-release-artifacts:
+    name: Promote Release Artifacts
+    needs: [ build-check, build-rustfs, release-certification ]
+    # Target-channel lock for the official R2 release path.
+    concurrency:
+      group: release-channel-r2-release
+      cancel-in-progress: false
+    if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      actions: read
+    env:
+      R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
+      R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
+      R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
+      R2_BUCKET: ${{ secrets.R2_BUCKET }}
+      AWS_EC2_METADATA_DISABLED: true
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Download build and certification artifacts
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          path: ./artifacts
+          pattern: rustfs-*
+          merge-multiple: true
+
+      - name: Download certification evidence
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: release-certification
+          path: ./certification
+
+      - name: Verify and promote staged release artifacts
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
+            echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
+            exit 1
+          fi
+
+          for zip_file in artifacts/rustfs-*.zip; do
+            [[ -f "$zip_file" ]] || continue
+            name="$(basename "$zip_file")"
+            python3 scripts/release/release_gate.py verify-promotion \
+              --plan .config/release-validation-plan.json \
+              --candidate certification/candidate.json \
+              --summary certification/release-gate-output \
+              --artifact "$zip_file" \
+              --artifact-name "$name"
+          done
+
+          if ! command -v aws >/dev/null 2>&1; then
+            sudo apt-get update && sudo apt-get install -y awscli
+          fi
+          export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
+          export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
+          export AWS_DEFAULT_REGION="auto"
+
+          STAGING_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
+          RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
+          for zip_file in artifacts/rustfs-*.zip; do
+            [[ -f "$zip_file" ]] || continue
+            name="$(basename "$zip_file")"
+            staged="$(mktemp)"
+            aws s3 cp "${STAGING_PATH}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+            staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
+            built_sha="$(sha256sum "$zip_file" | awk '{print $1}')"
+            if [[ "$staged_sha" != "$built_sha" ]]; then
+              echo "❌ Staged object ${name} does not match the verified bytes" >&2
+              exit 1
+            fi
+            aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+            rm -f "$staged"
+            echo "✅ Promoted ${name} (${built_sha})"
+          done
+
   # Create GitHub Release for every valid release tag, including previews
   create-release:
     name: Create GitHub Release
@@ -1028,6 +1248,10 @@ jobs:
   update-latest-version:
     name: Update Latest Version
     needs: [ build-check, publish-release ]
+    # Target-channel lock for the latest.json version pointer.
+    concurrency:
+      group: release-channel-oss-latest
+      cancel-in-progress: false
     if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
     runs-on: ubuntu-latest
     timeout-minutes: 30
@@ -1068,6 +1292,45 @@ jobs:
           rm -rf "$OSSUTIL_DIR" "$OSSUTIL_ZIP"
           chmod +x "$OSSUTIL_BIN"
 
+          # Candidate/pointer ordering: an older candidate must never overwrite
+          # a newer pointer, even if its promotion job finishes later.
+          CURRENT_LATEST="${RUNNER_TEMP}/current-latest.json"
+          if "$OSSUTIL_BIN" cp oss://rustfs-version/latest.json "$CURRENT_LATEST" 2>/dev/null; then
+            python3 - "$CURRENT_LATEST" "$VERSION" <<'PY'
+          import json
+          import re
+          import sys
+
+          def key(version):
+              match = re.match(r"^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$", version)
+              if not match:
+                  raise SystemExit(3)
+              core = tuple(int(part) for part in match.groups()[:3])
+              prerelease = match.group(4)
+              if prerelease is None:
+                  return core + (1,)
+              return core + (0, tuple(int(p) if p.isdigit() else p for p in re.split(r"[.-]", prerelease)))
+
+          try:
+              current = json.load(open(sys.argv[1], encoding="utf-8")).get("version", "")
+              proposed = sys.argv[2]
+              if current and key(current) > key(proposed):
+                  raise SystemExit(1)
+          except SystemExit:
+              raise
+          except (OSError, ValueError, TypeError):
+              raise SystemExit(3)
+          PY
+            guard_status=$?
+            if [[ "$guard_status" -eq 1 ]]; then
+              echo "Current latest.json points at a newer version; refusing to downgrade" >&2
+              exit 0
+            elif [[ "$guard_status" -ne 0 ]]; then
+              echo "Existing latest.json is unparsable; refusing to overwrite it blindly" >&2
+              exit 1
+            fi
+          fi
+
           # Create latest.json
           cat > latest.json << EOF
           {
@@ -1084,16 +1347,59 @@ jobs:
 
           echo "✅ Updated latest.json for ${RELEASE_TYPE} release $VERSION"
 
-  # Publish release (remove draft status)
+  # Publish release (remove draft status). Requires the release-certification
+  # aggregate: without a complete legal success there is no public write.
   publish-release:
     name: Publish Release
-    needs: [ build-check, create-release, upload-release-assets ]
+    needs: [ build-check, create-release, upload-release-assets, release-certification ]
+    # Target-channel lock: concurrent releases serialize on this channel so an
+    # older candidate cannot interleave its publish with a newer one.
+    concurrency:
+      group: release-channel-github-release
+      cancel-in-progress: false
     if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
     runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
     timeout-minutes: 30
     permissions:
       contents: write
+      actions: read
     steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Download certification evidence
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: release-certification
+          path: ./certification
+
+      - name: Verify release assets against certification
+        env:
+          GH_TOKEN: ${{ github.token }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          TAG="${{ needs.build-check.outputs.version }}"
+          mkdir -p "$RUNNER_TEMP/promotion-assets"
+          gh release download "$TAG" --dir "$RUNNER_TEMP/promotion-assets" --pattern '*.zip'
+          found=0
+          for zip_file in "$RUNNER_TEMP"/promotion-assets/*.zip; do
+            [[ -f "$zip_file" ]] || continue
+            found=1
+            python3 scripts/release/release_gate.py verify-promotion \
+              --plan .config/release-validation-plan.json \
+              --candidate certification/candidate.json \
+              --summary certification/release-gate-output \
+              --artifact "$zip_file" \
+              --artifact-name "$(basename "$zip_file")"
+          done
+          if [[ "$found" -ne 1 ]]; then
+            echo "No release zip assets to verify against the certification" >&2
+            exit 1
+          fi
+
       - name: Publish release
         env:
           GH_TOKEN: ${{ github.token }}

.github/workflows/cache-warm.yml

@@ -96,7 +96,7 @@ env:
   RUST_BACKTRACE: 1
 
 jobs:
-  # Readers: test-and-lint (workspace and ILM), build-rustfs-debug-binary,
+  # Readers: workspace-lint-and-build, workspace-tests (workspace and ILM), build-rustfs-debug-binary,
   # e2e-tests, e2e-full.
   warm-ci-dev:
     name: Warm ci-dev
@@ -146,7 +146,7 @@ jobs:
           if-no-files-found: error
 
       # --all-targets covers the test binaries nextest builds, including
-      # e2e_test, which test-and-lint's own run excludes. The second build adds
+      # e2e_test, which the workspace run excludes. The second build adds
       # the e2e-test-hooks feature resolution that build-rustfs-debug-binary uses
       # and that no lint lane enables.
       - name: Build ci-dev superset

.github/workflows/ci.yml

@@ -113,7 +113,7 @@ jobs:
             printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
             echo "Base CI policy unavailable; running the full matrix."
           fi
-          if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^rehearse_shards=' "$GITHUB_OUTPUT"; then
+          if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=3$' "$GITHUB_OUTPUT"; then
             printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
             echo "Base policy does not cover this workflow contract; retaining the full matrix."
           fi
@@ -160,7 +160,7 @@ jobs:
       - name: Run shared quick checks
         uses: ./.github/actions/quick-checks
 
-  test-and-lint:
+  workspace-lint-and-build:
     name: Workspace Lint and Build
     if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
     needs: [ quick-checks, classify-changes ]
@@ -193,13 +193,13 @@ jobs:
 
       - name: Prepare test evidence
         run: |
-          mkdir -p artifacts/test-and-lint
+          mkdir -p artifacts/workspace
           {
             echo "run_id=${GITHUB_RUN_ID}"
             echo "job=${GITHUB_JOB}"
             echo "runner=${RUNNER_NAME}"
             echo "started_at=$(date --utc --iso-8601=seconds)"
-          } > artifacts/test-and-lint/run-metadata.txt
+          } > artifacts/workspace/run-metadata.txt
 
       # Clippy gates every event, pull requests included. When it ran only after
       # merge, a lint regression merged freely and then failed every main run
@@ -233,11 +233,11 @@ jobs:
 
       - name: Run documentation tests
         run: |
-          mkdir -p artifacts/test-and-lint
+          mkdir -p artifacts/workspace
           set +e
           timeout --verbose --signal=TERM --kill-after=30s 15m \
             python3 scripts/ci_test_artifact.py cargo doc --configuration workspace \
-            2>&1 | tee artifacts/test-and-lint/doctest.log
+            2>&1 | tee artifacts/workspace/doctest.log
           status=${PIPESTATUS[0]}
           {
             echo "command=cargo test --all --exclude e2e_test --doc"
@@ -246,7 +246,7 @@ jobs:
             echo
             echo "Remaining test-related processes:"
             pgrep -af 'cargo|rustdoc|target/.*/deps/' || true
-          } > artifacts/test-and-lint/doctest-diagnostics.txt
+          } > artifacts/workspace/doctest-diagnostics.txt
           exit "${status}"
 
       - name: Upload test reports and diagnostics
@@ -255,7 +255,7 @@ jobs:
         with:
           name: junit-test-and-lint-${{ github.run_number }}-${{ github.run_attempt }}
           path: |
-            artifacts/test-and-lint
+            artifacts/workspace
           retention-days: 3
           if-no-files-found: error
 
@@ -268,7 +268,7 @@ jobs:
 
   workspace-tests:
     name: Workspace tests
-    needs: [ test-and-lint, classify-changes ]
+    needs: [ workspace-lint-and-build, classify-changes ]
     # The archive retains compile-time fixture paths and the large test set
     # requires the same runner image until the hosted-runtime pilot passes.
     runs-on: sm-standard-4
@@ -293,14 +293,14 @@ jobs:
 
       - name: Extract and list workspace tests
         run: |
-          mkdir -p artifacts/test-and-lint
+          mkdir -p artifacts/workspace
           cargo nextest list --profile ci --archive-file "$RUNNER_TEMP/workspace-tests.tar.zst" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json \
-            > artifacts/test-and-lint/core-test-listing.json
-          python3 scripts/check_test_wiring.py --check-core artifacts/test-and-lint/core-test-listing.json
+            > artifacts/workspace/core-test-listing.json
+          python3 scripts/check_test_wiring.py --check-core artifacts/workspace/core-test-listing.json
 
       - name: Run nextest tests
         run: |
-          mkdir -p artifacts/test-and-lint
+          mkdir -p artifacts/workspace
           rm -f target/nextest/ci/junit.xml
           ./scripts/ci/resource_sampler.sh start nextest
           trap './scripts/ci/resource_sampler.sh stop' EXIT
@@ -309,7 +309,7 @@ jobs:
             cargo nextest run --profile ci --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json \
             --workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" \
             --status-level all --final-status-level all \
-            2>&1 | tee artifacts/test-and-lint/nextest.log
+            2>&1 | tee artifacts/workspace/nextest.log
           status=${PIPESTATUS[0]}
           {
             echo "command=workspace archive execution"
@@ -321,13 +321,13 @@ jobs:
             echo
             echo "Kernel OOM / kill events:"
             dmesg -T 2>/dev/null | grep -iE 'oom|out of memory|killed process' | tail -20 || true
-          } > artifacts/test-and-lint/nextest-diagnostics.txt
+          } > artifacts/workspace/nextest-diagnostics.txt
           exit "${status}"
 
       - name: Verify workspace execution completeness
         run: |
-          python3 scripts/ci_test_artifact.py evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
-          ./scripts/check_migration_gate_count.sh evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
+          python3 scripts/ci_test_artifact.py evidence artifacts/workspace/core-test-listing.json target/nextest/ci/junit.xml
+          ./scripts/check_migration_gate_count.sh evidence artifacts/workspace/core-test-listing.json target/nextest/ci/junit.xml
 
       - name: Run ignored ILM integration tests serially
         if: needs.classify-changes.outputs.mode == 'full'
@@ -358,7 +358,7 @@ jobs:
           name: workspace-test-evidence-${{ github.run_attempt }}
           path: |
             target/nextest/ci/junit.xml
-            artifacts/test-and-lint
+            artifacts/workspace
             artifacts/ilm-integration
           retention-days: 3
           if-no-files-found: error
@@ -368,7 +368,7 @@ jobs:
         if: needs.classify-changes.outputs.rehearse_shards == 'true'
         run: |
           python3 scripts/ci_test_shards.py prepare \
-            --listing artifacts/test-and-lint/core-test-listing.json \
+            --listing artifacts/workspace/core-test-listing.json \
             --junit target/nextest/ci/junit.xml --output artifacts/workspace-shard-plan
       - name: Upload same-run shard plan
         if: needs.classify-changes.outputs.rehearse_shards == 'true'
@@ -449,7 +449,7 @@ jobs:
 
   connect-feature-boundaries:
     name: Connect Feature Boundaries
-    if: needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect != 'false' && (github.event_name != 'pull_request' || github.event.action != 'closed')
+    if: needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect != 'false' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
     needs: [ quick-checks, classify-changes ]
     runs-on: sm-standard-4
     timeout-minutes: 90
@@ -497,13 +497,72 @@ jobs:
       - name: Check offline enrollment E2E root boundary
         run: ./scripts/check_offline_enrollment_e2e.sh
 
+  fips-provider:
+    name: FIPS provider boundaries
+    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
+    needs: [ quick-checks, classify-changes ]
+    # This builds the RustFS library test target, which exceeds hosted-runner disk capacity.
+    runs-on: sm-standard-4
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        mode: [default, fips]
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+      CARGO_BUILD_JOBS: "2"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-fips-provider
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'
+
+      - name: Install FIPS build tools
+        if: matrix.mode == 'fips'
+        run: sudo apt-get install -y cmake golang clang libclang-dev
+
+      - name: Verify provider installation
+        shell: bash
+        run: |
+          args=()
+          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
+            args+=(--features fips)
+          fi
+          cargo nextest run --locked -p rustfs-tls-runtime --lib "${args[@]}" --no-tests fail -E 'test(provider_policy_isolated)'
+          cargo nextest run --locked -p rustfs-s3-client --lib "${args[@]}" --no-tests fail -E 'test(overflowing_closed_range_is_rejected_before_network_io)'
+          cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(validate_pulsar)'
+          cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(webhook_provider_policy_isolated)'
+          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
+            cargo nextest run --locked -p rustfs-protocols --lib --features ftps,webdav,fips --no-tests fail -E 'test(protocols_reject_incompatible_host_provider)'
+            cargo nextest run --locked -p rustfs-targets --lib --features fips --no-tests fail -E 'test(fips_postgres_rejects_incompatible_host_provider)'
+            cargo nextest run --locked -p rustfs-s3-client --lib --features fips --no-tests fail -E 'test(fips_transition_rejects_incompatible_host_provider)'
+          fi
+          tests=(provider_installation_isolated prepared_generation_shares_crypto_and_serves_sni_with_mtls)
+          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
+            tests+=(fips_provider_builds_fips_tls_configs fips_provider_tls_handshake_accepts_and_rejects_algorithms)
+          fi
+          for test_name in "${tests[@]}"; do
+            cargo nextest run --locked -p rustfs --lib "${args[@]}" --no-tests fail -E "test($test_name)"
+          done
+          cargo clippy --locked -p rustfs --lib "${args[@]}" -- -D warnings
+
   test-and-lint-rio-v2:
     name: Test and Lint (rio-v2)
-    # Alternate feature configurations run on the nightly schedule, merge
-    # groups and manual dispatch. The default workspace remains the PR gate and
-    # the per-push main baseline: repeating four feature builds on every merge
-    # bought nothing the nightly run does not report within a day.
-    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["merge_group", "schedule", "workflow_dispatch"]'), github.event_name)
+    # Specialized boundary, hardware and alternate-feature lanes run only on
+    # the nightly schedule and manual dispatch. They do not gate a merge: the
+    # nightly run reports a regression within a day, and repeating these builds
+    # on every PR revision and merge group bought no earlier signal worth the
+    # runner occupancy. Workspace lint and tests, the core contract, the debug
+    # build, E2E smoke and the S3 lanes remain the pull-request gate.
+    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
     needs: [ quick-checks, classify-changes ]
     runs-on: sm-standard-4
     timeout-minutes: 90
@@ -540,12 +599,12 @@ jobs:
           # --profile ci so the quarantine list (and its junit flaky markers)
           # covers this leg too; the default profile is the local no-retry
           # profile and silently ignored quarantined flakes here (rustfs#6703).
-          mkdir -p artifacts/test-and-lint
+          mkdir -p artifacts/workspace
           rm -f target/nextest/ci/junit.xml
           ./scripts/ci/resource_sampler.sh start nextest-rio-v2
           trap './scripts/ci/resource_sampler.sh stop' EXIT
           cargo nextest run --profile ci -p rustfs -p rustfs-ecstore --features rio-v2 \
-            2>&1 | tee artifacts/test-and-lint/nextest.log
+            2>&1 | tee artifacts/workspace/nextest.log
           cargo test -p rustfs --doc --features rio-v2
 
       - name: Upload rio-v2 test reports and diagnostics
@@ -555,14 +614,14 @@ jobs:
           name: junit-test-and-lint-rio-v2-${{ github.run_number }}-${{ github.run_attempt }}
           path: |
             target/nextest/ci/junit.xml
-            artifacts/test-and-lint
+            artifacts/workspace
           retention-days: 3
           if-no-files-found: error
 
   test-and-lint-protocols:
     name: "Test and Lint (${{ matrix.features.name }})"
     # Same cadence as test-and-lint-rio-v2 above.
-    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["merge_group", "schedule", "workflow_dispatch"]'), github.event_name)
+    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
     needs: [ quick-checks, classify-changes ]
     runs-on: sm-standard-4
     timeout-minutes: 90
@@ -631,7 +690,7 @@ jobs:
           path: |
             target/nextest/ci/junit.xml
             artifacts/protocol-tests
-            artifacts/test-and-lint
+            artifacts/workspace
           retention-days: 3
           if-no-files-found: warn
 
@@ -783,7 +842,7 @@ jobs:
     # job had neither, so each closed/merged PR really ran the whole io_uring
     # suite (measured 4m17s / 7m19s / 7m31s on runs 30678272341 / 30678117601 /
     # 30662728539) and kept the cancellation run in progress for minutes.
-    if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
+    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
     needs: [ quick-checks, classify-changes ]
     # GitHub-hosted ubuntu-latest runs a recent kernel with io_uring and, unlike
     # a container, applies no seccomp filter that would block io_uring_setup — so
@@ -978,7 +1037,7 @@ jobs:
           path: |
             ${{ runner.temp }}/rustfs-e2e-smoke-logs/
             ${{ runner.temp }}/rustfs-e2e-smoke-list.json
-            artifacts/test-and-lint
+            artifacts/workspace
           if-no-files-found: warn
 
       - name: Upload e2e smoke JUnit report
@@ -1389,12 +1448,13 @@ jobs:
       - classify-changes
       - typos
       - quick-checks
-      - test-and-lint
+      - workspace-lint-and-build
       - workspace-tests
       - workspace-shards
       - workspace-shard-evidence
       - core-contract
       - connect-feature-boundaries
+      - fips-provider
       - test-and-lint-rio-v2
       - test-and-lint-protocols
       - build-rustfs-debug-binary
@@ -1422,10 +1482,11 @@ jobs:
     needs:
       - classify-changes
       - connect-feature-boundaries
+      - fips-provider
       - required-checks
       - typos
       - quick-checks
-      - test-and-lint
+      - workspace-lint-and-build
       - workspace-tests
       - core-contract
       - test-and-lint-rio-v2

.github/workflows/docker.yml

@@ -329,13 +329,75 @@ jobs:
           echo "  - Create latest: $create_latest"
           echo "  - Source ref: $source_ref"
 
+  # Consume the trusted release certification for the exact version this
+  # workflow wraps and verify the certified input digests. A manual dispatch
+  # without that certification fails here and pushes no official tag; only
+  # results from the trusted verification sources carry release authority.
+  release-certification:
+    name: Release Certification
+    needs: build-check
+    if: needs.build-check.outputs.should_build == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      actions: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+          ref: ${{ needs.build-check.outputs.source_ref }}
+
+      - name: Download and verify release certification
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
+          TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
+          VERSION: ${{ needs.build-check.outputs.version }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p certification
+          run_id="$TRIGGERING_RUN_ID"
+          if [[ -z "$run_id" ]]; then
+            run_id="$(gh api --method GET \
+              "repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
+              -f branch="$VERSION" -f status=success -F per_page=1 \
+              --jq '.workflow_runs[0].id // empty' || true)"
+          fi
+          if [[ -z "$run_id" ]]; then
+            echo "No certified Build and Release run found for ${VERSION}; failing closed" >&2
+            exit 1
+          fi
+          if ! gh run download "$run_id" -n release-certification -D certification; then
+            echo "Build run ${run_id} published no release-certification artifact; failing closed" >&2
+            exit 1
+          fi
+          python3 scripts/release/release_gate.py verify-promotion \
+            --plan .config/release-validation-plan.json \
+            --candidate certification/candidate.json \
+            --summary certification/release-gate-output \
+            --artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
+            --artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
+
+          # Input digest binding: every zip this image wraps must be the
+          # certified byte-for-byte content, never a mutable latest pointer.
+          while IFS=$'\t' read -r name location expected; do
+            echo "Verifying certified input ${name}"
+            actual="$(curl -fsSL "$location" | sha256sum | awk '{print $1}')"
+            if [[ "$actual" != "$expected" ]]; then
+              echo "Certified input ${name} does not match its recorded digest" >&2
+              exit 1
+            fi
+          done < <(jq -r '.deliverables[] | select(.name | contains("linux")) | [.name, .location, .sha256] | @tsv' certification/candidate.json)
+
   # Build multi-arch Docker images
   # Strategy: Build images using pre-built binaries from dl.rustfs.com
   # Supports both release and dev channel binaries based on build context
   # Only runs when should_build is true (which includes workflow success check)
   build-docker:
     name: Build Docker Images
-    needs: build-check
+    needs: [ build-check, release-certification ]
     if: needs.build-check.outputs.should_build == 'true'
     runs-on: ubuntu-latest
     timeout-minutes: 60
@@ -443,6 +505,14 @@ jobs:
           # Output tags
           echo "tags=$TAGS" >> "$GITHUB_OUTPUT"
 
+          # Staging identity for this run's promotion: official tags are only
+          # created after the staged image verifies against the certification,
+          # and they point at this same digest (no rebuild-after-verify).
+          STAGING_TAG="staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${VARIANT_SUFFIX}"
+          STAGING_TAGS="${{ env.REGISTRY_DOCKERHUB }}:${STAGING_TAG},${{ env.REGISTRY_GHCR }}:${STAGING_TAG},${{ env.REGISTRY_QUAY }}:${STAGING_TAG}"
+          echo "staging_tags=$STAGING_TAGS" >> "$GITHUB_OUTPUT"
+          echo "staging_tag=$STAGING_TAG" >> "$GITHUB_OUTPUT"
+
           # Generate labels
           LABELS="org.opencontainers.image.title=RustFS"
           LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system"
@@ -467,7 +537,9 @@ jobs:
           file: ${{ matrix.file }}
           platforms: ${{ env.DOCKER_PLATFORMS }}
           push: ${{ needs.build-check.outputs.should_push == 'true' }}
-          tags: ${{ steps.meta.outputs.tags }}
+          # Push to staging tags only: official tags are created by the
+          # certification-gated promotion step below from this same digest.
+          tags: ${{ steps.meta.outputs.staging_tags }}
           labels: ${{ steps.meta.outputs.labels }}
           # No layer cache. This build compiles nothing — it downloads a
           # release zip and runs apk/apt — so the cache could only save the
@@ -493,6 +565,114 @@ jobs:
           no-cache: false
           pull: true
 
+      # A later-built wrapper: the staged image must complete this platform's
+      # own binary identity verification before any official tag is written.
+      # It may not inherit an unverified package's conclusion.
+      - name: Verify staged image against certification
+        if: needs.build-check.outputs.should_push == 'true'
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
+          TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
+          VERSION: ${{ needs.build-check.outputs.version }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p certification
+          run_id="$TRIGGERING_RUN_ID"
+          if [[ -z "$run_id" ]]; then
+            run_id="$(gh api --method GET \
+              "repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
+              -f branch="$VERSION" -f status=success -F per_page=1 \
+              --jq '.workflow_runs[0].id // empty' || true)"
+          fi
+          gh run download "$run_id" -n release-certification -D certification
+
+          STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
+          case "${{ matrix.variant }}" in
+            musl) libc="musl" ;;
+            glibc) libc="gnu" ;;
+          esac
+          for platform in linux/amd64 linux/arm64; do
+            case "$platform" in
+              linux/amd64) arch_sub="x86_64" ;;
+              linux/arm64) arch_sub="aarch64" ;;
+            esac
+            docker pull --platform "$platform" "$STAGING_REF"
+            container="$(docker create --platform "$platform" "$STAGING_REF")"
+            docker cp "$container:/usr/bin/rustfs" "$RUNNER_TEMP/staged-rustfs"
+            docker rm "$container" >/dev/null
+            actual="$(sha256sum "$RUNNER_TEMP/staged-rustfs" | awk '{print $1}')"
+            expected="$(jq -r --arg needle "linux-${arch_sub}-${libc}" \
+              '[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
+              certification/candidate.json)"
+            if [[ -z "$expected" || "$actual" != "$expected" ]]; then
+              echo "Staged image binary for ${platform} does not match the certified binary digest" >&2
+              exit 1
+            fi
+            python3 scripts/release/release_gate.py verify-promotion \
+              --plan .config/release-validation-plan.json \
+              --candidate certification/candidate.json \
+              --summary certification/release-gate-output \
+              --artifact "$RUNNER_TEMP/staged-rustfs" \
+              --artifact-role binary
+          done
+
+      - name: Promote staged image to official tags
+        if: needs.build-check.outputs.should_push == 'true'
+        shell: bash
+        run: |
+          set -euo pipefail
+          STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
+          IFS=',' read -r -a official_tags <<< "${{ steps.meta.outputs.tags }}"
+          for tag in "${official_tags[@]}"; do
+            docker buildx imagetools create -t "$tag" "$STAGING_REF"
+            echo "Promoted $tag -> $STAGING_REF"
+          done
+
+      # Downstream consumers (Helm) verify this record instead of inheriting
+      # an unverified wrapper's conclusion.
+      - name: Record image promotion
+        if: needs.build-check.outputs.should_push == 'true'
+        shell: bash
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
+          TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
+          VERSION: ${{ needs.build-check.outputs.version }}
+        run: |
+          set -euo pipefail
+          mkdir -p certification
+          run_id="$TRIGGERING_RUN_ID"
+          if [[ -z "$run_id" ]]; then
+            run_id="$(gh api --method GET \
+              "repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
+              -f branch="$VERSION" -f status=success -F per_page=1 \
+              --jq '.workflow_runs[0].id // empty' || true)"
+          fi
+          gh run download "$run_id" -n release-certification -D certification
+          STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
+          digest="$(docker buildx imagetools inspect "$STAGING_REF" --format '{{json .Manifest}}' | jq -r '.digest')"
+          candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
+          jq -n \
+            --arg version "$VERSION" \
+            --arg digest "$digest" \
+            --arg staging_ref "$STAGING_REF" \
+            --arg candidate_sha256 "$candidate_sha" \
+            --arg tags "${{ steps.meta.outputs.tags }}" \
+            --slurpfile candidate certification/candidate.json \
+            '{schema_version: 1, version: $version, image_digest: $digest, staging_ref: $staging_ref,
+              promoted_tags: ($tags | split(",")), candidate_sha256: $candidate_sha256,
+              binary_sha256s: ($candidate[0].deliverables | map(.binary_sha256) | unique)}' \
+            > image-promotion-record.json
+
+      - name: Upload image promotion record
+        if: needs.build-check.outputs.should_push == 'true'
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: image-promotion-record${{ matrix.suffix }}
+          path: image-promotion-record.json
+          if-no-files-found: error
+          retention-days: 30
+
   # Note: Manifest creation is no longer needed as we only build one variant
   # Multi-arch manifests are automatically created by docker/build-push-action
 

.github/workflows/e2e-distributed.yml

@@ -17,8 +17,8 @@
 # Each selected test starts a real localhost cluster via
 # `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
 # case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
-# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
-# nightly runs, and manual dispatches all execute the same fail-closed suite.
+# `[profile.e2e-distributed]` in `.config/nextest.toml`. The nightly schedule
+# and manual dispatches execute the same fail-closed suite.
 # Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
 #
 # Isolated pool filesystems: expand/decommission/rebalance cases require
@@ -33,32 +33,6 @@
 name: e2e-distributed
 
 on:
-  pull_request:
-    paths:
-      - "Cargo.lock"
-      - "Cargo.toml"
-      - ".config/nextest.toml"
-      - ".config/e2e-distributed-selection.txt"
-      - ".github/workflows/e2e-distributed.yml"
-      - "crates/audit/**"
-      - "crates/common/**"
-      - "crates/config/**"
-      - "crates/e2e_test/**"
-      - "crates/ecstore/**"
-      - "crates/filemeta/**"
-      - "crates/heal/**"
-      - "crates/iam/**"
-      - "crates/lock/**"
-      - "crates/madmin/**"
-      - "crates/notify/**"
-      - "crates/replication/**"
-      - "crates/s3-client/**"
-      - "crates/s3-ops/**"
-      - "crates/s3-types/**"
-      - "crates/scanner/**"
-      - "crates/storage-api/**"
-      - "crates/utils/**"
-      - "rustfs/**"
   workflow_dispatch:
     inputs:
       filter:

.github/workflows/e2e-upgrade.yml

@@ -12,30 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Single-node upgrade and rollback lane.
+#
+# The weekly Monday schedule, version tags, and workflow_dispatch run this
+# suite. Distributed upgrade cases stay on the nightly e2e-distributed lane.
+
 name: Upgrade Compatibility
 
 on:
-  pull_request:
-    paths:
-      - ".github/workflows/e2e-upgrade.yml"
-      - "crates/e2e_test/src/common.rs"
-      - "crates/e2e_test/src/distributed/**"
-      - "crates/e2e_test/src/fake_s3_target/**"
-      - "crates/e2e_test/src/lib.rs"
-      - "crates/e2e_test/src/replication_extension_test.rs"
-      - "crates/e2e_test/src/rpc_fault_proxy.rs"
-      - "crates/e2e_test/src/upgrade_compatibility_test.rs"
-      - "crates/ecstore/**"
-      - "crates/filemeta/**"
-      - "crates/heal/**"
-      - "crates/kms/**"
-      - "crates/storage-api/**"
-      - "rustfs/**"
-      - "Cargo.lock"
   push:
     tags:
       - "[0-9]*.[0-9]*.[0-9]*"
   schedule:
+    # 03:17 UTC every Monday.
     - cron: "17 3 * * 1"
   workflow_dispatch:
 
@@ -44,7 +33,7 @@ permissions:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+  cancel-in-progress: false
 
 env:
   CARGO_TERM_COLOR: always

.github/workflows/helm-package.yml

@@ -30,9 +30,102 @@ permissions:
   contents: read
 
 jobs:
+  # Consume the same trusted release certification as every other publication
+  # exit and verify the image promotion record of the Docker workflow. The
+  # chart wraps no binary of its own, so it may only reference an image whose
+  # platform verification already passed; it must not inherit an unverified
+  # package's conclusion. A run without that evidence fails closed here and
+  # never reaches the chart repository.
+  release-certification:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      actions: read
+    outputs:
+      version: ${{ steps.version.outputs.raw_tag }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Normalize release version
+        id: version
+        env:
+          RAW_INPUT: ${{ github.event.inputs.version }}
+          RAW_BRANCH: ${{ github.event.workflow_run.head_branch }}
+        run: |
+          set -euo pipefail
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            RAW="$RAW_INPUT"
+          else
+            RAW="$RAW_BRANCH"
+          fi
+          case "$RAW" in
+            refs/tags/*) RAW_TAG="${RAW#refs/tags/}" ;;
+            *) RAW_TAG="$RAW" ;;
+          esac
+          echo "raw_tag=$RAW_TAG" >> "$GITHUB_OUTPUT"
+
+      - name: Download and verify release certification and image record
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
+          TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
+          RAW_TAG: ${{ steps.version.outputs.raw_tag }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p certification image-record
+
+          build_run="$TRIGGERING_RUN_ID"
+          if [[ -z "$build_run" ]]; then
+            build_run="$(gh api --method GET \
+              "repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
+              -f branch="$RAW_TAG" -f status=success -F per_page=1 \
+              --jq '.workflow_runs[0].id // empty' || true)"
+          fi
+          if [[ -z "$build_run" ]]; then
+            echo "No certified Build and Release run found for ${RAW_TAG}; failing closed" >&2
+            exit 1
+          fi
+          if ! gh run download "$build_run" -n release-certification -D certification; then
+            echo "Build run ${build_run} published no release-certification artifact; failing closed" >&2
+            exit 1
+          fi
+          python3 scripts/release/release_gate.py verify-promotion \
+            --plan .config/release-validation-plan.json \
+            --candidate certification/candidate.json \
+            --summary certification/release-gate-output \
+            --artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
+            --artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
+
+          docker_run="$(gh api --method GET \
+            "repos/${GITHUB_REPOSITORY}/actions/workflows/docker.yml/runs" \
+            -f branch="$RAW_TAG" -f status=success -F per_page=1 \
+            --jq '.workflow_runs[0].id // empty' || true)"
+          if [[ -z "$docker_run" ]] || ! gh run download "$docker_run" -n image-promotion-record -D image-record; then
+            echo "No image promotion record for ${RAW_TAG}; a chart may not wrap an unverified image" >&2
+            exit 1
+          fi
+          candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
+          record_sha="$(jq -r '.candidate_sha256 // empty' image-record/image-promotion-record.json)"
+          if [[ "$record_sha" != "$candidate_sha" ]]; then
+            echo "Image promotion record does not belong to this certification" >&2
+            exit 1
+          fi
+          overlap="$(jq -r --slurpfile candidate certification/candidate.json \
+            '[.binary_sha256s[]] - ($candidate[0].deliverables | map(.binary_sha256)) | length' \
+            image-record/image-promotion-record.json)"
+          if [[ "$overlap" -ne 0 ]]; then
+            echo "Image promotion record carries binary digests outside the certification" >&2
+            exit 1
+          fi
+
   build-helm-package:
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    needs: [ release-certification ]
     if: |
       (github.event_name == 'workflow_dispatch' && !contains(github.event.inputs.version, '-preview')) ||
       (

.github/workflows/nightly-gnu.yml

@@ -87,18 +87,206 @@ jobs:
           install-build-packaging-tools: 'false'
           install-test-tools: 'false'
 
+      - name: Download static console assets
+        shell: bash
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          # Packaging tools are off on this lane, so setup does not install unzip.
+          if ! command -v unzip >/dev/null 2>&1; then
+            sudo apt-get update -qq
+            sudo apt-get install -y -qq unzip
+          fi
+          if ! command -v unzip >/dev/null 2>&1; then
+            echo "unzip is required to extract console assets" >&2
+            exit 1
+          fi
+
+          mkdir -p ./rustfs/static
+
+          verify_sha256() {
+            local expected="$1"
+            local file="$2"
+            local actual=""
+            local actual_line=""
+
+            if command -v sha256sum >/dev/null 2>&1; then
+              actual_line=$(sha256sum -- "$file") || return 1
+            elif command -v shasum >/dev/null 2>&1; then
+              actual_line=$(shasum -a 256 -- "$file") || return 1
+            else
+              echo "No SHA-256 verification tool found" >&2
+              return 1
+            fi
+            actual="${actual_line%%[[:space:]]*}"
+
+            if [ "$actual" = "$expected" ]; then
+              echo "SHA256 verified OK: $actual"
+              return 0
+            else
+              echo "SHA256 mismatch: expected=$expected actual=$actual" >&2
+              return 1
+            fi
+          }
+
+          download_console_assets() {
+            local curl_bin="curl"
+            local python_bin="python3"
+            local console_api="https://api.github.com/repos/rustfs/console/releases/latest"
+            local console_json="console-release.json"
+            local console_url
+            local console_sha256
+            local curl_auth_args=()
+            console_tag=""
+
+            chmod +w ./rustfs/static/LICENSE || true
+
+            if ! command -v "$python_bin" >/dev/null 2>&1; then
+              python_bin="python"
+            fi
+            if ! command -v "$python_bin" >/dev/null 2>&1; then
+              echo "No Python interpreter found for release metadata parsing" >&2
+              return 1
+            fi
+
+            if [[ -n "${GITHUB_TOKEN:-}" ]]; then
+              curl_auth_args=(-H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28")
+            fi
+
+            "$curl_bin" "${curl_auth_args[@]}" --fail -L "$console_api" \
+              -o "$console_json" --retry 3 --retry-delay 5 --max-time 300 || return 1
+
+            read -r console_tag console_url console_sha256 < <("$python_bin" - "$console_json" <<'PY'
+          import json
+          import re
+          import sys
+
+          with open(sys.argv[1], encoding="utf-8") as handle:
+              release = json.load(handle)
+
+          tag = release.get("tag_name", "") or "unknown"
+
+          for asset in release.get("assets", []):
+              name = asset.get("name", "")
+              digest = asset.get("digest", "")
+              url = asset.get("browser_download_url", "")
+              if name.startswith("rustfs-console-") and name.endswith(".zip") and digest.startswith("sha256:") and url:
+                  sha256 = digest.split(":", 1)[1]
+                  if not re.fullmatch(r"[0-9a-fA-F]{64}", sha256):
+                      raise SystemExit(f"console zip asset has invalid sha256 digest: {sha256}")
+                  sys.stdout.buffer.write(f"{tag} {url} {sha256}\n".encode("utf-8"))
+                  break
+          else:
+              raise SystemExit("no console zip asset with sha256 digest found")
+          PY
+            ) || return 1
+
+            if [[ -z "$console_url" || -z "$console_sha256" ]]; then
+              echo "Console release metadata is missing URL or SHA-256 digest" >&2
+              return 1
+            fi
+
+            echo "Console release: ${console_tag}"
+            echo "Downloading console asset: ${console_url}"
+            "$curl_bin" --fail -L "$console_url" -o console.zip --retry 3 --retry-delay 5 --max-time 300 || return 1
+            verify_sha256 "$console_sha256" console.zip || return 2
+            unzip -o console.zip -d ./rustfs/static || return 2
+          }
+
+          if download_console_assets; then
+            rm -f console.zip console-release.json
+          else
+            status=$?
+            rm -f console.zip console-release.json
+            if [[ "$status" -eq 2 ]]; then
+              echo "Console asset integrity verification failed" >&2
+            fi
+            echo "Failed to download verified console assets" >&2
+            exit 1
+          fi
+
+          if [[ ! -s ./rustfs/static/index.html ]]; then
+            echo "Console asset archive is missing static/index.html" >&2
+            exit 1
+          fi
+
+          asset_count=$(find ./rustfs/static -type f | wc -l | tr -d '[:space:]')
+          echo "Console assets ready: version=${console_tag:-unknown}, ${asset_count} files extracted to ./rustfs/static"
+
       - name: Build RustFS
         # Pin the glibc baseline (`<target>.<glibc>` for cargo-zigbuild) so
         # nightlies run on older distros such as RHEL/Rocky 9 (glibc 2.34); #8062.
-        run: cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bins
+        shell: bash
+        run: |
+          # Force rebuild by touching build.rs so a restored cache cannot
+          # reuse a rustfs artifact compiled before these console assets existed.
+          touch rustfs/build.rs
+          cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bin rustfs
 
       - name: Verify glibc symbol baseline
         run: |
           python3 scripts/release/check_glibc.py 2.31 \
-            target/x86_64-unknown-linux-gnu/release/rustfs \
-            target/x86_64-unknown-linux-gnu/release/rustfs-cli
+            target/x86_64-unknown-linux-gnu/release/rustfs
           target/x86_64-unknown-linux-gnu/release/rustfs --version | grep -Fx 'build target : x86_64-unknown-linux-gnu'
 
+      - name: Verify console
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          find_free_port() {
+            python3 - <<'PY'
+          import socket
+
+          with socket.socket() as sock:
+              sock.bind(("127.0.0.1", 0))
+              print(sock.getsockname()[1])
+          PY
+          }
+
+          binary="target/x86_64-unknown-linux-gnu/release/rustfs"
+          data_dir="$(mktemp -d)"
+          log_file="${RUNNER_TEMP}/rustfs-console-smoke.log"
+          server_pid=""
+          trap '
+            if [[ -n "${server_pid:-}" ]]; then
+              kill "$server_pid" 2>/dev/null || true
+              wait "$server_pid" 2>/dev/null || true
+            fi
+            rm -rf "$data_dir"
+          ' EXIT
+
+          if [[ ! -x "$binary" ]]; then
+            echo "rustfs binary not found: ${binary}" >&2
+            exit 1
+          fi
+
+          api_port="$(find_free_port)"
+          console_port="$(find_free_port)"
+          console_url="http://127.0.0.1:${console_port}/rustfs/console/"
+
+          "$binary" server \
+            --address "127.0.0.1:${api_port}" \
+            --console-enable \
+            --console-address "127.0.0.1:${console_port}" \
+            --access-key console-smoke \
+            --secret-key console-smoke-secret \
+            "$data_dir" >"$log_file" 2>&1 &
+          server_pid=$!
+
+          response=""
+          for _ in {1..40}; do
+            response="$(curl --silent --output /dev/null --write-out '%{http_code} %{content_type}' "$console_url" || true)"
+            if [[ "$response" == 200\ text/html* ]]; then
+              exit 0
+            fi
+            sleep 0.25
+          done
+
+          echo "Console endpoint did not return 200 text/html: $response" >&2
+          cat "$log_file"
+          exit 1
+
       - name: Build DEB package
         id: deb
         shell: bash
@@ -346,12 +534,12 @@ jobs:
           path: ${{ steps.publish.outputs.candidate_file }}
           if-no-files-found: error
 
-      # Publish the deb/rpm pair to the auto-testing repo's `assets` branch so
-      # engineers can download and install the nightly directly. The branch is
-      # a single-commit orphan rewritten on every build, which keeps the repo
-      # small while the latest files stay reachable at stable raw URLs.
       # Publish the deb/rpm pair as assets of the rolling `nightly` release on
       # rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
+      # The dispatch below runs only after this upload and the immutable R2
+      # candidate both succeed. It does not replace rustfs-functional-chain.yml,
+      # which still starts from this workflow's workflow_run completion.
+      # kms-vault-lane and kms-vault-ha-failover stay parallel with this job.
       - name: Publish packages to auto-testing release assets
         env:
           ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
@@ -360,6 +548,90 @@ jobs:
           DEB_DATE: ${{ steps.deb.outputs.deb_date }}
           BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
         run: bash scripts/release/publish_nightly_assets.sh
+
+      # Cross-repo repository_dispatch cannot use GITHUB_TOKEN. The secret is a
+      # PAT or GitHub App token for rustfs/auto-testing: classic `repo` scope,
+      # or fine-grained Contents write (the permission GitHub checks for
+      # POST /repos/{owner}/{repo}/dispatches). A dispatch failure fails this
+      # step, and therefore this job, so the scheduled failure alert fires.
+      # GitHub allows at most 10 top-level client_payload properties, so the
+      # schema 2 candidate object is nested under `candidate` and the release
+      # fields sit beside it.
+      - name: Dispatch nightly candidate to auto-testing
+        if: ${{ steps.publish.outputs.candidate_file != '' }}
+        env:
+          DISPATCH_TOKEN: ${{ secrets.AUTO_TESTING_DISPATCH_TOKEN }}
+          CANDIDATE_FILE: ${{ steps.publish.outputs.candidate_file }}
+          DEB_DATE: ${{ steps.deb.outputs.deb_date }}
+          SOURCE_REPO: ${{ github.repository }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          : "${DISPATCH_TOKEN:?AUTO_TESTING_DISPATCH_TOKEN is required to dispatch rustfs/auto-testing}"
+          : "${CANDIDATE_FILE:?nightly candidate manifest is required}"
+          : "${DEB_DATE:?DEB_DATE is required}"
+          [[ "${DEB_DATE}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]
+          [[ "${SOURCE_REPO}" == "rustfs/rustfs" ]]
+
+          payload="${RUNNER_TEMP}/nightly-candidate-dispatch-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
+          jq -n --slurpfile candidate "${CANDIDATE_FILE}" \
+            --arg event_type "nightly-candidate-ready" \
+            --arg source_repo "${SOURCE_REPO}" \
+            --arg latest_alias_url "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb" \
+            --arg auto_testing_release "https://github.com/rustfs/auto-testing/releases/tag/nightly" \
+            --arg channel "nightly" \
+            --arg deb_date "${DEB_DATE}" \
+            --arg nightly_workflow ".github/workflows/nightly-gnu.yml" \
+            '{
+              event_type: $event_type,
+              source_repo: $source_repo,
+              latest_alias_url: $latest_alias_url,
+              auto_testing_release: $auto_testing_release,
+              channel: $channel,
+              deb_date: $deb_date,
+              nightly_workflow: $nightly_workflow,
+              candidate: $candidate[0]
+            }' > "${payload}"
+          if ! jq -e '
+            .event_type == "nightly-candidate-ready"
+            and .source_repo == "rustfs/rustfs"
+            and .channel == "nightly"
+            and .deb_date == env.DEB_DATE
+            and .nightly_workflow == ".github/workflows/nightly-gnu.yml"
+            and .latest_alias_url == "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb"
+            and .auto_testing_release == "https://github.com/rustfs/auto-testing/releases/tag/nightly"
+            and .candidate.schema == 2
+            and ((.candidate | keys | sort) == ["build_run_attempt","build_run_id","package_sha256","package_url","schema","source_ref","source_sha","workflow_sha"])
+            and ((keys | sort) == ["auto_testing_release","candidate","channel","deb_date","event_type","latest_alias_url","nightly_workflow","source_repo"])
+          ' "${payload}" >/dev/null; then
+            echo "nightly dispatch payload does not match the nightly-candidate-ready contract" >&2
+            exit 1
+          fi
+
+          body="${RUNNER_TEMP}/nightly-candidate-dispatch-body-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
+          jq -n --slurpfile client_payload "${payload}" \
+            '{event_type: "nightly-candidate-ready", client_payload: $client_payload[0]}' > "${body}"
+          response="${RUNNER_TEMP}/nightly-candidate-dispatch-response-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.txt"
+          status="$(curl -sS --retry 3 --connect-timeout 15 --max-time 60 \
+            -o "${response}" -w '%{http_code}' \
+            -X POST \
+            -H "Authorization: Bearer ${DISPATCH_TOKEN}" \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            -H "Content-Type: application/json" \
+            --data-binary @"${body}" \
+            https://api.github.com/repos/rustfs/auto-testing/dispatches)" || {
+            echo "repository_dispatch request to rustfs/auto-testing failed before an HTTP status" >&2
+            exit 1
+          }
+          if [[ "${status}" != "204" ]]; then
+            echo "repository_dispatch to rustfs/auto-testing failed: HTTP ${status}" >&2
+            if [[ -s "${response}" ]]; then
+              cat "${response}" >&2
+            fi
+            exit 1
+          fi
+          echo "dispatched nightly-candidate-ready to rustfs/auto-testing (deb_date=${DEB_DATE}, run=${GITHUB_RUN_ID}, attempt=${GITHUB_RUN_ATTEMPT})"
   # Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
   #
   # RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and

.github/workflows/package.yml

@@ -225,10 +225,57 @@ jobs:
           echo "  Build run ID: $BUILD_RUN_ID"
           echo "  Build run number: $RUN_NUMBER"
 
+  # Consume the trusted release certification of the resolved build run and
+  # verify the certified input digests before any package is wrapped. A run
+  # without that certification fails closed here; packages built later must
+  # complete their own verification and cannot inherit an unverified
+  # conclusion.
+  release-certification:
+    name: Release Certification
+    needs: resolve
+    if: needs.resolve.result == 'success'
+    runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      actions: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Download and verify release certification
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
+          BUILD_RUN_ID: ${{ needs.resolve.outputs.build_run_id }}
+          BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p certification
+          if [[ "$BUILD_TYPE" == "development" ]]; then
+            # Development artifacts are explicitly uncertified and never touch
+            # official or latest channels; only the staging/dev paths remain.
+            jq -n '{schema_version: 1, certified: false, reason: "development artifacts are explicitly uncertified"}' \
+              > certification/development-uncertified.json
+            exit 0
+          fi
+          if ! gh run download "$BUILD_RUN_ID" -n release-certification -D certification; then
+            echo "Build run ${BUILD_RUN_ID} published no release-certification artifact; failing closed" >&2
+            exit 1
+          fi
+          python3 scripts/release/release_gate.py verify-promotion \
+            --plan .config/release-validation-plan.json \
+            --candidate certification/candidate.json \
+            --summary certification/release-gate-output \
+            --artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
+            --artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
+
   # Build DEB and RPM packages for each architecture and libc variant
   package:
     name: Package (${{ matrix.arch }}-${{ matrix.libc }})
-    needs: resolve
+    needs: [ resolve, release-certification ]
     runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
     timeout-minutes: 30
     strategy:
@@ -494,6 +541,55 @@ jobs:
           echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
           echo "✅ RPM built: $RPM_FILE"
 
+      - name: Download certification evidence
+        if: needs.resolve.outputs.build_type != 'development'
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: release-certification
+          path: ./certification
+
+      # Later-built wrapper verification: the binary inside each package must
+      # match the certified binary digest for this platform before anything is
+      # uploaded, and the package itself stays on staging until promotion.
+      - name: Verify wrapped binary against certification
+        if: needs.resolve.outputs.build_type != 'development'
+        shell: bash
+        env:
+          DEB_FILE: ${{ steps.deb.outputs.deb_file }}
+          RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
+          ARTIFACT_NAME: ${{ matrix.artifact_name }}
+        run: |
+          set -euo pipefail
+          expected="$(jq -r --arg needle "$ARTIFACT_NAME" \
+            '[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
+            certification/candidate.json)"
+          if [[ -z "$expected" ]]; then
+            echo "No certified binary digest recorded for ${ARTIFACT_NAME}" >&2
+            exit 1
+          fi
+          if ! command -v rpm2cpio >/dev/null 2>&1; then
+            sudo apt-get update && sudo apt-get install -y rpm2cpio
+          fi
+          if ! command -v cpio >/dev/null 2>&1; then
+            sudo apt-get update && sudo apt-get install -y cpio
+          fi
+          for package_file in "$DEB_FILE" "$RPM_FILE"; do
+            case "$package_file" in
+              *.deb)
+                dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
+                ;;
+              *.rpm)
+                rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
+                ;;
+            esac
+            python3 scripts/release/release_gate.py verify-promotion \
+              --plan .config/release-validation-plan.json \
+              --candidate certification/candidate.json \
+              --summary certification/release-gate-output \
+              --artifact "$RUNNER_TEMP/wrapped-rustfs" \
+              --artifact-role binary
+          done
+
       - name: Upload packages to artifacts
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
@@ -531,10 +627,14 @@ jobs:
           export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
           export AWS_DEFAULT_REGION="auto"
 
+          # Build-phase writes stay on staging for release types: the official
+          # packages/release and packages/latest paths are written only by the
+          # certification-gated promote-packages job, from these same staged
+          # objects. Development packages keep the uncertified dev channel.
           if [[ "$BUILD_TYPE" == "development" ]]; then
             R2_PREFIX="artifacts/rustfs/packages/dev"
           else
-            R2_PREFIX="artifacts/rustfs/packages/release"
+            R2_PREFIX="artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}"
           fi
           R2_PATH="s3://${R2_BUCKET}/${R2_PREFIX}/"
 
@@ -549,18 +649,6 @@ jobs:
 
           echo "✅ Upload complete"
 
-          # Also upload as latest for release/prerelease
-          if [[ "$BUILD_TYPE" == "release" || "$BUILD_TYPE" == "prerelease" ]]; then
-            LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
-            for f in "$DEB_FILE" "$RPM_FILE"; do
-              if [[ -n "$f" && -f "$f" ]]; then
-                echo "Uploading latest: $(basename "$f")"
-                aws s3 cp "$f" "$LATEST_PATH" --endpoint-url "$R2_ENDPOINT" --only-show-errors
-              fi
-            done
-            echo "✅ Latest packages updated"
-          fi
-
       - name: Upload packages to GitHub Release
         if: needs.resolve.outputs.tag != ''
         env:
@@ -614,6 +702,114 @@ jobs:
 
           echo "✅ GitHub release assets updated"
 
+  # The only writer of the official packages/release and packages/latest R2
+  # paths. Promotion re-checks each package's wrapped binary against the
+  # passed certification list and copies the staged objects byte-for-byte:
+  # no rebuild-after-verify, no mutable latest resolution downstream.
+  promote-packages:
+    name: Promote Packages
+    needs: [ resolve, package, release-certification ]
+    # Target-channel lock for the official package paths.
+    concurrency:
+      group: release-channel-r2-packages
+      cancel-in-progress: false
+    if: needs.resolve.outputs.tag != '' && (needs.resolve.outputs.build_type == 'release' || needs.resolve.outputs.build_type == 'prerelease')
+    runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      actions: read
+    env:
+      R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
+      R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
+      R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
+      R2_BUCKET: ${{ secrets.R2_BUCKET }}
+      AWS_EC2_METADATA_DISABLED: true
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Download packages and certification
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          path: ./packages
+          pattern: packages-*
+          merge-multiple: true
+
+      - name: Download certification evidence
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: release-certification
+          path: ./certification
+
+      - name: Verify and promote staged packages
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
+            echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
+            exit 1
+          fi
+
+          STAGING_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
+          RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/release/"
+          LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
+
+          if ! command -v aws >/dev/null 2>&1; then
+            sudo apt-get update && sudo apt-get install -y awscli
+          fi
+          export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
+          export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
+          export AWS_DEFAULT_REGION="auto"
+
+          if ! command -v rpm2cpio >/dev/null 2>&1; then
+            sudo apt-get install -y rpm2cpio
+          fi
+          if ! command -v cpio >/dev/null 2>&1; then
+            sudo apt-get install -y cpio
+          fi
+
+          promoted=0
+          for package_file in packages/*.deb packages/*.rpm; do
+            [[ -f "$package_file" ]] || continue
+            name="$(basename "$package_file")"
+            case "$package_file" in
+              *.deb)
+                dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
+                ;;
+              *.rpm)
+                rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
+                ;;
+            esac
+            python3 scripts/release/release_gate.py verify-promotion \
+              --plan .config/release-validation-plan.json \
+              --candidate certification/candidate.json \
+              --summary certification/release-gate-output \
+              --artifact "$RUNNER_TEMP/wrapped-rustfs" \
+              --artifact-role binary
+
+            staged="$(mktemp)"
+            aws s3 cp "${STAGING_PREFIX}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+            staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
+            built_sha="$(sha256sum "$package_file" | awk '{print $1}')"
+            if [[ "$staged_sha" != "$built_sha" ]]; then
+              echo "❌ Staged package ${name} does not match the verified bytes" >&2
+              exit 1
+            fi
+            aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+            aws s3 cp "$staged" "${LATEST_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+            rm -f "$staged"
+            promoted=$((promoted + 1))
+            echo "✅ Promoted ${name} (${built_sha})"
+          done
+          if [[ "$promoted" -eq 0 ]]; then
+            echo "No staged packages found to promote" >&2
+            exit 1
+          fi
+
   # Summary
   summary:
     name: Summary

.github/workflows/rustfs-validation-sweep.yml

@@ -0,0 +1,457 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: RustFS Validation Sweep
+
+# Executes the migrated canonical validation cases (rustfs/auto-testing#113)
+# on the shared functional fleet. Everything with a lane executor is swept in
+# rotating nightly batches so the whole set is covered roughly weekly; a
+# single case or one module can be dispatched manually. That is the
+# implemented catalog (66 today) plus the eight planned cases whose dedicated
+# executors already exist and have live PASS evidence - fresh nightly
+# evidence feeds their acceptance review. heal.* cases stay on the
+# interactive validation flow: they require reviewed native/tool collectors
+# built by separate validation-tools runs before fleet execution.
+
+on:
+  schedule:
+    # After the nightly functional chain settles (chain starts ~16:48 UTC and
+    # runs for about two hours).
+    - cron: "30 19 * * *"
+  workflow_dispatch:
+    inputs:
+      case:
+        description: 'Single canonical validation case id (overrides module and rotation), e.g. pool.decommission'
+        type: string
+        required: false
+        default: ''
+      module:
+        description: 'Restrict the sweep to one catalog module, e.g. bucket-replication'
+        type: string
+        required: false
+        default: ''
+      limit:
+        description: 'Maximum cases per sweep (0 = run everything selected)'
+        type: number
+        required: false
+        default: 12
+      build_run_id:
+        description: 'Successful nightly package run; empty resolves the latest'
+        type: string
+        required: false
+        default: ''
+      build_run_attempt:
+        description: 'Attempt of that package run'
+        type: string
+        required: false
+        default: '1'
+
+permissions:
+  contents: read
+
+concurrency:
+  group: rustfs-validation-sweep
+  cancel-in-progress: false
+
+env:
+  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES || 'vm000 vm001 vm002 vm003' }}
+  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER || 'azureuser' }}
+  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
+  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
+  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
+  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+jobs:
+  sweep:
+    if: github.repository == 'rustfs/rustfs'
+    runs-on: smoke-testing
+    timeout-minutes: 150
+    steps:
+      - name: Checkout chain tooling
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Resolve nightly candidate
+        id: candidate
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
+          CHAIN_OUTPUT: ${{ runner.temp }}/validation-sweep-candidate.json
+        run: |
+          set -euo pipefail
+          # Scheduled runs carry no inputs and the resolver only derives a
+          # build from a workflow_run payload (which a schedule cannot
+          # provide), so resolve the newest successful nightly build
+          # explicitly before invoking it.
+          if [ -z "${BUILD_RUN_ID:-}" ]; then
+            LATEST="$(gh api "repos/rustfs/rustfs/actions/workflows/nightly-gnu.yml/runs?branch=main&status=success&per_page=1" --jq '.workflow_runs[0]')"
+            BUILD_RUN_ID="$(printf '%s' "${LATEST}" | jq -r '.id')"
+            BUILD_RUN_ATTEMPT="$(printf '%s' "${LATEST}" | jq -r '.run_attempt')"
+            export BUILD_RUN_ID BUILD_RUN_ATTEMPT
+            echo "scheduled sweep pinned to nightly ${BUILD_RUN_ID} attempt ${BUILD_RUN_ATTEMPT}"
+          fi
+          python3 scripts/resolve_functional_candidate.py
+          python3 - <<'PY'
+          import json
+          import os
+          from pathlib import Path
+          root = Path(os.environ["RUNNER_TEMP"]) / "validation-sweep"
+          root.mkdir(parents=True, exist_ok=True)
+          chain = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
+          manifest = chain["candidate"]["manifest"]
+          (root / "candidate.json").write_text(json.dumps({
+              "package_url": manifest["package_url"],
+              "package_sha256": manifest["package_sha256"],
+              "source_sha": manifest["source_sha"],
+          }, sort_keys=True) + "\n")
+          with open(os.environ["GITHUB_OUTPUT"], "a") as output:
+              output.write("testing_sha=" + chain["testing_sha"] + "\n")
+              output.write("package_url=" + manifest["package_url"] + "\n")
+              output.write("source_sha=" + manifest["source_sha"] + "\n")
+          PY
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.candidate.outputs.testing_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
+
+      - name: Prepare validation Python runtime
+        # The pool / ilm / replication validation runners require a managed
+        # Python, exactly like the chain's validation job provides.
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.12"
+
+      - name: Install validation Python requirements
+        # Case scripts call python3 and import boto3 at startup. Install the
+        # functional chain's client into a per-run env so that interpreter
+        # sees it without changing the runner's shared Python.
+        run: |
+          set -euo pipefail
+          validation_python="${RUNNER_TEMP}/rustfs-validation-python-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          test ! -e "${validation_python}"
+          python3 -m venv "${validation_python}"
+          "${validation_python}/bin/python" -m pip install \
+            -r auto-testing/scripts/validation_ilm/requirements.txt
+          printf '%s\n' "${validation_python}/bin" >> "${GITHUB_PATH}"
+
+      - name: Select sweep cases
+        id: select
+        env:
+          SELECTED_CASE: ${{ inputs.case }}
+          SELECTED_MODULE: ${{ inputs.module }}
+          CASE_LIMIT: ${{ inputs.limit }}
+          SWEEP_EVENT: ${{ github.event_name }}
+          SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
+        run: |
+          set -euo pipefail
+          python3 - <<'PY'
+          import datetime
+          import json
+          import os
+          from pathlib import Path
+          sweep_root = Path(os.environ["SWEEP_ROOT"])
+
+          catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
+          def sweepable(case):
+              status = case.get("source_status")
+              if status == "implemented":
+                  return True
+              # The remaining planned non-heal cases route to the lane
+              # executors and already have live PASS evidence; sweeping them
+              # nightly produces fresh evidence for their acceptance review.
+              return status == "planned" and not case["id"].startswith("heal.")
+
+          selected = [(case["id"], module["id"]) for module in catalog["modules"]
+                      for case in module["cases"]
+                      if sweepable(case)]
+          if not selected:
+              raise SystemExit("catalog has no sweepable cases")
+
+          one_case = os.environ.get("SELECTED_CASE", "").strip()
+          module_filter = os.environ.get("SELECTED_MODULE", "").strip()
+          known = {case_id for case_id, _ in selected}
+          modules = {module_id for _, module_id in selected}
+          if one_case:
+              if one_case not in known:
+                  raise SystemExit(f"unknown case or no lane executor for it: {one_case}")
+              selected = [(one_case, dict(selected)[one_case])]
+          elif module_filter:
+              if module_filter not in modules:
+                  raise SystemExit(f"unknown module or no implemented cases: {module_filter}")
+              selected = [pair for pair in selected if pair[1] == module_filter]
+          elif os.environ.get("SWEEP_EVENT") == "schedule":
+              # Stateless rotation: every night take one window of the catalog
+              # so a full sweep needs len(selected)/limit nights and no state
+              # has to be carried between runs.
+              limit = max(1, int(os.environ.get("CASE_LIMIT") or 12))
+              day = datetime.datetime.now(datetime.timezone.utc).timetuple().tm_yday
+              start = (day * limit) % len(selected)
+              selected = (selected[start:] + selected[:start])[:limit]
+          limit = int(os.environ.get("CASE_LIMIT") or 0)
+          if one_case or module_filter or (not os.environ.get("SWEEP_EVENT") == "schedule"):
+              if limit > 0:
+                  selected = selected[:limit]
+
+          (sweep_root / "sweep-cases.txt").write_text("".join(f"{case_id}\n" for case_id, _ in selected))
+          with open(os.environ["GITHUB_OUTPUT"], "a") as output:
+              output.write(f"count={len(selected)}\n")
+          print(f"selected {len(selected)} case(s):", ", ".join(c for c, _ in selected))
+          PY
+
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
+        run: python3 scripts/functional_environment.py enter --suite heal
+
+      - name: Run validation cases
+        id: run_cases
+        timeout-minutes: 120
+        env:
+          AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
+          SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
+        run: |
+          set -uo pipefail
+          mkdir -p "${SWEEP_ROOT}/results"
+          : > "${SWEEP_ROOT}/results.txt"
+          while read -r CASE_ID; do
+            [ -n "${CASE_ID}" ] || continue
+            echo "::group::case ${CASE_ID}"
+            rm -rf "${SWEEP_ROOT}/run"
+            if ./auto-testing/scripts/run_validation_case.sh \
+                 "${CASE_ID}" "${SWEEP_ROOT}/candidate.json" \
+                 "${AUTO_TESTING_SHA}" "${SWEEP_ROOT}/run" \
+                 > "${SWEEP_ROOT}/results/${CASE_ID//\//_}.log" 2>&1; then
+              printf '%s\tPASS\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
+              echo "case ${CASE_ID}: PASS"
+            else
+              printf '%s\tFAIL\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
+              echo "case ${CASE_ID}: FAIL (log retained in the sweep artifact)"
+            fi
+            if [ -f "${SWEEP_ROOT}/run/report.md" ]; then
+              cp "${SWEEP_ROOT}/run/report.md" "${SWEEP_ROOT}/results/${CASE_ID//\//_}.report.md" 2>/dev/null || true
+            fi
+            echo "::endgroup::"
+            # Fleet hygiene between cases: the case runners set up their own
+            # topology, this purge guarantees the next case starts clean even
+            # when the previous one died mid-run.
+            for node in ${RUSTFS_NODES}; do
+              # -n keeps ssh from consuming the case list this loop reads from stdin
+              ssh -n -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
+                SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+                ${SUDO} systemctl stop rustfs 2>/dev/null || true
+                if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                  ${SUDO} dpkg -P rustfs
+                fi
+                for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+                ${SUDO} rm -f /etc/default/rustfs
+              ' || echo "inter-case cleanup failed on ${node} (continuing)"
+            done
+          done < "${SWEEP_ROOT}/sweep-cases.txt"
+          echo "sweep finished: $(wc -l < "${SWEEP_ROOT}/results.txt") case(s) executed"
+
+      - name: Generate sweep report
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        env:
+          SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
+          PACKAGE_URL: ${{ steps.candidate.outputs.package_url }}
+          SOURCE_SHA: ${{ steps.candidate.outputs.source_sha }}
+          AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
+        run: |
+          set -euo pipefail
+          python3 - <<'PY'
+          import json
+          import os
+          from pathlib import Path
+
+          root = Path(os.environ["SWEEP_ROOT"])
+          catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
+          module_of = {case["id"]: module["id"] for module in catalog["modules"] for case in module["cases"]}
+          rows = []
+          for line in (root / "results.txt").read_text().splitlines():
+            if not line:
+                continue
+            case_id, status = line.split("\t")
+            rows.append((case_id, module_of.get(case_id, "?"), status))
+          passed = sum(1 for _, _, s in rows if s == "PASS")
+          failed = len(rows) - passed
+          run_url = (f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/"
+                     f"{os.environ.get('GITHUB_REPOSITORY', 'rustfs/rustfs')}/actions/runs/"
+                     f"{os.environ.get('GITHUB_RUN_ID', '')}")
+
+          report = [
+              "# RustFS validation sweep report", "",
+              f"- Run: {run_url}",
+              f"- Attempt: {os.environ.get('GITHUB_RUN_ATTEMPT', '')}",
+              f"- Trigger: {os.environ.get('GITHUB_EVENT_NAME', '')}",
+              f"- Package: {os.environ.get('PACKAGE_URL', 'nightly (R2 latest)')}",
+              f"- Source SHA: {os.environ.get('SOURCE_SHA', '')}",
+              f"- auto-testing: {os.environ.get('AUTO_TESTING_SHA', '')}", "",
+              "Canonical validation cases (rustfs/auto-testing#113) executed by the sweep, grouped by catalog module.", "",
+          ]
+          # Group the executed cases by catalog module (the major category:
+          # pool lifecycle, heal, ILM expiration, bucket/site replication, kms,
+          # data services, identity, s3 compatibility). Per-module subtotals
+          # deliberately use "- Subtotal:" so the dashboard overview regexes
+          # (which anchor on "- Total:") keep reading the grand totals below.
+          by_module = {}
+          module_order = []
+          for case_id, module, status in rows:
+              if module not in by_module:
+                  by_module[module] = []
+                  module_order.append(module)
+              by_module[module].append((case_id, status))
+          report += ["## Case Summary", ""]
+          for module in module_order:
+              cases = by_module[module]
+              mod_pass = sum(1 for _, st in cases if st == "PASS")
+              mod_fail = len(cases) - mod_pass
+              report += [f"### {module} ({len(cases)})", "",
+                         "| Case | Status |", "| --- | --- |"]
+              report += [f"| {case_id} | {status} |" for case_id, status in cases]
+              report += ["", f"- Subtotal: {mod_pass} passed, {mod_fail} failed", ""]
+          report += [
+              "## Totals", "",
+              f"- Total: {len(rows)}",
+              f"- PASS: {passed}",
+              f"- FAIL: {failed}",
+              "- UNSUPPORTED: 0", "",
+              "Failing cases keep this workflow red only as a report; the sweep artifact",
+              "retains every case log and per-case report for diagnosis.", "",
+          ]
+          (root / "report.md").write_text("\n".join(report))
+          (root / "cases.md").write_text("\n".join(f"| {c} | {m} | {s} |" for c, m, s in rows) + "\n")
+          PY
+          cat "${SWEEP_ROOT}/report.md" >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Upload functional report to dashboard
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 2
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: ${{ runner.temp }}/validation-sweep/report.md
+          SUITE: validation
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
+            exit 0
+          fi
+          DATE="$(date -u +%Y-%m-%d)"
+          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
+          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${SHA}" ]; then
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          else
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          fi
+          echo "dashboard updated: ${REPORT_PATH}"
+
+      - name: Retain sweep evidence
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: rustfs-validation-sweep-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            ${{ runner.temp }}/validation-sweep/report.md
+            ${{ runner.temp }}/validation-sweep/results.txt
+            ${{ runner.temp }}/validation-sweep/sweep-cases.txt
+            ${{ runner.temp }}/validation-sweep/candidate.json
+            ${{ runner.temp }}/validation-sweep/results/
+            ${{ runner.temp }}/validation-sweep/run/
+          # warn (not error): an early infra failure legitimately leaves no
+          # per-case evidence, and the failing step is already red.
+          if-no-files-found: warn
+          retention-days: 30
+
+      - name: Cleanup environment (after)
+        id: environment_cleanup
+        timeout-minutes: 5
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        run: |
+          set -uo pipefail
+          cleanup_status=0
+          for node in ${RUSTFS_NODES}; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -f /etc/default/rustfs
+            ' || cleanup_status=$?
+          done
+          exit "${cleanup_status}"
+
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 scripts/functional_environment.py finish --suite heal --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Notify on failure
+        if: failure()
+        run: echo "RustFS validation sweep failed (harness/environment breakdown); see the sweep artifact for case logs."
+
+      - name: Enforce sweep result
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        env:
+          SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
+        run: |
+          set -euo pipefail
+          if grep -q $'\tFAIL$' "${SWEEP_ROOT}/results.txt"; then
+            echo "validation sweep has failing cases:"
+            grep $'\tFAIL$' "${SWEEP_ROOT}/results.txt"
+            exit 1
+          fi
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    # A scheduled sweep failure usually means a canonical validation case
+    # regressed or the fleet environment broke; surface it as a tracking
+    # issue instead of leaving it in the Actions tab.
+    needs: [sweep]
+    if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

CHANGELOG.md

@@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - IAM snapshot/deletion replay, target-assigned delete-marker purges, timestamp ordering, and best-effort peer broadcast now close the control-plane gaps found by the R6 review ([#7195](https://github.com/rustfs/rustfs/pull/7195)).
 - A `set-remote-target?update=true&creds=true` request on a site-replication peer's bucket target is now rejected with `400 InvalidRequest`, naming the target and why it counts as peer-owned. It used to return success while keeping the stored endpoint, credentials, and TLS settings, so an endpoint edit from the Console's bucket replication form appeared to apply but did not. Change a peer's address with a site-replication edit instead; see "Changing a site's address" in `docs/operations/site-replication-operations.md`. The `sync`, `bandwidth`, `proxy`, and `path` update groups are unchanged.
 - Site-replication edits that omit the site name, such as `mc admin replicate update --deployment-id <ID> --endpoint <URL>`, now keep the stored name on the editing site and on every receiver instead of renaming the site after the new endpoint host. No persisted format changes.
+- The persisted replication MRF journal is now replayed every 5 minutes while the server runs, not only at startup. Delete-marker purges whose targets were unreachable, and replications that overflowed the worker queues, now converge within one replay interval after the target returns instead of waiting for a restart. A target that stays down is re-attempted once per retained entry per interval across the cluster: the start time of the last pass is shared through a small `config/replication/mrf.bin.replay` object, so nodes do not each replay once per interval. Older versions ignore that object; existing persisted formats are unchanged.
 - Upgrade and rollback: upgrade every node in one site consecutively and verify reconciliation before moving to the next site; do not intentionally run a site mixed-version. Target-version ledger keys are harmless on rollback, although old code cannot use their routing. Before rolling back past [#7307](https://github.com/rustfs/rustfs/pull/7307), drain or repair every pending version purge: older code can free a retained version's data directory before its remote purge is acknowledged. See `docs/operations/site-replication-operations.md`.
 
 ### Security

Cargo.toml

@@ -147,7 +147,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
 async-channel = "2.5.0"
 async_zip = { default-features = false, version = "0.0.19" }
 mysql_async = { default-features = false, version = "0.37.1" }
-async-compression = { version = "0.4.51" }
+async-compression = { version = "0.4.52" }
 async-trait = "0.1.92"
 async-nats = { version = "0.51.0", default-features = false }
 axum = "0.8.9"
@@ -168,16 +168,16 @@ http-body-util = "0.1.5"
 memchr = "2.8.3"
 minlz = "1.2.3"
 reqwest = "0.13.5"
-rustfs-kafka-async = { version = "1.4.0" }
+rustfs-kafka-async = { version = "1.4.1" }
 socket2 = { version = "0.6.5" }
 tokio = { version = "1.54.1" }
 tokio-rustls = { default-features = false, version = "0.26.6" }
 tokio-stream = { version = "0.1.19" }
 tokio-test = "0.4.6"
 tokio-util = { version = "0.7.20" }
-tonic = { version = "0.14.8" }
-tonic-prost = { version = "0.14.8" }
-tonic-prost-build = { version = "0.14.8" }
+tonic = { version = "0.14.9" }
+tonic-prost = { version = "0.14.9" }
+tonic-prost-build = { version = "0.14.9" }
 tower = { version = "0.5.3" }
 tower-http = { version = "0.7.1" }
 h3 = "0.0.8"
@@ -212,12 +212,13 @@ serde_urlencoded = "0.7.1"
 # releases.
 aes-gcm = { version = "0.11.1" }
 argon2 = { version = "0.6.0" }
+aws-lc-rs = { version = "1.19.0", default-features = false }
 blake2 = "0.11.0"
 chacha20poly1305 = { version = "0.11.0" }
 crc-fast = "1.10.0"
 hmac = { version = "0.13.0" }
 jsonwebtoken = { version = "11.1.0" }
-openidconnect = { default-features = false, version = "4.0" }
+openidconnect = { default-features = false, version = "4.1.0" }
 pbkdf2 = "0.13.0"
 p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
 rsa = { version = "=0.10.0-rc.18" }
@@ -234,7 +235,7 @@ proptest = "1"
 # Time and Date
 chrono = { version = "0.4.45" }
 humantime = "2.4.0"
-jiff = { version = "0.2.38" }
+jiff = { version = "0.2.39" }
 time = { version = "0.3.55" }
 
 # Database
@@ -253,9 +254,9 @@ atoi = "3.1.1"
 atomic_enum = "0.3.0"
 aws-config = { version = "1.13.0" }
 aws-credential-types = { version = "1.3.1" }
-aws-sdk-kms = { default-features = false, version = "1.128.0" }
-aws-sdk-s3 = { default-features = false, version = "1.156.0" }
-aws-sdk-sts = { default-features = false, version = "1.123.0" }
+aws-sdk-kms = { default-features = false, version = "1.129.0" }
+aws-sdk-s3 = { default-features = false, version = "1.157.0" }
+aws-sdk-sts = { default-features = false, version = "1.124.0" }
 aws-smithy-async = { version = "1.3.1" }
 aws-smithy-http-client = { default-features = false, version = "1.5.2" }
 aws-smithy-runtime-api = { version = "1.19.2" }
@@ -319,7 +320,7 @@ rustfs-erasure-codec = { version = "9.0.0" }
 reed-solomon-simd = "3.1.0"
 regex = { version = "1.13.1" }
 rumqttc = { package = "rumqttc-next", version = "0.34.0" }
-redis = { version = "1.8.0" }
+redis = { version = "1.8.1" }
 rustify = { version = "0.7", default-features = false }
 rustix = { version = "1.1.6" }
 rust-embed = { version = "8.13.0" }
@@ -334,10 +335,10 @@ compact_str = "0.10.0"
 snap = "1.1.2"
 starshard = { version = "2.4.0" }
 strum = { version = "0.28.0" }
-sysinfo = "0.40.0"
+sysinfo = "0.40.1"
 temp-env = "0.3.6"
 tempfile = "3.27.0"
-test-case = "3.4.0"
+test-case = "3.4.1"
 thiserror = "2.0.21"
 tracing = { version = "0.1.44" }
 tracing-appender = "0.2.5"
@@ -356,13 +357,13 @@ winapi-util = "0.1.11"
 windows = { version = "0.62.2" }
 windows-sys = "0.61.2"
 xxhash-rust = { version = "0.8.19" }
-zip = "9.1.3"
+zip = "9.3.0"
 zstd = "0.14.3"
 
 # Observability and Metrics
 metrics = "0.24.6"
 metrics-util = "0.20"
-dial9 = { version = "0.5.6", default-features = false }
+dial9 = { version = "0.5.7", default-features = false }
 opentelemetry = { version = "0.33.1" }
 opentelemetry-appender-tracing = { version = "0.33.1" }
 opentelemetry-otlp = { version = "0.33.1" }
@@ -377,16 +378,16 @@ libunftp = { version = "0.24.0" }
 unftp-core = "0.1.0"
 suppaftp = { version = "12.2.1" }
 rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
-russh = { version = "0.65.0" }
+russh = { version = "0.65.1" }
 russh-sftp = "3.0.1"
 
 # WebDAV
 dav-server = "0.11.0"
 
 # Performance Analysis and Memory Profiling
-rustfs-mimalloc = { version = "0.7.2" }
+rustfs-mimalloc = { version = "0.7.3" }
 # Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
-hotpath = { version = "0.30.1", default-features = false }
+hotpath = { version = "0.30.3", default-features = false }
 # Snapshot testing for output format regression detection
 insta = { version = "1.49" }
 
@@ -427,7 +428,3 @@ strip = "none"
 # Pin upstream fixes until compatible crates.io releases are available.
 [patch.crates-io]
 zstd = { path = "crates/zstd-compat" }
-# Preserve RustFS OIDC flows while migrating the old elliptic-curve dependency chain.
-# Upstream: openidconnect-rs#244 and oauth2-rs#352.
-openidconnect = { git = "https://github.com/ramosbugs/openidconnect-rs", rev = "2f77ee73bc4554e1cb6279adb386005429fb4e41" }
-oauth2 = { git = "https://github.com/ramosbugs/oauth2-rs", rev = "89585f4a280afd7b285eaf1c9be0068874e03d22" }

Dockerfile

@@ -16,6 +16,7 @@ FROM alpine:3.24.2 AS build
 
 ARG TARGETARCH
 ARG RELEASE=latest
+ARG RC_RELEASE=latest
 
 RUN apk add --no-cache ca-certificates curl jq unzip
 WORKDIR /build
@@ -69,6 +70,9 @@ RUN set -eux; \
     chmod +x /build/rustfs; \
     rm -rf rustfs.zip /build/.tmp || true
 
+COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
+RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /build/rc
+
 
 FROM alpine:3.24.2
 
@@ -95,14 +99,18 @@ RUN apk upgrade --no-cache && \
     ca-certificates \
     coreutils \
     curl \
+    jq \
     tzdata \
     && test "$(TZ=Asia/Kolkata date +%z)" = "+0530"
 
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=build /build/rustfs /usr/bin/rustfs
-COPY entrypoint.sh /entrypoint.sh
+COPY --from=build /build/rc /usr/bin/rc
+COPY entrypoint.sh /etc/rustfs/entrypoint.sh
+COPY bootstrap/ /etc/rustfs/bootstrap/
 
-RUN chmod +x /usr/bin/rustfs /entrypoint.sh
+RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
+    ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
 
 RUN addgroup -g 10001 -S rustfs && \
     adduser -u 10001 -G rustfs -S rustfs -D && \
@@ -124,6 +132,6 @@ VOLUME ["/data"]
 
 USER rustfs
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
 
 CMD ["rustfs"]

Dockerfile.decommission-local

@@ -12,7 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM rust:1.97.1-trixie
+FROM rust:1.99.1-trixie
+
+ARG TARGETARCH
+ARG RC_RELEASE=latest
 
 RUN set -eux; \
     export DEBIAN_FRONTEND=noninteractive; \
@@ -21,6 +24,7 @@ RUN set -eux; \
     build-essential \
     ca-certificates \
     curl \
+    jq \
     git \
     pkg-config \
     libssl-dev \
@@ -36,6 +40,7 @@ COPY . .
 RUN ./scripts/static.sh
 RUN cargo run --bin gproto
 RUN cargo build --release --locked --bin rustfs
+RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
 
 RUN set -eux; \
     groupadd -g 10001 rustfs; \
@@ -50,8 +55,12 @@ RUN set -eux; \
     chown -R rustfs:rustfs /data /logs /app; \
     chmod 0750 /data /logs
 
-COPY entrypoint.sh /entrypoint.sh
-RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && chmod +x /entrypoint.sh
+COPY entrypoint.sh /etc/rustfs/entrypoint.sh
+COPY bootstrap/ /etc/rustfs/bootstrap/
+RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && \
+    install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
+    chmod +x /etc/rustfs/entrypoint.sh && \
+    ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
 
 ENV RUSTFS_VOLUMES="/data" \
     RUST_LOG="warn" \
@@ -63,5 +72,5 @@ ENV RUSTFS_VOLUMES="/data" \
 
 EXPOSE 9000 9001
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
 CMD ["/usr/bin/rustfs"]

Dockerfile.glibc

@@ -16,6 +16,7 @@ FROM ubuntu:26.04 AS build
 
 ARG TARGETARCH
 ARG RELEASE=latest
+ARG RC_RELEASE=latest
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     ca-certificates \
@@ -76,6 +77,9 @@ RUN set -eux; \
     chmod +x /build/rustfs; \
     rm -rf rustfs.zip /build/.tmp || true
 
+COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
+RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" gnu "$RC_RELEASE" /build/rc
+
 FROM ubuntu:26.04
 
 ARG RELEASE=latest
@@ -99,14 +103,18 @@ RUN apt-get update && apt-get upgrade -y \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
     ca-certificates \
     curl \
+    jq \
     tzdata \
     && test "$(TZ=Asia/Kolkata date +%z)" = "+0530" \
     && rm -rf /var/lib/apt/lists/*
 
 COPY --from=build /build/rustfs /usr/bin/rustfs
-COPY entrypoint.sh /entrypoint.sh
+COPY --from=build /build/rc /usr/bin/rc
+COPY entrypoint.sh /etc/rustfs/entrypoint.sh
+COPY bootstrap/ /etc/rustfs/bootstrap/
 
-RUN chmod +x /usr/bin/rustfs /entrypoint.sh
+RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
+    ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
 
 RUN groupadd -g 10001 rustfs && \
     useradd -u 10001 -g rustfs -m -s /sbin/nologin rustfs && \
@@ -131,6 +139,6 @@ VOLUME ["/data"]
 
 USER rustfs
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
 
 CMD ["rustfs"]

Dockerfile.source

@@ -28,17 +28,19 @@ ARG TARGETPLATFORM
 ARG BUILDPLATFORM
 ARG TARGETARCH
 ARG RUSTFS_BUILD_FEATURES=""
+ARG RC_RELEASE=latest
 
 # -----------------------------
 # Build stage
 # -----------------------------
-FROM rust:1.97.1-trixie AS builder
+FROM rust:1.99.1-trixie AS builder
 
 # Re-declare args after FROM
 ARG TARGETPLATFORM
 ARG BUILDPLATFORM
 ARG TARGETARCH
 ARG RUSTFS_BUILD_FEATURES
+ARG RC_RELEASE
 
 # Debug: print platforms
 RUN echo "Build info -> BUILDPLATFORM=${BUILDPLATFORM}, TARGETPLATFORM=${TARGETPLATFORM}"
@@ -52,6 +54,7 @@ RUN set -eux; \
     build-essential \
     ca-certificates \
     curl \
+    jq \
     git \
     pkg-config \
     libssl-dev \
@@ -166,6 +169,8 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
     ;; \
     esac
 
+RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
+
 # -----------------------------
 # Development stage (keeps toolchain)
 # -----------------------------
@@ -199,10 +204,13 @@ ENV RUSTFS_VOLUMES="/data" \
 # We rely on cargo run to build and run
 EXPOSE 9000 9001
 
-COPY entrypoint.sh /entrypoint.sh
-RUN chmod +x /entrypoint.sh
+COPY entrypoint.sh /etc/rustfs/entrypoint.sh
+COPY bootstrap/ /etc/rustfs/bootstrap/
+RUN install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
+    chmod +x /etc/rustfs/entrypoint.sh && \
+    ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
 CMD ["cargo", "run", "--bin", "rustfs", "--"]
 
 # -----------------------------
@@ -227,6 +235,7 @@ RUN set -eux; \
     apt-get install -y --no-install-recommends \
     ca-certificates \
     curl \
+    jq \
     tzdata \
     coreutils; \
     rm -rf /var/lib/apt/lists/*
@@ -248,8 +257,11 @@ RUN set -eux; \
 
 # Copy the freshly built binary and the entrypoint
 COPY --from=builder /usr/local/bin/rustfs /usr/bin/rustfs
-COPY entrypoint.sh /entrypoint.sh
-RUN chmod +x /usr/bin/rustfs /entrypoint.sh
+COPY --from=builder /usr/local/bin/rc /usr/bin/rc
+COPY entrypoint.sh /etc/rustfs/entrypoint.sh
+COPY bootstrap/ /etc/rustfs/bootstrap/
+RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
+    ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
 
 # Default environment (override in docker run/compose as needed)
 ENV RUSTFS_VOLUMES="/data" \
@@ -263,5 +275,5 @@ EXPOSE 9000
 VOLUME ["/data"]
 
 # Keep root here; entrypoint will drop privileges using chroot --userspec
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
 CMD ["/usr/bin/rustfs"]

bootstrap/README.md

@@ -0,0 +1,21 @@
+# RustFS image bootstrap
+
+The official image bundles this directory at `/etc/rustfs/bootstrap`. The default `buckets.json` is an empty JSON array, so the image starts RustFS directly and does not require a bootstrap endpoint or bootstrap credentials.
+
+To create buckets, replace `buckets.json` in a derived image with a JSON array of bucket names. The entrypoint creates each bucket if it does not exist and applies `public-access-block.json`, whose default blocks all four public access paths. Edit that file in the derived image only when public access is an explicit requirement.
+
+Optional per-bucket files use the bucket name as the filename without an extension:
+
+| Directory | File format | Applied operation |
+| --- | --- | --- |
+| `policies/` | S3 bucket policy JSON | `rc bucket anonymous set-json` |
+| `versioning/` | JSON boolean (`true` or `false`) | Enable or suspend bucket versioning |
+| `lifecycle/` | `rc bucket lifecycle rule import` JSON | Replace lifecycle rules |
+| `cors/` | JSON or XML accepted by `rc bucket cors set` | Replace CORS rules |
+| `notifications/` | JSON array of `{ "arn": "...", "events": ["..."] }` | Add or replace event rules by ARN (prefix/suffix filters are not currently accepted) |
+
+Every per-bucket file must correspond to a bucket listed in `buckets.json`. Applying these files is repeatable: bucket creation ignores existing buckets, while each configuration operation replaces or reconciles that bucket's requested state.
+
+When `buckets.json` is non-empty, set `RUSTFS_BOOTSTRAP_ENDPOINT` to the RustFS S3 endpoint reachable from the container, for example `http://rustfs:9000` for a Compose service or the internal service DNS name in Kubernetes. The entrypoint waits on that endpoint's `/health/ready`; it never assumes `localhost`. `RUSTFS_BOOTSTRAP_TIMEOUT` controls the readiness wait in seconds (default `120`, maximum `3600`). `RUSTFS_BOOTSTRAP_DIR` can point to a different in-image configuration directory; it defaults to `/etc/rustfs/bootstrap`. Set `RUSTFS_ACCESS_KEY` and `RUSTFS_SECRET_KEY`, or their `_FILE` variants, for bootstrap authentication. The endpoint and credentials are read at runtime and are not baked into the image.
+
+RustFS built-in IAM policies remain unchanged; the default bootstrap installs no custom bucket policies, lifecycle rules, versioning, CORS rules, or notification rules. The bundled RustFS CLI is resolved from a published release and its release checksum is verified. Set the Docker build argument `RC_RELEASE` to a version to pin it. Build the RustFS image after a CLI release containing the `bucket public-access-block` commands is available. For Kubernetes, the configured service must route traffic to the starting pod before its readiness probe succeeds (for example, with `publishNotReadyAddresses: true`); otherwise the service endpoint cannot reach the pod while the entrypoint waits for readiness.

bootstrap/buckets.json

@@ -0,0 +1 @@
+[]

bootstrap/cors/.gitkeep

@@ -0,0 +1 @@
+

bootstrap/lifecycle/.gitkeep

@@ -0,0 +1 @@
+

bootstrap/notifications/.gitkeep

@@ -0,0 +1 @@
+

bootstrap/policies/.gitkeep

@@ -0,0 +1 @@
+

bootstrap/public-access-block.json

@@ -0,0 +1,6 @@
+{
+  "block_public_acls": true,
+  "ignore_public_acls": true,
+  "block_public_policy": true,
+  "restrict_public_buckets": true
+}

bootstrap/versioning/.gitkeep

@@ -0,0 +1 @@
+

crates/config/src/constants/proxy.rs

@@ -30,10 +30,10 @@ pub const ENV_TRUSTED_PROXY_VALIDATION_MODE: &str = "RUSTFS_TRUSTED_PROXY_VALIDA
 /// Default validation mode is "hop_by_hop".
 pub const DEFAULT_TRUSTED_PROXY_VALIDATION_MODE: &str = "hop_by_hop";
 
-/// Environment variable to enable RFC 7239 "Forwarded" header support.
+/// Select RFC 7239 "Forwarded" instead of "X-Forwarded-For" in legacy mode.
 pub const ENV_TRUSTED_PROXY_ENABLE_RFC7239: &str = "RUSTFS_TRUSTED_PROXY_ENABLE_RFC7239";
-/// RFC 7239 support is enabled by default.
-pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = true;
+/// Legacy mode uses X-Forwarded-For unless RFC 7239 is explicitly selected.
+pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = false;
 
 /// Environment variable for the maximum allowed proxy hops.
 pub const ENV_TRUSTED_PROXY_MAX_HOPS: &str = "RUSTFS_TRUSTED_PROXY_MAX_HOPS";

crates/config/src/constants/targets.rs

@@ -146,3 +146,9 @@ pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";
 
 /// Queue-store compression is enabled by default to reduce disk footprint.
 pub const DEFAULT_TARGET_STORE_COMPRESS: bool = true;
+
+/// Maximum on-disk bytes in each target queue, including failed entries and pending writes.
+pub const ENV_TARGET_STORE_MAX_BYTES: &str = "RUSTFS_TARGET_STORE_MAX_BYTES";
+
+/// Queue writes fail at one GiB unless the operator sets another positive byte budget.
+pub const DEFAULT_TARGET_STORE_MAX_BYTES: u64 = 1024 * 1024 * 1024;

crates/e2e_test/README.md

@@ -26,7 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
 | **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
 | **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
 | **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
-| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
+| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
 | **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
 | **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
 
@@ -194,9 +194,9 @@ the same profile for membership and execution with one nightly worker.
 | `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
 | ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
 | KMS suite | `e2e-full` job, merge queue + main | **Active** |
-| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
+| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, release tags + weekly schedule + manual dispatch | **Active** |
 | Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
-| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
+| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (nightly / dispatch) |
 | Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
 | Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
 | Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |

crates/e2e_test/src/create_bucket_region_test.rs

@@ -502,10 +502,163 @@ mod tests {
                 assert!(response.contains(if expected == reqwest::StatusCode::FORBIDDEN {
                     "<Code>AccessDenied</Code>"
                 } else {
-                    "<Code>BucketAlreadyExists</Code>"
+                    "<Code>BucketAlreadyOwnedByYou</Code>"
                 }));
             }
         }
         Ok(())
     }
+
+    /// A deployment owns a single bucket namespace, so an IAM user that recreates
+    /// an existing bucket is the owner, not another account. In the default region
+    /// every such request returns the legacy idempotent `200 OK` (AWS CreateBucket
+    /// semantics), including buckets created by root and by another IAM user.
+    #[tokio::test]
+    async fn test_create_bucket_duplicate_in_default_region_is_idempotent_for_iam_user()
+    -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec!["--region", "us-east-1"]).await?;
+        let root = env.create_s3_client();
+
+        admin_create_user(&env, "duplicate-iam", "duplicate-iam-secret").await?;
+        admin_attach_user_policy_via(
+            AdminTransport::Signed,
+            &env.url,
+            &env.access_key,
+            &env.secret_key,
+            "readwrite",
+            "duplicate-iam",
+        )
+        .await?;
+        let iam = Client::from_conf(build_test_s3_config(
+            &env.url,
+            "duplicate-iam",
+            "duplicate-iam-secret",
+            None,
+            "duplicate-create",
+        ));
+
+        root.create_bucket()
+            .bucket("duplicate-root-owned")
+            .send()
+            .await
+            .expect("root creates the first bucket");
+        iam.create_bucket()
+            .bucket("duplicate-root-owned")
+            .send()
+            .await
+            .expect("an IAM user recreating a root-owned bucket in us-east-1 gets 200 OK");
+
+        iam.create_bucket()
+            .bucket("duplicate-iam-owned")
+            .send()
+            .await
+            .expect("the IAM user creates its own bucket");
+        iam.create_bucket()
+            .bucket("duplicate-iam-owned")
+            .send()
+            .await
+            .expect("the same IAM user recreating its bucket gets 200 OK");
+        root.create_bucket()
+            .bucket("duplicate-iam-owned")
+            .send()
+            .await
+            .expect("root recreating an IAM-created bucket gets 200 OK");
+
+        Ok(())
+    }
+
+    /// Outside the default region the same ownership answer becomes
+    /// `409 BucketAlreadyOwnedByYou` for every authorized requester. Requests that
+    /// lack `s3:CreateBucket` still fail authorization before the duplicate check.
+    #[tokio::test]
+    async fn test_create_bucket_duplicate_outside_default_region_reports_owned_by_you() -> Result<(), Box<dyn Error + Send + Sync>>
+    {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec!["--region", "eu-west-1"]).await?;
+        let region = "eu-west-1";
+        let root = Client::from_conf(
+            env.create_s3_client()
+                .config()
+                .to_builder()
+                .region(Region::new(region))
+                .build(),
+        );
+
+        admin_create_user(&env, "duplicate-eu-iam", "duplicate-eu-iam-secret").await?;
+        admin_attach_user_policy_via(
+            AdminTransport::Signed,
+            &env.url,
+            &env.access_key,
+            &env.secret_key,
+            "readwrite",
+            "duplicate-eu-iam",
+        )
+        .await?;
+        let iam = Client::from_conf(
+            build_test_s3_config(&env.url, "duplicate-eu-iam", "duplicate-eu-iam-secret", None, "duplicate-create")
+                .to_builder()
+                .region(Region::new(region))
+                .build(),
+        );
+
+        admin_create_user(&env, "duplicate-eu-readonly", "duplicate-eu-readonly-secret").await?;
+        admin_attach_user_policy_via(
+            AdminTransport::Signed,
+            &env.url,
+            &env.access_key,
+            &env.secret_key,
+            "readonly",
+            "duplicate-eu-readonly",
+        )
+        .await?;
+        let readonly = Client::from_conf(
+            build_test_s3_config(
+                &env.url,
+                "duplicate-eu-readonly",
+                "duplicate-eu-readonly-secret",
+                None,
+                "duplicate-create",
+            )
+            .to_builder()
+            .region(Region::new(region))
+            .build(),
+        );
+
+        for (client, bucket) in [(&root, "duplicate-eu-root"), (&iam, "duplicate-eu-iam")] {
+            client
+                .create_bucket()
+                .bucket(bucket)
+                .send()
+                .await
+                .unwrap_or_else(|error| panic!("{bucket} must be created: {error:?}"));
+            // Both the creator and the other authenticated identity own the namespace.
+            for creator in [&root, &iam] {
+                let error = creator
+                    .create_bucket()
+                    .bucket(bucket)
+                    .send()
+                    .await
+                    .expect_err("a duplicate create outside us-east-1 must report ownership");
+                assert_eq!(
+                    error.as_service_error().and_then(ProvideErrorMetadata::code),
+                    Some("BucketAlreadyOwnedByYou"),
+                    "{bucket}"
+                );
+                assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(409), "{bucket}");
+            }
+        }
+
+        let denied = readonly
+            .create_bucket()
+            .bucket("duplicate-eu-root")
+            .send()
+            .await
+            .expect_err("a requester without s3:CreateBucket must be denied before the duplicate check");
+        assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
+
+        Ok(())
+    }
 }

crates/e2e_test/src/delete_marker_migration_semantics_test.rs

@@ -16,8 +16,193 @@
 mod tests {
     use crate::common::{RustFSTestEnvironment, init_logging};
     use aws_sdk_s3::Client;
+    use aws_sdk_s3::config::http::HttpResponse;
+    use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
     use aws_sdk_s3::primitives::ByteStream;
-    use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
+    use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
+
+    fn assert_not_found<E: ProvideErrorMetadata + std::fmt::Debug>(error: &SdkError<E, HttpResponse>) {
+        assert_eq!(error.raw_response().map(|r| r.status().as_u16()), Some(404), "{error:?}");
+        assert_eq!(
+            error.as_service_error().and_then(ProvideErrorMetadata::code),
+            Some("NoSuchKey"),
+            "{error:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn conditional_writes_cannot_replace_delete_marker_with_if_match() {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
+        env.start_rustfs_server(vec![]).await.expect("start RustFS");
+        let client = env.create_s3_client();
+        let bucket = "conditional-delete-marker-proof";
+        create_versioned_bucket(&client, bucket).await;
+        client
+            .put_object()
+            .bucket(bucket)
+            .key("source")
+            .body(ByteStream::from_static(b"source"))
+            .send()
+            .await
+            .expect("write copy source");
+
+        for operation in ["put", "copy", "multipart"] {
+            let old = client
+                .put_object()
+                .bucket(bucket)
+                .key(operation)
+                .body(ByteStream::from_static(b"old"))
+                .send()
+                .await
+                .expect("write historical version");
+            let old_etag = old.e_tag().expect("historical ETag");
+            let marker = client
+                .delete_object()
+                .bucket(bucket)
+                .key(operation)
+                .send()
+                .await
+                .expect("create delete marker");
+            let marker_id = marker.version_id().expect("delete marker version");
+
+            let (upload_id, completed) = if operation == "multipart" {
+                let upload = client
+                    .create_multipart_upload()
+                    .bucket(bucket)
+                    .key(operation)
+                    .send()
+                    .await
+                    .expect("initiate multipart upload");
+                let upload_id = upload.upload_id().expect("upload ID").to_owned();
+                let part = client
+                    .upload_part()
+                    .bucket(bucket)
+                    .key(operation)
+                    .upload_id(&upload_id)
+                    .part_number(1)
+                    .body(ByteStream::from_static(b"new"))
+                    .send()
+                    .await
+                    .expect("upload part");
+                let completed = CompletedMultipartUpload::builder()
+                    .parts(
+                        CompletedPart::builder()
+                            .part_number(1)
+                            .set_e_tag(part.e_tag().map(str::to_owned))
+                            .build(),
+                    )
+                    .build();
+                (Some(upload_id), Some(completed))
+            } else {
+                (None, None)
+            };
+
+            for etag in [old_etag, "\"00000000000000000000000000000000\""] {
+                match operation {
+                    "put" => {
+                        let err = client
+                            .put_object()
+                            .bucket(bucket)
+                            .key(operation)
+                            .if_match(etag)
+                            .body(ByteStream::from_static(b"new"))
+                            .send()
+                            .await
+                            .expect_err("If-Match must reject marker");
+                        assert_not_found(&err);
+                    }
+                    "copy" => {
+                        let err = client
+                            .copy_object()
+                            .bucket(bucket)
+                            .key(operation)
+                            .copy_source(format!("{bucket}/source"))
+                            .if_match(etag)
+                            .send()
+                            .await
+                            .expect_err("If-Match copy must reject marker");
+                        assert_not_found(&err);
+                    }
+                    _ => {
+                        let err = client
+                            .complete_multipart_upload()
+                            .bucket(bucket)
+                            .key(operation)
+                            .upload_id(upload_id.as_deref().expect("upload ID"))
+                            .multipart_upload(completed.clone().expect("completed parts"))
+                            .if_match(etag)
+                            .send()
+                            .await
+                            .expect_err("If-Match completion must reject marker");
+                        assert_not_found(&err);
+                    }
+                }
+                let versions = client
+                    .list_object_versions()
+                    .bucket(bucket)
+                    .prefix(operation)
+                    .send()
+                    .await
+                    .expect("list versions after rejected write");
+                assert_eq!(versions.versions().len(), 1, "{operation}: failed write created a version");
+                assert_eq!(versions.delete_markers().len(), 1);
+                assert_eq!(versions.delete_markers()[0].version_id(), Some(marker_id));
+                assert_eq!(versions.delete_markers()[0].is_latest(), Some(true));
+            }
+
+            match operation {
+                "put" => {
+                    client
+                        .put_object()
+                        .bucket(bucket)
+                        .key(operation)
+                        .if_none_match("*")
+                        .body(ByteStream::from_static(b"new"))
+                        .send()
+                        .await
+                        .expect("If-None-Match PUT over marker");
+                }
+                "copy" => {
+                    client
+                        .copy_object()
+                        .bucket(bucket)
+                        .key(operation)
+                        .copy_source(format!("{bucket}/source"))
+                        .if_none_match("*")
+                        .send()
+                        .await
+                        .expect("If-None-Match copy over marker");
+                }
+                _ => {
+                    client
+                        .complete_multipart_upload()
+                        .bucket(bucket)
+                        .key(operation)
+                        .upload_id(upload_id.as_deref().expect("upload ID"))
+                        .multipart_upload(completed.expect("completed parts"))
+                        .if_none_match("*")
+                        .send()
+                        .await
+                        .expect("If-None-Match completion over marker");
+                }
+            }
+            let current = client
+                .get_object()
+                .bucket(bucket)
+                .key(operation)
+                .send()
+                .await
+                .expect("read successful conditional write")
+                .body
+                .collect()
+                .await
+                .expect("read object body")
+                .into_bytes();
+            let expected: &[u8] = if operation == "copy" { b"source" } else { b"new" };
+            assert_eq!(current.as_ref(), expected, "{operation}: unexpected current object");
+        }
+    }
 
     async fn create_versioned_bucket(client: &Client, bucket: &str) {
         client

crates/e2e_test/src/distributed/concurrency_stability_test.rs

@@ -12,11 +12,249 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
+use super::harness::{DistCluster, DistLayout, TestResult, payload_for, s3_probe_client, sha256_hex, unique_bucket};
 use crate::common::init_logging;
+use aws_sdk_s3::{
+    Client,
+    error::{ProvideErrorMetadata, SdkError},
+    primitives::ByteStream,
+};
+use std::any::TypeId;
 use std::collections::BTreeSet;
+use std::fmt::Debug;
+use std::future::Future;
 use std::sync::Arc;
+use std::time::Duration;
 use tokio::sync::Barrier;
+use tokio::time::Instant;
+
+const LOCK_ADMISSION_BUDGET: Duration = Duration::from_secs(45);
+
+async fn after_lock_contention<T, E, F, Fut>(operation: &str, mut call: F) -> TestResult<Result<T, SdkError<E>>>
+where
+    E: Debug + ProvideErrorMetadata + 'static,
+    F: FnMut() -> Fut,
+    Fut: Future<Output = Result<T, SdkError<E>>>,
+{
+    let started = Instant::now();
+    let mut attempts = 0;
+    // The workload uses unique unversioned keys. Repeat only explicitly
+    // advertised lock backpressure; every step still needs its real result.
+    let outcome = tokio::time::timeout(LOCK_ADMISSION_BUDGET, async {
+        loop {
+            attempts += 1;
+            let result = call().await;
+            let retry = result.as_ref().err().is_some_and(|error| {
+                let Some(response) = error.raw_response() else {
+                    return false;
+                };
+                let Some(service_error) = error.as_service_error() else {
+                    return false;
+                };
+                // HEAD has no XML error body, so its SDK service error can
+                // lack a code even when the server supplies retry headers.
+                let namespace_code = service_error.code() == Some("ServiceUnavailable")
+                    || (service_error.code().is_none()
+                        && TypeId::of::<E>() == TypeId::of::<aws_sdk_s3::operation::head_object::HeadObjectError>());
+                response.status().as_u16() == 503
+                    && namespace_code
+                    && response.headers().get("x-rustfs-error-cause") == Some("namespace_lock_timeout")
+                    && response.headers().get("retry-after") == Some("1")
+            });
+            if !retry {
+                return result;
+            }
+            tokio::time::sleep(Duration::from_secs(1)).await;
+        }
+    })
+    .await;
+    if attempts > 1 || !matches!(&outcome, Ok(Ok(_))) {
+        eprintln!(
+            "mixed workload request: operation={operation}, attempts={attempts}, elapsed={:?}",
+            started.elapsed()
+        );
+    }
+    outcome.map_err(|_| {
+        format!("{operation} exceeded its {LOCK_ADMISSION_BUDGET:?} request budget after {attempts} attempts").into()
+    })
+}
+
+async fn assert_workload_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
+    let output = after_lock_contention(&format!("GET {key}"), || client.get_object().bucket(bucket).key(key).send()).await??;
+    let actual = output.body.collect().await?.into_bytes();
+    if actual.as_ref() != expected {
+        return Err(format!(
+            "object {key} bytes mismatch: expected {} bytes sha256={} got {} bytes sha256={}",
+            expected.len(),
+            sha256_hex(expected),
+            actual.len(),
+            sha256_hex(&actual)
+        )
+        .into());
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use aws_smithy_http_client::test_util::infallible_client_fn;
+    use aws_smithy_types::body::SdkBody;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+
+    type Reply = (u16, &'static str, Option<&'static str>, Option<&'static str>, &'static str);
+
+    fn client(respond: impl Fn(usize) -> Reply + Send + Sync + 'static) -> (Client, Arc<AtomicUsize>) {
+        let requests = Arc::new(AtomicUsize::new(0));
+        let seen = Arc::clone(&requests);
+        let transport = infallible_client_fn(move |_| {
+            let (status, code, cause, retry_after, body) = respond(seen.fetch_add(1, Ordering::SeqCst));
+            let body = if code.is_empty() {
+                body.to_string()
+            } else {
+                format!("<Error><Code>{code}</Code></Error>")
+            };
+            let mut response = http::Response::builder()
+                .status(status)
+                .header("content-type", "application/xml")
+                .header("content-length", body.len());
+            if let Some(cause) = cause {
+                response = response.header("x-rustfs-error-cause", cause);
+            }
+            if let Some(retry_after) = retry_after {
+                response = response.header("retry-after", retry_after);
+            }
+            response.body(SdkBody::from(body)).expect("scripted S3 response")
+        });
+        let config = crate::common::build_test_s3_config(
+            "http://contention.example.invalid",
+            "synthetic-access",
+            "synthetic-secret",
+            None,
+            "contention-test",
+        );
+        let client = Client::from_conf(config.to_builder().http_client(transport).build());
+        (s3_probe_client(&client), requests)
+    }
+
+    fn contention() -> Reply {
+        (503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("1"), "")
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn advertised_contention_recovers_after_standard_sdk_attempts_are_exhausted() -> TestResult {
+        let (client, requests) = client(|attempt| {
+            if attempt < 3 {
+                contention()
+            } else {
+                (200, "", None, None, "")
+            }
+        });
+        let standard = Client::from_conf(
+            client
+                .config()
+                .to_builder()
+                .retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(3))
+                .build(),
+        );
+        standard
+            .put_object()
+            .bucket("bucket")
+            .key("key")
+            .body(ByteStream::from_static(b"body"))
+            .send()
+            .await
+            .expect_err("three advertised lock responses exhaust the standard SDK policy");
+        assert_eq!(requests.load(Ordering::SeqCst), 3);
+        requests.store(0, Ordering::SeqCst);
+        let started = Instant::now();
+        after_lock_contention("PUT recovery", || {
+            client
+                .put_object()
+                .bucket("bucket")
+                .key("key")
+                .body(ByteStream::from_static(b"body"))
+                .send()
+        })
+        .await??;
+        assert_eq!(requests.load(Ordering::SeqCst), 4);
+        assert!(started.elapsed() >= Duration::from_secs(3), "honor each Retry-After response");
+        Ok(())
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn bodyless_head_contention_uses_explicit_retry_headers() -> TestResult {
+        let (client, requests) = client(|attempt| {
+            if attempt == 0 {
+                (503, "", Some("namespace_lock_timeout"), Some("1"), "")
+            } else {
+                (200, "", None, None, "")
+            }
+        });
+        after_lock_contention("HEAD recovery", || client.head_object().bucket("bucket").key("key").send()).await??;
+        assert_eq!(requests.load(Ordering::SeqCst), 2);
+        Ok(())
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn unrelated_or_malformed_responses_are_not_retried() -> TestResult {
+        for reply in [
+            (500, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
+            (403, "AccessDenied", None, None, ""),
+            (404, "NoSuchKey", None, None, ""),
+            (503, "ServiceUnavailable", Some("storage_write_quorum"), None, ""),
+            (503, "ServiceUnavailable", Some("namespace_lock_unavailable"), Some("1"), ""),
+            (503, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
+            (503, "", Some("namespace_lock_timeout"), Some("1"), ""),
+            (503, "ServiceUnavailable", None, Some("1"), ""),
+            (503, "ServiceUnavailable", Some("namespace_lock_timeout"), None, ""),
+            (503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("2"), ""),
+        ] {
+            let (client, requests) = client(move |_| reply);
+            let result = after_lock_contention("PUT fatal response", || {
+                client
+                    .put_object()
+                    .bucket("bucket")
+                    .key("key")
+                    .body(ByteStream::from_static(b"body"))
+                    .send()
+            })
+            .await?;
+            assert!(result.is_err(), "fatal response cannot become success: {reply:?}");
+            assert_eq!(requests.load(Ordering::SeqCst), 1, "fatal response must remain visible: {reply:?}");
+        }
+        Ok(())
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn persistent_lock_contention_exhausts_the_request_budget() {
+        let (client, requests) = client(|_| contention());
+        let started = Instant::now();
+        let result = after_lock_contention("PUT unavailable", || {
+            client
+                .put_object()
+                .bucket("bucket")
+                .key("key")
+                .body(ByteStream::from_static(b"body"))
+                .send()
+        })
+        .await;
+        assert!(result.is_err(), "persistent contention cannot acknowledge an operation");
+        assert!(started.elapsed() >= LOCK_ADMISSION_BUDGET);
+        assert!(requests.load(Ordering::SeqCst) > 1);
+        assert!(requests.load(Ordering::SeqCst) <= LOCK_ADMISSION_BUDGET.as_secs() as usize + 1);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn acknowledged_wrong_bytes_fail_without_retry() {
+        let (client, requests) = client(|_| (200, "", None, None, "wrong bytes"));
+        let error = assert_workload_bytes(&client, "bucket", "key", b"expected bytes")
+            .await
+            .expect_err("a successful but incorrect read is a consistency failure");
+        assert!(error.to_string().contains("bytes mismatch"));
+        assert_eq!(requests.load(Ordering::SeqCst), 1);
+    }
+}
 
 #[tokio::test]
 async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
@@ -24,7 +262,9 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
     let dist = DistCluster::start(DistLayout::FourByFour).await?;
     let bucket = unique_bucket("conc");
     dist.create_bucket(&bucket).await?;
-    let clients = Arc::new(dist.clients()?);
+    // This consistency probe owns its admission policy; unrelated responses
+    // must not be hidden by the SDK's general-purpose retries.
+    let clients = Arc::new(dist.clients()?.iter().map(s3_probe_client).collect::<Vec<_>>());
     let worker_count = 24;
     let rounds = 4;
     let barrier = Arc::new(Barrier::new(worker_count));
@@ -44,31 +284,46 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
                 let key = format!("source/worker-{idx:02}-round-{round}.bin");
                 let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
                 let body = payload_for(&key, 64 * 1024);
-                put_object(writer, &bucket, &key, body.clone()).await?;
+                let context = format!("worker={idx} round={round}");
+                after_lock_contention(&format!("{context} PUT source"), || {
+                    writer
+                        .put_object()
+                        .bucket(&bucket)
+                        .key(&key)
+                        .body(ByteStream::from(body.clone()))
+                        .send()
+                })
+                .await??;
 
-                let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
+                let head = after_lock_contention(&format!("{context} HEAD source"), || {
+                    reader.head_object().bucket(&bucket).key(&key).send()
+                })
+                .await??;
                 if head.content_length() != Some(body.len() as i64) {
                     return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
                 }
-                assert_object_bytes(reader, &bucket, &key, &body).await?;
+                assert_workload_bytes(reader, &bucket, &key, &body).await?;
 
-                copier
-                    .copy_object()
-                    .bucket(&bucket)
-                    .key(&copy_key)
-                    .copy_source(format!("{bucket}/{key}"))
-                    .send()
-                    .await?;
-                assert_object_bytes(writer, &bucket, &copy_key, &body).await?;
+                after_lock_contention(&format!("{context} COPY retained"), || {
+                    copier
+                        .copy_object()
+                        .bucket(&bucket)
+                        .key(&copy_key)
+                        .copy_source(format!("{bucket}/{key}"))
+                        .send()
+                })
+                .await??;
+                assert_workload_bytes(writer, &bucket, &copy_key, &body).await?;
 
-                writer.delete_object().bucket(&bucket).key(&key).send().await?;
-                let missing = reader
-                    .head_object()
-                    .bucket(&bucket)
-                    .key(&key)
-                    .send()
-                    .await
-                    .expect_err("deleted source key must not remain visible");
+                after_lock_contention(&format!("{context} DELETE source"), || {
+                    writer.delete_object().bucket(&bucket).key(&key).send()
+                })
+                .await??;
+                let missing = after_lock_contention(&format!("{context} HEAD deleted source"), || {
+                    reader.head_object().bucket(&bucket).key(&key).send()
+                })
+                .await?
+                .expect_err("deleted source key must not remain visible");
                 if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
                     return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
                 }
@@ -82,14 +337,22 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
     for handle in handles {
         inventory.extend(handle.await??);
     }
+    assert_eq!(
+        inventory.len(),
+        worker_count * rounds,
+        "every worker round must retain its acknowledged copy"
+    );
 
     let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
     for (node_idx, client) in clients.iter().enumerate() {
-        let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
+        let listed = after_lock_contention(&format!("node={node_idx} LIST retained"), || {
+            client.list_objects_v2().bucket(&bucket).prefix("retained/").send()
+        })
+        .await??;
         let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
         assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
         for (key, body) in &inventory {
-            assert_object_bytes(client, &bucket, key, body)
+            assert_workload_bytes(client, &bucket, key, body)
                 .await
                 .map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
         }

crates/e2e_test/src/distributed/scanner_heal_topology_fixture.rs

@@ -25,7 +25,8 @@
 
 use super::harness::{
     DistCluster, DistLayout, TestResult, cluster_admin, cluster_admin_ok, decommission_status_json, get_object_bytes,
-    payload_for, put_object, s3_probe_client, s3_probe_error_code, sha256_hex, unique_bucket, wait_until,
+    payload_for, put_object, rebalance_complete, rebalance_status_json, s3_probe_client, s3_probe_error_code, sha256_hex,
+    unique_bucket, wait_until,
 };
 use crate::chaos::{VersionShardCensus, census_object_version_on_disk};
 use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment};
@@ -47,6 +48,7 @@ use tokio::time::{Instant, sleep};
 use uuid::Uuid;
 
 pub(crate) const MOVEMENT_BARRIER_DIR_ENV: &str = "RUSTFS_E2E_DATA_MOVEMENT_BARRIER_DIR";
+pub(crate) const STAGE_AFTER_ENTRY_ADMISSION: &str = "after_entry_admission";
 pub(crate) const STAGE_BEFORE_TARGET_COMMIT: &str = "before_target_commit";
 pub(crate) const STAGE_BEFORE_SOURCE_CLEANUP: &str = "before_source_cleanup";
 pub(crate) const STAGE_AFTER_RESPONSIBILITY_PERSISTED: &str = "after_responsibility_persisted";
@@ -1166,8 +1168,21 @@ pub(crate) async fn clear_decommission(cluster: &RustFSTestClusterEnvironment, p
     .await
 }
 
-pub(crate) async fn stop_rebalance(cluster: &RustFSTestClusterEnvironment) -> TestResult<String> {
-    cluster_admin_ok(cluster, Method::POST, "/rustfs/admin/v3/rebalance/stop", None).await
+pub(crate) async fn stop_rebalance(cluster: &RustFSTestClusterEnvironment, expected_id: &str) -> TestResult<String> {
+    let path = "/rustfs/admin/v3/rebalance/stop";
+    let (status, response) = cluster_admin(cluster, Method::POST, path, None).await?;
+    if status.is_success() {
+        return Ok(response);
+    }
+    // The run can finish between the active observation and the stop request.
+    // Only completion of that same run makes a not-started response acceptable.
+    if status == StatusCode::NOT_FOUND
+        && response.contains("<Code>NoSuchResource</Code>")
+        && rebalance_complete(&rebalance_status_json(cluster).await?, expected_id)?
+    {
+        return Ok(response);
+    }
+    Err(format!("POST {path} failed: {status} {response}").into())
 }
 
 pub(crate) fn decommission_pool_entry(status: &Value, pool_id: usize) -> TestResult<Value> {
@@ -1585,6 +1600,129 @@ pub(crate) async fn assert_every_node_serves(dist: &DistCluster, bucket: &str, l
 mod tests {
     use super::*;
 
+    const REBALANCE_NOT_STARTED: &str =
+        "<Error><Code>NoSuchResource</Code><Message>pool rebalance is not started</Message></Error>";
+
+    async fn mock_rebalance_stop(
+        stop_status: StatusCode,
+        stop_body: &str,
+        run_status: Value,
+    ) -> TestResult<(TestResult<String>, Vec<String>)> {
+        use bytes::Bytes;
+        use http_body_util::Full;
+        use hyper::service::service_fn;
+        use hyper_util::rt::TokioIo;
+        use tokio::net::TcpListener;
+        use tokio::sync::mpsc;
+        use tokio::task::JoinSet;
+
+        let listener = TcpListener::bind("127.0.0.1:0").await?;
+        let mut cluster = RustFSTestClusterEnvironment::new(1).await?;
+        cluster.nodes[0].url = format!("http://{}", listener.local_addr()?);
+        let stop_body = stop_body.to_string();
+        let (requests_tx, mut requests_rx) = mpsc::unbounded_channel();
+        let mut server = JoinSet::new();
+        server.spawn(async move {
+            loop {
+                let (stream, _) = match listener.accept().await {
+                    Ok(connection) => connection,
+                    Err(error) => return Err::<(), Box<dyn std::error::Error + Send + Sync>>(error.into()),
+                };
+                let stop_body = stop_body.clone();
+                let run_status = run_status.clone();
+                let requests_tx = requests_tx.clone();
+                let handler = service_fn(move |request: hyper::Request<hyper::body::Incoming>| {
+                    let route = format!("{} {}", request.method(), request.uri().path());
+                    requests_tx.send(route.clone()).expect("record mock admin request");
+                    let (status, body) = match route.as_str() {
+                        "POST /rustfs/admin/v3/rebalance/stop" => (stop_status, stop_body.clone()),
+                        "GET /rustfs/admin/v3/rebalance/status" => (StatusCode::OK, run_status.to_string()),
+                        _ => (StatusCode::BAD_REQUEST, "unexpected admin request".to_string()),
+                    };
+                    async move { http::Response::builder().status(status).body(Full::new(Bytes::from(body))) }
+                });
+                hyper::server::conn::http1::Builder::new()
+                    .keep_alive(false)
+                    .serve_connection(TokioIo::new(stream), handler)
+                    .await?;
+            }
+        });
+        let outcome = tokio::time::timeout(Duration::from_secs(5), stop_rebalance(&cluster, "run-1")).await?;
+        server.abort_all();
+        let requests = std::iter::from_fn(|| requests_rx.try_recv().ok()).collect();
+        Ok((outcome, requests))
+    }
+
+    fn completed_rebalance() -> Value {
+        serde_json::json!({
+            "id": "run-1",
+            "pools": [
+                { "id": 0, "status": "Completed", "progress": { "versions": 113 } },
+                { "id": 1, "status": "None" }
+            ]
+        })
+    }
+
+    #[tokio::test]
+    async fn rebalance_stop_success_does_not_query_completion() -> TestResult {
+        let (outcome, requests) = mock_rebalance_stop(StatusCode::OK, "", Value::Null).await?;
+        assert_eq!(outcome?, "");
+        assert_eq!(requests, ["POST /rustfs/admin/v3/rebalance/stop"]);
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn rebalance_stop_accepts_completed_run_after_not_started_response() -> TestResult {
+        let (outcome, requests) =
+            mock_rebalance_stop(StatusCode::NOT_FOUND, REBALANCE_NOT_STARTED, completed_rebalance()).await?;
+        outcome?;
+        assert_eq!(
+            requests,
+            [
+                "POST /rustfs/admin/v3/rebalance/stop",
+                "GET /rustfs/admin/v3/rebalance/status"
+            ]
+        );
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn rebalance_stop_rejects_unproven_completion_and_other_errors() -> TestResult {
+        let complete = completed_rebalance();
+        let mut invalid = vec![Value::Null];
+        for (pointer, value) in [
+            ("/id", serde_json::json!("run-2")),
+            ("/pools/0/status", serde_json::json!("Started")),
+            ("/pools/0/status", serde_json::json!("Failed")),
+            ("/pools/0/status", serde_json::json!("Stopped")),
+            ("/pools/0/progress/versions", serde_json::json!(0)),
+        ] {
+            let mut status = complete.clone();
+            *status.pointer_mut(pointer).expect("completion fixture field") = value;
+            invalid.push(status);
+        }
+        let mut pending = complete.clone();
+        pending["stopPropagation"] = serde_json::json!({ "pendingTerminalReload": true });
+        invalid.push(pending);
+        let mut failed = complete.clone();
+        failed["pools"][0]["lastError"] = serde_json::json!("migration failed");
+        invalid.push(failed);
+        for status in invalid {
+            let (outcome, _) = mock_rebalance_stop(StatusCode::NOT_FOUND, REBALANCE_NOT_STARTED, status.clone()).await?;
+            assert!(outcome.is_err(), "unproven completion accepted: {status}");
+        }
+        for (code, body) in [
+            (StatusCode::FORBIDDEN, REBALANCE_NOT_STARTED),
+            (StatusCode::INTERNAL_SERVER_ERROR, REBALANCE_NOT_STARTED),
+            (StatusCode::NOT_FOUND, "<Error><Code>NoSuchBucket</Code></Error>"),
+        ] {
+            let (outcome, requests) = mock_rebalance_stop(code, body, complete.clone()).await?;
+            assert!(outcome.is_err(), "unrelated error accepted: {code} {body}");
+            assert_eq!(requests, ["POST /rustfs/admin/v3/rebalance/stop"]);
+        }
+        Ok(())
+    }
+
     #[test]
     fn version_ids_normalize_null_spellings() {
         for spelling in [None, Some(""), Some("null"), Some("00000000-0000-0000-0000-000000000000")] {

crates/e2e_test/src/distributed/scanner_heal_topology_test.rs

@@ -28,7 +28,7 @@ use super::harness::{
     wait_for_rebalance_active, wait_for_rebalance_complete,
 };
 use super::scanner_heal_topology_fixture::{
-    LiveWorkload, MOVEMENT_ACTIVE_TIMEOUT, MOVEMENT_COMPLETE_TIMEOUT, MovementOperation, SEED_POOL,
+    LiveWorkload, MOVEMENT_ACTIVE_TIMEOUT, MOVEMENT_COMPLETE_TIMEOUT, MovementOperation, SEED_POOL, STAGE_AFTER_ENTRY_ADMISSION,
     STAGE_AFTER_RESPONSIBILITY_PERSISTED, STAGE_BEFORE_SOURCE_CLEANUP, STAGE_BEFORE_TARGET_COMMIT, TopologyManifest, TopologyRun,
     assert_bucket_durability_strict, assert_every_node_serves, assert_probe_objects, assert_process_durability_strict,
     assert_seed_pool_census_unchanged, assert_single_pool_placement, cancel_decommission, capture_seed_pool_census,
@@ -274,6 +274,7 @@ async fn wait_for_rebalance_stopped_or_completed(run: &TopologyRun, rebalance_id
 #[tokio::test]
 async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart() -> TestResult {
     init_logging();
+    require_test_hooks();
     let mut run = TopologyRun::start("rebalance").await?;
     let manifest = seed_manifest(&run).await?;
     assert_bucket_durability_strict(&run.dist.cluster, &manifest.buckets()).await?;
@@ -288,23 +289,52 @@ async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart()
         .map(|entry| (entry.key.clone(), entry.sha256.clone()))
         .collect();
 
-    let first_id = start_rebalance(&run.dist.cluster).await?;
-    wait_for_rebalance_active(&run.dist.cluster, &first_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
-    let during_rebalance = put_probe_objects(&run.dist, &scratch, "during-rebalance", 4).await?;
-    let workload = tokio::spawn(run_live_workload(LiveWorkload {
-        clients: run.dist.clients()?,
-        bucket: scratch.clone(),
-        prefix: "live".to_string(),
-        writers: LIVE_WRITERS,
-        rounds: LIVE_ROUNDS,
-        baseline_bucket: manifest.plain_bucket.clone(),
-        baseline: baseline.clone(),
-        pause: Duration::from_millis(100),
-    }));
-    sleep(Duration::from_secs(2)).await;
-    stop_rebalance(&run.dist.cluster).await?;
-    let (first_outcome, first_status) = wait_for_rebalance_stopped_or_completed(&run, &first_id).await?;
-    let ops = workload.await??;
+    // Keep an admitted entry on the source until stop cancels admission. Start
+    // propagation may await that entry, so observe and stop before awaiting ACK.
+    let first_key = manifest.plain_key("external", 0)?;
+    run.movement_barrier
+        .arm_movement("rebalance", &manifest.plain_bucket, &first_key, STAGE_AFTER_ENTRY_ADMISSION)?;
+    let observe_first = async {
+        let (observed_id, reached) = tokio::time::timeout(MOVEMENT_ACTIVE_TIMEOUT, async {
+            let reached = run.movement_barrier.wait_reached(MOVEMENT_ACTIVE_TIMEOUT).await?;
+            let status = rebalance_status_json(&run.dist.cluster).await?;
+            let observed_id = status
+                .get("id")
+                .and_then(Value::as_str)
+                .ok_or("rebalance status omitted id")?;
+            wait_for_rebalance_active(&run.dist.cluster, observed_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
+            Ok::<_, Box<dyn std::error::Error + Send + Sync>>((observed_id.to_string(), reached))
+        })
+        .await??;
+        let probes = put_probe_objects(&run.dist, &scratch, "during-rebalance", 4).await?;
+        let workload = run_live_workload(LiveWorkload {
+            clients: run.dist.clients()?,
+            bucket: scratch.clone(),
+            prefix: "live".to_string(),
+            writers: LIVE_WRITERS,
+            rounds: LIVE_ROUNDS,
+            baseline_bucket: manifest.plain_bucket.clone(),
+            baseline: baseline.clone(),
+            pause: Duration::from_millis(100),
+        });
+        let stop = async {
+            sleep(Duration::from_secs(2)).await;
+            stop_rebalance(&run.dist.cluster, &observed_id).await?;
+            let (outcome, status) = wait_for_rebalance_stopped_or_completed(&run, &observed_id).await?;
+            if outcome != "stopped" {
+                return Err("rebalance completed before stop; mid-run stop was not exercised".into());
+            }
+            Ok::<_, Box<dyn std::error::Error + Send + Sync>>((outcome, status))
+        };
+        let (ops, (outcome, status)) = tokio::try_join!(workload, stop)?;
+        Ok::<_, Box<dyn std::error::Error + Send + Sync>>((observed_id, reached, probes, ops, outcome, status))
+    };
+    let (first_id, (observed_id, first_reached, during_rebalance, ops, first_outcome, first_status)) =
+        tokio::try_join!(start_rebalance(&run.dist.cluster), observe_first)?;
+    if first_id != observed_id {
+        return Err(format!("rebalance start acknowledged {first_id}, but the stopped run was {observed_id}").into());
+    }
+    run.movement_barrier.disarm()?;
     let live_state = expected_live_state(&ops);
 
     for node_idx in 0..run.dist.cluster.nodes.len() {
@@ -333,8 +363,40 @@ async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart()
     verify_from_every_node(&run.dist, &manifest, "after restart following the stop").await?;
     verify_live_state(&run.dist.client(0)?, &scratch, &live_state, "after restart following the stop").await?;
 
-    let second_id = start_rebalance(&run.dist.cluster).await?;
-    wait_for_rebalance_active(&run.dist.cluster, &second_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
+    // A short second run can finish before the start response reaches us.
+    // Hold one remaining source entry until its active state is observed.
+    let mut second_key = None;
+    for entry in manifest.plain_entries() {
+        if pools_holding(&run.dist, &manifest.plain_bucket, &entry.key, None)?.contains(&SEED_POOL) {
+            second_key = Some(entry.key.clone());
+            break;
+        }
+    }
+    let second_key = second_key.ok_or("no seed-pool object remains for the second rebalance")?;
+    let second_barrier = run.movement_barrier.next_phase();
+    second_barrier.arm_movement("rebalance", &manifest.plain_bucket, &second_key, STAGE_BEFORE_SOURCE_CLEANUP)?;
+    let observe_second = async {
+        let reached = second_barrier.wait_reached(MOVEMENT_ACTIVE_TIMEOUT).await?;
+        let status = rebalance_status_json(&run.dist.cluster).await?;
+        let observed_id = status
+            .get("id")
+            .and_then(Value::as_str)
+            .ok_or("rebalance status omitted id")?;
+        if observed_id == first_id {
+            return Err("the second rebalance reused the stopped run id".into());
+        }
+        wait_for_rebalance_active(&run.dist.cluster, observed_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
+        second_barrier.disarm()?;
+        second_barrier.release_movement()?;
+        Ok::<_, Box<dyn std::error::Error + Send + Sync>>((observed_id.to_string(), reached))
+    };
+    // Worker propagation can hold the start response until movement drains.
+    let (second_id, (observed_id, second_reached)) = tokio::try_join!(start_rebalance(&run.dist.cluster), async {
+        tokio::time::timeout(MOVEMENT_ACTIVE_TIMEOUT, observe_second).await?
+    })?;
+    if second_id != observed_id {
+        return Err(format!("rebalance start acknowledged {second_id}, but the active run was {observed_id}").into());
+    }
     wait_for_rebalance_complete(&run.dist.cluster, &second_id, MOVEMENT_COMPLETE_TIMEOUT).await?;
     verify_from_every_node(&run.dist, &manifest, "after the second rebalance completed").await?;
     verify_live_state(&run.dist.client(3)?, &scratch, &live_state, "after the second rebalance completed").await?;
@@ -351,8 +413,8 @@ async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart()
         &json!({
             "gate": "G-REB",
             "manifest": manifest_summary(&manifest),
-            "first_run": { "id": first_id, "outcome": first_outcome, "status": first_status, "versions_held_by_two_pools_after_stop": retained_after_stop },
-            "second_run": { "id": second_id, "versions_off_seed_pool": moved },
+            "first_run": { "id": first_id, "outcome": first_outcome, "status": first_status, "versions_held_by_two_pools_after_stop": retained_after_stop, "barrier_object": first_key, "barrier": first_reached },
+            "second_run": { "id": second_id, "versions_off_seed_pool": moved, "barrier_object": second_key, "barrier": second_reached },
             "live_ops": ops,
             "probe_placement_during_rebalance": during_placement,
             "process_durability_strict_nodes": strict_nodes,

crates/e2e_test/src/distributed/scanner_peer_bucket_test.rs

@@ -1636,20 +1636,53 @@ async fn hot_window_with_old_followers(
     let mixed_initial_physical = if mixed { json!(converged_shards(&run)?) } else { Value::Null };
     let scheduler_config = run.assert_default_cadence().await?;
     let baseline = run.quiet_baseline().await?;
-    let leader = run.leader().await?;
-    let leader_pid = run.pid(leader)?;
+    // A leader-lock refresh gap during old-binary replacement can make
+    // `leader()` briefly observe a different node. Poll until the coordinator
+    // is stable before comparing against the measured fleet. A persistent
+    // change to an old-binary node is a genuine election outcome and fails.
+    let (leader, leader_pid) = if mixed {
+        let expected_leader = usize::try_from(
+            mixed_fleet["leader"]
+                .as_u64()
+                .ok_or("mixed fleet omitted its measured leader")?,
+        )?;
+        let expected_pid = mixed_fleet["leader_pid"]
+            .as_u64()
+            .ok_or("mixed fleet omitted its measured leader pid")?;
+        let mut resolved = None;
+        for _ in 0..20 {
+            let candidate = run.leader().await?;
+            let candidate_pid = u64::from(run.pid(candidate)?);
+            if Some(u64::try_from(candidate)?) == mixed_fleet["leader"].as_u64() && Some(candidate_pid) == Some(expected_pid) {
+                resolved = Some((candidate, run.pid(candidate)?));
+                break;
+            }
+            tokio::time::sleep(std::time::Duration::from_millis(500)).await;
+        }
+        match resolved {
+            Some(pair) => pair,
+            None => {
+                let current = match run.leader().await {
+                    Ok(node) => Some((node, run.pid(node).ok())),
+                    Err(_) => None,
+                };
+                return Err(format!(
+                    "the hot window did not retain its measured new coordinator after stabilization \
+                     (expected node {expected_leader} pid {expected_pid}, observed {current:?})"
+                )
+                .into());
+            }
+        }
+    } else {
+        let leader = run.leader().await?;
+        (leader, run.pid(leader)?)
+    };
     let producer = (leader + 1) % 4;
     let producers = if fault.is_some() {
         vec![producer, (leader + 2) % 4]
     } else {
         vec![producer]
     };
-    if mixed
-        && (mixed_fleet["leader"].as_u64() != Some(u64::try_from(leader)?)
-            || mixed_fleet["leader_pid"].as_u64() != Some(u64::from(leader_pid)))
-    {
-        return Err("the hot window did not retain its measured new coordinator".into());
-    }
     let producer_pid = run.pid(producer)?;
     let producer_address = run.peer_address(producer);
     let [cold_a, cold_b, hot] = run.buckets.clone();

crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs

@@ -33,7 +33,7 @@ mod tests {
     use sha2::{Digest, Sha256};
     use std::collections::HashSet;
     use std::error::Error;
-    use std::io::{Read, Write};
+    use std::io::{Read, Seek, SeekFrom, Write};
     use std::net::SocketAddr;
     use std::path::{Path, PathBuf};
     use std::process::Command;
@@ -1654,6 +1654,89 @@ mod tests {
         TargetEndpointBlackhole,
     }
 
+    async fn release_heal_commit_barrier_after_shutdown(
+        log: &Path,
+        log_offset: u64,
+        barrier: &Path,
+    ) -> Result<serde_json::Value, Box<dyn Error + Send + Sync>> {
+        // Observe within the existing graceful process-stop budget.
+        timeout(Duration::from_secs(45), async {
+            loop {
+                let mut file = std::fs::File::open(log)?;
+                file.seek(SeekFrom::Start(log_offset))?;
+                let mut tail = String::new();
+                file.read_to_string(&mut tail)?;
+                if let Some(event) = tail
+                    .lines()
+                    .filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
+                    .find(|event| {
+                        event["event"] == "server_shutdown_state"
+                            && event["component"] == "main"
+                            && event["subsystem"] == "startup"
+                            && event["state"] == "stopping"
+                    })
+                {
+                    // Local heal has stopped and HTTP shutdown has been signalled.
+                    // Let admitted peer RPCs finish; keeping this synthetic barrier
+                    // until process exit would make the fixture block its own drain.
+                    std::fs::remove_file(barrier)?;
+                    return Ok(event);
+                }
+                sleep(Duration::from_millis(10)).await;
+            }
+        })
+        .await?
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn heal_commit_barrier_waits_for_new_shutdown_event() -> Result<(), Box<dyn Error + Send + Sync>> {
+        let directory = tempfile::tempdir()?;
+        let log = directory.path().join("server.log");
+        let barrier = directory.path().join("barrier");
+        let event = serde_json::json!({
+            "event": "server_shutdown_state", "component": "main", "subsystem": "startup", "state": "stopping"
+        });
+        std::fs::write(&log, format!("{event}\n"))?;
+        std::fs::write(&barrier, "bucket/prefix/")?;
+        let log_offset = log.metadata()?.len();
+        let mut writer = std::fs::OpenOptions::new().append(true).open(&log)?;
+        writeln!(writer, "{}", serde_json::json!({"event": "shutdown_signal_received"}))?;
+        writeln!(writer, "{}", serde_json::json!({"event": "server_shutdown_state", "state": "stopping"}))?;
+
+        let release = release_heal_commit_barrier_after_shutdown(&log, log_offset, &barrier);
+        tokio::pin!(release);
+        assert!(
+            futures::poll!(release.as_mut()).is_pending(),
+            "old or unrelated events must not release the barrier"
+        );
+        assert!(barrier.is_file(), "the partial-rebuild barrier must remain armed before shutdown");
+        tokio::time::advance(Duration::from_secs(6)).await;
+        assert!(
+            futures::poll!(release.as_mut()).is_pending(),
+            "background shutdown may take longer than five seconds within the process-stop budget"
+        );
+        writeln!(writer, "{event}")?;
+        tokio::time::advance(Duration::from_millis(10)).await;
+        assert_eq!(release.await?, event, "retain the actual new shutdown witness");
+        assert!(!barrier.exists(), "shutdown must release the synthetic hold");
+        Ok(())
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn heal_commit_barrier_missing_shutdown_fails_without_releasing() -> Result<(), Box<dyn Error + Send + Sync>> {
+        let directory = tempfile::tempdir()?;
+        let log = directory.path().join("server.log");
+        let barrier = directory.path().join("barrier");
+        std::fs::write(&log, "")?;
+        std::fs::write(&barrier, "bucket/prefix/")?;
+        let error = release_heal_commit_barrier_after_shutdown(&log, 0, &barrier)
+            .await
+            .expect_err("missing shutdown evidence must fail the fixture");
+        assert!(error.is::<tokio::time::error::Elapsed>());
+        assert!(barrier.is_file(), "a missing shutdown witness must not release repairs");
+        Ok(())
+    }
+
     async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
         let server_binary = rustfs_binary_path();
         let evidence_case = match scenario {
@@ -1791,6 +1874,7 @@ mod tests {
         let mut created_online_objects = Vec::with_capacity(online_object_count);
         let mut expected_manifests = Vec::with_capacity(online_object_count);
         let mut unclean_shutdown_marker_observed = None;
+        let mut shutdown_barrier_release = None;
         let mut attempt_count = 0usize;
         let max_online_attempts = online_object_count.saturating_mul(topology.total_drives().max(1));
         while expected_manifests.len() < online_object_count {
@@ -2373,7 +2457,21 @@ mod tests {
                     | InterruptionScenario::BackgroundTargetRestartEc84MultiSet
                     | InterruptionScenario::BackgroundCoordinatorRestart
             ) {
-                cluster.stop_node_gracefully(interruption_node).await?;
+                if interruption_node == 1
+                    && let Some(barrier) = &commit_barrier
+                {
+                    let target_log = PathBuf::from(format!("{log_dir}/node1.log"));
+                    let log_offset = target_log.metadata()?.len();
+                    let (stopped, released) = tokio::join!(
+                        cluster.stop_node_gracefully(interruption_node),
+                        release_heal_commit_barrier_after_shutdown(&target_log, log_offset, barrier),
+                    );
+                    // Await both branches even on error so the child is reaped.
+                    stopped?;
+                    shutdown_barrier_release = Some(released?);
+                } else {
+                    cluster.stop_node_gracefully(interruption_node).await?;
+                }
             } else {
                 cluster.stop_node(interruption_node)?;
             }
@@ -2415,7 +2513,9 @@ mod tests {
                     }
                 }
             }
-            if let Some(barrier) = &commit_barrier {
+            if shutdown_barrier_release.is_none()
+                && let Some(barrier) = &commit_barrier
+            {
                 std::fs::remove_file(barrier)?;
             }
             cluster.start_node_from_binary(interruption_node, &server_binary).await?;
@@ -2746,6 +2846,7 @@ mod tests {
                 "all_peers_bound_to_generation_window": true,
                 "pid_before": target_pid, "pid_after": restarted_pid,
                 "unclean_shutdown_marker": unclean_shutdown_marker_observed.unwrap_or(false),
+                "graceful_barrier_release": shutdown_barrier_release,
                 "objects": evidence_objects, "node_listings": node_listings,
             });
             if let Some(outage_write) = outage_write_diagnostic {

crates/e2e_test/src/lib.rs

@@ -283,6 +283,9 @@ mod ssec_copy_test;
 #[cfg(test)]
 mod multipart_storage_class_test;
 
+#[cfg(test)]
+mod multipart_upload_id_test;
+
 #[cfg(test)]
 mod storage_class_capability_test;
 

crates/e2e_test/src/multipart_upload_id_test.rs

@@ -0,0 +1,175 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Multipart upload ID regression tests.
+//!
+//! Every operation that addresses an existing multipart upload must report an
+//! upload ID that does not name a live upload as `404 NoSuchUpload`, whether
+//! the ID is malformed or is a well-formed ID of an aborted upload.
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging};
+    use aws_sdk_s3::Client;
+    use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
+    use aws_sdk_s3::primitives::ByteStream;
+    use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
+
+    type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
+
+    const BUCKET: &str = "multipart-upload-id";
+    const KEY: &str = "target.bin";
+    const COPY_SOURCE_KEY: &str = "copy-source.bin";
+    // RustFS upload IDs are URL-safe base64; '.' is outside that alphabet, so
+    // the object layer rejects this ID as malformed instead of looking it up.
+    const MALFORMED_UPLOAD_ID: &str = "malformed.upload.id";
+    const PART_NUMBER: i32 = 1;
+    const PART_BODY: &[u8] = b"multipart upload id regression part";
+    const PART_ETAG: &str = "\"00000000000000000000000000000000\"";
+    const NO_SUCH_UPLOAD_STATUS: u16 = 404;
+    const NO_SUCH_UPLOAD_CODE: &str = "NoSuchUpload";
+    const OBJECT_NOT_FOUND_STATUS: u16 = 404;
+
+    /// Describes how `result` differs from `404 NoSuchUpload`, or returns `None` when it matches.
+    fn no_such_upload_mismatch<T, E>(operation: &str, result: Result<T, SdkError<E>>) -> Option<String>
+    where
+        E: ProvideErrorMetadata + std::fmt::Debug,
+    {
+        let Err(error) = result else {
+            return Some(format!("{operation}: succeeded"));
+        };
+        let status = error.raw_response().map(|response| response.status().as_u16());
+        let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
+        (status != Some(NO_SUCH_UPLOAD_STATUS) || code != Some(NO_SUCH_UPLOAD_CODE))
+            .then(|| format!("{operation}: status {status:?}, code {code:?}: {error:?}"))
+    }
+
+    async fn start_server_with_copy_source() -> Result<(RustFSTestEnvironment, Client), Box<dyn std::error::Error + Send + Sync>>
+    {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(Vec::new()).await?;
+        env.create_test_bucket(BUCKET).await?;
+        let client = env.create_s3_client();
+        client
+            .put_object()
+            .bucket(BUCKET)
+            .key(COPY_SOURCE_KEY)
+            .body(ByteStream::from_static(PART_BODY))
+            .send()
+            .await?;
+        Ok((env, client))
+    }
+
+    /// Sends every operation that addresses an existing upload and asserts that all of them report `404 NoSuchUpload`.
+    async fn assert_every_upload_operation_reports_no_such_upload(client: &Client, upload_id: &str) {
+        let upload_part = client
+            .upload_part()
+            .bucket(BUCKET)
+            .key(KEY)
+            .upload_id(upload_id)
+            .part_number(PART_NUMBER)
+            .body(ByteStream::from_static(PART_BODY))
+            .send()
+            .await;
+        let upload_part_copy = client
+            .upload_part_copy()
+            .bucket(BUCKET)
+            .key(KEY)
+            .upload_id(upload_id)
+            .part_number(PART_NUMBER)
+            .copy_source(format!("/{BUCKET}/{COPY_SOURCE_KEY}"))
+            .send()
+            .await;
+        let list_parts = client.list_parts().bucket(BUCKET).key(KEY).upload_id(upload_id).send().await;
+        let complete = client
+            .complete_multipart_upload()
+            .bucket(BUCKET)
+            .key(KEY)
+            .upload_id(upload_id)
+            .multipart_upload(
+                CompletedMultipartUpload::builder()
+                    .parts(CompletedPart::builder().part_number(PART_NUMBER).e_tag(PART_ETAG).build())
+                    .build(),
+            )
+            .send()
+            .await;
+        let abort = client
+            .abort_multipart_upload()
+            .bucket(BUCKET)
+            .key(KEY)
+            .upload_id(upload_id)
+            .send()
+            .await;
+
+        let mismatches: Vec<String> = [
+            no_such_upload_mismatch("UploadPart", upload_part),
+            no_such_upload_mismatch("UploadPartCopy", upload_part_copy),
+            no_such_upload_mismatch("ListParts", list_parts),
+            no_such_upload_mismatch("CompleteMultipartUpload", complete),
+            no_such_upload_mismatch("AbortMultipartUpload", abort),
+        ]
+        .into_iter()
+        .flatten()
+        .collect();
+        assert!(
+            mismatches.is_empty(),
+            "upload ID {upload_id:?}: operations not reporting {NO_SUCH_UPLOAD_STATUS} {NO_SUCH_UPLOAD_CODE}:\n{}",
+            mismatches.join("\n")
+        );
+    }
+
+    /// Asserts that the rejected operations left neither a multipart upload nor an object under `KEY`.
+    async fn assert_no_upload_or_object_published(client: &Client) -> TestResult {
+        let uploads = client.list_multipart_uploads().bucket(BUCKET).prefix(KEY).send().await?;
+        assert!(uploads.uploads().is_empty(), "rejected operations must not leave an upload: {uploads:?}");
+        let head = client.head_object().bucket(BUCKET).key(KEY).send().await;
+        let status = head
+            .as_ref()
+            .err()
+            .and_then(|error| error.raw_response())
+            .map(|response| response.status().as_u16());
+        assert_eq!(
+            status,
+            Some(OBJECT_NOT_FOUND_STATUS),
+            "rejected operations must not publish {KEY}: {head:?}"
+        );
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn malformed_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
+        let (_env, client) = start_server_with_copy_source().await?;
+
+        assert_every_upload_operation_reports_no_such_upload(&client, MALFORMED_UPLOAD_ID).await;
+        assert_no_upload_or_object_published(&client).await
+    }
+
+    #[tokio::test]
+    async fn aborted_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
+        let (_env, client) = start_server_with_copy_source().await?;
+        let created = client.create_multipart_upload().bucket(BUCKET).key(KEY).send().await?;
+        let upload_id = created.upload_id().ok_or("CreateMultipartUpload returned no upload ID")?;
+        client
+            .abort_multipart_upload()
+            .bucket(BUCKET)
+            .key(KEY)
+            .upload_id(upload_id)
+            .send()
+            .await?;
+
+        assert_every_upload_operation_reports_no_such_upload(&client, upload_id).await;
+        assert_no_upload_or_object_published(&client).await
+    }
+}

crates/e2e_test/src/replication_extension_test.rs

@@ -31,8 +31,8 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{
-    BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DeleteMarkerEntry, ObjectVersion, ServerSideEncryption,
-    VersioningConfiguration,
+    BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Delete, DeleteMarkerEntry, ObjectIdentifier, ObjectVersion,
+    ServerSideEncryption, VersioningConfiguration,
 };
 use base64_simd::STANDARD as BASE64_STANDARD;
 use bytes::Bytes;
@@ -3729,6 +3729,8 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
     init_logging();
 
     let mut source_env = RustFSTestEnvironment::new().await?;
+    let source_log = format!("{}/source.log", source_env.temp_dir);
+    source_env.capture_log_path = Some(source_log.clone());
     let mut source_env_vars = replication_fast_env();
     source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
     source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
@@ -3791,19 +3793,66 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
         "target did not preserve the source delete-marker version ID"
     );
 
-    source_client
-        .delete_object()
+    let wait_for_source_writeback = || async {
+        let metrics_url = format!("{}/{source_bucket}?replication-metrics", source_env.url);
+        timeout(Duration::from_secs(30), async {
+            loop {
+                let response = signed_request(
+                    http::Method::GET,
+                    &metrics_url,
+                    &source_env.access_key,
+                    &source_env.secret_key,
+                    None,
+                    None,
+                )
+                .await?;
+                let metrics: serde_json::Value = response.error_for_status()?.json().await?;
+                let queued = metrics["queued"]["curr"]["count"]
+                    .as_f64()
+                    .ok_or("missing replication queue count")?;
+                if metrics["provider_available"] == true && queued == 0.0 {
+                    return Ok::<(), Box<dyn Error + Send + Sync>>(());
+                }
+                sleep(Duration::from_millis(50)).await;
+            }
+        })
+        .await?
+    };
+    wait_for_source_writeback().await?;
+
+    let purged = source_client
+        .delete_objects()
         .bucket(source_bucket)
-        .key(object_key)
-        .version_id(&purged_version_id)
+        .delete(
+            Delete::builder()
+                .objects(
+                    ObjectIdentifier::builder()
+                        .key(object_key)
+                        .version_id(&purged_version_id)
+                        .build()?,
+                )
+                .objects(
+                    ObjectIdentifier::builder()
+                        .key(object_key)
+                        .version_id(delete_marker_version_id)
+                        .build()?,
+                )
+                .build()?,
+        )
         .send()
         .await?;
+    assert!(purged.errors().is_empty(), "batch version purge failed: {:?}", purged.errors());
+    assert_eq!(purged.deleted().len(), 2, "both the data version and marker must be purged");
     assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
     let target_state = list_replication_state(&target_client, target_bucket).await?;
     assert!(
         target_state.iter().all(|entry| entry.version_id != purged_version_id),
         "target retained the explicitly purged object version"
     );
+    assert!(
+        target_state.iter().all(|entry| entry.version_id != delete_marker_version_id),
+        "target retained the explicitly purged delete marker"
+    );
     assert!(
         target_state
             .iter()
@@ -3819,6 +3868,19 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
         .await?;
     assert_eq!(retained.body.collect().await?.into_bytes().as_ref(), b"versioned replication payload v2");
 
+    // Wait for the source workers to finish their final writeback, which runs
+    // after the remote purge has already become visible (rustfs#9121).
+    wait_for_source_writeback().await?;
+    let logs = fs::read_to_string(source_log).await?;
+    let writeback_failures: Vec<_> = logs
+        .lines()
+        .filter(|line| line.contains("apply_replication_delete_state"))
+        .collect();
+    assert!(
+        writeback_failures.is_empty(),
+        "a completed batch purge must not report a source writeback failure: {writeback_failures:?}"
+    );
+
     Ok(())
 }
 

crates/e2e_test/src/rpc_fault_proxy.rs

@@ -2146,3 +2146,4 @@ mod tests {
         );
     }
 }
+// event-drill: isolated harness-only change

crates/e2e_test/src/upgrade_compatibility_test.rs

@@ -48,6 +48,7 @@ const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
 const G09_EVIDENCE_DIR_ENV: &str = "RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR";
 const RC5_COMMIT: &str = "40a2470feb567201165a5b809b7598bb4b1f68f5";
 const STABLE_1_0_1_COMMIT: &str = "6de965ae3c965a78ff819fbcd7acd4aa44177d92";
+const STABLE_1_0_4_COMMIT: &str = "46337942d55f3a8e6a97b147ca98d9ea8115555d";
 const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
 const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI=";
 const PLAIN_BUCKET: &str = "upgrade-plain-data";
@@ -954,7 +955,8 @@ async fn exercise_mixed_cluster(
     phase: &str,
     current_node: usize,
     previous_node: usize,
-) -> TestResult {
+    previous_has_delete_recovery: bool,
+) -> Result<Option<String>, BoxError> {
     let clients = cluster.create_all_clients()?;
     wait_for_upgrade_write_readiness(&clients, MIXED_BUCKET, phase, LISTING_CONVERGENCE_TIMEOUT).await?;
     wait_for_drive_readmission(&clients, MIXED_BUCKET, phase, DRIVE_READMISSION_TIMEOUT).await?;
@@ -1009,6 +1011,30 @@ async fn exercise_mixed_cluster(
         vec![u8::try_from(MULTIPART_WORKERS - 1)?; 64 * 1024]
     );
 
+    // 1.0.4 already implements the delete decision protocol. Use a separate
+    // old-writer key so the original seed still spans the entire rollout.
+    if previous_has_delete_recovery {
+        let key = format!(".upgrade-delete/{phase}/previous-writer");
+        previous_client
+            .put_object()
+            .bucket(MIXED_BUCKET)
+            .key(&key)
+            .body(ByteStream::from_static(UPGRADE_READINESS_BODY))
+            .send()
+            .await?;
+        for client in &clients {
+            assert_eq!(read_object(client, MIXED_BUCKET, &key, None).await?.1, UPGRADE_READINESS_BODY);
+        }
+        upgrade_probe_client(current_client)
+            .delete_object()
+            .bucket(MIXED_BUCKET)
+            .key(&key)
+            .send()
+            .await?;
+        assert_mixed_deletes_visible(&clients, std::slice::from_ref(&key)).await?;
+        return Ok(Some(key));
+    }
+
     // Every online participant must understand the delete decision protocol.
     // An upgrade phase can serve reads/writes while intentionally refusing a
     // modern DELETE; rejecting it must leave the old writer's seed intact.
@@ -1029,6 +1055,31 @@ async fn exercise_mixed_cluster(
         );
     }
 
+    Ok(None)
+}
+
+async fn assert_mixed_deletes_visible(clients: &[Client], keys: &[String]) -> TestResult {
+    for (node, client) in clients.iter().enumerate() {
+        for key in keys {
+            let error = upgrade_probe_client(client)
+                .get_object()
+                .bucket(MIXED_BUCKET)
+                .key(key)
+                .send()
+                .await
+                .expect_err("a successful mixed-version DELETE must remain visible through every reader");
+            assert_eq!(
+                error.raw_response().map(|response| response.status().as_u16()),
+                Some(404),
+                "node {node}: {key}"
+            );
+            assert_eq!(
+                error.as_service_error().and_then(ProvideErrorMetadata::code),
+                Some("NoSuchKey"),
+                "node {node}: {key}"
+            );
+        }
+    }
     Ok(())
 }
 
@@ -1112,8 +1163,9 @@ async fn wait_for_upgrade_writes(
             match response {
                 Ok(Ok(_)) => break,
                 Ok(Err(error)) => {
+                    let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
                     if error.raw_response().map(|response| response.status().as_u16()) != Some(503)
-                        || error.as_service_error().and_then(ProvideErrorMetadata::code) != Some("ServiceUnavailable")
+                        || !matches!(code, Some("ServiceUnavailable" | "SlowDown" | "SlowDownRead"))
                     {
                         return Err(format!("{phase}: node {node} {probe} failed: {error:?}").into());
                     }
@@ -2239,6 +2291,9 @@ async fn direct_upgrade_from_previous_release_preserves_object_contracts() -> Te
 async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts() -> TestResult {
     init_logging();
     let previous_binary = source_binary()?;
+    // Bind the expected protocol to the verified release source, never to the
+    // DELETE result itself. The older CI source pins retain their 503 contract.
+    let previous_has_delete_recovery = binary_source_revision(&previous_binary).await? == STABLE_1_0_4_COMMIT;
     let evidence_context = g09_evidence_context(&previous_binary).await?;
     let current_binary = rustfs_binary_path();
     let mut cluster = RustFSTestClusterEnvironment::new(MIXED_NODE_COUNT).await?;
@@ -2259,7 +2314,10 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
 
     cluster.stop_node(0)?;
     cluster.start_node_from_binary(0, &current_binary).await?;
-    exercise_mixed_cluster(&cluster, "one-current-node", 0, 1).await?;
+    let mut mixed_deleted_keys = Vec::new();
+    if let Some(key) = exercise_mixed_cluster(&cluster, "one-current-node", 0, 1, previous_has_delete_recovery).await? {
+        mixed_deleted_keys.push(key);
+    }
     // One controlled out-of-policy attempt proves fail-closed behavior below
     // native write quorum; positive MPU writers remain paused while mixed.
     assert_mixed_pending_refusal(&cluster, &pending, &sources).await?;
@@ -2267,7 +2325,9 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
     for (node_idx, phase) in [(1, "two-current-nodes"), (2, "one-previous-node")] {
         cluster.stop_node(node_idx)?;
         cluster.start_node_from_binary(node_idx, &current_binary).await?;
-        exercise_mixed_cluster(&cluster, phase, 0, 3).await?;
+        if let Some(key) = exercise_mixed_cluster(&cluster, phase, 0, 3, previous_has_delete_recovery).await? {
+            mixed_deleted_keys.push(key);
+        }
     }
 
     cluster.stop_node(3)?;
@@ -2277,6 +2337,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
     let completed_pending = finish_pending_multipart(&clients, &pending, pending_parts.concat()).await?;
     exercise_homogeneous_multipart(&cluster, "homogeneous-current").await?;
     verify_load_bodies(&clients, "homogeneous-previous", "multipart").await?;
+    assert_mixed_deletes_visible(&clients, &mixed_deleted_keys).await?;
 
     for (node_idx, client) in clients.iter().enumerate() {
         assert_eq!(
@@ -2325,6 +2386,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
     cluster.start_with_binary(&previous_binary).await?;
     prepare_release_baseline(&mut cluster, &previous_binary, "rollback").await?;
     let old_clients = cluster.create_all_clients()?;
+    assert_mixed_deletes_visible(&old_clients, &mixed_deleted_keys).await?;
     assert_resumed_multipart(&old_clients, &completed_pending).await?;
     for phase in ["homogeneous-previous", "homogeneous-current"] {
         verify_load_bodies(&old_clients, phase, "multipart").await?;
@@ -2389,6 +2451,8 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
                 "phases": phases,
                 "objects_per_phase": objects_per_phase,
                 "mixed_load": "plain-put",
+                "mixed_delete_protocol_supported": previous_has_delete_recovery,
+                "mixed_deleted_keys": mixed_deleted_keys,
                 "homogeneous_multipart_phases": ["homogeneous-previous", "homogeneous-current", "homogeneous-rollback"],
                 "multipart_objects_per_homogeneous_phase": MULTIPART_WORKERS * MULTIPART_UPLOADS_PER_WORKER,
                 "assertions": [
@@ -2421,7 +2485,11 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
                     "16x16 multipart load runs in homogeneous previous, current and rollback phases",
                     "one-current-node Complete returns typed 503 and preserves actual source authority and parts",
                     "the same legacy upload completes and replays once fully upgraded without adding a version",
-                    "modern DELETE fails closed and preserves every byte while old peers remain"
+                    if previous_has_delete_recovery {
+                        "mixed-version DELETE succeeds and stays absent through every reader after upgrade and rollback"
+                    } else {
+                        "modern DELETE fails closed and preserves every byte while old peers remain"
+                    }
                 ],
             }),
         )?;

crates/ecstore/Cargo.toml

@@ -31,6 +31,7 @@ workspace = true
 
 [features]
 default = []
+fips = ["rustfs-s3-client/fips", "rustfs-utils/fips"]
 diagnostic-read-version-caller = ["rustfs-io-metrics/diagnostic-read-version-caller"]
 # Sampled wall/CPU diagnostics only; does not change the read backend.
 diagnostic-local-batch = ["rustix/time"]

crates/ecstore/src/bucket/metadata_sys.rs

@@ -683,6 +683,7 @@ pub(crate) async fn update_validated_in<F>(
     bucket: &str,
     config_file: &str,
     data: Vec<u8>,
+    expected_incarnation_id: Option<Uuid>,
     validate: F,
 ) -> Result<OffsetDateTime>
 where
@@ -690,7 +691,7 @@ where
 {
     Box::pin(async {
         let sys = require_bucket_metadata_sys_in(ctx)?;
-        let guard = acquire_config_write_guard(sys.clone(), bucket).await?;
+        let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?;
         validate()?;
         update_under_config_write_guard(sys, &guard, config_file, data, None).await
     })
@@ -1901,6 +1902,15 @@ impl BucketMetadataSys {
         self.api.upgrade()
     }
 
+    /// Whether the backing `ECStore` has been dropped.
+    ///
+    /// Used by `InstanceContext::clear_bucket_metadata_sys_if_stale` during
+    /// `ECStore::drop` so a shared bootstrap context does not retain a
+    /// `BucketMetadataSys` whose weak handle is dangling.
+    pub(crate) fn is_store_stale(&self) -> bool {
+        self.api.upgrade().is_none()
+    }
+
     fn metadata_publish_lock(&self, bucket: &str) -> Arc<Mutex<MetadataPublishLockState>> {
         Self::bucket_lock_in(&self.metadata_publish_locks, bucket)
     }
@@ -5200,6 +5210,61 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn validated_config_update_rejects_a_different_bucket_incarnation() {
+        let (_dirs, store) = isolated_store_over_temp_disks().await;
+        init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let bucket = "catalog-cleanup-incarnation";
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create bucket");
+        let incarnation = store
+            .bucket_incarnation_id_from_disk(bucket)
+            .await
+            .expect("read bucket incarnation");
+        let called = std::sync::atomic::AtomicBool::new(false);
+        let result = store
+            .update_bucket_metadata_config_for_incarnation_validated(
+                bucket,
+                "quota.json",
+                br#"{"quota":1}"#.to_vec(),
+                Uuid::new_v4(),
+                || {
+                    called.store(true, std::sync::atomic::Ordering::SeqCst);
+                    Ok(())
+                },
+            )
+            .await;
+        assert!(matches!(result, Err(Error::BucketNotFound(name)) if name == bucket));
+        assert!(
+            !called.load(std::sync::atomic::Ordering::SeqCst),
+            "incarnation must be checked before the external fence"
+        );
+        let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
+            .await
+            .expect("read unchanged bucket metadata");
+        assert!(
+            metadata.quota_config_json.is_empty(),
+            "a stale cleanup must not modify the replacement bucket"
+        );
+        store
+            .update_bucket_metadata_config_for_incarnation_validated(
+                bucket,
+                "quota.json",
+                br#"{"quota":1}"#.to_vec(),
+                incarnation,
+                || Ok(()),
+            )
+            .await
+            .expect("the current incarnation permits the validated update");
+        let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
+            .await
+            .expect("read updated bucket metadata");
+        assert_eq!(metadata.quota_config_json, br#"{"quota":1}"#);
+    }
+
     #[tokio::test]
     #[serial]
     async fn quota_admission_rejects_stale_cached_no_quota() {

crates/ecstore/src/bucket/replication/replication_metadata_boundary.rs

@@ -33,6 +33,7 @@ pub(crate) struct ReplicationMetadataStore;
 impl ReplicationMetadataStore {
     pub(crate) const MRF_REPLICATION_FILE: &'static str = "config/replication/mrf.bin";
     pub(crate) const MRF_REPLICATION_RECOVERY_LOCK: &'static str = "config/replication/mrf.bin.recovery";
+    pub(crate) const MRF_REPLICATION_REPLAY_STAMP: &'static str = "config/replication/mrf.bin.replay";
     pub(crate) const FORCE_DELETE_REPLICATION_FILE: &'static str = "config/replication/force-delete.bin";
     pub(crate) const FORCE_DELETE_REPLICATION_TRANSACTION_LOCK: &'static str = "config/replication/force-delete.bin.transaction";
 
@@ -116,6 +117,10 @@ mod tests {
             ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
             "config/replication/mrf.bin.recovery"
         );
+        assert_eq!(
+            ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
+            "config/replication/mrf.bin.replay"
+        );
         assert_eq!(
             ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE,
             "config/replication/force-delete.bin"

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -86,6 +86,10 @@ const MRF_CORRUPT_FILE_PREFIX: &str = "config/replication/mrf.corrupt";
 const MRF_PENDING_CAP: usize = 200_000;
 const MRF_RETRY_INITIAL_DELAY: Duration = Duration::from_millis(100);
 const MRF_RETRY_MAX_DELAY: Duration = Duration::from_secs(5);
+/// Interval between runtime replays of the MRF journal, cluster-wide. A pass
+/// re-attempts every retained entry, so a target that stays down costs one
+/// attempt per entry per interval.
+const MRF_REPLAY_INTERVAL: Duration = Duration::from_secs(300);
 
 #[derive(Debug, Default)]
 pub struct DurableMrfBacklog {
@@ -615,7 +619,7 @@ async fn read_mrf_entries<S: ReplicationObjectIO>(storage: Arc<S>) -> Result<Vec
 }
 
 /// Acknowledge only the generation read by the recovery leader. Entries appended
-/// after that generation are retained as a suffix and replayed on the next startup.
+/// after that generation are retained as a suffix and replayed on the next pass.
 /// Lock order: recovery leader lock -> MRF journal object lock.
 async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
     storage: Arc<S>,
@@ -667,7 +671,7 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
     Err(EcstoreError::PreconditionFailed)
 }
 
-/// Acquires the MRF recovery leader lock for the startup replay.
+/// Acquires the MRF recovery leader lock for one journal replay pass.
 /// Returns `None` (after logging) when the lock cannot be created or another
 /// node is already processing the backlog.
 async fn acquire_mrf_recovery_guard<S: ReplicationStorage>(storage: &Arc<S>) -> Option<rustfs_lock::NamespaceLockGuard> {
@@ -1018,7 +1022,7 @@ fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_t
 }
 
 /// Acknowledges the replayed MRF prefix and returns the retained backlog.
-/// On acknowledgement failure the backlog is preserved for the next startup and
+/// On acknowledgement failure the backlog is preserved for the next replay pass and
 /// re-read (falling back to the replayed snapshot) so the published summary stays accurate.
 async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
     storage: &Arc<S>,
@@ -1033,7 +1037,7 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
                 component = LOG_COMPONENT_ECSTORE,
                 subsystem = LOG_SUBSYSTEM_REPLICATION,
                 error = %error,
-                "Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
+                "Failed to acknowledge the MRF recovery prefix; preserving it for the next replay pass"
             );
             match read_mrf_entries(storage.clone()).await {
                 Ok(current) => current,
@@ -1051,6 +1055,120 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
     }
 }
 
+fn unix_millis_now() -> i64 {
+    i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000).unwrap_or(i64::MAX)
+}
+
+/// Whether a periodic pass may run, given the start time of the last pass on
+/// any node. A stamp more than one interval ahead of this node's clock is skew
+/// or a bad value, so it does not suppress replay.
+fn mrf_replay_due(last_started_ms: Option<i64>, now_ms: i64, interval: Duration) -> bool {
+    let Some(last_started_ms) = last_started_ms else {
+        return true;
+    };
+    let interval_ms = i64::try_from(interval.as_millis()).unwrap_or(i64::MAX);
+    now_ms.saturating_sub(last_started_ms) >= interval_ms || last_started_ms.saturating_sub(now_ms) >= interval_ms
+}
+
+async fn read_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>) -> Option<i64> {
+    let data = ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP)
+        .await
+        .ok()?;
+    std::str::from_utf8(&data).ok()?.trim().parse().ok()
+}
+
+async fn write_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>, started_ms: i64) {
+    if let Err(error) = ReplicationConfigStore::save(
+        storage.clone(),
+        ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
+        started_ms.to_string().into_bytes(),
+    )
+    .await
+    {
+        warn!(
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REPLICATION,
+            error = %error,
+            "Failed to record the MRF replay start time"
+        );
+    }
+}
+
+/// One MRF journal replay pass. The recovery leader reads the journal,
+/// re-drives every entry, and acknowledges the replayed prefix; entries that
+/// could not be admitted, and entries appended meanwhile, stay for the next pass.
+///
+/// Every node runs the periodic loop, and the leader lock only serializes
+/// passes. With `min_interval`, a pass is skipped when a pass on any node
+/// started within that interval, read from the shared replay stamp, so a
+/// cluster replays once per interval rather than once per node. The startup
+/// pass passes `None` and always runs. A pass that finds entries records its
+/// start time. Returns whether the journal was read.
+/// Lock order: recovery leader lock -> replay stamp object lock.
+async fn replay_mrf_journal<S: ReplicationStorage>(storage: &Arc<S>, min_interval: Option<Duration>) -> bool {
+    let Some(recovery_guard) = acquire_mrf_recovery_guard(storage).await else {
+        return false;
+    };
+    let started_ms = unix_millis_now();
+    if let Some(interval) = min_interval
+        && !mrf_replay_due(read_mrf_replay_stamp(storage).await, started_ms, interval)
+    {
+        return false;
+    }
+
+    let Some(entries) = load_mrf_recovery_entries(storage).await else {
+        return true;
+    };
+    if !entries.is_empty() {
+        write_mrf_replay_stamp(storage, started_ms).await;
+    }
+    set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
+
+    let total = entries.len();
+    let mut queued_count = 0usize;
+    let mut retry_entries = Vec::new();
+
+    for entry in entries.iter() {
+        let Some(admission) = replay_mrf_entry(entry, storage, &mut retry_entries).await else {
+            continue;
+        };
+
+        if admission == ReplicationQueueAdmission::Missed {
+            retry_entries.push(entry.clone());
+        } else if admission == ReplicationQueueAdmission::Queued {
+            queued_count += 1;
+        }
+    }
+
+    let retained = resolve_retained_mrf_entries(storage, &recovery_guard, &entries, &retry_entries).await;
+    let retained_count = retained.len();
+    set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
+
+    if queued_count > 0 {
+        info!(
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REPLICATION,
+            recovered = queued_count,
+            total,
+            retained = retained_count,
+            "Replayed MRF entries admitted for retry"
+        );
+    }
+    true
+}
+
+/// Re-runs [`replay_mrf_journal`] every `period`. The recovery leader lock keeps
+/// one pass at a time across the cluster, including the startup pass, and the
+/// shared replay stamp spaces passes from different nodes by `period`.
+fn spawn_mrf_replay_loop<S: ReplicationStorage>(storage: Arc<S>, period: Duration) -> JoinHandle<()> {
+    tokio::spawn(async move {
+        loop {
+            tokio::time::sleep(period).await;
+            replay_mrf_journal(&storage, Some(period)).await;
+        }
+    })
+}
+
 #[derive(Debug, thiserror::Error)]
 #[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
 struct ResyncActiveConflictError {
@@ -1171,6 +1289,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
         // Start background tasks
         pool.start_mrf_persister().await;
         pool.start_mrf_processor().await;
+        pool.start_mrf_replayer().await;
         pool.start_force_delete_processor().await;
 
         pool
@@ -1639,55 +1758,26 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 
     /// Starts the MRF processor — one-shot at startup.
     ///
-    /// Reads the on-disk MRF file, re-injects admitted entries as Heal operations, and
-    /// rewrites any entries that could not be admitted for a later startup retry.
+    /// Replays the on-disk MRF journal once; [`Self::start_mrf_replayer`] repeats
+    /// the pass while the server runs.
     async fn start_mrf_processor(&self) {
         let storage = self.storage.clone();
-
         let handle = tokio::spawn(async move {
-            let Some(recovery_guard) = acquire_mrf_recovery_guard(&storage).await else {
-                return;
-            };
-
-            let Some(entries) = load_mrf_recovery_entries(&storage).await else {
-                return;
-            };
-            set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
-
-            let total = entries.len();
-            let mut queued_count = 0usize;
-            let mut retry_entries = Vec::new();
-
-            for entry in entries.iter() {
-                let Some(admission) = replay_mrf_entry(entry, &storage, &mut retry_entries).await else {
-                    continue;
-                };
-
-                if admission == ReplicationQueueAdmission::Missed {
-                    retry_entries.push(entry.clone());
-                } else if admission == ReplicationQueueAdmission::Queued {
-                    queued_count += 1;
-                }
-            }
-
-            let retained = resolve_retained_mrf_entries(&storage, &recovery_guard, &entries, &retry_entries).await;
-            let retained_count = retained.len();
-            set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
-
-            if queued_count > 0 {
-                info!(
-                    component = LOG_COMPONENT_ECSTORE,
-                    subsystem = LOG_SUBSYSTEM_REPLICATION,
-                    recovered = queued_count,
-                    total,
-                    retained = retained_count,
-                    "Replayed MRF entries admitted for retry"
-                );
-            }
+            replay_mrf_journal(&storage, None).await;
         });
         self.task_handles.lock().await.push(handle);
     }
 
+    /// Starts the runtime MRF replayer — ongoing background task.
+    ///
+    /// Entries reach the journal while the server runs (worker-queue overflow,
+    /// delete-marker purge intents whose watch window drained with targets still
+    /// dirty). Replaying only at startup left them waiting for a restart.
+    async fn start_mrf_replayer(&self) {
+        let handle = spawn_mrf_replay_loop(self.storage.clone(), MRF_REPLAY_INTERVAL);
+        self.task_handles.lock().await.push(handle);
+    }
+
     async fn start_force_delete_processor(&self) {
         let storage = self.storage.clone();
         let handle =
@@ -1755,7 +1845,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
     /// writes them to the on-disk MRF file every flush interval (default 10s,
     /// overridable via `RUSTFS_REPL_MRF_FLUSH_INTERVAL_MS`) or when 1 000 new
     /// entries accumulate. Runtime writers append under the journal lock; only the
-    /// startup recovery leader may remove an acknowledged prefix.
+    /// recovery leader of a replay pass may remove an acknowledged prefix.
     async fn start_mrf_persister(&self) {
         let Some(mut rx) = self.mrf_save_rx.lock().await.take() else {
             return;
@@ -3421,6 +3511,8 @@ mod tests {
         allow_first_read: Notify,
         read_count: AtomicUsize,
         delete_storage_calls: AtomicUsize,
+        delete_error: StdMutex<EcstoreError>,
+        replay_stamp: StdMutex<Option<Vec<u8>>>,
         reported_size: StdMutex<Option<i64>>,
         stream_read_bytes: Arc<AtomicUsize>,
         write_count: AtomicUsize,
@@ -3488,6 +3580,24 @@ mod tests {
             _h: Self::HeaderMap,
             _opts: &Self::ObjectOptions,
         ) -> Result<Self::GetObjectReader, Self::Error> {
+            if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
+                let stamp = self
+                    .shared
+                    .replay_stamp
+                    .lock()
+                    .expect("test replay stamp lock should not be poisoned")
+                    .clone()
+                    .ok_or(EcstoreError::FileNotFound)?;
+                return Ok(Self::GetObjectReader {
+                    object_info: ObjectInfo {
+                        size: i64::try_from(stamp.len()).expect("test stamp length should fit i64"),
+                        ..Default::default()
+                    },
+                    stream: Box::new(Cursor::new(stamp)),
+                    buffered_body: None,
+                    body_source: Default::default(),
+                });
+            }
             if !object.ends_with("/.replication/resync.bin")
                 && !object.ends_with("config/replication/mrf.bin")
                 && !object.ends_with("config/replication/force-delete.bin")
@@ -3551,6 +3661,16 @@ mod tests {
                 let lock = self.new_ns_lock(bucket, object).await?;
                 Some(lock.get_write_lock(Duration::from_secs(10)).await?)
             };
+            if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
+                let mut stamp = Vec::new();
+                data.stream.read_to_end(&mut stamp).await.map_err(EcstoreError::from)?;
+                *self
+                    .shared
+                    .replay_stamp
+                    .lock()
+                    .expect("test replay stamp lock should not be poisoned") = Some(stamp);
+                return Ok(ObjectInfo::default());
+            }
             if opts.http_preconditions.is_some()
                 && let Some(replacement) = self
                     .shared
@@ -3672,7 +3792,12 @@ mod tests {
             _opts: Self::ObjectOptions,
         ) -> Result<Self::ObjectInfo, Self::Error> {
             self.shared.delete_storage_calls.fetch_add(1, Ordering::SeqCst);
-            Err(EcstoreError::NotImplemented)
+            Err(self
+                .shared
+                .delete_error
+                .lock()
+                .expect("delete error lock should not be poisoned")
+                .clone())
         }
 
         async fn delete_objects(
@@ -4171,6 +4296,8 @@ mod tests {
             allow_first_read: Notify::new(),
             read_count: AtomicUsize::new(0),
             delete_storage_calls: AtomicUsize::new(0),
+            delete_error: StdMutex::new(EcstoreError::NotImplemented),
+            replay_stamp: StdMutex::new(None),
             reported_size: StdMutex::new(None),
             stream_read_bytes: Arc::new(AtomicUsize::new(0)),
             write_count: AtomicUsize::new(0),
@@ -4221,6 +4348,103 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn delete_worker_already_purged_source_completes_without_retry() {
+        let arn = "arn:test";
+        for delete_marker_version in [false, true] {
+            for version_id in [Uuid::new_v4(), Uuid::nil()] {
+                for error in [
+                    EcstoreError::VersionNotFound("source".to_string(), "purged".to_string(), version_id.to_string()),
+                    EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
+                    EcstoreError::FileVersionNotFound,
+                    EcstoreError::FileNotFound,
+                ] {
+                    let shared = empty_resync_shared_state();
+                    *shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
+                    let storage = Arc::new(LoadResyncNodeStore::new("purged-source", shared.clone()));
+                    let delete = DeletedObjectReplicationInfo {
+                        bucket: "source".to_string(),
+                        delete_object: ReplicationDeletedObject {
+                            object_name: "purged".to_string(),
+                            version_id: (!delete_marker_version).then_some(version_id),
+                            delete_marker_version_id: delete_marker_version.then_some(version_id),
+                            replication_state: Some(ReplicationState {
+                                replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
+                                purge_targets: HashMap::from([(arn.to_string(), VersionPurgeStatusType::Complete)]),
+                                ..Default::default()
+                            }),
+                            ..Default::default()
+                        },
+                        op_type: ReplicationType::Delete,
+                        ..Default::default()
+                    };
+
+                    assert!(
+                        replicate_delete_with_outcome(delete, storage).await,
+                        "a confirmed target purge must complete when the source is already absent (marker={delete_marker_version}, version={version_id})"
+                    );
+                    assert_eq!(
+                        shared.delete_storage_calls.load(Ordering::SeqCst),
+                        2 + usize::from(!delete_marker_version && !version_id.is_nil()),
+                        "the worker must lock and attempt writeback, reading a non-null data version first"
+                    );
+                }
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn delete_worker_missing_source_does_not_hide_unfinished_replication() {
+        let arn = "arn:test";
+        for (version_purge, target_status, error) in [
+            (
+                true,
+                VersionPurgeStatusType::Pending,
+                EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
+            ),
+            (true, VersionPurgeStatusType::Complete, EcstoreError::ErasureWriteQuorum),
+            (
+                false,
+                VersionPurgeStatusType::Empty,
+                EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
+            ),
+        ] {
+            let shared = empty_resync_shared_state();
+            *shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
+            let storage = Arc::new(LoadResyncNodeStore::new("unfinished-delete", shared));
+            let delete = DeletedObjectReplicationInfo {
+                bucket: "source".to_string(),
+                delete_object: ReplicationDeletedObject {
+                    object_name: "purged".to_string(),
+                    version_id: version_purge.then(Uuid::new_v4),
+                    delete_marker: !version_purge,
+                    replication_state: Some(ReplicationState {
+                        replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
+                        targets: if version_purge {
+                            HashMap::new()
+                        } else {
+                            HashMap::from([(arn.to_string(), ReplicationStatusType::Completed)])
+                        },
+                        purge_targets: if version_purge {
+                            HashMap::from([(arn.to_string(), target_status)])
+                        } else {
+                            HashMap::new()
+                        },
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                },
+                op_type: ReplicationType::Delete,
+                ..Default::default()
+            };
+
+            assert!(
+                !replicate_delete_with_outcome(delete, storage).await,
+                "unfinished target work, source I/O failures, and marker creation must remain retryable"
+            );
+        }
+    }
+
     #[tokio::test]
     async fn delete_worker_preserves_persisted_admission_without_live_config() {
         let arn = "arn:test";
@@ -5304,6 +5528,8 @@ mod tests {
                 allow_first_read: Notify::new(),
                 read_count: AtomicUsize::new(0),
                 delete_storage_calls: AtomicUsize::new(0),
+                delete_error: StdMutex::new(EcstoreError::NotImplemented),
+                replay_stamp: StdMutex::new(None),
                 reported_size: StdMutex::new(None),
                 stream_read_bytes: Arc::new(AtomicUsize::new(0)),
                 write_count: AtomicUsize::new(0),
@@ -6268,6 +6494,108 @@ mod tests {
         assert_eq!(retained.target_arns, entry.target_arns);
     }
 
+    #[tokio::test]
+    async fn mrf_replay_loop_drains_entries_persisted_after_startup() {
+        let shared = empty_resync_shared_state();
+        let storage = Arc::new(LoadResyncNodeStore::new("mrf-runtime-replay", shared.clone()));
+        // A force-delete intent from a superseded generation replays as
+        // Skipped, so a pass that reaches it acknowledges it off the journal.
+        let stale_intent = |object: &str| MrfReplicateEntry {
+            bucket: "mrf-runtime-replay".to_string(),
+            object: object.to_string(),
+            op: MrfOpKind::Delete,
+            force_delete_id: Some(uuid::Uuid::new_v4()),
+            force_delete_local_commit: false,
+            target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
+            ..Default::default()
+        };
+        let journal_drained = || async {
+            tokio::time::timeout(Duration::from_secs(5), async {
+                while !shared.data.lock().expect("test data lock should not be poisoned").is_empty() {
+                    tokio::time::sleep(Duration::from_millis(10)).await;
+                }
+            })
+            .await
+            .is_ok()
+        };
+
+        let handle = spawn_mrf_replay_loop(storage, Duration::from_millis(20));
+        for object in ["first", "second"] {
+            *shared.data.lock().expect("test data lock should not be poisoned") =
+                encode_mrf_file(&[stale_intent(object)]).expect("MRF entry should encode");
+            assert!(
+                journal_drained().await,
+                "an MRF entry persisted while the server runs must be replayed without a restart ({object})"
+            );
+        }
+        handle.abort();
+    }
+
+    #[tokio::test]
+    async fn mrf_periodic_replay_runs_once_per_interval_across_nodes() {
+        let shared = empty_resync_shared_state();
+        let node_a = Arc::new(LoadResyncNodeStore::new("mrf-node-a", shared.clone()));
+        let node_b = Arc::new(LoadResyncNodeStore::new("mrf-node-b", shared.clone()));
+        let interval = Duration::from_secs(1);
+        let persist_backlog = |object: &str| {
+            let entry = MrfReplicateEntry {
+                bucket: "mrf-node-replay".to_string(),
+                object: object.to_string(),
+                op: MrfOpKind::Delete,
+                force_delete_id: Some(uuid::Uuid::new_v4()),
+                force_delete_local_commit: false,
+                target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
+                ..Default::default()
+            };
+            *shared.data.lock().expect("test data lock should not be poisoned") =
+                encode_mrf_file(&[entry]).expect("MRF entry should encode");
+        };
+        let journal_is_empty = || shared.data.lock().expect("test data lock should not be poisoned").is_empty();
+
+        persist_backlog("first");
+        assert!(replay_mrf_journal(&node_a, Some(interval)).await, "node A's periodic pass should replay");
+        assert!(journal_is_empty(), "node A's pass should acknowledge the replayed entry");
+
+        // Node B's timer fires within the same interval with a backlog present.
+        persist_backlog("second");
+        assert!(
+            !replay_mrf_journal(&node_b, Some(interval)).await,
+            "a node must not replay within one interval of another node's pass"
+        );
+        assert!(!journal_is_empty(), "the backlog should wait for the next interval");
+
+        assert!(
+            replay_mrf_journal(&node_b, None).await,
+            "the startup pass should replay regardless of the shared stamp"
+        );
+        assert!(journal_is_empty(), "the startup pass should acknowledge the replayed entry");
+
+        persist_backlog("third");
+        tokio::time::sleep(interval).await;
+        assert!(
+            replay_mrf_journal(&node_a, Some(interval)).await,
+            "a periodic pass should replay once the interval has elapsed"
+        );
+        assert!(journal_is_empty(), "the due pass should acknowledge the replayed entry");
+    }
+
+    #[test]
+    fn mrf_replay_due_tolerates_missing_and_skewed_stamps() {
+        let interval = Duration::from_secs(300);
+        let now = 1_000_000_000;
+        assert!(mrf_replay_due(None, now, interval));
+        assert!(!mrf_replay_due(Some(now - 299_999), now, interval));
+        assert!(mrf_replay_due(Some(now - 300_000), now, interval));
+        assert!(
+            !mrf_replay_due(Some(now + 1_000), now, interval),
+            "small clock skew should still space passes"
+        );
+        assert!(
+            mrf_replay_due(Some(now + 300_000), now, interval),
+            "a stamp an interval ahead must not suppress replay"
+        );
+    }
+
     #[test]
     fn mrf_entry_delete_marker_roundtrip() {
         let dm_vid = Uuid::new_v4();

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -2809,17 +2809,22 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
             true
         }
         Err(e) => {
-            error!(
-                event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
-                component = LOG_COMPONENT_ECSTORE,
-                subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
-                bucket = %bucket,
-                arn = %dobj.target_arn,
-                object = %dobj.delete_object.object_name,
-                operation = "apply_replication_delete_state",
-                error = %e,
-                "Replication target operation failed"
-            );
+            // A batch purge or another worker may have removed this version
+            // already. Remote failures still control the final outcome below.
+            let source_already_purged = is_version_purge && (is_err_object_not_found(&e) || is_err_version_not_found(&e));
+            if !source_already_purged {
+                error!(
+                    event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
+                    bucket = %bucket,
+                    arn = %dobj.target_arn,
+                    object = %dobj.delete_object.object_name,
+                    operation = "apply_replication_delete_state",
+                    error = %e,
+                    "Replication target operation failed"
+                );
+            }
             send_event(EventArgs {
                 event_name,
                 bucket_name: bucket.clone(),
@@ -2832,7 +2837,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
                 },
                 ..Default::default()
             });
-            false
+            source_already_purged
         }
     };
 
@@ -3010,7 +3015,7 @@ const DELETE_MARKER_PURGE_WATCH_INTERVAL: TokioDuration = TokioDuration::from_se
 /// If the marker disappears (deleted before or while the replica landed),
 /// purge the replicated marker from the targets, retrying failed targets on
 /// later rounds. When the window drains with targets still dirty, persist the
-/// purge intent as a durable MRF entry so the next startup replays it through
+/// purge intent as a durable MRF entry so the next MRF replay pass re-drives it through
 /// `purge_stale_delete_marker_targets`.
 async fn watch_and_purge_source_delete_marker<S: ReplicationStorage>(
     bucket: String,
@@ -3101,7 +3106,7 @@ async fn enqueue_delete_marker_purge_mrf(dobj: &DeletedObjectReplicationInfo, fa
 /// Heal callers retry for the full watch window because the startup MRF
 /// processor runs before bucket metadata (and thus target clients) finishes
 /// initializing — the first pass can see `target_client_missing` and a later
-/// round resolves the client; the replay loop is serial and startup-only, so
+/// round resolves the client; replay passes are serial background work, so
 /// blocking it for up to the window per dirty entry is acceptable. Live
 /// callers run on replication workers where a down target would pin a worker
 /// for the whole window, so they attempt once and lean on the durable intent

crates/ecstore/src/cache_value/metacache_set.rs

@@ -747,6 +747,16 @@ async fn list_path_raw_inner(
             // Start every missing head read in the same round so one stalled
             // disk cannot multiply the wait budget by the erasure-set width.
             // Outcomes are still consumed below in stable disk-index order.
+            if !revjob_rx.is_cancelled() {
+                for (i, reader) in readers.iter_mut().enumerate() {
+                    if errs[i].is_none()
+                        && pending_entries[i].is_none()
+                        && let Some(Ok(Some(entry))) = reader.try_peek_buffered()
+                    {
+                        pending_entries[i] = Some(entry);
+                    }
+                }
+            }
             let concurrent_peeks = readers.iter_mut().enumerate().filter_map(|(i, reader)| {
                 if errs[i].is_some() || pending_entries[i].is_some() {
                     return None;
@@ -878,7 +888,7 @@ async fn list_path_raw_inner(
                 }
                 if entry.name == current.name {
                     // If exact match, we agree.
-                    if current.matches_strict(&entry) {
+                    if current.matches_strict_borrowed(&entry) {
                         top_entries[i] = Some(entry);
                         agree += 1;
                     } else {
@@ -986,7 +996,9 @@ async fn list_path_raw_inner(
 
             if agree == readers.len() && !opts.preserve_replica_metadata {
                 for r in readers.iter_mut() {
-                    let _ = r.skip(1).await;
+                    if !r.skip_current() {
+                        let _ = r.skip(1).await;
+                    }
                 }
 
                 if let Some(agreed_fn) = opts.agreed.as_ref() {
@@ -1009,7 +1021,7 @@ async fn list_path_raw_inner(
             }
 
             for (i, r) in readers.iter_mut().enumerate() {
-                if top_entries[i].is_some() {
+                if top_entries[i].is_some() && !r.skip_current() {
                     let _ = r.skip(1).await;
                 }
             }

crates/ecstore/src/cluster/rpc/http_auth.rs

@@ -32,7 +32,6 @@ use crate::storage_api_contracts::internode::{
     PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_CAPABILITY_VERSION,
 };
 
-use hmac::{Hmac, KeyInit, Mac};
 use http::uri::Authority;
 use http::{HeaderMap, HeaderValue, Method, Uri};
 #[cfg(test)]
@@ -47,6 +46,7 @@ use rustfs_io_metrics::internode_metrics::{
     global_internode_metrics,
 };
 use rustfs_object_data_cache::{MemoryBasis, resolve_effective_memory};
+use rustfs_utils::crypto::{HmacSha256, KeyInit, Mac};
 use rustfs_utils::get_env_bool;
 use sha2::Digest as _;
 use sha2::Sha256;
@@ -58,8 +58,6 @@ use time::OffsetDateTime;
 use tracing::{error, info, warn};
 use uuid::Uuid;
 
-type HmacSha256 = Hmac<Sha256>;
-
 const SIGNATURE_HEADER: &str = "x-rustfs-signature";
 pub const RPC_BACKGROUND_ADMISSION_HEADER: &str = "x-rustfs-rpc-background-admission";
 const RPC_BACKGROUND_ADMISSION_V1: &str = "v1";

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -1637,6 +1637,7 @@ impl RemoteDisk {
         let initial_probe_ok = Self::perform_connectivity_check(&addr).await.is_ok();
         if initial_probe_ok {
             health.record_operation_success(&endpoint, "connectivity_probe_success");
+            rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
         } else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
             warn!(
                 event = EVENT_REMOTE_DISK_HEALTH,
@@ -1708,6 +1709,7 @@ impl RemoteDisk {
                     // Perform basic connectivity check
                     if Self::perform_connectivity_check(&addr).await.is_ok() {
                         health.record_operation_success(&endpoint, "connectivity_probe_success");
+                        rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
                     } else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
                         warn!(
                             event = EVENT_REMOTE_DISK_HEALTH,
@@ -1778,6 +1780,7 @@ impl RemoteDisk {
                 }
                 _ = interval.tick() => {
                     if Self::perform_recovery_probe(&addr, &endpoint).await.is_ok() {
+                        rustfs_io_metrics::internode_metrics::record_peer_reachable(&addr);
                         let became_online = health.mark_recovery_success(&endpoint, "disk_info_probe_success");
                         debug!(
                             event = EVENT_REMOTE_DISK_HEALTH,
@@ -2009,6 +2012,7 @@ impl RemoteDisk {
             let operation_result = operation().await;
             if operation_result.is_ok() {
                 self.health.record_operation_success(&self.endpoint, "operation_success");
+                rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
             }
             self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
                 .await;
@@ -2023,6 +2027,7 @@ impl RemoteDisk {
                 // Log success; the waiting guard balances every exit path.
                 if operation_result.is_ok() {
                     self.health.record_operation_success(&self.endpoint, "operation_success");
+                    rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
                 }
                 self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
                     .await;
@@ -5740,6 +5745,7 @@ mod tests {
         object_read_all_disks: Arc<StdMutex<Vec<String>>>,
         format_data: Bytes,
         read_all_data: Bytes,
+        fail_format_reads: Arc<AtomicBool>,
         next_format_read_gate: Arc<StdMutex<Option<Arc<TestFormatReadGate>>>>,
         unsupported_methods: Arc<StdMutex<Vec<String>>>,
     }
@@ -5759,6 +5765,7 @@ mod tests {
                 object_read_all_disks: Arc::default(),
                 format_data,
                 read_all_data,
+                fail_format_reads: Arc::default(),
                 next_format_read_gate: Arc::default(),
                 unsupported_methods: Arc::default(),
             }
@@ -5882,6 +5889,9 @@ mod tests {
                                         gate.entered.notify_one();
                                         gate.release.cancelled().await;
                                     }
+                                    if peer.fail_format_reads.load(Ordering::Acquire) {
+                                        return Err(Status::unavailable("format read temporarily unavailable"));
+                                    }
                                     peer.format_data.clone()
                                 } else {
                                     peer.object_read_all_disks
@@ -8847,6 +8857,124 @@ mod tests {
         peer.stop().await;
     }
 
+    #[tokio::test]
+    #[serial(remote_disk_recovery_probe)]
+    async fn endpoint_reconnect_failure_preserves_registered_remote_recovery() {
+        temp_env::async_with_vars(
+            [
+                (crate::disk::disk_store::ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, Some("true")),
+                (rustfs_config::ENV_DRIVE_RETURNING_SUCCESS_THRESHOLD, Some("1")),
+            ],
+            async {
+                runtime_sources::ensure_test_rpc_secret();
+                let mut reference = crate::layout::format::FormatV3::new(1, 1);
+                reference.erasure.this = reference.erasure.sets[0][0];
+                let peer = TestGrpcPeer::spawn(
+                    Bytes::from(reference.to_json().expect("format should serialize")),
+                    Bytes::from_static(b"recovered-peer-data"),
+                )
+                .await
+                .expect("authenticated regression peer is required");
+                let set = missing_read_set(&peer, reference).await;
+                let disk = crate::disk::new_disk(
+                    &set.set_endpoints[0],
+                    &DiskOption {
+                        cleanup: false,
+                        health_check: true,
+                    },
+                )
+                .await
+                .expect("registered remote disk should construct");
+                let crate::disk::Disk::Remote(remote) = disk.as_ref() else {
+                    panic!("fixture must create a remote disk");
+                };
+                assert!(remote.health_check);
+                remote.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
+                set.disks.write().await[0] = Some(Arc::clone(&disk));
+
+                peer.peer.fail_format_reads.store(true, Ordering::Release);
+                set.connect_disks().await;
+                peer.peer.fail_format_reads.store(false, Ordering::Release);
+                let cancelled_after_failure = remote.cancel_token.is_cancelled();
+                assert!(peer.peer.read_all_calls() > 0, "reconnect must attempt the failing format RPC");
+                assert!(
+                    Arc::ptr_eq(&disk, set.disks.read().await[0].as_ref().expect("existing slot must remain")),
+                    "a failed reconnect must not publish its unvalidated candidate"
+                );
+                remote.spawn_recovery_monitor_if_needed();
+                let recovery = time::timeout(Duration::from_secs(5), async {
+                    while !disk.is_online().await {
+                        tokio::task::yield_now().await;
+                    }
+                    disk.read_all("bucket", "object").await
+                })
+                .await;
+                disk.close().await.expect("stop the registered disk's monitoring");
+                peer.stop().await;
+                assert!(!cancelled_after_failure, "failed replacement must retain the registered monitor");
+                assert_eq!(
+                    recovery
+                        .expect("failed reconnect must leave the registered disk able to recover without another sweep")
+                        .expect("the recovered registered disk must serve real I/O"),
+                    Bytes::from_static(b"recovered-peer-data")
+                );
+            },
+        )
+        .await;
+    }
+
+    #[tokio::test]
+    #[serial(remote_disk_recovery_probe)]
+    async fn endpoint_reconnect_stops_old_remote_monitor_only_after_publication() {
+        runtime_sources::ensure_test_rpc_secret();
+        let mut reference = crate::layout::format::FormatV3::new(1, 1);
+        reference.erasure.this = reference.erasure.sets[0][0];
+        let peer = TestGrpcPeer::spawn(
+            Bytes::from(reference.to_json().expect("format should serialize")),
+            Bytes::from_static(b"replacement-peer-data"),
+        )
+        .await
+        .expect("authenticated regression peer is required");
+        let set = missing_read_set(&peer, reference).await;
+        let disk = crate::disk::new_disk(
+            &set.set_endpoints[0],
+            &DiskOption {
+                cleanup: false,
+                health_check: true,
+            },
+        )
+        .await
+        .expect("registered remote disk should construct");
+        let crate::disk::Disk::Remote(remote) = disk.as_ref() else {
+            panic!("fixture must create a remote disk");
+        };
+        remote.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
+        set.disks.write().await[0] = Some(Arc::clone(&disk));
+        let gate = peer.peer.pause_next_format_read();
+        let pending_set = Arc::clone(&set);
+        let pending = tokio::spawn(async move { pending_set.connect_disks().await });
+        time::timeout(Duration::from_secs(5), gate.entered.notified())
+            .await
+            .expect("replacement must reach the format validation barrier");
+        let cancelled_before_publication = remote.cancel_token.is_cancelled();
+        gate.release.cancel();
+        pending.await.expect("validated replacement must finish publication");
+        let replacement = set.disks.read().await[0].clone().expect("replacement must occupy the slot");
+        let readback = replacement.read_all("bucket", "object").await;
+        replacement.close().await.expect("stop replacement monitoring");
+        peer.stop().await;
+        assert!(
+            !cancelled_before_publication,
+            "the registered monitor must survive pending format validation"
+        );
+        assert!(!Arc::ptr_eq(&disk, &replacement), "validated replacement must be published");
+        assert!(remote.cancel_token.is_cancelled(), "the displaced monitor must stop after publication");
+        assert_eq!(
+            readback.expect("replacement must serve real I/O"),
+            Bytes::from_static(b"replacement-peer-data")
+        );
+    }
+
     #[tokio::test]
     #[serial(remote_disk_recovery_probe)]
     async fn endpoint_reconnect_admits_ready_drive_while_another_format_probe_is_blocked() {

crates/ecstore/src/cluster/rpc/remote_locker.rs

@@ -445,6 +445,10 @@ impl RemoteClient {
                 }
             }
         });
+        // A successful lock RPC over a cached or fresh channel proves the peer is
+        // reachable; reconcile the internode peer-health gauge so it does not stay
+        // stale when no fresh dial fires (issue #9119).
+        rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
     }
 
     fn admit_late_release(&self) -> bool {
@@ -672,6 +676,7 @@ impl RemoteClient {
                     with_lock_peer_health(&addr, |health| {
                         health.last_success = Some(Instant::now());
                     });
+                    rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
                     rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "success");
                     if let Some(late) = late {
                         late(response).await;

crates/ecstore/src/config/com.rs

@@ -3506,6 +3506,7 @@ mod tests {
                 replication_decision: String::new(),
                 checksum: None,
                 multipart_completion_replayed: false,
+                backfilled_old_current_size: None,
             }
         }
     }

crates/ecstore/src/core/pool_meta_native_integration_tests.rs

@@ -18,6 +18,7 @@ use crate::storage_api_contracts::namespace::NamespaceLocking as _;
 use codec::{fixture_inline_pool_meta_replicas as replicas, fixture_split_inline_pool_meta_replicas as split};
 use std::path::PathBuf;
 use tokio::fs;
+use uuid::Uuid;
 
 struct PhysicalFixture {
     _directories: Vec<tempfile::TempDir>,
@@ -216,6 +217,264 @@ async fn pool_meta_native_physical_split_without_ordinary_commit_performs_no_mut
         "all current heads, identities and complete backups must remain unchanged"
     );
     assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
+    state
+        .ensure_write_safe("ordinary authority temporarily unavailable")
+        .expect("missing ordinary authority alone must not install a permanent corruption latch");
+}
+
+async fn interrupted_first_commit_fixture() -> (PhysicalFixture, PoolMetaCommittedCandidate) {
+    let fixture = fixture(3).await;
+    let PoolMetaReplica::Valid {
+        previous: Some(previous),
+        committed: false,
+        ..
+    } = decode_pool_meta_replica(fixture.pending.clone())
+    else {
+        panic!("fixture must retain its acknowledged predecessor");
+    };
+    let physical = disks(&fixture.store).await;
+    let prepared = replicas(&fixture.pending, OffsetDateTime::now_utc()).await;
+    for (index, disk) in physical[0].iter().enumerate() {
+        fs::write(object_path(disk).join("xl.meta"), &prepared[index])
+            .await
+            .expect("one pool must retain an ordinary-readable PREPARE");
+    }
+    (fixture, *previous)
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_physical_interrupted_first_commit_restores_proven_predecessor() {
+    let (mut fixture, previous) = interrupted_first_commit_fixture().await;
+    let before = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+    assert!(
+        before[1..]
+            .iter()
+            .all(|read| matches!(read.replica, PoolMetaReplica::Unreadable(_)))
+    );
+    assert!(matches!(before[0].replica, PoolMetaReplica::Valid { committed: false, .. }));
+    let mut state = clean_state(&fixture);
+    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
+    assert!(
+        repair(&fixture.store, &mut state, &fence)
+            .await
+            .expect("ordinary PREPARE must authorize restoring only its acknowledged predecessor")
+    );
+    state
+        .ensure_write_safe("first commit rollback")
+        .expect("rollback must finish its transaction");
+    assert_ne!(
+        previous.canonical, fixture.canonical,
+        "the interrupted COMMIT must not become authoritative"
+    );
+    fixture.canonical = previous.canonical;
+    fixture.revision = previous.revision;
+    assert_committed(&fixture).await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_physical_interrupted_rollback_retains_authority_and_resumes() {
+    let (mut fixture, previous) = interrupted_first_commit_fixture().await;
+    let pause = Arc::new(NativeApplyPause::default());
+    let reached = pause.reached.notified();
+    tokio::pin!(reached);
+    reached.as_mut().enable();
+    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
+    let mut state = clean_state(&fixture);
+    let recovery = tokio::spawn({
+        let store = Arc::clone(&fixture.store);
+        let pause = Arc::clone(&pause);
+        async move {
+            AFTER_NATIVE_POOL_APPLY
+                .scope(pause, async move { repair(&store, &mut state, &fence).await })
+                .await
+        }
+    });
+    tokio::time::timeout(std::time::Duration::from_secs(10), reached)
+        .await
+        .expect("real rollback should pause after one pool is durably restored");
+    let partial = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+    assert!(matches!(partial[0].replica, PoolMetaReplica::Valid { committed: false, .. }));
+    assert!(matches!(&partial[1].replica, PoolMetaReplica::Valid { canonical, committed: true, .. }
+        if canonical == &previous.canonical));
+    assert!(matches!(partial[2].replica, PoolMetaReplica::Unreadable(_)));
+    recovery.abort();
+    assert!(
+        recovery
+            .await
+            .expect_err("the paused repair must be cancelled")
+            .is_cancelled()
+    );
+    let mut state = clean_state(&fixture);
+    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
+    assert!(
+        repair(&fixture.store, &mut state, &fence)
+            .await
+            .expect("restart must resume from the retained ordinary PREPARE")
+    );
+    fixture.canonical = previous.canonical;
+    fixture.revision = previous.revision;
+    assert_committed(&fixture).await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_physical_rollback_resumes_after_last_prepare_loses_read_quorum() {
+    let (mut fixture, previous) = interrupted_first_commit_fixture().await;
+    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
+    let inspected = inspect(&fixture.store.pools, &fence)
+        .await
+        .expect("physical rollback inventory");
+    let replacements = plan(
+        &inspected,
+        &fixture.canonical,
+        fixture.revision,
+        &fixture.identity,
+        Some(&previous.canonical),
+    )
+    .await
+    .expect("the ordinary PREPARE must authorize its predecessor");
+    for (pool, count) in [(1, 4), (2, 4), (0, 2)] {
+        for (index, replacement) in replacements[pool].iter().enumerate().take(count) {
+            let snapshot = &inspected[pool].snapshots[index];
+            assert_eq!(
+                inspected[pool].disks[index]
+                    .pool_meta_native_recovery(Request::Apply {
+                        object: PoolMetaNativeObject::PoolBin,
+                        expected_disk_id: snapshot.disk_id,
+                        expected_current_sha256: snapshot.current.as_ref().map(|raw| Sha256::digest(raw).into()),
+                        expected_backup_inventory_sha256: snapshot.inventory_sha256().expect("original inventory hash"),
+                        replacement: replacement.clone(),
+                    })
+                    .await
+                    .expect("partial rollback must use the real conditional publication and sync path"),
+                Response::Applied
+            );
+        }
+    }
+    let partial = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+    assert!(matches!(partial[0].replica, PoolMetaReplica::Unreadable(_)));
+    assert!(partial[1..].iter().all(|read| matches!(&read.replica,
+        PoolMetaReplica::Valid { canonical, committed: true, .. } if canonical == &previous.canonical)));
+    let mut state = clean_state(&fixture);
+    assert!(
+        repair(&fixture.store, &mut state, &fence)
+            .await
+            .expect("restored ordinary COMMIT must finish rollback after the last PREPARE becomes split")
+    );
+    fixture.canonical = previous.canonical;
+    fixture.revision = previous.revision;
+    assert_committed(&fixture).await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_physical_rollback_backup_does_not_poison_reused_generation() {
+    let (mut fixture, previous) = interrupted_first_commit_fixture().await;
+    let mut state = clean_state(&fixture);
+    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
+    assert!(repair(&fixture.store, &mut state, &fence).await.expect("initial rollback"));
+    let revision = PoolMetaRevision {
+        transaction_id: Some(Uuid::new_v4()),
+        ..fixture.revision
+    };
+    let mut meta = previous.meta.clone();
+    meta.pools[0].last_update += Duration::seconds(1);
+    let canonical = encode_pool_meta_v3_envelope(&meta, revision, true, None).expect("later COMMIT");
+    let pending = encode_pool_meta_v3_envelope(&meta, revision, false, Some(&previous)).expect("later PREPARE");
+    let modified = OffsetDateTime::now_utc() + Duration::seconds(1);
+    let split = split(&pending, &canonical, modified).await;
+    let commit = replicas(&canonical, modified).await;
+    let prepared = replicas(&pending, modified).await;
+    let directory = Uuid::new_v4();
+    let retained = snapshots(&fixture.store).await;
+    for (pool_index, pool) in disks(&fixture.store).await.into_iter().enumerate() {
+        for (disk_index, disk) in pool.iter().enumerate() {
+            assert!(
+                !retained[pool_index][disk_index].backups.is_empty(),
+                "the abandoned PREPARE must remain as real on-disk evidence"
+            );
+            let object = object_path(disk);
+            fs::write(object.join("xl.meta"), &if pool_index < 2 { &split } else { &commit }[disk_index])
+                .await
+                .expect("install later partial COMMIT without clearing old backups");
+            fs::create_dir(object.join(directory.to_string()))
+                .await
+                .expect("later backup directory");
+            fs::write(object.join(directory.to_string()).join("xl.meta.bkp"), &prepared[disk_index])
+                .await
+                .expect("later PREPARE backup");
+        }
+    }
+    assert!(
+        repair(&fixture.store, &mut state, &fence)
+            .await
+            .expect("an ordinary later COMMIT must survive the abandoned sibling PREPARE")
+    );
+    fixture.canonical = canonical;
+    fixture.revision = revision;
+    assert_committed(&fixture).await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_physical_rollback_rejects_retained_commit_without_mutation() {
+    let (fixture, _) = interrupted_first_commit_fixture().await;
+    let physical = disks(&fixture.store).await;
+    let commit = replicas(&fixture.canonical, OffsetDateTime::now_utc()).await;
+    let directory = Uuid::new_v4();
+    let object = object_path(&physical[0][0]);
+    fs::create_dir(object.join(directory.to_string()))
+        .await
+        .expect("retained COMMIT directory");
+    fs::write(object.join(directory.to_string()).join("xl.meta.bkp"), &commit[0])
+        .await
+        .expect("retain evidence that the interrupted transaction may already have committed");
+    let before = snapshots(&fixture.store).await;
+    let mut state = clean_state(&fixture);
+    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
+    assert!(repair(&fixture.store, &mut state, &fence).await.is_err());
+    assert_eq!(snapshots(&fixture.store).await, before, "ambiguous COMMIT evidence must remain untouched");
+    assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_physical_rollback_rejects_competing_prepares_without_mutation() {
+    let (fixture, previous) = interrupted_first_commit_fixture().await;
+    let pending = encode_pool_meta_v3_envelope(
+        &previous.meta,
+        PoolMetaRevision {
+            transaction_id: Some(Uuid::new_v4()),
+            ..fixture.revision
+        },
+        false,
+        Some(&previous),
+    )
+    .expect("competing PREPARE must remain structurally valid");
+    let physical = disks(&fixture.store).await;
+    let prepared = replicas(&pending, OffsetDateTime::now_utc()).await;
+    for (index, disk) in physical[1].iter().enumerate() {
+        fs::write(object_path(disk).join("xl.meta"), &prepared[index])
+            .await
+            .expect("second pool must expose the competing transaction through ordinary reads");
+    }
+    let before = snapshots(&fixture.store).await;
+    let mut state = clean_state(&fixture);
+    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
+    let error = repair(&fixture.store, &mut state, &fence)
+        .await
+        .expect_err("two distinct prepared transactions must not authorize a guessed rollback");
+    assert_eq!(
+        error
+            .pool_metadata_failure()
+            .expect("transaction conflict must be typed")
+            .kind,
+        crate::error::PoolMetadataFailure::RecoveryRequired
+    );
+    assert_eq!(snapshots(&fixture.store).await, before, "competing transactions must remain untouched");
+    assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
 }
 
 struct IdentityEpochFork {
@@ -467,7 +726,7 @@ async fn pool_meta_native_physical_partial_apply_resumes_after_conditional_rejec
     let inspected = inspect(&fixture.store.pools, &fence)
         .await
         .expect("real native snapshots should be obtained");
-    let replacements = plan(&inspected, &fixture.canonical, fixture.revision, &fixture.identity)
+    let replacements = plan(&inspected, &fixture.canonical, fixture.revision, &fixture.identity, None)
         .await
         .expect("physical recovery plan should validate");
     let snapshot = &inspected[0].snapshots[0];
@@ -628,3 +887,559 @@ async fn pool_meta_native_physical_three_plus_one_checkpoint_converges_all_shard
         );
     }
 }
+
+async fn owner_repair_no_backups_fixture() -> PhysicalFixture {
+    let fixture = fixture(0).await;
+    for pool in disks(&fixture.store).await {
+        for disk in pool {
+            let object = object_path(&disk);
+            let mut entries = fs::read_dir(&object).await.expect("fixture metadata directory");
+            while let Some(entry) = entries.next_entry().await.expect("fixture directory entry") {
+                if entry.file_type().await.expect("fixture entry type").is_dir() {
+                    fs::remove_dir_all(entry.path())
+                        .await
+                        .expect("remove task-owned fixture rollback directory");
+                }
+            }
+        }
+    }
+    fixture
+}
+
+async fn owner_repair_identity_heads(store: &ECStore) -> Vec<Vec<(Uuid, Bytes)>> {
+    let mut all = Vec::new();
+    for pool in &store.pools {
+        let set = pool.get_disks_by_key(POOL_META_IDENTITY_NAME);
+        let members = set
+            .disks
+            .read()
+            .await
+            .iter()
+            .cloned()
+            .map(|disk| disk.expect("identity fixture disk must be online"))
+            .collect::<Vec<_>>();
+        let mut heads = Vec::new();
+        for disk in members {
+            match disk
+                .pool_meta_native_recovery(Request::InspectIdentity {})
+                .await
+                .expect("physical identity inspection")
+            {
+                Response::IdentityInspected { disk_id, raw } => heads.push((disk_id, raw)),
+                _ => panic!("identity inspection must return its physical head"),
+            }
+        }
+        all.push(heads);
+    }
+    all
+}
+
+async fn legacy_native_owner_fixture(version: u16, stale: bool) -> (PhysicalFixture, Vec<u8>) {
+    let fixture = owner_repair_no_backups_fixture().await;
+    let mut meta = match decode_pool_meta_replica(fixture.canonical.clone()) {
+        PoolMetaReplica::Valid {
+            meta, committed: true, ..
+        } => meta,
+        _ => panic!("fixture must provide valid committed pool metadata"),
+    };
+    meta.version = version;
+    let raw = meta
+        .encode_config_data_for_v2_gate(version == POOL_META_VERSION)
+        .expect("legacy fixture should encode in its original version");
+    assert_eq!(u16::from_le_bytes([raw[2], raw[3]]), version);
+    let all = disks(&fixture.store).await;
+    let now = OffsetDateTime::now_utc();
+    for pool in &all {
+        let heads = replicas(&raw, now).await;
+        for (index, disk) in pool.iter().enumerate() {
+            fs::write(object_path(disk).join("xl.meta"), &heads[index])
+                .await
+                .expect("install exact legacy head");
+        }
+    }
+    if stale {
+        let old = replicas(&raw, now - Duration::seconds(10)).await;
+        fs::write(object_path(&all[0][0]).join("xl.meta"), &old[0])
+            .await
+            .expect("install stale same-raw legacy head");
+    }
+    (fixture, raw)
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_v1_with_all_upgrade_gates_disabled_keeps_exact_raw() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            let (fixture, raw) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+            let guard = owner(&fixture.store).await;
+            let mut state = fixture.store.pool_meta_save_gate.lock().await;
+            let result = tokio::time::timeout(
+                std::time::Duration::from_secs(30),
+                fixture.store.heal_pool_meta_replica_under_fence(0, guard, &mut state),
+            )
+            .await
+            .expect("legacy native heal must terminate");
+            assert!(result.is_ok(), "fully online legacy native heal should succeed: {result:?}");
+            drop(state);
+            let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            for read in &reads {
+                assert!(
+                    matches!(&read.replica, PoolMetaReplica::Valid { raw: bytes, revision, .. }
+                    if bytes == &raw && revision.version == POOL_META_V1_VERSION),
+                    "native heal must not upgrade a V1 raw payload when both upgrade gates are disabled"
+                );
+            }
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_legacy_source_with_trailing_bytes_without_writes() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            for version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
+                let (fixture, raw) = legacy_native_owner_fixture(version, false).await;
+                let mut source_raw = raw.clone();
+                // A second MessagePack value is readable by legacy decoding but
+                // must not be copied as part of an unverified repair payload.
+                source_raw.push(0xc0);
+                assert!(
+                    matches!(decode_pool_meta_replica(source_raw.clone()), PoolMetaReplica::Valid { raw, .. } if raw == source_raw),
+                    "ordinary legacy reads must remain compatible with trailing bytes"
+                );
+                let all_disks = disks(&fixture.store).await;
+                let source_heads = replicas(&source_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+                for (index, disk) in all_disks[0].iter().enumerate() {
+                    fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
+                        .await
+                        .expect("install readable legacy source with trailing bytes");
+                }
+                let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
+                    .await
+                    .expect("legacy semantic selection must remain readable");
+                assert_eq!(selection.legacy_raw.as_deref(), Some(source_raw.as_slice()));
+                let before = snapshots(&fixture.store).await;
+                let identities = owner_repair_identity_heads(&fixture.store).await;
+                let guard = owner(&fixture.store).await;
+                let mut state = fixture.store.pool_meta_save_gate.lock().await;
+                let result = tokio::time::timeout(
+                    std::time::Duration::from_secs(30),
+                    fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
+                )
+                .await
+                .expect("legacy native heal must terminate");
+                drop(state);
+                assert!(result.is_err(), "version {version}: unparsed trailing source bytes must not be republished");
+                assert_eq!(snapshots(&fixture.store).await, before, "version {version}: no pool.bin head or backup may change");
+                assert_eq!(
+                    owner_repair_identity_heads(&fixture.store).await,
+                    identities,
+                    "version {version}: no identity shard may change"
+                );
+                let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+                assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &source_raw));
+                assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw: target, .. } if target == &raw));
+            }
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_mixed_v1_v2_preserves_observed_format_floor() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            for first_version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
+                for target_pool in [0, 1] {
+                    let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+                    let mut meta = match decode_pool_meta_replica(raw_v1.clone()) {
+                        PoolMetaReplica::Valid { meta, committed: true, .. } => meta,
+                        _ => panic!("initial V1 metadata must decode"),
+                    };
+                    meta.version = POOL_META_VERSION;
+                    let raw_v2 = meta.encode_config_data_for_v2_gate(true).expect("equivalent V2 payload must encode");
+                    assert_eq!(u16::from_le_bytes([raw_v1[2], raw_v1[3]]), POOL_META_V1_VERSION);
+                    assert_eq!(u16::from_le_bytes([raw_v2[2], raw_v2[3]]), POOL_META_VERSION);
+                    let semantic_v1 = match decode_pool_meta_replica(raw_v1.clone()) {
+                        PoolMetaReplica::Valid { canonical, .. } => canonical,
+                        _ => panic!("V1 payload must remain valid"),
+                    };
+                    let semantic_v2 = match decode_pool_meta_replica(raw_v2.clone()) {
+                        PoolMetaReplica::Valid { canonical, .. } => canonical,
+                        _ => panic!("V2 payload must remain valid"),
+                    };
+                    assert_eq!(semantic_v1, semantic_v2, "the two formats must carry equivalent metadata");
+                    let other_version = if first_version == POOL_META_V1_VERSION { POOL_META_VERSION } else { POOL_META_V1_VERSION };
+                    // Pool two follows pool zero; pools zero and one cover both
+                    // orderings and both target formats in separate runs.
+                    let versions = [first_version, other_version, first_version];
+                    let physical = disks(&fixture.store).await;
+                    let timestamp = OffsetDateTime::now_utc();
+                    for (pool_index, pool) in physical.iter().enumerate() {
+                        let raw = if versions[pool_index] == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
+                        let heads = replicas(raw, timestamp).await;
+                        for (index, disk) in pool.iter().enumerate() {
+                            fs::write(object_path(disk).join("xl.meta"), &heads[index]).await.expect("install mixed legacy pool head");
+                        }
+                    }
+                    let before = snapshots(&fixture.store).await;
+                    let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
+                        .await.expect("equivalent legacy pools should be selectable");
+                    assert_eq!(selection.revision.version, POOL_META_VERSION, "the observed V2 format floor must be retained");
+                    assert_eq!(selection.canonical.as_deref(), Some(semantic_v2.as_slice()));
+                    let guard = owner(&fixture.store).await;
+                    let mut state = fixture.store.pool_meta_save_gate.lock().await;
+                    tokio::time::timeout(std::time::Duration::from_secs(30),
+                        fixture.store.heal_pool_meta_replica_under_fence(target_pool, guard, &mut state))
+                        .await.expect("mixed legacy heal must terminate")
+                        .expect("equivalent legacy metadata should heal without downgrading the format floor");
+                    drop(state);
+                    let after = snapshots(&fixture.store).await;
+                    let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+                    for (pool_index, read) in reads.iter().enumerate() {
+                        let expected_version = if pool_index == target_pool { POOL_META_VERSION } else { versions[pool_index] };
+                        let expected_raw = if expected_version == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
+                        assert!(matches!(&read.replica,
+                            PoolMetaReplica::Valid { raw, canonical, revision, committed: true, .. }
+                                if raw == expected_raw && canonical == &semantic_v2 && revision.version == expected_version
+                        ), "pool {pool_index}, first={first_version}, target={target_pool}: preserve equivalent payload and expected legacy version");
+                        for (index, snapshot) in after[pool_index].iter().enumerate() {
+                            codec::validate_inline_pool_meta_shard(snapshot.current.as_ref().expect("physical head"), index, expected_raw)
+                                .await.expect("every physical shard must match its selected legacy payload");
+                        }
+                        if pool_index != target_pool {
+                            assert_eq!(after[pool_index], before[pool_index], "healing one pool must not modify other pools");
+                        }
+                    }
+                }
+            }
+        },
+    ).await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_v1_and_v3_prepare_in_either_pool_order() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            for prepare_pool in [0, 1] {
+                let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+                let (legacy_meta, legacy_canonical) = match decode_pool_meta_replica(raw_v1.clone()) {
+                    PoolMetaReplica::Valid {
+                        meta,
+                        canonical,
+                        revision,
+                        committed: true,
+                        ..
+                    } if revision.version == POOL_META_V1_VERSION => (meta, canonical),
+                    _ => panic!("baseline V1 metadata must be committed and readable"),
+                };
+                let previous = PoolMetaCommittedCandidate {
+                    canonical: legacy_canonical,
+                    meta: legacy_meta.clone(),
+                    revision: PoolMetaRevision::legacy(POOL_META_V1_VERSION),
+                };
+                let mut next = legacy_meta;
+                next.version = POOL_META_GENERATION_VERSION;
+                next.pools[0].last_update += Duration::seconds(1);
+                let revision = PoolMetaRevision {
+                    version: POOL_META_GENERATION_VERSION,
+                    cluster_id: fixture.revision.cluster_id,
+                    epoch: fixture.revision.epoch,
+                    generation: 1,
+                    transaction_id: Some(Uuid::new_v4()),
+                };
+                let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&previous))
+                    .expect("V3 PREPARE with V1 predecessor must encode");
+                assert!(matches!(decode_pool_meta_replica(pending.clone()),
+                    PoolMetaReplica::Valid { committed: false, revision: actual, .. } if actual == revision));
+                let heads = replicas(&pending, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+                let physical = disks(&fixture.store).await;
+                for (index, disk) in physical[prepare_pool].iter().enumerate() {
+                    fs::write(object_path(disk).join("xl.meta"), &heads[index])
+                        .await
+                        .expect("install task-owned V3 PREPARE evidence");
+                }
+                let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+                assert!(matches!(&reads[prepare_pool].replica,
+                    PoolMetaReplica::Valid { raw, committed: false, revision: actual, .. }
+                        if raw == &pending && *actual == revision));
+                let other_pool = 1 - prepare_pool;
+                assert!(matches!(&reads[other_pool].replica,
+                    PoolMetaReplica::Valid { raw, committed: true, revision: actual, .. }
+                        if raw == &raw_v1 && actual.version == POOL_META_V1_VERSION));
+                let before = snapshots(&fixture.store).await;
+                let identities = owner_repair_identity_heads(&fixture.store).await;
+                let guard = owner(&fixture.store).await;
+                let mut state = fixture.store.pool_meta_save_gate.lock().await;
+                let result = tokio::time::timeout(
+                    std::time::Duration::from_secs(30),
+                    fixture
+                        .store
+                        .heal_pool_meta_replica_under_fence(prepare_pool, guard, &mut state),
+                )
+                .await
+                .expect("mixed V1/V3 PREPARE heal must terminate");
+                drop(state);
+                assert!(
+                    result.is_err(),
+                    "pool {prepare_pool}: native heal must not overwrite a V3 PREPARE with legacy data"
+                );
+                assert_eq!(
+                    snapshots(&fixture.store).await,
+                    before,
+                    "pool {prepare_pool}: rejection must retain all physical head and backup evidence"
+                );
+                assert_eq!(
+                    owner_repair_identity_heads(&fixture.store).await,
+                    identities,
+                    "pool {prepare_pool}: rejection must not mutate identity evidence"
+                );
+            }
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_v1_legacy_dont_save_true_without_writes() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+            let meta = match decode_pool_meta_replica(safe_v1.clone()) {
+                PoolMetaReplica::Valid {
+                    meta, committed: true, ..
+                } => meta,
+                _ => panic!("baseline V1 metadata must decode"),
+            };
+            let statuses = meta
+                .pools
+                .iter()
+                .map(|pool| {
+                    assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
+                    LegacyPoolStatus {
+                        id: pool.id,
+                        cmd_line: pool.cmd_line.clone(),
+                        last_update: pool.last_update,
+                        decommission: None,
+                    }
+                })
+                .collect();
+            let legacy = LegacyPoolMeta {
+                version: POOL_META_V1_VERSION,
+                pools: statuses,
+                dont_save: true,
+            };
+            let mut dangerous_raw = Vec::new();
+            dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
+            dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
+            dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
+            let old_reader: LegacyPoolMeta =
+                rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode source");
+            assert!(old_reader.dont_save, "the old reader would treat this source as permanently do-not-save");
+            assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
+            let source = match decode_pool_meta_replica(dangerous_raw.clone()) {
+                PoolMetaReplica::Valid {
+                    canonical,
+                    revision,
+                    committed: true,
+                    ..
+                } => {
+                    assert_eq!(revision.version, POOL_META_V1_VERSION);
+                    canonical
+                }
+                _ => panic!("current reader must recognize the legacy source"),
+            };
+            assert!(matches!(decode_pool_meta_replica(safe_v1.clone()),
+                PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
+                    if canonical == source && revision.version == POOL_META_V1_VERSION));
+            let physical = disks(&fixture.store).await;
+            let source_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+            for (index, disk) in physical[0].iter().enumerate() {
+                fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
+                    .await
+                    .expect("install legacy source on pool zero");
+            }
+            let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
+            assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
+            let before = snapshots(&fixture.store).await;
+            let identities = owner_repair_identity_heads(&fixture.store).await;
+            let guard = owner(&fixture.store).await;
+            let mut state = fixture.store.pool_meta_save_gate.lock().await;
+            let result = tokio::time::timeout(
+                std::time::Duration::from_secs(30),
+                fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
+            )
+            .await
+            .expect("unsafe legacy heal must terminate");
+            drop(state);
+            let error = result.expect_err("native heal must reject a source whose old-reader dont_save is true");
+            assert!(
+                error.to_string().contains("dont_save"),
+                "rejection must identify the unsafe legacy field: {error}"
+            );
+            assert_eq!(
+                snapshots(&fixture.store).await,
+                before,
+                "rejection must preserve every original head and backup"
+            );
+            assert_eq!(
+                owner_repair_identity_heads(&fixture.store).await,
+                identities,
+                "rejection must not write identity shards"
+            );
+            let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw));
+            assert!(
+                matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1),
+                "the dangerous source bytes must not be copied into the target pool"
+            );
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_target_v1_dont_save_true_without_writes() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+            let meta = match decode_pool_meta_replica(safe_v1.clone()) {
+                PoolMetaReplica::Valid {
+                    meta, committed: true, ..
+                } => meta,
+                _ => panic!("baseline V1 metadata must decode"),
+            };
+            let statuses = meta
+                .pools
+                .iter()
+                .map(|pool| {
+                    assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
+                    LegacyPoolStatus {
+                        id: pool.id,
+                        cmd_line: pool.cmd_line.clone(),
+                        last_update: pool.last_update,
+                        decommission: None,
+                    }
+                })
+                .collect();
+            let legacy = LegacyPoolMeta {
+                version: POOL_META_V1_VERSION,
+                pools: statuses,
+                dont_save: true,
+            };
+            let mut dangerous_raw = Vec::new();
+            dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
+            dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
+            dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
+            let old_reader: LegacyPoolMeta =
+                rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode target");
+            assert!(old_reader.dont_save, "the old reader interprets the target as do-not-save");
+            assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
+            let source_canonical = match decode_pool_meta_replica(safe_v1.clone()) {
+                PoolMetaReplica::Valid {
+                    canonical,
+                    revision,
+                    committed: true,
+                    ..
+                } => {
+                    assert_eq!(revision.version, POOL_META_V1_VERSION);
+                    canonical
+                }
+                _ => panic!("safe pool-zero source must decode"),
+            };
+            assert!(
+                matches!(decode_pool_meta_replica(dangerous_raw.clone()),
+                PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
+                    if canonical == source_canonical && revision.version == POOL_META_V1_VERSION),
+                "the dangerous target must remain semantically equivalent to the safe source"
+            );
+            let physical = disks(&fixture.store).await;
+            let target_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+            for (index, disk) in physical[1].iter().enumerate() {
+                fs::write(object_path(disk).join("xl.meta"), &target_heads[index])
+                    .await
+                    .expect("install legacy dangerous target head");
+            }
+            let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
+            assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
+            let before = snapshots(&fixture.store).await;
+            let identities = owner_repair_identity_heads(&fixture.store).await;
+            let guard = owner(&fixture.store).await;
+            let mut state = fixture.store.pool_meta_save_gate.lock().await;
+            let result = tokio::time::timeout(
+                std::time::Duration::from_secs(30),
+                fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
+            )
+            .await
+            .expect("unsafe legacy target heal must terminate");
+            drop(state);
+            let error = result.expect_err("native heal must reject a target whose old-reader dont_save is true");
+            assert!(
+                error.to_string().contains("dont_save"),
+                "rejection must identify the unsafe legacy field: {error}"
+            );
+            assert_eq!(
+                snapshots(&fixture.store).await,
+                before,
+                "rejection must preserve all physical heads and backups"
+            );
+            assert_eq!(
+                owner_repair_identity_heads(&fixture.store).await,
+                identities,
+                "rejection must not write identity shards"
+            );
+            let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1));
+            assert!(
+                matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw),
+                "unsafe target evidence must not be overwritten by the safe source"
+            );
+        },
+    )
+    .await;
+}

crates/ecstore/src/core/pool_meta_native_recovery.rs

@@ -30,6 +30,7 @@ pub(super) struct NativeApplyPause {
 #[cfg(test)]
 tokio::task_local! {
     pub(super) static AFTER_NATIVE_APPLY: Arc<NativeApplyPause>;
+    pub(super) static AFTER_NATIVE_POOL_APPLY: Arc<NativeApplyPause>;
 }
 
 struct InspectedPool {
@@ -168,6 +169,7 @@ async fn plan(
     canonical: &[u8],
     revision: PoolMetaRevision,
     identity: &[u8],
+    rollback: Option<&[u8]>,
 ) -> Result<Vec<Vec<Bytes>>> {
     let mut pending = Vec::<Vec<u8>>::new();
     let mut predecessors = Vec::<Vec<u8>>::new();
@@ -223,23 +225,83 @@ async fn plan(
             }
         }
     }
-    let mut obsolete = Vec::<Vec<u8>>::new();
-    for payload in &decoded {
-        match decode_pool_meta_replica(payload.clone()) {
-            PoolMetaReplica::Valid {
+    // The last ordinary PREPARE can itself become split during rollback. In
+    // that case another pool already exposes the restored ordinary COMMIT.
+    // Its authority permits continuing the rollback, never promoting a child
+    // transaction reconstructed from disk evidence.
+    let mut continuation: Option<(Vec<u8>, PoolMetaRevision)> = None;
+    if rollback.is_none() {
+        let mut outstanding = Vec::new();
+        for pool in inspected {
+            for (index, snapshot) in pool.snapshots.iter().enumerate() {
+                let raw = snapshot.current.as_ref().ok_or_else(invalid)?;
+                if codec::validate_inline_pool_meta_shard(raw, index, canonical).await.is_err() {
+                    outstanding.push((index, raw));
+                }
+            }
+        }
+        for payload in &decoded {
+            if let PoolMetaReplica::Valid {
                 canonical: candidate,
                 revision: candidate_revision,
                 committed: false,
                 previous: Some(previous),
                 ..
-            } if candidate_revision == revision && candidate == canonical => {
-                if !pending.contains(payload) {
-                    pending.push(payload.clone());
+            } = decode_pool_meta_replica(payload.clone())
+                && previous.revision == revision
+                && previous.canonical == canonical
+            {
+                let mut matches = true;
+                for (index, raw) in &outstanding {
+                    matches &= codec::validate_inline_pool_meta_shard(raw, *index, payload).await.is_ok()
+                        || codec::validate_inline_pool_meta_shard(raw, *index, &candidate).await.is_ok();
                 }
-                if !predecessors.contains(&previous.canonical) {
-                    predecessors.push(previous.canonical);
+                if matches {
+                    if !outstanding.is_empty()
+                        && continuation.as_ref().is_some_and(|(selected, selected_revision)| {
+                            selected != &candidate || *selected_revision != candidate_revision
+                        })
+                    {
+                        return Err(known_validation(invalid(), "native_transaction_validation"));
+                    }
+                    continuation.get_or_insert((candidate, candidate_revision));
                 }
             }
+        }
+    }
+    let (canonical, revision, rollback) = match &continuation {
+        Some((transaction, next_revision)) => (transaction.as_slice(), *next_revision, Some(canonical)),
+        None => (canonical, revision, rollback),
+    };
+    for payload in &decoded {
+        if let PoolMetaReplica::Valid {
+            canonical: candidate,
+            revision: candidate_revision,
+            committed: false,
+            previous: Some(previous),
+            ..
+        } = decode_pool_meta_replica(payload.clone())
+            && candidate_revision == revision
+            && candidate == canonical
+        {
+            pending.push(payload.clone());
+            if !predecessors.contains(&previous.canonical) {
+                predecessors.push(previous.canonical);
+            }
+        }
+    }
+    if pending.len() > 1 {
+        return Err(known_validation(
+            Error::other("native V3 prepare records disagree on the predecessor"),
+            "native_transaction_validation",
+        ));
+    }
+    let mut obsolete = Vec::<Vec<u8>>::new();
+    for payload in &decoded {
+        if pending.contains(payload) {
+            continue;
+        }
+        match decode_pool_meta_replica(payload.clone()) {
             PoolMetaReplica::Valid { revision: older, .. }
                 if older.is_generation_protocol()
                     && older.cluster_id == revision.cluster_id
@@ -251,6 +313,23 @@ async fn plan(
                 // fully reconstructed and bound into the native CAS.
                 obsolete.push(payload.clone());
             }
+            PoolMetaReplica::Valid {
+                revision: abandoned,
+                committed: false,
+                previous: Some(previous),
+                ..
+            } if abandoned.is_generation_protocol()
+                && abandoned.cluster_id == revision.cluster_id
+                && abandoned.epoch == revision.epoch
+                && abandoned.generation == revision.generation
+                && abandoned.transaction_id != revision.transaction_id
+                && predecessors.contains(&previous.canonical) =>
+            {
+                // A rollback retains the failed PREPARE as evidence. A later
+                // transaction may reuse that generation, but only a backup of
+                // an uncommitted sibling with the same predecessor is obsolete.
+                obsolete.push(payload.clone());
+            }
             PoolMetaReplica::Valid {
                 canonical: candidate,
                 revision: candidate_revision,
@@ -272,13 +351,7 @@ async fn plan(
             _ => {}
         }
     }
-    if pending.len() > 1 {
-        return Err(known_validation(
-            Error::other("native V3 prepare records disagree on the predecessor"),
-            "native_transaction_validation",
-        ));
-    }
-    if pending.is_empty() {
+    if pending.is_empty() || rollback.is_some_and(|previous| !predecessors.iter().any(|candidate| candidate == previous)) {
         return Err(invalid());
     }
     for payload in &decoded {
@@ -294,13 +367,25 @@ async fn plan(
         for (index, snapshot) in pool.snapshots.iter().enumerate() {
             let raw = snapshot.current.as_ref().ok_or_else(invalid)?;
             let mut matched = false;
-            for payload in std::iter::once(canonical).chain(pending.iter().map(Vec::as_slice)) {
+            for payload in std::iter::once(canonical)
+                .chain(pending.iter().map(Vec::as_slice))
+                .chain(rollback)
+            {
                 matched |= codec::validate_inline_pool_meta_shard(raw, index, payload).await.is_ok();
             }
             if !matched {
                 return Err(invalid());
             }
             for backup in &snapshot.backups {
+                if rollback.is_some()
+                    && codec::validate_inline_pool_meta_shard(&backup.data, index, canonical)
+                        .await
+                        .is_ok()
+                {
+                    // A retained COMMIT might have been authoritative before
+                    // this interruption. Never roll it back from PREPARE alone.
+                    return Err(invalid());
+                }
                 let mut matched = false;
                 for payload in std::iter::once(canonical).chain(
                     pending
@@ -323,7 +408,7 @@ async fn plan(
     let mut replacements = Vec::with_capacity(inspected.len());
     for pool in inspected {
         replacements.push(
-            codec::encode_inline_pool_meta_replicas(&current(pool)?, canonical, modified)
+            codec::encode_inline_pool_meta_replicas(&current(pool)?, rollback.unwrap_or(canonical), modified)
                 .await
                 .map_err(|error| codec_validation(error, "native_metadata_validation"))?,
         );
@@ -338,10 +423,12 @@ pub(super) async fn repair(
 ) -> Result<bool> {
     fence.ensure_held()?;
     let reads = read_pool_meta_replicas(store.pools.clone(), true).await;
-    if !reads
+    let unreadable = reads
         .iter()
-        .any(|read| matches!(read.replica, PoolMetaReplica::Unreadable(_)))
-    {
+        .enumerate()
+        .filter_map(|(index, read)| matches!(read.replica, PoolMetaReplica::Unreadable(_)).then_some(index))
+        .collect::<Vec<_>>();
+    if unreadable.is_empty() {
         return Ok(false);
     }
     state.ensure_write_safe("native pool metadata recovery")?;
@@ -360,8 +447,9 @@ pub(super) async fn repair(
     {
         return Err(invalid());
     }
-    // Only an ordinary, successfully read COMMIT grants authority. Rebuilding
-    // two erasure shards below proves integrity, never a commit decision.
+    // Ordinary reads grant authority: a COMMIT selects its own state, while a
+    // PREPARE can select only its acknowledged predecessor. Reconstruction
+    // below proves integrity and never promotes an interrupted transaction.
     let ordinary_commits = reads
         .iter()
         .filter_map(|read| match &read.replica {
@@ -375,7 +463,22 @@ pub(super) async fn repair(
             _ => None,
         })
         .collect::<Vec<_>>();
-    if ordinary_commits.is_empty() {
+    let ordinary_prepares = reads
+        .iter()
+        .filter_map(|read| match &read.replica {
+            PoolMetaReplica::Valid {
+                canonical,
+                revision,
+                committed: false,
+                previous: Some(previous),
+                ..
+            } if revision.is_generation_protocol() && previous.revision.is_generation_protocol() => {
+                Some((canonical.clone(), *revision, previous.canonical.clone()))
+            }
+            _ => None,
+        })
+        .collect::<Vec<_>>();
+    if ordinary_commits.is_empty() && ordinary_prepares.is_empty() {
         return Err(invalid());
     }
     let selection = select_pool_meta_replica_reads(reads)
@@ -395,10 +498,20 @@ pub(super) async fn repair(
         return Err(invalid());
     }
     let canonical = selection.canonical.as_ref().ok_or_else(invalid)?;
+    let mut rollback = None;
+    for (pending, revision, previous) in &ordinary_prepares {
+        if previous == canonical {
+            if rollback.is_some_and(|candidate| candidate != (pending, *revision)) {
+                return Err(latch_validation(state, known_validation(invalid(), "native_transaction_validation")));
+            }
+            rollback = Some((pending, *revision));
+        }
+    }
     if !selection.revision.is_generation_protocol()
-        || !ordinary_commits
-            .iter()
-            .any(|(raw, revision)| raw == canonical && *revision == selection.revision)
+        || (rollback.is_none()
+            && !ordinary_commits
+                .iter()
+                .any(|(raw, revision)| raw == canonical && *revision == selection.revision))
     {
         return Err(invalid());
     }
@@ -445,13 +558,18 @@ pub(super) async fn repair(
         .await
         .map_err(|error| latch_validation(state, error))?;
     let inspected = inspect(&store.pools, fence).await?;
-    let replacements = plan(&inspected, canonical, selection.revision, &identity_bytes)
+    let (transaction, revision) = rollback.unwrap_or((canonical, selection.revision));
+    let replacements = plan(&inspected, transaction, revision, &identity_bytes, rollback.map(|_| canonical.as_slice()))
         .await
         .map_err(|error| latch_validation(state, error))?;
     fence.ensure_held()?;
     let mut arm = state.arm_transaction();
     arm.phase = Some("native_repair_apply");
-    for (pool, replacements) in inspected.iter().zip(replacements) {
+    let mut apply_order = inspected.iter().zip(replacements).enumerate().collect::<Vec<_>>();
+    // Preserve ordinary authority until all unreadable pools are repaired.
+    // Cancellation can then resume from a PREPARE or a restored COMMIT.
+    apply_order.sort_by_key(|(index, _)| !unreadable.contains(index));
+    for (_, (pool, replacements)) in apply_order {
         for ((disk, snapshot), replacement) in pool.disks.iter().zip(&pool.snapshots).zip(replacements) {
             fence.ensure_held()?;
             let request = Request::Apply {
@@ -466,6 +584,11 @@ pub(super) async fn repair(
                 _ => return Err(invalid()),
             }
         }
+        #[cfg(test)]
+        if let Ok(pause) = AFTER_NATIVE_POOL_APPLY.try_with(Arc::clone) {
+            pause.reached.notify_one();
+            pause.resume.notified().await;
+        }
     }
     #[cfg(test)]
     if let Ok(pause) = AFTER_NATIVE_APPLY.try_with(Arc::clone) {
@@ -627,7 +750,7 @@ mod tests {
     #[tokio::test]
     async fn pool_meta_native_plan_restores_physical_split_with_fresh_common_identity() {
         let f = fixture().await;
-        let repaired = plan(&f.pools, &f.commit, f.revision, &f.identity).await.unwrap();
+        let repaired = plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.unwrap();
         assert_eq!(repaired.len(), 4);
         for pool in repaired {
             assert_eq!(
@@ -648,7 +771,7 @@ mod tests {
         let mut f = fixture().await;
         let wrong = replicas(b"different identity", f.modified).await;
         f.pools[0].snapshots[0].identity = Some(wrong[0].clone());
-        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
+        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
     }
 
     #[tokio::test]
@@ -671,10 +794,10 @@ mod tests {
         .unwrap();
         let wrong = replicas(&higher, f.modified + Duration::seconds(2)).await;
         f.pools[0].snapshots[0].current = Some(wrong[0].clone());
-        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
+        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
         f.pools[0].snapshots[0].current = Some(replicas(&f.pending, f.modified).await[0].clone());
         f.pools[0].snapshots[0].backups[0].data = wrong[0].clone();
-        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
+        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
     }
 
     #[tokio::test]
@@ -717,17 +840,17 @@ mod tests {
             }
         }
         assert!(older_pending != f.pending);
-        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_ok());
+        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_ok());
     }
 
     #[tokio::test]
     async fn pool_meta_native_plan_rejects_missing_or_corrupt_evidence() {
         let mut f = fixture().await;
         let before = f.pools[1].snapshots[2].current.take();
-        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
+        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
         f.pools[1].snapshots[2].current = before;
         f.pools[1].snapshots[2].backups[0].data = Bytes::from_static(b"invalid metadata");
-        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
+        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
     }
 }
 

crates/ecstore/src/core/pools.rs

@@ -94,7 +94,6 @@ use serde::{Deserialize, Serialize};
 use sha2::{Digest, Sha256};
 use std::collections::{HashMap, HashSet};
 use std::fmt::Display;
-#[cfg(test)]
 use std::io::Cursor;
 use std::io::Write;
 use std::path::PathBuf;
@@ -3720,33 +3719,45 @@ fn is_decommission_copy_quorum_deferral(err: &Error) -> bool {
     data_movement::data_movement_stage_source_as::<DecommissionCopyQuorumFailure>(err).is_some()
 }
 
-/// The storage error of a version copy that kept timing out on a namespace
-/// lock through its in-place retries, kept typed so the entry failure policy
-/// defers it.
+/// A copy whose object namespace lock remained unavailable through its
+/// in-place retries, retained as a typed deferral.
 #[derive(Debug, thiserror::Error)]
 #[error("{0}")]
-struct DecommissionCopyLockTimeout(#[source] Error);
+struct DecommissionCopyLockUnavailable(#[source] Error);
 
-fn is_decommission_copy_lock_timeout_error(err: &Error) -> bool {
-    let err = data_movement::data_movement_stage_source(err).unwrap_or(err);
-    matches!(err, Error::Lock(rustfs_lock::LockError::Timeout { .. }))
+fn is_decommission_copy_lock_unavailable_error(err: &Error, bucket: &str, object: &str) -> bool {
+    let source = data_movement::data_movement_stage_source(err).unwrap_or(err);
+    match source {
+        Error::Lock(rustfs_lock::LockError::Timeout { .. }) => true,
+        Error::NamespaceLockQuorumUnavailable {
+            mode: "read",
+            bucket: lock_bucket,
+            object: lock_object,
+            ..
+        } => {
+            data_movement::data_movement_error_is_stage(err, "prepare_put_object")
+                && lock_bucket == bucket
+                && lock_object == object
+        }
+        _ => false,
+    }
 }
 
-/// A version copy that timed out acquiring a namespace lock leaves the source
+/// A version copy that could not acquire its namespace lock leaves the source
 /// in place; the next attempt copies the version again or finds an equivalent
 /// committed target. The lock is held by another mutation of the key or, when
 /// this node restarted mid-copy, by its previous incarnation until that lock's
 /// lease expires on the peers. Either clears by waiting, so the entry is
 /// deferred instead of being counted as a permanent per-object failure.
-fn decommission_copy_lock_timeout_deferral(stage: &str, bucket: &str, object: &str, err: Error) -> Error {
+fn decommission_copy_lock_unavailable_deferral(stage: &str, bucket: &str, object: &str, err: Error) -> Error {
     data_movement::data_movement_context_error(
-        format!("decommission {stage} timed out on a namespace lock for {bucket}/{object}: {err}"),
-        DecommissionCopyLockTimeout(err),
+        format!("decommission {stage} could not acquire a namespace lock for {bucket}/{object}: {err}"),
+        DecommissionCopyLockUnavailable(err),
     )
 }
 
-fn is_decommission_copy_lock_timeout_deferral(err: &Error) -> bool {
-    data_movement::data_movement_stage_source_as::<DecommissionCopyLockTimeout>(err).is_some()
+fn is_decommission_copy_lock_unavailable_deferral(err: &Error) -> bool {
+    data_movement::data_movement_stage_source_as::<DecommissionCopyLockUnavailable>(err).is_some()
 }
 
 /// The deferral of a version copy whose in-place retries ended on `error`, if
@@ -3756,8 +3767,8 @@ fn take_decommission_copy_deferral(stage: &str, bucket: &str, object: &str, erro
         return Some(decommission_copy_quorum_deferral(stage, bucket, object, err));
     }
     error
-        .take_if(|err| is_decommission_copy_lock_timeout_error(err))
-        .map(|err| decommission_copy_lock_timeout_deferral(stage, bucket, object, err))
+        .take_if(|err| is_decommission_copy_lock_unavailable_error(err, bucket, object))
+        .map(|err| decommission_copy_lock_unavailable_deferral(stage, bucket, object, err))
 }
 
 fn resolve_decommission_entry_reload_result(result: Result<()>, bucket: &str, object_name: &str) -> Result<()> {
@@ -5822,13 +5833,13 @@ enum DecommissionEntryOutcome {
 /// entry is done and must not be counted as a failure or escalated. A quorum
 /// failure of the source cleanup is deferred too: the copied source is still
 /// in place and the next attempt cleans it up again. So is a version copy that
-/// lost drive quorum or timed out on a namespace lock through its in-place
+/// lost drive quorum or could not acquire an object namespace lock through its in-place
 /// retries: the source still holds the version and the next attempt copies it
 /// again. Every other failure stays terminal.
 fn resolve_decommission_entry_failure(err: Error) -> Result<DecommissionEntryOutcome> {
     if is_decommission_source_cleanup_retryable_error(&err)
         || is_decommission_copy_quorum_deferral(&err)
-        || is_decommission_copy_lock_timeout_deferral(&err)
+        || is_decommission_copy_lock_unavailable_deferral(&err)
     {
         return Ok(DecommissionEntryOutcome::Deferred(err));
     }
@@ -6827,11 +6838,47 @@ impl Drop for PoolMetaTransactionArm {
     }
 }
 
+fn ensure_legacy_pool_meta_republish_safe(data: &[u8]) -> Result<()> {
+    if data.len() >= 4
+        && LittleEndian::read_u16(&data[2..4]) == POOL_META_V1_VERSION
+        // The selection already validates the payload. The runtime-only
+        // flag can only appear in the legacy three-field sequence or map.
+        && rmp::decode::read_array_len(&mut &data[4..])
+            .or_else(|_| rmp::decode::read_map_len(&mut &data[4..]))
+            .ok()
+            == Some(3)
+        && let Ok(legacy) = rmp_serde::from_slice::<LegacyPoolMeta>(&data[4..])
+        && legacy.dont_save
+    {
+        return Err(Error::other("pool metadata legacy source contains dont_save=true"));
+    }
+    Ok(())
+}
+
+// Ordinary legacy reads retain their compatibility behavior, but bytes that
+// will be copied to another pool must not contain an unparsed extension.
+fn ensure_legacy_pool_meta_payload_complete(data: &[u8]) -> Result<()> {
+    let payload = data
+        .get(4..)
+        .ok_or_else(|| Error::other("pool metadata legacy source is too short"))?;
+    let mut decoder = rmp_serde::Deserializer::new(Cursor::new(payload));
+    serde::de::IgnoredAny::deserialize(&mut decoder).map_err(|_| Error::other("pool metadata legacy source is invalid"))?;
+    let payload_len = u64::try_from(payload.len()).map_err(|_| Error::other("pool metadata legacy source is too large"))?;
+    if decoder.get_ref().position() != payload_len {
+        return Err(Error::other("pool metadata legacy source has trailing bytes"));
+    }
+    Ok(())
+}
+
 #[derive(Debug)]
 struct PoolMetaSelection {
     meta: PoolMeta,
     revision: PoolMetaRevision,
     canonical: Option<Vec<u8>>,
+    // Legacy semantic comparison canonicalizes to V2; a heal must instead
+    // preserve verified raw bytes at the highest observed format floor.
+    legacy_raw: Option<Vec<u8>>,
+    legacy_has_unsafe_flag: bool,
     replica_state: PoolMetaReplicaState,
     cas_tokens: Vec<PoolMetaCasToken>,
     absent: bool,
@@ -7222,6 +7269,8 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
             meta: PoolMeta::default(),
             revision: PoolMetaRevision::legacy(0),
             canonical: None,
+            legacy_raw: None,
+            legacy_has_unsafe_flag: false,
             replica_state: PoolMetaReplicaState {
                 needs_repair: false,
                 repair_write_safe: true,
@@ -7276,6 +7325,8 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
             meta: selected.meta.clone(),
             revision: selected.revision,
             canonical: Some(selected.canonical.clone()),
+            legacy_raw: None,
+            legacy_has_unsafe_flag: false,
             replica_state: PoolMetaReplicaState {
                 needs_repair,
                 repair_write_safe,
@@ -7289,6 +7340,10 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
 
     // Legacy V1/V2 has no durable generation. Pool zero remains the commit
     // record; divergent backups without it are ambiguous and fail closed.
+    let legacy_has_unsafe_flag = committed
+        .iter()
+        .any(|(_, raw, _)| ensure_legacy_pool_meta_republish_safe(raw).is_err());
+    let mut floor_raw = None;
     let mut selected: Option<(usize, Vec<u8>, PoolMetaCommittedCandidate)> = None;
     for (idx, raw, candidate) in committed {
         if let Some((selected_idx, selected_raw, selected_candidate)) = selected.as_ref() {
@@ -7302,19 +7357,36 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
                 }
             } else {
                 needs_repair |= selected_raw != &raw;
+                if floor_raw.is_none()
+                    && candidate.revision.version == observed_version
+                    && raw
+                        .get(2..4)
+                        .is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
+                {
+                    floor_raw = Some(raw);
+                }
             }
         } else {
             selected = Some((idx, raw, candidate));
         }
     }
-    let (_, _, mut selected) =
+    let (_, selected_raw, mut selected) =
         selected.ok_or_else(|| Error::other("pool metadata recovery required: no valid legacy replica is available"))?;
+    if selected.revision.version == observed_version
+        && selected_raw
+            .get(2..4)
+            .is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
+    {
+        floor_raw = Some(selected_raw);
+    }
     selected.meta.version = observed_version;
     selected.revision.version = observed_version;
     Ok(PoolMetaSelection {
         meta: selected.meta,
         revision: selected.revision,
         canonical: Some(selected.canonical),
+        legacy_raw: floor_raw,
+        legacy_has_unsafe_flag,
         replica_state: PoolMetaReplicaState {
             needs_repair,
             repair_write_safe,
@@ -8308,8 +8380,9 @@ where
         })
     });
     let result = save_config_with_opts_and_metadata(pool, object, data, &opts).await;
-    let may_have_mutated = !matches!(&result, Err(Error::PreconditionFailed));
-    if !may_have_mutated {
+    let write_not_started = matches!(&result, Err(Error::Io(inner)) if inner.get_ref().is_some_and(|source| source.is::<crate::set_disk::SystemMetadataWriteNotStarted>()));
+    let may_have_mutated = !write_not_started && !matches!(&result, Err(Error::PreconditionFailed));
+    if matches!(&result, Err(Error::PreconditionFailed)) {
         record_pool_meta_stale_write_rejection(phase);
     }
     let mut source = None;
@@ -8320,6 +8393,8 @@ where
             source = Some(Arc::clone(&original));
             if matches!(original.as_ref(), Error::PreconditionFailed) {
                 Err(Error::PreconditionFailed)
+            } else if write_not_started {
+                Err(Error::other(original))
             } else {
                 Err(Error::other(pool_metadata_error(
                     crate::error::PoolMetadataFailure::TransactionUnknown,
@@ -14534,6 +14609,23 @@ impl ECStore {
             .canonical
             .as_ref()
             .ok_or_else(|| Error::other("pool metadata heal has no committed source"))?;
+        if selection.generation_protocol_observed && !selection.revision.is_generation_protocol() {
+            return Err(Error::other("pool metadata heal requires recovery of the observed generation protocol"));
+        }
+        if selection.legacy_has_unsafe_flag {
+            return Err(Error::other("pool metadata legacy replica contains dont_save=true"));
+        }
+        let repair_data = if selection.revision.is_generation_protocol() {
+            canonical.clone()
+        } else {
+            let raw = selection
+                .legacy_raw
+                .clone()
+                .ok_or_else(|| Error::other("pool metadata heal cannot prove the legacy format floor"))?;
+            ensure_legacy_pool_meta_payload_complete(&raw)?;
+            raw
+        };
+        ensure_legacy_pool_meta_republish_safe(&repair_data)?;
         let pool = self
             .pools
             .get(pool_index)
@@ -14544,10 +14636,10 @@ impl ECStore {
             .get(pool_index)
             .ok_or_else(|| Error::other("pool metadata heal has no target revision"))?;
         let mut arm = state.arm_transaction();
-        save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, canonical.clone(), token, &fence, "native_heal", &mut arm)
+        save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, repair_data.clone(), token, &fence, "native_heal", &mut arm)
             .await?;
         let (confirmed, _) = read_config_no_lock_preserve_empty_with_metadata(pool, POOL_META_NAME).await?;
-        if confirmed != *canonical {
+        if confirmed != repair_data {
             return Err(Error::other("pool metadata native heal readback diverged"));
         }
         fence.ensure_held()?;
@@ -28353,7 +28445,7 @@ mod tests {
             "one version must commit before the next version times out"
         );
         assert!(
-            matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_timeout_deferral(err)),
+            matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
             "{outcome:?}"
         );
         assert_eq!(
@@ -28387,6 +28479,88 @@ mod tests {
         assert_eq!(actual, b"written after versioning was enabled");
     }
 
+    /// A target lookup that loses its read-lock quorum must retain the source
+    /// and resume an entry after an earlier version has already committed.
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn decommission_defers_target_lookup_read_lock_quorum_and_resumes() {
+        use tokio::io::AsyncReadExt as _;
+
+        let fixture = NullVersionDecommission::new(NullVersionTop::NewerVersion).await;
+        let object = fixture.object;
+        let attempted = Arc::new(AtomicUsize::new(0));
+        let fault = DecommissionTestFaultGuard::install_with_error(
+            Arc::new({
+                let attempted = Arc::clone(&attempted);
+                move |stage, _, faulted, _, success| {
+                    stage == "before_target_commit"
+                        && faulted == object
+                        && success
+                        && attempted.fetch_add(1, Ordering::SeqCst) > 0
+                }
+            }),
+            Some(Arc::new(|_, bucket, object| {
+                data_movement::data_movement_stage_error_for_test(
+                    "decommission_object",
+                    "prepare_put_object",
+                    bucket,
+                    object,
+                    Error::NamespaceLockQuorumUnavailable {
+                        mode: "read",
+                        bucket: bucket.to_owned(),
+                        object: object.to_owned(),
+                        required: 1,
+                        achieved: 0,
+                    },
+                )
+            })),
+        );
+        let (outcome, _) = fixture.run_entry().await;
+        assert!(
+            matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
+            "{outcome:?}"
+        );
+        fixture.assert_source_retained().await;
+        assert_eq!(
+            fixture
+                .target_versions()
+                .await
+                .expect("first version committed")
+                .versions
+                .len(),
+            1
+        );
+        assert_eq!(
+            fixture.store.pool_meta.read().await.pools[0]
+                .decommission
+                .as_ref()
+                .expect("active decommission")
+                .items_decommission_failed,
+            0
+        );
+
+        drop(fault);
+        let (outcome, _) = fixture.run_entry().await;
+        assert!(matches!(outcome, DecommissionEntryOutcome::Complete), "{outcome:?}");
+        fixture.assert_moved().await;
+        let mut reader = fixture.store.pools[1]
+            .get_object_reader(
+                fixture.bucket,
+                fixture.object,
+                None,
+                HeaderMap::new(),
+                &ObjectOptions {
+                    version_id: Some(fixture.seeded.versions[0].version_id.expect("newer version").to_string()),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("the newer version must remain readable");
+        let mut actual = Vec::new();
+        reader.stream.read_to_end(&mut actual).await.expect("read the newer version");
+        assert_eq!(actual, b"written after versioning was enabled");
+    }
+
     /// A namespace lock still held on the target key, as by a copy whose node
     /// was killed mid-write, makes target selection fail with a typed
     /// object-lock timeout. A real held lock, not an injected error, must defer
@@ -28429,7 +28603,7 @@ mod tests {
 
             let (outcome, _) = fixture.run_entry().await;
             assert!(
-                matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_timeout_deferral(err)),
+                matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
                 "{outcome:?}"
             );
             assert_eq!(
@@ -28838,17 +29012,17 @@ mod tests {
             timeout(),
         );
         for err in [timeout(), prepare] {
-            assert!(is_decommission_copy_lock_timeout_error(&err), "{err}");
+            assert!(is_decommission_copy_lock_unavailable_error(&err, "bucket-a", "object-a"), "{err}");
             assert!(!is_decommission_copy_quorum_error(&err), "{err}");
             let mut error = Some(err);
             let deferred = take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error)
                 .expect("a lock timeout copy must be deferred");
             assert!(error.is_none());
-            assert!(is_decommission_copy_lock_timeout_deferral(&deferred));
+            assert!(is_decommission_copy_lock_unavailable_deferral(&deferred));
             assert!(!is_decommission_copy_quorum_deferral(&deferred));
-            let failure = data_movement::data_movement_stage_source_as::<DecommissionCopyLockTimeout>(&deferred)
+            let failure = data_movement::data_movement_stage_source_as::<DecommissionCopyLockUnavailable>(&deferred)
                 .expect("the deferred error must keep its lock-timeout origin");
-            assert!(is_decommission_copy_lock_timeout_error(&failure.0));
+            assert!(is_decommission_copy_lock_unavailable_error(&failure.0, "bucket-a", "object-a"));
             assert!(matches!(
                 resolve_decommission_entry_failure(deferred),
                 Ok(DecommissionEntryOutcome::Deferred(_))
@@ -28873,6 +29047,43 @@ mod tests {
         }
     }
 
+    #[test]
+    fn decommission_copy_defers_only_the_matching_object_read_lock_quorum() {
+        let quorum = |mode, bucket: &str, object: &str| Error::NamespaceLockQuorumUnavailable {
+            mode,
+            bucket: bucket.to_owned(),
+            object: object.to_owned(),
+            required: 1,
+            achieved: 0,
+        };
+        let wrap = |stage, err| {
+            data_movement::data_movement_stage_error_for_test("decommission_object", stage, "bucket-a", "object-a", err)
+        };
+        let mut error = Some(wrap("prepare_put_object", quorum("read", "bucket-a", "object-a")));
+        let deferred = take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error)
+            .expect("matching read-lock quorum must defer");
+        assert!(error.is_none());
+        assert!(matches!(
+            resolve_decommission_entry_failure(deferred),
+            Ok(DecommissionEntryOutcome::Deferred(_))
+        ));
+        for err in [
+            quorum("read", "bucket-a", "object-a"),
+            wrap("source_recheck", quorum("read", "bucket-a", "object-a")),
+            wrap("prepare_put_object", quorum("write", "bucket-a", "object-a")),
+            wrap(
+                "prepare_put_object",
+                quorum("exact_delete_capacity_reconciliation", "bucket-a", "object-a"),
+            ),
+            wrap("prepare_put_object", quorum("read", "other-bucket", "object-a")),
+            wrap("prepare_put_object", quorum("read", "bucket-a", "other-object")),
+        ] {
+            let mut error = Some(err);
+            assert!(take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error).is_none());
+            assert!(error.is_some(), "unrelated lock failures must stay terminal");
+        }
+    }
+
     #[test]
     fn decommission_target_capacity_error_accepts_wrapped_capacity_errors() {
         let disk_full = Error::other(format!("decommission_object: put_object failed for bucket/object: {}", Error::DiskFull));
@@ -29351,6 +29562,53 @@ mod tests {
         );
     }
 
+    #[test]
+    fn legacy_republish_guard_checks_only_three_field_sequence_or_map() {
+        let frame = |payload: Vec<u8>| {
+            let mut data = Vec::new();
+            data.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
+            data.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
+            data.extend_from_slice(&payload);
+            data
+        };
+        let two_fields = PersistedPoolMetaV1 {
+            version: POOL_META_V1_VERSION,
+            pools: Vec::new(),
+        };
+        for payload in [rmp_serde::to_vec(&two_fields), rmp_serde::to_vec_named(&two_fields)] {
+            let data = frame(payload.expect("two-field V1 payload must serialize"));
+            assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
+            assert!(ensure_legacy_pool_meta_republish_safe(&data).is_ok());
+            assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
+            let mut trailing = data;
+            trailing.push(0xc0);
+            assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
+        }
+        for dont_save in [false, true] {
+            let legacy = LegacyPoolMeta {
+                version: POOL_META_V1_VERSION,
+                pools: Vec::new(),
+                dont_save,
+            };
+            for payload in [rmp_serde::to_vec(&legacy), rmp_serde::to_vec_named(&legacy)] {
+                let data = frame(payload.expect("three-field V1 payload must serialize"));
+                assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
+                assert_eq!(ensure_legacy_pool_meta_republish_safe(&data).is_err(), dont_save);
+                assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
+                let mut trailing = data;
+                trailing.push(0xc0);
+                assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
+            }
+        }
+        // This guard is only called for already-decoded replicas; malformed
+        // containers must fail selection rather than becoming repair sources.
+        let truncated = frame(vec![0x93, 0x01]);
+        assert!(matches!(
+            decode_pool_meta_replica(truncated),
+            PoolMetaReplica::Corrupt(_) | PoolMetaReplica::Incompatible(_)
+        ));
+    }
+
     #[test]
     fn pool_meta_decode_supports_legacy_payload() {
         let start_time = OffsetDateTime::now_utc();
@@ -30571,6 +30829,154 @@ mod pools_tests {
         );
     }
 
+    #[tokio::test]
+    async fn pool_meta_prewrite_quorum_rejection_allows_retry_after_disks_return() {
+        let (dirs, disks, pool) = crate::set_disk::hermetic_set_disks_isolated(4).await;
+        pool.disks.write().await[2..].fill(None);
+        let state = super::PoolMetaWriteState::default();
+        let data =
+            super::initialized_pool_meta_identity_for_test(uuid::Uuid::new_v4(), 1).expect("encode the deployment identity");
+        let mut transaction = state.arm_transaction();
+        save_pool_meta_object_cas(
+            pool.clone(),
+            POOL_META_IDENTITY_NAME,
+            data.clone(),
+            &PoolMetaCasToken::Missing,
+            &PoolMetaPersistenceFence::OfflineRecovery,
+            "identity_cas",
+            &mut transaction,
+        )
+        .await
+        .expect_err("two of four disks cannot satisfy the identity write quorum");
+        drop(transaction);
+        for dir in &dirs {
+            assert!(
+                !dir.path()
+                    .join(crate::disk::RUSTFS_META_BUCKET)
+                    .join(POOL_META_IDENTITY_NAME)
+                    .exists(),
+                "the rejected write must not create an identity replica"
+            );
+        }
+        state
+            .ensure_write_safe("retry after disks return")
+            .expect("a rejected inline write must not poison the startup transaction");
+
+        let ordinary_error = crate::config::com::save_config_with_opts(
+            pool.clone(),
+            "ordinary-config",
+            data.clone(),
+            &ObjectOptions {
+                max_parity: true,
+                no_lock: true,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect_err("ordinary writes must still fail without quorum");
+        assert!(
+            matches!(ordinary_error, Error::InsufficientWriteQuorum(_, _)),
+            "ordinary callers must retain their existing quorum error"
+        );
+
+        *pool.disks.write().await = disks.into_iter().map(Some).collect();
+        let mut retry = state.arm_transaction();
+        save_pool_meta_object_cas(
+            pool.clone(),
+            POOL_META_IDENTITY_NAME,
+            data.clone(),
+            &PoolMetaCasToken::Missing,
+            &PoolMetaPersistenceFence::OfflineRecovery,
+            "identity_cas",
+            &mut retry,
+        )
+        .await
+        .expect("the same transaction must succeed when the disks return");
+        retry.disarm();
+        drop(retry);
+        assert_eq!(
+            crate::config::com::read_config(pool, POOL_META_IDENTITY_NAME)
+                .await
+                .expect("read back the committed identity"),
+            data
+        );
+        state
+            .ensure_write_safe("after successful retry")
+            .expect("retry must leave writes enabled");
+    }
+
+    #[tokio::test]
+    async fn pool_meta_prewrite_quorum_rejection_keeps_an_earlier_write_armed() {
+        let (_dirs, disks, pool) = crate::set_disk::hermetic_set_disks_isolated(4).await;
+        let state = super::PoolMetaWriteState::default();
+        let data =
+            super::initialized_pool_meta_identity_for_test(uuid::Uuid::new_v4(), 1).expect("encode the deployment identity");
+        let mut transaction = state.arm_transaction();
+        let written = save_pool_meta_object_cas(
+            pool.clone(),
+            POOL_META_IDENTITY_NAME,
+            data.clone(),
+            &PoolMetaCasToken::Missing,
+            &PoolMetaPersistenceFence::OfflineRecovery,
+            "prepare_cas",
+            &mut transaction,
+        )
+        .await
+        .expect("the earlier phase must really commit");
+        pool.disks.write().await[2..].fill(None);
+        save_pool_meta_object_cas(
+            pool.clone(),
+            POOL_META_IDENTITY_NAME,
+            data.clone(),
+            &PoolMetaCasToken::Existing(written.etag.expect("committed identity has an ETag")),
+            &PoolMetaPersistenceFence::OfflineRecovery,
+            "identity_cas",
+            &mut transaction,
+        )
+        .await
+        .expect_err("the next phase must fail without a write quorum");
+        drop(transaction);
+        *pool.disks.write().await = disks.into_iter().map(Some).collect();
+        assert_eq!(
+            crate::config::com::read_config(pool, POOL_META_IDENTITY_NAME)
+                .await
+                .expect("the earlier commit must survive the rejected write"),
+            data
+        );
+        let error = state
+            .ensure_write_safe("after a committed phase")
+            .expect_err("a later rejection cannot discharge the earlier write");
+        assert_eq!(
+            error.pool_metadata_failure().expect("retain typed uncertainty").kind,
+            crate::error::PoolMetadataFailure::TransactionUnknown
+        );
+        assert_eq!(error.pool_metadata_failure().expect("retain the earlier mutation").phase, "prepare_cas");
+    }
+
+    #[tokio::test]
+    async fn pool_meta_prewrite_noninline_failure_keeps_transaction_armed() {
+        let (_dirs, _disks, pool) = crate::set_disk::hermetic_set_disks_isolated(4).await;
+        pool.disks.write().await[2..].fill(None);
+        let state = super::PoolMetaWriteState::default();
+        let mut transaction = state.arm_transaction();
+        save_pool_meta_object_cas(
+            pool,
+            POOL_META_IDENTITY_NAME,
+            vec![0x51; 2 * 1024 * 1024],
+            &PoolMetaCasToken::Missing,
+            &PoolMetaPersistenceFence::OfflineRecovery,
+            "identity_cas",
+            &mut transaction,
+        )
+        .await
+        .expect_err("the noninline write must fail without quorum");
+        drop(transaction);
+        assert!(
+            state.ensure_write_safe("after writer setup").is_err(),
+            "a failure after disk writers were created must remain fail-closed"
+        );
+    }
+
     #[derive(Clone, Copy, Debug, PartialEq, Eq)]
     enum IdentityRepairFault {
         Healthy,

crates/ecstore/src/data_movement/e2e_barrier.rs

@@ -31,6 +31,8 @@ use tokio::fs;
 use tokio::time::Instant;
 
 pub(crate) const BARRIER_DIR_ENV: &str = "RUSTFS_E2E_DATA_MOVEMENT_BARRIER_DIR";
+/// Rebalance holds its admission fence; no version of this entry has moved.
+pub(crate) const STAGE_AFTER_ENTRY_ADMISSION: &str = "after_entry_admission";
 /// Every target part is staged and the target version is not yet published.
 pub(crate) const STAGE_BEFORE_TARGET_COMMIT: &str = "before_target_commit";
 /// The target commit is durable and the source preflight matched; the source

crates/ecstore/src/data_movement/mod.rs

@@ -605,6 +605,7 @@ fn resolve_data_movement_abort_result(
 #[derive(Debug)]
 struct DataMovementStageError {
     rendered: String,
+    stage: Option<String>,
     source: Box<dyn std::error::Error + Send + Sync>,
 }
 
@@ -626,6 +627,7 @@ where
 {
     Error::other(DataMovementStageError {
         rendered,
+        stage: None,
         source: Box::new(err),
     })
 }
@@ -634,7 +636,21 @@ fn data_movement_stage_error<E>(op_label: &str, stage: &str, bucket: &str, objec
 where
     E: std::error::Error + Send + Sync + 'static,
 {
-    data_movement_context_error(format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"), err)
+    Error::other(DataMovementStageError {
+        rendered: format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"),
+        stage: Some(stage.to_owned()),
+        source: Box::new(err),
+    })
+}
+
+pub(crate) fn data_movement_error_is_stage(err: &Error, stage: &str) -> bool {
+    let Error::Io(io_err) = err else {
+        return false;
+    };
+    io_err
+        .get_ref()
+        .and_then(|source| source.downcast_ref::<DataMovementStageError>())
+        .is_some_and(|source| source.stage.as_deref() == Some(stage))
 }
 
 #[cfg(test)]

crates/ecstore/src/disk/disk_store.rs

@@ -853,6 +853,8 @@ pub(crate) struct DiskHealthMetricEpoch {
     fallback_operation_metrics: StdRwLock<HashMap<&'static str, Arc<DiskOperationMetrics>>>,
     #[cfg(test)]
     multipart_cleanup_observer: std::sync::Mutex<Option<std::sync::Weak<ScopedMultipartCleanupCompletions>>>,
+    #[cfg(test)]
+    operation_timeout_override: std::sync::Mutex<Option<Duration>>,
     /// Caller API operations currently executing through the disk health wrapper.
     api_waiting: AtomicU32,
     /// Operations rejected because the disk was unavailable for the caller.
@@ -880,6 +882,8 @@ impl Default for DiskHealthMetricEpoch {
             fallback_operation_metrics: StdRwLock::new(HashMap::new()),
             #[cfg(test)]
             multipart_cleanup_observer: std::sync::Mutex::new(None),
+            #[cfg(test)]
+            operation_timeout_override: std::sync::Mutex::new(None),
             api_waiting: AtomicU32::new(0),
             total_errors_availability: AtomicU64::new(0),
             total_errors_timeout: AtomicU64::new(0),
@@ -1573,6 +1577,20 @@ impl Default for DiskHealthTracker {
     }
 }
 
+#[cfg(test)]
+#[derive(Debug)]
+pub(crate) struct ScopedDiskOperationTimeout {
+    metrics: Arc<DiskHealthMetricEpoch>,
+    previous: Option<Duration>,
+}
+
+#[cfg(test)]
+impl Drop for ScopedDiskOperationTimeout {
+    fn drop(&mut self) {
+        *self.metrics.operation_timeout_override.lock().expect("test timeout lock") = self.previous;
+    }
+}
+
 /// LocalDiskWrapper wraps a DiskStore with health tracking capabilities.
 /// This is similar to Go's xlStorageDiskIDCheck.
 #[derive(Debug, Clone)]
@@ -1914,6 +1932,23 @@ impl LocalDiskWrapper {
         self.health_check
     }
 
+    #[cfg(test)]
+    pub(crate) fn override_operation_timeout_for_test(&self, timeout: Duration) -> ScopedDiskOperationTimeout {
+        assert!(!timeout.is_zero(), "fault injection needs a positive timeout");
+        // The metrics epoch follows this drive across clones and reconnects.
+        // Its scoped override cannot shorten unrelated healthy drives' I/O.
+        let previous = self
+            .metrics
+            .operation_timeout_override
+            .lock()
+            .expect("test timeout lock")
+            .replace(timeout);
+        ScopedDiskOperationTimeout {
+            metrics: Arc::clone(&self.metrics),
+            previous,
+        }
+    }
+
     /// Enable health monitoring after disk creation.
     /// Used to defer health checks until after startup format loading completes.
     pub fn enable_health_check(&self) {
@@ -2398,6 +2433,10 @@ impl LocalDiskWrapper {
         .await
     }
 
+    pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
+        self.disk.has_clean_active_multipart_source(upload_or_part_path, part_path)
+    }
+
     pub(crate) async fn try_publish_prepared_delete_local(
         &self,
         volume: &str,
@@ -2634,6 +2673,13 @@ impl LocalDiskWrapper {
             }
             return result;
         }
+        #[cfg(test)]
+        let timeout_duration = self
+            .metrics
+            .operation_timeout_override
+            .lock()
+            .expect("test timeout lock")
+            .map_or(timeout_duration, |injected| timeout_duration.min(injected));
         // Execute the operation with timeout
         let result = tokio::time::timeout(timeout_duration, operation).await;
 
@@ -6040,6 +6086,67 @@ mod tests {
         assert!(health.offline_duration().is_none());
     }
 
+    #[tokio::test]
+    async fn scoped_operation_timeout_is_per_disk_and_preserves_caller_policy() {
+        let directories = [
+            tempfile::tempdir().expect("faulted drive"),
+            tempfile::tempdir().expect("healthy drive"),
+        ];
+        let mut wrappers = Vec::new();
+        for directory in &directories {
+            let endpoint = Endpoint::try_from(directory.path().to_str().expect("UTF-8 drive path")).expect("drive endpoint");
+            let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local drive"));
+            wrappers.push(LocalDiskWrapper::new(disk, false));
+        }
+        let reconnected = LocalDiskWrapper::new_with_reconnect_state(
+            Arc::clone(&wrappers[0].disk),
+            false,
+            Some(wrappers[0].health_tracker_epoch_for_reconnect()),
+        );
+        let scope = wrappers[0].override_operation_timeout_for_test(Duration::from_secs(1));
+        tokio::time::pause();
+        let operation = || async {
+            tokio::time::sleep(Duration::from_secs(2)).await;
+            Ok(())
+        };
+        let (faulted, healthy) = tokio::join!(
+            reconnected.track_disk_health_with_op_and_timeout_action(
+                "read_metadata",
+                operation,
+                Duration::from_secs(3),
+                TimeoutHealthAction::IgnoreFailure,
+            ),
+            wrappers[1].track_disk_health_with_op_and_timeout_action(
+                "read_metadata",
+                operation,
+                Duration::from_secs(3),
+                TimeoutHealthAction::IgnoreFailure,
+            ),
+        );
+        assert_eq!(faulted.expect_err("only the selected drive must time out"), DiskError::Timeout);
+        healthy.expect("healthy drive keeps its caller's deadline");
+        let shorter_started = tokio::time::Instant::now();
+        let shorter = wrappers[0]
+            .track_disk_health_with_op_and_timeout_action(
+                "read_metadata",
+                operation,
+                Duration::from_millis(500),
+                TimeoutHealthAction::IgnoreFailure,
+            )
+            .await;
+        assert_eq!(shorter.expect_err("fault injection cannot extend a caller deadline"), DiskError::Timeout);
+        assert!(shorter_started.elapsed() < Duration::from_secs(1));
+        wrappers[0]
+            .track_disk_health_with_op("read_metadata", operation, Duration::ZERO)
+            .await
+            .expect("explicit unbounded physical ownership cannot acquire a test deadline");
+        drop(scope);
+        reconnected
+            .track_disk_health_with_op("read_metadata", operation, Duration::from_secs(3))
+            .await
+            .expect("dropping the scope restores reconnected callers' deadlines");
+    }
+
     #[tokio::test]
     async fn ignored_timeout_does_not_mark_drive_failure() {
         let dir = tempfile::tempdir().expect("temp dir should be created");

crates/ecstore/src/disk/fd_cache_tests.rs

@@ -5,6 +5,7 @@ use super::{FdCache, FdCacheEntry, FdKey};
 use std::fs::File;
 use std::os::unix::fs::FileExt;
 use std::sync::Arc;
+use std::sync::atomic::AtomicBool;
 
 fn key(volume: &str, path: &str, direct: bool) -> FdKey {
     FdKey {
@@ -52,7 +53,11 @@ async fn uring_fd_cache_exact_invalidation_removes_both_modes_and_preserves_othe
 async fn uring_fd_cache_exact_invalidation_bumps_one_generation_and_rejects_old_epoch_inserts() {
     let (_directory, file) = fixture();
     let cache = FdCache::new();
-    let entry = Arc::new(FdCacheEntry { file, len: 3 });
+    let entry = Arc::new(FdCacheEntry {
+        file,
+        len: 3,
+        invalidated: AtomicBool::new(false),
+    });
     let old_generation = cache.generation();
     cache.invalidate_exact("v", "part").await;
     let current_generation = cache.generation();
@@ -125,7 +130,7 @@ async fn uring_fd_cache_volume_and_clear_invalidation_cover_both_modes() {
         cache.insert(key("v", "part", direct), Arc::clone(&file)).await;
         assert!(
             cache.get(&key("v", "part", direct)).await.is_some(),
-            "new entries must survive the earlier volume predicate"
+            "new entries must survive the earlier volume invalidation"
         );
     }
     cache.clear();
@@ -164,6 +169,7 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
     let replacement = Arc::new(FdCacheEntry {
         file: Arc::new(File::open(directory.path().join("part")).expect("open replacement inode")),
         len: 11,
+        invalidated: AtomicBool::new(false),
     });
     for direct in [false, true] {
         let key = key("v", "part", direct);
@@ -177,3 +183,459 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
         assert_eq!(fresh.len, 11);
     }
 }
+
+#[tokio::test]
+async fn fd_cache_tracker_skips_unmatched_invalidations_and_handles_deep_prefixes() {
+    let (_directory, file) = fixture();
+    let mut cache = FdCache::new();
+
+    // Unmatched invalidations on an empty cache must not spawn the maintenance task.
+    cache.invalidate_under("v", "missing/prefix");
+    cache.invalidate_volume("v");
+    cache.invalidate_exact("v", "missing/prefix/part.1").await;
+    cache.clear();
+    assert!(
+        cache.maintenance_task.get().is_none(),
+        "empty-cache invalidations must not spawn maintenance"
+    );
+
+    // Populate a normal entry and verify matched vs unmatched filter gates directly,
+    // and ensure unmatched invalidations do not enqueue pending removals.
+    let cached_key = key("v", "obj-1/0198a3f0/part.1", false);
+    cache.insert(cached_key.clone(), Arc::clone(&file)).await;
+    assert_eq!(cache.tracker.live_entries(), 1);
+    assert!(cache.tracker.may_match_volume("v"));
+    assert!(!cache.tracker.may_match_volume("other-volume"));
+    assert!(cache.tracker.may_match_prefix("v", "obj-1"));
+    assert!(cache.tracker.may_match_prefix("v", "obj-1/0198a3f0"));
+    assert!(!cache.tracker.may_match_prefix("v", "obj-2/0198a3f0"));
+    assert!(cache.tracker.may_match_exact("v", "obj-1/0198a3f0/part.1"));
+    assert!(!cache.tracker.may_match_exact("v", "obj-1/0198a3f0/xl.meta"));
+
+    cache.invalidate_volume("other-volume");
+    cache.invalidate_under("v", "obj-2/0198a3f0");
+    cache.invalidate_exact("v", "obj-1/0198a3f0/xl.meta").await;
+    assert!(
+        !cache
+            .tracker
+            .has_pending_invalidations
+            .load(std::sync::atomic::Ordering::SeqCst),
+        "unmatched invalidations must not enqueue maintenance removals"
+    );
+    assert!(cache.get(&cached_key).await.is_some());
+
+    // Deeply nested path (> FD_CACHE_MAX_INDEXED_PREFIXES slashes) must fall back
+    // to per-volume unindexed_counts without degrading other volumes.
+    let other_vol_key = key("v2", "shallow/part.1", false);
+    cache.insert(other_vol_key.clone(), Arc::clone(&file)).await;
+    let deep_path = "c1/c2/c3/c4/c5/c6/c7/c8/c9/c10/c11/c12/c13/c14/part.1";
+    let deep_key = key("v", deep_path, false);
+    cache.insert(deep_key.clone(), Arc::clone(&file)).await;
+    assert_eq!(
+        cache
+            .tracker
+            .unindexed_counts
+            .iter()
+            .map(|c| c.load(std::sync::atomic::Ordering::Acquire))
+            .sum::<u32>(),
+        1
+    );
+    assert!(
+        !cache.tracker.may_match_prefix("v2", "unmatched/prefix"),
+        "deep key in volume v must not disable prefix filtering in volume v2"
+    );
+    cache.invalidate_under("v", "c1/c2/c3/c4/c5/c6/c7/c8/c9/c10/c11/c12/c13");
+    assert!(cache.get(&deep_key).await.is_none());
+    cache.invalidate_volume("v2");
+    assert!(cache.get(&other_vol_key).await.is_none());
+
+    // Replacing an invalidated key before maintenance drains pending_invalidations
+    // must release the old descriptor immediately and keep the replacement live
+    // even when a strong reference to the old CachedFdEntry shell survives until
+    // maintenance runs its Arc::ptr_eq check.
+    let old_file = Arc::new(tempfile::tempfile().expect("create replaced descriptor"));
+    let old_weak = Arc::downgrade(&old_file);
+    cache.insert(cached_key.clone(), old_file).await;
+    let held_old_shell = cache.cache.get(&cached_key).expect("cached shell must exist");
+    cache.invalidate_under("v", "obj-1/0198a3f0");
+    assert!(cache.get(&cached_key).await.is_none());
+    cache.insert(cached_key.clone(), Arc::clone(&file)).await;
+    assert!(
+        old_weak.upgrade().is_none(),
+        "replacing an entry must retire and drop the previous descriptor immediately"
+    );
+    tokio::time::timeout(std::time::Duration::from_secs(5), async {
+        while cache
+            .tracker
+            .has_pending_invalidations
+            .load(std::sync::atomic::Ordering::SeqCst)
+        {
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("maintenance should drain pending invalidations");
+    drop(held_old_shell);
+    assert!(
+        cache.get(&cached_key).await.is_some(),
+        "stale pending invalidation must not evict a newer replacement"
+    );
+    assert_eq!(
+        cache
+            .tracker
+            .unindexed_counts
+            .iter()
+            .map(|c| c.load(std::sync::atomic::Ordering::Acquire))
+            .sum::<u32>(),
+        0
+    );
+
+    // Dropping a cache while a live non-invalidated entry is present (even if a
+    // cloned Moka handle outlives FdCache and the maintenance task is aborted)
+    // must retire the descriptor and release FdCacheTracker without cycles.
+    let live_drop_file = Arc::new(tempfile::tempfile().expect("create live drop descriptor"));
+    let live_drop_weak = Arc::downgrade(&live_drop_file);
+    let shutdown_file = Arc::new(tempfile::tempfile().expect("create shutdown descriptor"));
+    let shutdown_weak = Arc::downgrade(&shutdown_file);
+    let tracker_weak = Arc::downgrade(&cache.tracker);
+    let cloned_moka = cache.cache.clone();
+    cache.insert(key("v", "live-on-drop/part.1", false), live_drop_file).await;
+    cache.insert(key("v", "shutdown/part.1", false), shutdown_file).await;
+    cache.invalidate_under("v", "shutdown");
+    if let Some(handle) = cache.maintenance_task.take() {
+        handle.abort();
+        let _ = handle.await;
+    }
+    drop(cache);
+    assert!(
+        live_drop_weak.upgrade().is_none(),
+        "Drop for FdCache must retire live descriptors even if a cloned Moka handle survives"
+    );
+    assert!(
+        shutdown_weak.upgrade().is_none(),
+        "dropping cache when maintenance is aborted must still release descriptors"
+    );
+    assert!(
+        tracker_weak.upgrade().is_none(),
+        "dropping cache must release FdCacheTracker without reference cycles"
+    );
+    drop(cloned_moka);
+
+    // Short-TTL expired entries (including mixed expired and fresh entries) must
+    // be retired by invalidate_under and clear() even though Moka's iterator skips
+    // expired entries.
+    let ttl_cache = FdCache::with_ttl(std::time::Duration::from_millis(15));
+    let expired_file = Arc::new(tempfile::tempfile().expect("create short-ttl descriptor"));
+    let expired_weak = Arc::downgrade(&expired_file);
+    ttl_cache.insert(key("v", "ttl-obj/part.1", false), expired_file).await;
+    tokio::time::sleep(std::time::Duration::from_millis(30)).await;
+    let fresh_file = Arc::new(tempfile::tempfile().expect("create fresh descriptor"));
+    let fresh_weak = Arc::downgrade(&fresh_file);
+    ttl_cache.insert(key("v", "ttl-obj/part.2", false), fresh_file).await;
+    ttl_cache.invalidate_under("v", "ttl-obj");
+    tokio::time::timeout(std::time::Duration::from_secs(5), async {
+        while expired_weak.upgrade().is_some() || fresh_weak.upgrade().is_some() {
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("invalidate_under must retire both expired and fresh matching descriptors");
+    assert_eq!(ttl_cache.tracker.live_entries(), 0);
+
+    let clear_expired_file = Arc::new(tempfile::tempfile().expect("create clear short-ttl descriptor"));
+    let clear_expired_weak = Arc::downgrade(&clear_expired_file);
+    ttl_cache
+        .insert(key("v", "ttl-clear/part.1", false), clear_expired_file)
+        .await;
+    tokio::time::sleep(std::time::Duration::from_millis(30)).await;
+    let clear_live_file = Arc::new(tempfile::tempfile().expect("create clear live descriptor"));
+    let clear_live_weak = Arc::downgrade(&clear_live_file);
+    ttl_cache.insert(key("v", "ttl-clear/part.2", false), clear_live_file).await;
+    ttl_cache.clear();
+    assert!(
+        clear_expired_weak.upgrade().is_none() && clear_live_weak.upgrade().is_none(),
+        "clear() must synchronously retire both TTL-expired and live descriptors"
+    );
+    assert_eq!(ttl_cache.tracker.live_entries(), 0);
+}
+
+#[tokio::test]
+async fn fd_cache_lockstep_oracle_equivalence_and_counter_conservation() {
+    use std::collections::HashMap;
+    use std::sync::atomic::Ordering;
+
+    let (_directory, file) = fixture();
+    let cache = FdCache::with_ttl(std::time::Duration::from_secs(3600));
+    let mut oracle: HashMap<FdKey, Arc<FdCacheEntry>> = HashMap::new();
+
+    let volumes = ["vol-a", "vol-b", "vol-c", ".rustfs.sys/multipart"];
+    let paths = [
+        "a/b",
+        "a/b/part.1",
+        "a/b/dir/part.2",
+        "a/bc",
+        "a/bc/part.1",
+        "prefix/obj-0001.bin/0198a3f0/part.1",
+        "prefix/obj-0002.bin/0198a3f0/part.1",
+        "d1/d2/d3/d4/d5/d6/d7/d8/d9/d10/d11/d12/d13/part.1",
+    ];
+    let prefixes = [
+        "a",
+        "a/",
+        "a/b",
+        "a/b/",
+        "a/b/dir",
+        "a/bc",
+        "prefix",
+        "prefix/obj-0001.bin",
+        "prefix/obj-0001.bin/0198a3f0",
+        "d1/d2/d3/d4/d5/c_missing",
+        "d1/d2/d3/d4/d5/d6/d7/d8/d9/d10/d11/d12/d13",
+        "nonexistent/prefix",
+    ];
+
+    let mut candidate_keys = Vec::new();
+    for v in volumes {
+        for p in paths {
+            for direct in [false, true] {
+                candidate_keys.push(key(v, p, direct));
+            }
+        }
+    }
+
+    let mut rng = 0x9e37_79b9_7f4a_7c15u64;
+    let mut next_u64 = || {
+        rng ^= rng << 13;
+        rng ^= rng >> 7;
+        rng ^= rng << 17;
+        rng
+    };
+
+    for step in 0..1000 {
+        let op = next_u64() % 100;
+        if op < 45 {
+            // Insert or replace (either fresh or stale generation)
+            let k = candidate_keys[(next_u64() as usize) % candidate_keys.len()].clone();
+            let entry = Arc::new(FdCacheEntry {
+                file: Arc::clone(&file),
+                len: step,
+                invalidated: AtomicBool::new(false),
+            });
+            let cur_gen = cache.generation();
+            let stale = (next_u64() % 5) == 0 && cur_gen > 0;
+            let gen_to_use = if stale { cur_gen - 1 } else { cur_gen };
+            cache.insert_if_fresh(k.clone(), Arc::clone(&entry), gen_to_use).await;
+            if !stale {
+                oracle.insert(k, entry);
+            }
+        } else if op < 65 {
+            // Exact invalidation
+            let v = volumes[(next_u64() as usize) % volumes.len()];
+            let p = paths[(next_u64() as usize) % paths.len()];
+            cache.invalidate_exact(v, p).await;
+            oracle.remove(&key(v, p, false));
+            oracle.remove(&key(v, p, true));
+        } else if op < 88 {
+            // Prefix invalidation
+            let v = volumes[(next_u64() as usize) % volumes.len()];
+            let raw_prefix = prefixes[(next_u64() as usize) % prefixes.len()];
+            cache.invalidate_under(v, raw_prefix);
+            let trimmed = raw_prefix.trim_end_matches('/');
+            oracle.retain(|k, _| {
+                !(k.volume == v && (k.path == trimmed || k.path.strip_prefix(trimmed).is_some_and(|r| r.starts_with('/'))))
+            });
+        } else if op < 97 {
+            // Volume invalidation
+            let v = volumes[(next_u64() as usize) % volumes.len()];
+            cache.invalidate_volume(v);
+            oracle.retain(|k, _| k.volume != v);
+        } else {
+            // Full clear
+            cache.clear();
+            oracle.clear();
+        }
+
+        // Verify lockstep equivalence against the reference oracle after every operation.
+        for k in &candidate_keys {
+            let got = cache.get(k).await;
+            let expected = oracle.get(k);
+            match (got, expected) {
+                (Some(g), Some(e)) => assert!(
+                    Arc::ptr_eq(&g, e),
+                    "step {step}: pointer mismatch for key ({}, {}, {})",
+                    k.volume,
+                    k.path,
+                    k.direct
+                ),
+                (None, None) => {}
+                (g, e) => panic!(
+                    "step {step}: oracle mismatch for key ({}, {}, {}): got={}, expected={}",
+                    k.volume,
+                    k.path,
+                    k.direct,
+                    g.is_some(),
+                    e.is_some()
+                ),
+            }
+        }
+    }
+
+    // Clear and wait for maintenance to drain all entries, then verify exact counter conservation.
+    cache.clear();
+    tokio::time::timeout(std::time::Duration::from_secs(5), async {
+        while cache.tracker.live_entries() != 0 || cache.tracker.has_pending_invalidations.load(Ordering::SeqCst) {
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("clear must drain all live entries and pending invalidations");
+
+    assert_eq!(cache.tracker.live_entries(), 0);
+    assert!(cache.tracker.unindexed_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.volume_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.exact_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.prefix_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.prefix_shards.iter().all(|s| s.lock().is_empty()));
+    assert!(cache.tracker.entry_shards.iter().all(|s| s.lock().is_empty()));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn fd_cache_concurrent_insert_and_invalidation_stress() {
+    use std::sync::atomic::{AtomicU64, Ordering};
+
+    let (_directory, file) = fixture();
+    let cache = Arc::new(FdCache::new());
+
+    // Deterministic Dekker-partner interleaving check: pause `CachedFdEntry::new`
+    // at `entry_shards[shard].lock()` after it has passed the initial generation
+    // check and incremented `volume_counts`, bump generation via an unmatched
+    // invalidation, and verify the post-`new()` recheck refuses publication and
+    // preserves the existing entry.
+    let seam_key = key("v-seam", "obj-1/part.1", false);
+    cache.insert(seam_key.clone(), Arc::clone(&file)).await;
+    let initial_entry = cache.get(&seam_key).await.expect("initial entry must exist");
+    let (vol_hash, vol_slot) = super::fd_cache_volume_hash_and_slot("v-seam");
+    let entry_shard = (vol_hash as usize) & (super::FD_CACHE_PREFIX_SHARDS - 1);
+    let stale_gen = cache.generation();
+    let stale_entry = Arc::new(FdCacheEntry {
+        file: Arc::clone(&file),
+        len: 999_999,
+        invalidated: AtomicBool::new(false),
+    });
+    let blocked_insert = {
+        let hold = cache.tracker.entry_shards[entry_shard].lock();
+        let cache_clone = Arc::clone(&cache);
+        let seam_key_clone = seam_key.clone();
+        let task = tokio::spawn(async move {
+            cache_clone.insert_if_fresh(seam_key_clone, stale_entry, stale_gen).await;
+        });
+        let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
+        while cache.tracker.volume_counts[usize::from(vol_slot)].load(Ordering::SeqCst) < 2 {
+            assert!(
+                std::time::Instant::now() < deadline,
+                "insert never reached CachedFdEntry::new with the previous entry still live"
+            );
+            std::thread::yield_now();
+        }
+        cache.invalidate_under("v-seam", "unmatched-prefix");
+        drop(hold);
+        task
+    };
+    blocked_insert.await.expect("blocked insert task should complete");
+    let preserved = cache
+        .get(&seam_key)
+        .await
+        .expect("refused stale insert must not retire a valid previous entry");
+    assert!(Arc::ptr_eq(&preserved, &initial_entry));
+    cache.invalidate_volume("v-seam");
+
+    let floor_v1 = Arc::new(AtomicU64::new(0));
+    let floor_v2 = Arc::new(AtomicU64::new(0));
+    let mut handles = Vec::new();
+
+    for t in 0..8usize {
+        let cache = Arc::clone(&cache);
+        let file = Arc::clone(&file);
+        let floor_v1 = Arc::clone(&floor_v1);
+        let floor_v2 = Arc::clone(&floor_v2);
+        handles.push(tokio::spawn(async move {
+            for i in 0..250u64 {
+                let is_v1 = (t + i as usize).is_multiple_of(2);
+                let vol = if is_v1 { "v1" } else { "v2" };
+                let path = format!("dir-{}/obj-{}/part.1", t % 4, i % 16);
+                let k = key(vol, &path, i.is_multiple_of(2));
+                let g = cache.generation();
+                let entry = Arc::new(FdCacheEntry {
+                    file: Arc::clone(&file),
+                    len: g,
+                    invalidated: AtomicBool::new(false),
+                });
+                cache.insert_if_fresh(k.clone(), entry, g).await;
+                let min_gen = if is_v1 {
+                    floor_v1.load(Ordering::SeqCst)
+                } else {
+                    floor_v2.load(Ordering::SeqCst)
+                };
+                if let Some(hit) = cache.get(&k).await {
+                    assert!(
+                        hit.len >= min_gen,
+                        "stale descriptor served after volume invalidation: opened_gen={}, floor_gen={min_gen}",
+                        hit.len
+                    );
+                }
+            }
+        }));
+    }
+
+    for t in 0..4usize {
+        let cache = Arc::clone(&cache);
+        let floor_v1 = Arc::clone(&floor_v1);
+        let floor_v2 = Arc::clone(&floor_v2);
+        handles.push(tokio::spawn(async move {
+            for i in 0..250u64 {
+                let is_v1 = (t + i as usize).is_multiple_of(2);
+                let vol = if is_v1 { "v1" } else { "v2" };
+                match i % 4 {
+                    0 => cache.invalidate_under(vol, &format!("dir-{}/obj-{}", t % 4, i % 16)),
+                    1 => {
+                        let p = format!("dir-{}/obj-{}/part.1", t % 4, i % 16);
+                        cache.invalidate_exact(vol, &p).await;
+                    }
+                    2 => {
+                        let before = cache.generation();
+                        cache.invalidate_volume(vol);
+                        let floor = before + 1;
+                        if is_v1 {
+                            floor_v1.fetch_max(floor, Ordering::SeqCst);
+                        } else {
+                            floor_v2.fetch_max(floor, Ordering::SeqCst);
+                        }
+                    }
+                    _ => cache.invalidate_under("uncached-vol", "uncached/prefix"),
+                }
+                tokio::task::yield_now().await;
+            }
+        }));
+    }
+
+    for h in handles {
+        h.await.expect("worker task should succeed");
+    }
+
+    cache.clear();
+    tokio::time::timeout(std::time::Duration::from_secs(5), async {
+        while cache.tracker.live_entries() != 0 || cache.tracker.has_pending_invalidations.load(Ordering::SeqCst) {
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("clear after concurrent stress must drain all live entries");
+
+    assert_eq!(cache.tracker.live_entries(), 0);
+    assert!(cache.tracker.unindexed_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.volume_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.exact_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.prefix_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.prefix_shards.iter().all(|s| s.lock().is_empty()));
+    assert!(cache.tracker.entry_shards.iter().all(|s| s.lock().is_empty()));
+}

crates/ecstore/src/disk/local/commit.rs

@@ -175,17 +175,30 @@ pub(super) fn create_local_inline_rollback_backup(
     dst_file_path: &Path,
     staging_file_path: &Path,
     old_metadata: &[u8],
+    sync: bool,
 ) -> std::io::Result<PathBuf> {
     let Some(staging_parent) = staging_file_path.parent() else {
         return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing staging metadata parent"));
     };
     let backup_path = staging_parent.join(STORAGE_FORMAT_FILE_BACKUP);
     remove_file_if_exists(&backup_path)?;
-    if (should_fail_local_inline_rollback_hardlink(dst_file_path) || std::fs::hard_link(dst_file_path, &backup_path).is_err())
-        && let Err(err) = std::fs::write(&backup_path, old_metadata)
-    {
-        let _ = remove_file_if_exists(&backup_path);
-        return Err(err);
+    if should_fail_local_inline_rollback_hardlink(dst_file_path) || std::fs::hard_link(dst_file_path, &backup_path).is_err() {
+        let write_res = (|| -> std::io::Result<()> {
+            #[cfg(test)]
+            run_owned_file_before_open(&backup_path);
+            let mut file = std::fs::File::create(&backup_path)?;
+            std::io::Write::write_all(&mut file, old_metadata)?;
+            if sync && !old_metadata.is_empty() {
+                file.sync_data()?;
+            }
+            Ok(())
+        })();
+        if let Err(err) = write_res {
+            let _ = remove_file_if_exists(&backup_path);
+            return Err(err);
+        }
+    } else if sync && !old_metadata.is_empty() {
+        std::fs::File::open(&backup_path)?.sync_data()?;
     }
     Ok(backup_path)
 }
@@ -356,9 +369,12 @@ fn prepare_rename_commit_and_destination_metadata_std(
 
 #[cfg(target_os = "linux")]
 enum FusedCommitAttempt {
-    Committed(RenameDataResp),
+    Committed(Box<RenameDataResp>),
     CommitErr(DiskError),
-    Fallback(os::RenameCommitGuard, Option<Bytes>, FileInfo),
+    RestoreDataSource(os::RenameCommitGuard, DiskError),
+    Fallback(os::RenameCommitGuard, Option<Bytes>, Box<FileInfo>),
+    #[cfg(test)]
+    ResumePanic(Box<dyn std::any::Any + Send + 'static>),
 }
 
 async fn prepare_rename_commit_and_destination_metadata(
@@ -540,7 +556,9 @@ impl LocalDisk {
             None
         };
         let multipart_publication = if let Some(staging) = multipart_staging.as_ref() {
-            Some(pin_multipart_future(|| self.prepare_native_multipart_publication(src_path, &mut fi, staging)).await?)
+            Some(Arc::new(
+                pin_multipart_future(|| self.prepare_native_multipart_publication_deferred(src_path, &mut fi, staging)).await?,
+            ))
         } else {
             None
         };
@@ -741,11 +759,6 @@ impl LocalDisk {
             && !fi_healing
             && !os::dst_dir_fsync_group_commit_enabled()
             && !os::file_fdatasync_group_commit_enabled()
-            && !super::has_rename_data_test_failpoints(dst_path)
-            && !crash_inject::is_armed(CrashPoint::RenameAfterDataRename, dst_path)
-            && !crash_inject::is_armed(CrashPoint::RenameAfterBackupBeforeMetaCommit, dst_path)
-            && !crash_inject::is_armed(CrashPoint::RenameAfterMetaCommit, dst_path)
-            && !super::has_delete_version_test_failpoints()
         {
             #[cfg(test)]
             if !no_inline {
@@ -785,10 +798,11 @@ impl LocalDisk {
                     &dst_volume_dir_owned,
                     &publication_root_owned,
                 )?;
-                // An existing xl.meta can share versions or require rollback
-                // restoration when a post-rename directory sync fails.
-                if is_cancelled() || (sync_commit_metadata && has_dst_buf.is_some()) {
-                    return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, fi_moved));
+                // An existing xl.meta that replaces an existing version slot
+                // requires a rollback directory; appending a new version keeps
+                // an inline rollback backup for post-rename directory sync errors.
+                if is_cancelled() {
+                    return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
                 }
                 let mut xlmeta = FileMeta::new();
                 let mut dst_meta_unparsable = false;
@@ -811,7 +825,19 @@ impl LocalDisk {
                     }
                 });
                 if rollback_data_dir.is_some() {
-                    return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, fi_moved));
+                    return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
+                }
+                #[cfg(test)]
+                let probe_active = os::file_sync_probe::is_active();
+                #[cfg(not(test))]
+                let probe_active = false;
+                if no_inline
+                    && sync_data_shards
+                    && let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref()
+                    && (probe_active || !super::is_presynced_tmp_data_dir(src_data_path))
+                    && os::regular_files(src_data_path).is_ok_and(|files| files.len() >= os::PARALLEL_FILE_SYNC_THRESHOLD)
+                {
+                    return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
                 }
                 let (old_current_size, old_current_source_checked, old_current_source) = observe_old_current(
                     has_dst_buf.is_some(),
@@ -830,13 +856,31 @@ impl LocalDisk {
                     skip_parent = dst_parent_owned.clone();
                 }
 
+                #[cfg(test)]
+                run_owned_file_before_open(&src_file_path_owned);
                 let mut prepared_metadata_source =
                     os::create_prepared_rename_source_with_commit_guard(&src_file_path_owned, &dst_file_path_owned, &guard)?;
-                prepared_metadata_source.write_all(&new_dst_buf, sync_commit_metadata)?;
+                prepared_metadata_source.write_all(&new_dst_buf, false)?;
+                if !no_inline {
+                    run_inline_preparation_before_backup(&dst_path_str);
+                }
+
+                let mut staged_rollback_path = None;
+                if let Some(ref old_metadata) = has_dst_buf
+                    && (sync_commit_metadata || cfg!(test))
+                {
+                    let backup_path = create_local_inline_rollback_backup(
+                        &dst_file_path_owned,
+                        &src_file_path_owned,
+                        old_metadata,
+                        sync_commit_metadata,
+                    )?;
+                    staged_rollback_path = Some(StagedRollbackBackup(backup_path));
+                }
 
                 if no_inline {
                     if sync_data_shards && let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref() {
-                        let sync_result = if super::take_presynced_tmp_data_dir(src_data_path) {
+                        let sync_result = if !probe_active && super::take_presynced_tmp_data_dir(src_data_path) {
                             // The payload was synced, but its part.1 directory entry
                             // must also be persisted before moving the data directory.
                             os::fsync_dir_std(src_data_path)
@@ -846,49 +890,157 @@ impl LocalDisk {
                         if let Err(err) = sync_result
                             && err.kind() != ErrorKind::NotFound
                         {
+                            if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
+                                let _ = remove_file_if_exists(backup_path);
+                            }
                             return Err(err);
                         }
                     }
+                    if sync_commit_metadata && let Err(err) = prepared_metadata_source.sync_data() {
+                        if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
+                            let _ = remove_file_if_exists(backup_path);
+                        }
+                        return Err(err);
+                    }
+                } else if sync_commit_metadata && let Err(err) = prepared_metadata_source.sync_data() {
+                    if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
+                        let _ = remove_file_if_exists(backup_path);
+                    }
+                    return Err(err);
+                }
+
+                if is_cancelled() {
+                    drop(prepared_metadata_source);
+                    if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
+                        let _ = remove_file_if_exists(backup_path);
+                    }
+                    return Err(std::io::Error::new(ErrorKind::Interrupted, "commit cancelled"));
+                }
+
+                let guard = match super::remove_dst_base_before_commit_std(
+                    &dst_path_str,
+                    guard,
+                    &src_parent_owned,
+                    &dst_parent_owned,
+                    &dst_volume_dir_owned,
+                    &publication_root_owned,
+                ) {
+                    Ok(guard) => guard,
+                    Err(err) => {
+                        if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
+                            let _ = remove_file_if_exists(backup_path);
+                        }
+                        return Ok(FusedCommitAttempt::CommitErr(err));
+                    }
+                };
+                if should_remove_staged_meta_before_commit(&dst_path_str) {
+                    drop(prepared_metadata_source);
+                    if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
+                        let _ = remove_file_if_exists(backup_path);
+                    }
+                    std::fs::remove_file(&src_file_path_owned)?;
+                    return Ok(FusedCommitAttempt::CommitErr(DiskError::FileNotFound));
+                }
+
+                if no_inline {
                     if let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref()
                         && let Err(err) = os::rename_all_with_commit_guard_on_thread(src_data_path, dst_data_path, &guard)
                     {
-                        return Ok(FusedCommitAttempt::CommitErr(err));
+                        if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
+                            let _ = remove_file_if_exists(backup_path);
+                        }
+                        return Ok(FusedCommitAttempt::RestoreDataSource(guard, err));
                     }
                     #[cfg(test)]
-                    if has_data_dir_path_owned.is_some() {
-                        run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
+                    if has_data_dir_path_owned.is_some()
+                        && let Err(panic_payload) = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+                            run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
+                        }))
+                    {
+                        return Ok(FusedCommitAttempt::ResumePanic(panic_payload));
+                    }
+                    if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, &dst_path_str) {
+                        return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
+                    }
+                    if should_fail_before_old_metadata_backup(&dst_path_str) {
+                        if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
+                            let _ = remove_file_if_exists(backup_path);
+                        }
+                        return Ok(FusedCommitAttempt::RestoreDataSource(guard, DiskError::Unexpected));
+                    }
+                    if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, &dst_path_str) {
+                        return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
                     }
                 }
 
-                if let Err(err) = os::rename_prepared_source_on_thread(
-                    prepared_metadata_source,
-                    &src_file_path_owned,
-                    &dst_file_path_owned,
-                    &skip_parent,
-                    &publication_root_owned,
-                    &guard,
-                ) {
-                    if no_inline && let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
-                        let _ = std::fs::rename(dst_data_path, src_data_path);
+                let rename_res = if should_fail_commit_rename(&dst_path_str) {
+                    drop(prepared_metadata_source);
+                    Err(DiskError::other("test fail during metadata commit rename"))
+                } else {
+                    os::rename_prepared_source_on_thread(
+                        prepared_metadata_source,
+                        &src_file_path_owned,
+                        &dst_file_path_owned,
+                        &skip_parent,
+                        &publication_root_owned,
+                        &guard,
+                    )
+                };
+                if let Err(err) = rename_res {
+                    if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
+                        let _ = remove_file_if_exists(backup_path);
+                    }
+                    if no_inline {
+                        return Ok(FusedCommitAttempt::RestoreDataSource(guard, err));
                     }
                     return Ok(FusedCommitAttempt::CommitErr(err));
                 }
 
-                if sync_commit_metadata
-                    && let Err(err) =
-                        fsync_inline_commit_std(&dst_file_path_owned, &dst_volume_dir_owned, destination_was_absent, None, &guard)
-                {
-                    if no_inline && let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
-                        let _ = std::fs::rename(dst_data_path, src_data_path);
+                if should_fail_after_metadata_commit(&dst_path_str) {
+                    if no_inline {
+                        rollback_inline_metadata_std(&dst_file_path_owned, None, staged_rollback_path.as_ref())?;
+                        if let Some((_, new_data_path)) = has_data_dir_path_owned.as_ref() {
+                            remove_dir_all_if_exists(new_data_path)?;
+                        }
+                        return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
                     }
-                    return Err(err);
+                    let err = rollback_inline_metadata_std(&dst_file_path_owned, None, staged_rollback_path.as_ref())
+                        .err()
+                        .unwrap_or_else(|| std::io::Error::other("test fail after metadata commit"));
+                    return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
+                }
+
+                if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, &dst_path_str) {
+                    return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
+                }
+
+                if sync_commit_metadata
+                    && let Err(err) = fsync_inline_commit_std(
+                        &dst_file_path_owned,
+                        &dst_volume_dir_owned,
+                        destination_was_absent,
+                        staged_rollback_path.as_ref(),
+                        &guard,
+                    )
+                {
+                    if no_inline {
+                        if let Some((_, new_data_path)) = has_data_dir_path_owned.as_ref() {
+                            remove_dir_all_if_exists(new_data_path)?;
+                        }
+                        return Ok(FusedCommitAttempt::CommitErr(to_file_error(err).into()));
+                    }
+                    return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
+                }
+
+                if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
+                    let _ = remove_file_if_exists(backup_path);
                 }
 
                 if let Some(parent) = src_file_path_owned.parent() {
                     let _ = std::fs::remove_dir(parent);
                 }
 
-                Ok(FusedCommitAttempt::Committed(RenameDataResp {
+                Ok(FusedCommitAttempt::Committed(Box::new(RenameDataResp {
                     multipart_source: None,
                     old_data_dir: None,
                     rollback_data_dir: None,
@@ -897,7 +1049,7 @@ impl LocalDisk {
                     old_current_size,
                     old_current_source_checked,
                     old_current_source,
-                }))
+                })))
             };
 
             let attempt_res = if let Some(admission) = file_sync_admission.as_ref() {
@@ -917,7 +1069,23 @@ impl LocalDisk {
                     if let Some(receipt_mutation) = receipt_mutation.as_mut() {
                         receipt_mutation.complete();
                     }
-                    return Ok(resp);
+                    return Ok(*resp);
+                }
+                Ok(FusedCommitAttempt::RestoreDataSource(directory_guard, err)) => {
+                    let commit = RenameCommitContext {
+                        directory_guard,
+                        publication_root: &self.publication_root,
+                        mutation_lease,
+                        source_metadata: &src_file_path,
+                        destination_metadata: &dst_file_path,
+                        source_volume_dir: &src_volume_dir,
+                        data_paths: has_data_dir_path.as_ref(),
+                    };
+                    commit.restore_published_data_source().await?;
+                    for part_path in &invalidate_part_paths {
+                        self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
+                    }
+                    return Err(err);
                 }
                 Ok(FusedCommitAttempt::CommitErr(err)) | Err(err) => {
                     for part_path in &invalidate_part_paths {
@@ -925,11 +1093,269 @@ impl LocalDisk {
                     }
                     return Err(err);
                 }
+                #[cfg(test)]
+                Ok(FusedCommitAttempt::ResumePanic(panic_payload)) => {
+                    std::panic::resume_unwind(panic_payload);
+                }
                 Ok(FusedCommitAttempt::Fallback(guard, dst_buf, restored_fi)) => {
-                    fi = restored_fi;
+                    fi = *restored_fi;
                     (guard, dst_buf)
                 }
             }
+        } else if let Some(publication) = multipart_publication.as_ref()
+            && no_inline
+            && !preserve_upload_metadata
+            && !publication.legacy
+            && !fi_healing
+            && !os::dst_dir_fsync_group_commit_enabled()
+        {
+            enum MultipartFusedCommitOutcome {
+                Committed(Box<RenameDataResp>),
+                AbortBeforeDataRename(DiskError),
+                RestoreDataSource(os::RenameCommitGuard, DiskError),
+                RestoreTargetAfterError(DiskError, bool),
+            }
+            let publication_owned = Arc::clone(publication);
+            let src_parent_owned = src_file_parent.to_path_buf();
+            let dst_parent_owned = dst_file_parent.to_path_buf();
+            let dst_file_path_owned = dst_file_path.clone();
+            let src_file_path_owned = src_file_path.clone();
+            let dst_volume_dir_owned = dst_volume_dir.clone();
+            let publication_root_owned = self.publication_root.clone();
+            let has_data_dir_path_owned = has_data_dir_path.clone();
+            let sync_data_shards = durability.syncs_data_shards();
+            let sync_commit_metadata = durability.syncs_commit_metadata();
+            let staging_cleanup = multipart_staging.as_ref().map(|s| s.cleanup_handle());
+            let dst_volume_str = dst_volume.to_string();
+            let dst_path_str = dst_path.to_string();
+            let fi_moved = fi;
+            #[cfg(test)]
+            let root_owned = self.root.clone();
+
+            let outcome = os::run_blocking_namespace_operation_with_sync(
+                mutation_lease.clone(),
+                true,
+                move || -> std::io::Result<MultipartFusedCommitOutcome> {
+                    let (directory_guard_owned, has_dst_buf_owned) = match prepare_rename_commit_and_destination_metadata_std(
+                        false,
+                        &src_parent_owned,
+                        &dst_parent_owned,
+                        &dst_file_path_owned,
+                        &dst_volume_dir_owned,
+                        &publication_root_owned,
+                    ) {
+                        Ok(res) => res,
+                        Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into())),
+                    };
+                    let mut xlmeta = FileMeta::new();
+                    let mut dst_meta_unparsable = false;
+                    if let Some(dst_buf) = has_dst_buf_owned.as_ref() {
+                        if FileMeta::is_xl2_v1_format(dst_buf)
+                            && let Ok(nmeta) = FileMeta::load(dst_buf)
+                        {
+                            xlmeta = nmeta;
+                        } else {
+                            dst_meta_unparsable = true;
+                        }
+                    }
+                    let skip_parent_owned = if has_dst_buf_owned.is_some() {
+                        dst_parent_owned.clone()
+                    } else {
+                        dst_volume_dir_owned.clone()
+                    };
+                    let version_id = fi_moved.version_id.unwrap_or_default();
+                    let (old_current_size, old_current_source_checked, old_current_source) = observe_old_current(
+                        has_dst_buf_owned.is_some(),
+                        dst_meta_unparsable,
+                        &xlmeta,
+                        &dst_volume_str,
+                        &dst_path_str,
+                        version_id,
+                    );
+                    let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id));
+                    if let Some(old_data_dir) = has_old_data_dir.as_ref() {
+                        let _ = xlmeta.data.remove_two(version_id, *old_data_dir);
+                    }
+                    if let Err(err) = xlmeta.add_version(fi_moved) {
+                        return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err.into()));
+                    }
+                    let version_signature = rename_data_versions_signature(&xlmeta);
+                    let new_dst_buf_owned = match xlmeta.marshal_msg() {
+                        Ok(buf) => buf,
+                        Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err.into())),
+                    };
+
+                    let prepared_metadata_source = match publication_owned.prepare_and_install_head_on_thread(
+                        &new_dst_buf_owned,
+                        has_dst_buf_owned.as_deref(),
+                        &dst_file_path_owned,
+                        &directory_guard_owned,
+                    ) {
+                        Ok(source) => source,
+                        Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err)),
+                    };
+                    #[cfg(test)]
+                    let probe_active = os::file_sync_probe::is_active();
+                    #[cfg(not(test))]
+                    let probe_active = false;
+                    // When presynced, shard payloads were synced during UploadPart and
+                    // prepare_and_install_head_on_thread just fsynced src_data_path after
+                    // writing RETIREMENT_ANCHOR.
+                    if sync_data_shards
+                        && let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref()
+                        && (probe_active || !super::take_presynced_tmp_data_dir(src_data_path))
+                        && let Err(err) = os::sync_dir_files_std(src_data_path)
+                        && err.kind() != ErrorKind::NotFound
+                    {
+                        return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into()));
+                    }
+                    let directory_guard_owned = match super::remove_dst_base_before_commit_std(
+                        &dst_path_str,
+                        directory_guard_owned,
+                        &src_parent_owned,
+                        &dst_parent_owned,
+                        &dst_volume_dir_owned,
+                        &publication_root_owned,
+                    ) {
+                        Ok(guard) => guard,
+                        Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err)),
+                    };
+                    if should_remove_staged_meta_before_commit(&dst_path_str) {
+                        drop(prepared_metadata_source);
+                        if let Err(err) = std::fs::remove_file(&src_file_path_owned) {
+                            return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into()));
+                        }
+                        return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::FileNotFound));
+                    }
+                    if let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
+                        let Some(src_parent) = src_data_path.parent() else {
+                            return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::InvalidPath));
+                        };
+                        let payload_guard = match os::multipart_payload_commit_guard_std(&directory_guard_owned, src_parent) {
+                            Ok(guard) => guard,
+                            Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into())),
+                        };
+                        if let Err(err) = os::rename_all_with_commit_guard_on_thread(src_data_path, dst_data_path, &payload_guard)
+                        {
+                            return Ok(MultipartFusedCommitOutcome::RestoreDataSource(directory_guard_owned, err));
+                        }
+                        #[cfg(test)]
+                        run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
+                        if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, &dst_path_str) {
+                            return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
+                        }
+                        if should_fail_before_old_metadata_backup(&dst_path_str) {
+                            return Ok(MultipartFusedCommitOutcome::RestoreDataSource(
+                                directory_guard_owned,
+                                DiskError::Unexpected,
+                            ));
+                        }
+                        if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, &dst_path_str) {
+                            return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
+                        }
+                    }
+                    let rename_res = if should_fail_commit_rename(&dst_path_str) {
+                        drop(prepared_metadata_source);
+                        Err(DiskError::other("test fail during metadata commit rename"))
+                    } else {
+                        os::rename_prepared_source_on_thread(
+                            prepared_metadata_source,
+                            &src_file_path_owned,
+                            &dst_file_path_owned,
+                            &skip_parent_owned,
+                            &publication_root_owned,
+                            &directory_guard_owned,
+                        )
+                    };
+                    if let Err(err) = rename_res {
+                        return Ok(MultipartFusedCommitOutcome::RestoreDataSource(directory_guard_owned, err));
+                    }
+                    if should_fail_after_metadata_commit(&dst_path_str) {
+                        return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(DiskError::Unexpected, false));
+                    }
+                    if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, &dst_path_str) {
+                        return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
+                    }
+                    if sync_commit_metadata {
+                        if let Some(parent) = dst_file_path_owned.parent()
+                            && let Err(err) = os::fsync_commit_directory_limited_std(parent, &directory_guard_owned)
+                        {
+                            return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(to_file_error(err).into(), true));
+                        }
+                        if has_dst_buf_owned.is_none() {
+                            let mut ancestor = dst_file_path_owned.parent().and_then(|object_dir| object_dir.parent());
+                            while let Some(dir) = ancestor {
+                                if !dir.starts_with(&dst_volume_dir_owned) {
+                                    break;
+                                }
+                                if let Err(err) = os::fsync_dir_limited_std(dir) {
+                                    return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(
+                                        to_file_error(err).into(),
+                                        true,
+                                    ));
+                                }
+                                if dir == dst_volume_dir_owned.as_path() {
+                                    break;
+                                }
+                                ancestor = dir.parent();
+                            }
+                        }
+                    }
+                    match publication_owned.finish_on_thread(sync_commit_metadata, staging_cleanup) {
+                        Ok(result) => Ok(MultipartFusedCommitOutcome::Committed(Box::new(RenameDataResp {
+                            multipart_source: Some(Box::new(result)),
+                            old_data_dir: has_old_data_dir,
+                            rollback_data_dir: None,
+                            cleanup_data_dir: has_old_data_dir,
+                            sign: version_signature,
+                            old_current_size,
+                            old_current_source_checked,
+                            old_current_source,
+                        }))),
+                        Err(err) => Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(err, false)),
+                    }
+                },
+            )
+            .await
+            .map_err(to_file_error)
+            .map_err(DiskError::from)?;
+
+            match outcome {
+                MultipartFusedCommitOutcome::Committed(resp) => {
+                    for part_path in &invalidate_part_paths {
+                        self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
+                    }
+                    return Ok(*resp);
+                }
+                MultipartFusedCommitOutcome::AbortBeforeDataRename(err) => return Err(err),
+                MultipartFusedCommitOutcome::RestoreDataSource(directory_guard, err) => {
+                    let commit = RenameCommitContext {
+                        directory_guard,
+                        publication_root: &self.publication_root,
+                        mutation_lease,
+                        source_metadata: &src_file_path,
+                        destination_metadata: &dst_file_path,
+                        source_volume_dir: &src_volume_dir,
+                        data_paths: has_data_dir_path.as_ref(),
+                    };
+                    commit.restore_published_data_source().await?;
+                    return Err(err);
+                }
+                MultipartFusedCommitOutcome::RestoreTargetAfterError(err, invalidate_before_return) => {
+                    if !invalidate_before_return {
+                        for part_path in &invalidate_part_paths {
+                            self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
+                        }
+                    }
+                    pin_multipart_future(|| publication.restore_target_after_error(mutation_lease.clone())).await?;
+                    if invalidate_before_return {
+                        for part_path in &invalidate_part_paths {
+                            self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
+                        }
+                    }
+                    return Err(err);
+                }
+            }
         } else {
             prepare_rename_commit_and_destination_metadata(
                 !no_inline && !preserve_upload_metadata,
@@ -1053,71 +1479,81 @@ impl LocalDisk {
             // after the commit rename, and a crash before the rename means the
             // PUT was never acknowledged. A metadata commit: relaxed tiers
             // leave it to the page cache.
-            if let Some(publication) = multipart_publication.as_ref() {
-                pin_multipart_future(|| publication.prepare_metadata(&new_dst_buf, has_dst_buf.as_deref())).await?;
-            }
-            let tmp_meta_sync = if durability.syncs_commit_metadata() || multipart_publication.is_some() {
-                SyncMode::FileOnly
-            } else {
-                SyncMode::None
-            };
-            // The tmp xl.meta write and the shard-file fdatasync are independent
-            // (disjoint paths) and both only need to be durable before the commit
-            // renames below, so run them concurrently to drop a blocking
-            // round-trip from the PUT commit critical path (rustfs/backlog#922
-            // step 2). The "contents durable -> rename -> dst dir fsync" ordering
-            // is unchanged — both futures complete before any rename — which the
-            // rename_data crash-consistency harness (backlog#935) exercises.
-            //
-            // Shard durability: once rename_data succeeds the write is
-            // acknowledged, so data must not live only in the page cache.
-            // Multipart parts were already synced during rename_part, so their
-            // fdatasync here is a cheap no-op. A missing source dir is left for the
-            // rename below to report through the existing rollback path. Payload
-            // durability is kept by both strict and relaxed.
-            let tmp_meta_write = {
-                let src_file_path = multipart_publication
-                    .as_ref()
-                    .map_or_else(|| commit.source_metadata.to_path_buf(), |publication| publication.prepared_path.clone());
-                let dst_file_path = dst_file_path.clone();
-                let owned_directory_guard = commit.directory_guard.clone();
-                let owned_mutation_lease = commit.mutation_lease.clone();
-                async move {
-                    os::run_blocking_namespace_operation(owned_mutation_lease, move || {
-                        #[cfg(test)]
-                        run_owned_file_before_open(&src_file_path);
-                        let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard(
-                            &src_file_path,
-                            &dst_file_path,
-                            &owned_directory_guard,
-                        )?;
-                        prepared_metadata_source.write_all(&new_dst_buf, tmp_meta_sync != SyncMode::None)?;
-                        Ok(prepared_metadata_source)
-                    })
-                    .await
-                    .map_err(to_file_error)
-                    .map_err(DiskError::from)
-                }
-            };
+            #[cfg(all(test, target_os = "linux"))]
+            let probe_active = os::file_sync_probe::is_active();
+            #[cfg(all(not(test), target_os = "linux"))]
+            let probe_active = false;
             let shard_sync = async {
                 if durability.syncs_data_shards()
                     && let Some((src_data_path, _)) = has_data_dir_path.as_ref()
-                    && let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await
-                    && err.kind() != ErrorKind::NotFound
                 {
-                    return Err::<(), DiskError>(to_file_error(err).into());
+                    #[cfg(target_os = "linux")]
+                    if !probe_active && super::take_presynced_tmp_data_dir(src_data_path) {
+                        if multipart_publication.is_none()
+                            && let Err(err) = os::fsync_dir_with_owner(src_data_path, Some(commit.mutation_lease.clone())).await
+                            && err.kind() != ErrorKind::NotFound
+                        {
+                            return Err::<(), DiskError>(to_file_error(err).into());
+                        }
+                        return Ok(());
+                    }
+                    if let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await
+                        && err.kind() != ErrorKind::NotFound
+                    {
+                        return Err::<(), DiskError>(to_file_error(err).into());
+                    }
                 }
                 Ok(())
             };
-            let (tmp_meta_res, shard_sync_res) = tokio::join!(tmp_meta_write, shard_sync);
-            // Surface a tmp-meta failure first (its prior serial position), then a
-            // shard-sync failure; either aborts before any rename, exactly as the
-            // sequential version did.
-            let prepared_metadata_source = tmp_meta_res?;
-            shard_sync_res?;
-            let prepared_metadata_source = match multipart_publication.as_ref() {
-                Some(publication) => pin_multipart_future(|| publication.install_head(prepared_metadata_source)).await?,
-                None => prepared_metadata_source,
+            let prepared_metadata_source = if let Some(publication) = multipart_publication.as_ref() {
+                let dst_file_path_ref = &dst_file_path;
+                let dir_guard_ref = &commit.directory_guard;
+                let lease_clone = commit.mutation_lease.clone();
+                let prepare_fut = pin_multipart_future(|| {
+                    publication.prepare_and_install_head(
+                        &new_dst_buf,
+                        has_dst_buf.as_deref(),
+                        dst_file_path_ref,
+                        dir_guard_ref,
+                        lease_clone,
+                    )
+                });
+                let (tmp_meta_res, shard_sync_res) = tokio::join!(prepare_fut, shard_sync);
+                let prepared_metadata_source = tmp_meta_res?;
+                shard_sync_res?;
+                prepared_metadata_source
+            } else {
+                let tmp_meta_sync = if durability.syncs_commit_metadata() {
+                    SyncMode::FileOnly
+                } else {
+                    SyncMode::None
+                };
+                let tmp_meta_write = {
+                    let src_file_path = commit.source_metadata.to_path_buf();
+                    let dst_file_path = dst_file_path.clone();
+                    let owned_directory_guard = commit.directory_guard.clone();
+                    let owned_mutation_lease = commit.mutation_lease.clone();
+                    async move {
+                        os::run_blocking_namespace_operation(owned_mutation_lease, move || {
+                            #[cfg(test)]
+                            run_owned_file_before_open(&src_file_path);
+                            let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard(
+                                &src_file_path,
+                                &dst_file_path,
+                                &owned_directory_guard,
+                            )?;
+                            prepared_metadata_source.write_all(&new_dst_buf, tmp_meta_sync != SyncMode::None)?;
+                            Ok(prepared_metadata_source)
+                        })
+                        .await
+                        .map_err(to_file_error)
+                        .map_err(DiskError::from)
+                    }
+                };
+                let (tmp_meta_res, shard_sync_res) = tokio::join!(tmp_meta_write, shard_sync);
+                let prepared_metadata_source = tmp_meta_res?;
+                shard_sync_res?;
+                prepared_metadata_source
             };
             commit.directory_guard = remove_dst_base_before_commit(
                 dst_path,
@@ -1457,13 +1893,22 @@ impl LocalDisk {
             }
 
             let multipart_source = match multipart_publication.as_ref() {
-                Some(publication) => match pin_multipart_future(|| publication.finish(commit.mutation_lease.clone())).await {
-                    Ok(result) => Some(Box::new(result)),
-                    Err(error) => {
-                        pin_multipart_future(|| publication.restore_target_after_error(commit.mutation_lease.clone())).await?;
-                        return Err(error);
+                Some(publication) => {
+                    let target_already_synced = durability.syncs_commit_metadata();
+                    let staging_cleanup = multipart_staging.as_ref().map(|s| s.cleanup_handle());
+                    match pin_multipart_future(|| {
+                        publication.finish(commit.mutation_lease.clone(), target_already_synced, staging_cleanup)
+                    })
+                    .await
+                    {
+                        Ok(result) => Some(Box::new(result)),
+                        Err(error) => {
+                            pin_multipart_future(|| publication.restore_target_after_error(commit.mutation_lease.clone()))
+                                .await?;
+                            return Err(error);
+                        }
                     }
-                },
+                }
                 None => None,
             };
             Ok(RenameDataResp {
@@ -1663,13 +2108,8 @@ impl LocalDisk {
                     #[cfg(not(windows))]
                     let backup_path = {
                         let _backup = os::StageTimer::new(native_timing, "pool_meta_native_backup_create");
-                        create_local_inline_rollback_backup(&dst, &src, old_metadata)?
+                        create_local_inline_rollback_backup(&dst, &src, old_metadata, sync)?
                     };
-                    #[cfg(not(windows))]
-                    if sync {
-                        let _backup_sync = os::StageTimer::new(native_timing, "pool_meta_native_backup_file_sync");
-                        std::fs::File::open(&backup_path)?.sync_data()?;
-                    }
                     staged_rollback_path = Some(StagedRollbackBackup(backup_path));
                 }
 

crates/ecstore/src/disk/local/delete_recovery.rs

@@ -1000,6 +1000,21 @@ pub(super) fn inspection_with_markers(
     Ok((inspection, has_physical_data_dirs_or_markers))
 }
 
+type PreparedDeleteReceiptEntry = (DeleteRecoveryJournal, bool);
+
+const PREPARED_DELETE_RECEIPTS_PER_SHARD: usize = 64;
+
+static PREPARED_DELETE_RECEIPTS: std::sync::LazyLock<os::ShardedPathCache<PreparedDeleteReceiptEntry>> =
+    std::sync::LazyLock::new(|| os::ShardedPathCache::new(PREPARED_DELETE_RECEIPTS_PER_SHARD));
+
+fn put_prepared_delete_receipt(rollback: impl AsRef<Path>, entry: PreparedDeleteReceiptEntry) {
+    PREPARED_DELETE_RECEIPTS.insert(rollback.as_ref(), entry);
+}
+
+fn take_prepared_delete_receipt(rollback: &Path) -> Option<PreparedDeleteReceiptEntry> {
+    PREPARED_DELETE_RECEIPTS.remove(rollback)
+}
+
 impl LocalDisk {
     // Native MPU source closure has its own conditional protocol. Generic
     // object recovery must never restore its head or acquire SourceRW under
@@ -1272,9 +1287,15 @@ impl LocalDisk {
         let volume = volume.to_owned();
         let object = object.to_owned();
         let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner).await;
-        os::run_blocking_namespace_operation(lease, move || Ok(inspection_with_markers(&root, &volume, &object, transaction)))
-            .await
-            .map_err(to_file_error)?
+        // Journal inspections ride the durable-mutation lane (the fused delete
+        // prepare already does): a bulk delete publishes one transaction per
+        // key, and the one-worker foreground namespace lane would serialize
+        // those per-drive publishes behind every other mutation.
+        os::run_blocking_namespace_operation_with_sync(lease, true, move || {
+            Ok(inspection_with_markers(&root, &volume, &object, transaction))
+        })
+        .await
+        .map_err(to_file_error)?
     }
 
     async fn activate_delete_preparation(
@@ -1523,6 +1544,9 @@ impl LocalDisk {
         owner: Option<Arc<dyn Send + Sync>>,
     ) -> Result<DeleteRecoveryInspection> {
         self.reject_multipart_delete_recovery_mutation(volume, object)?;
+        if let Ok(root) = self.io_get_object_path(volume, object) {
+            let _ = take_prepared_delete_receipt(&root.join(transaction.to_string()));
+        }
 
         let (mut observed, mut has_data_dir_work) = self
             .delete_recovery_inspect_owned_with_markers(volume, object, transaction, owner.clone())
@@ -1667,7 +1691,10 @@ impl LocalDisk {
             let plan_root = root.clone();
             let missing = observed.current_digest.is_none();
             let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner.clone()).await;
-            let plan = os::run_blocking_namespace_operation(lease, move || {
+            // Same durable-mutation lane as the journal inspection above: this
+            // plan closure runs inside every settle on the bulk-delete publish
+            // path.
+            let plan = os::run_blocking_namespace_operation_with_sync(lease, true, move || {
                 Ok((|| {
                     let before =
                         regular_metadata_bytes(&plan_root.join(transaction.to_string()).join(STORAGE_FORMAT_FILE_BACKUP))?
@@ -1911,8 +1938,14 @@ impl LocalDisk {
                     return Err(DiskError::DiskOngoingReq);
                 }
                 let bucket_path = self.io_get_bucket_path(volume)?;
-                let mut file_leases = Vec::with_capacity(3);
-                for name in [TARGET, PREPARED, FINISHED] {
+                let mut file_leases = Vec::with_capacity(5);
+                for name in [
+                    STORAGE_FORMAT_FILE_BACKUP,
+                    DELETE_MARKER_ROLLBACK_FILE,
+                    TARGET,
+                    PREPARED,
+                    FINISHED,
+                ] {
                     let path = rollback.join(name);
                     let lease = os::acquire_namespace_mutation_lease_with_owner(&path, owner.clone()).await;
                     file_leases.push((path, lease));
@@ -2132,119 +2165,151 @@ impl LocalDisk {
                     Err(err) if err.kind() == ErrorKind::NotFound => return Ok(Ok(None)),
                     Err(err) => return Err(err),
                 };
-                let (old_dir, journal_raw) = match (|| -> Result<Option<(Option<Uuid>, Vec<u8>)>> {
-                    let mut meta = FileMeta::load(&buf)?;
-                    let deleted_versions: Vec<_> = meta
-                        .versions
-                        .iter()
-                        .map(|v| v.header.version_id.unwrap_or_default())
-                        .collect();
-                    let original_versions: Vec<_> = meta
-                        .versions
-                        .iter()
-                        .map(|v| generation(v.into_fileinfo(&volume_str, &object_str, true)?))
-                        .collect::<Result<_>>()?;
-                    let old_dir = meta.delete_version(&fi_clone)?;
-                    if old_dir.is_some_and(|dir| dir == transaction || dir.is_nil()) {
-                        return Err(DiskError::FileCorrupt);
-                    }
-                    if let Some(uuid) = old_dir {
-                        meta.data.remove(vec![fi_clone.version_id.unwrap_or_default(), uuid])?;
-                        let old_path = path_join(&[file_path.as_path(), Path::new(uuid.to_string().as_str())]);
-                        check_path_length(old_path.to_string_lossy().as_ref())?;
-                        if old_path.join(STORAGE_FORMAT_FILE).exists() {
-                            return Ok(None);
+                type PreparedDeleteTuple = (Option<Uuid>, Option<Vec<u8>>, Vec<u8>, DeleteRecoveryJournal, bool);
+                let (old_dir, target, journal_raw, journal, sole_transaction) =
+                    match (|| -> Result<Option<PreparedDeleteTuple>> {
+                        let mut meta = FileMeta::load(&buf)?;
+                        let mut deleted_headers: Vec<_> = meta
+                            .versions
+                            .iter()
+                            .map(|v| (v.header.version_id, std::mem::discriminant(&v.header.version_type)))
+                            .collect();
+                        let original_versions: Vec<_> = meta
+                            .versions
+                            .iter()
+                            .map(|v| generation(v.into_fileinfo(&volume_str, &object_str, true)?))
+                            .collect::<Result<_>>()?;
+                        let old_dir = meta.delete_version(&fi_clone)?;
+                        if old_dir.is_some_and(|dir| dir == transaction || dir.is_nil()) {
+                            return Err(DiskError::FileCorrupt);
                         }
-                    }
-                    if !meta.versions.is_empty() {
-                        return Ok(None);
-                    }
-                    let mut predecessors = BTreeSet::new();
-                    if let Ok(entries) = std::fs::read_dir(&file_path) {
-                        for (visited, entry) in entries.enumerate() {
-                            if visited >= 65536 {
-                                return Err(DiskError::DiskOngoingReq);
+                        if let Some(uuid) = old_dir {
+                            meta.data.remove(vec![fi_clone.version_id.unwrap_or_default(), uuid])?;
+                            let old_path = path_join(&[file_path.as_path(), Path::new(uuid.to_string().as_str())]);
+                            check_path_length(old_path.to_string_lossy().as_ref())?;
+                            if old_path.join(STORAGE_FORMAT_FILE).exists() {
+                                return Ok(None);
                             }
-                            let entry = entry.map_err(to_file_error)?;
-                            let Some(id) = entry
-                                .file_name()
-                                .to_str()
-                                .and_then(|name| Uuid::parse_str(name).ok())
-                                .filter(|id| !id.is_nil() && *id != transaction)
-                            else {
-                                continue;
-                            };
-                            if !entry.file_type().map_err(to_file_error)?.is_dir() {
-                                return Err(DiskError::InvalidPath);
-                            }
-                            let prepared = regular_bytes(&entry.path().join(PREPARED))?;
-                            let receipt = regular_bytes(&entry.path().join(DELETE_QUORUM_COMMITTED_FILE))?;
-                            let mut records = Vec::new();
-                            if let Some(bytes) = prepared {
-                                records.push(
-                                    serde_json::from_slice::<DeleteRecoveryJournal>(&bytes)
-                                        .map_err(|_| DiskError::FileCorrupt)?,
-                                );
-                            }
-                            if let Some(bytes) = receipt.filter(|bytes| !bytes.is_empty()) {
-                                records.push(
-                                    serde_json::from_slice::<CommitReceipt>(&bytes)
-                                        .map_err(|_| DiskError::FileCorrupt)?
-                                        .journal,
-                                );
-                            }
-                            for record in records {
-                                if record.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION
-                                    || record.volume != volume_str
-                                    || record.object != object_str
-                                    || record.transaction != id
-                                    || incarnation.zip(record.bucket_incarnation).is_some_and(|(a, b)| a != b)
-                                    || record.predecessors.iter().any(|before| before.is_nil() || *before == id)
-                                {
-                                    return Err(DiskError::FileCorrupt);
+                        }
+                        let surviving: HashSet<_> = meta
+                            .versions
+                            .iter()
+                            .map(|v| (v.header.version_id, std::mem::discriminant(&v.header.version_type)))
+                            .collect();
+                        deleted_headers.retain(|header| !surviving.contains(header));
+                        let deleted_versions = deleted_headers.into_iter().map(|(id, _)| id.unwrap_or_default()).collect();
+                        let target = (!meta.versions.is_empty()).then(|| meta.marshal_msg()).transpose()?;
+                        let mut predecessors = BTreeSet::new();
+                        let mut sole_transaction = true;
+                        if let Ok(entries) = std::fs::read_dir(&file_path) {
+                            for (visited, entry) in entries.enumerate() {
+                                if visited >= 65536 {
+                                    return Err(DiskError::DiskOngoingReq);
+                                }
+                                let entry = entry.map_err(to_file_error)?;
+                                let file_name = entry.file_name();
+                                let Some(name_str) = file_name.to_str() else {
+                                    sole_transaction = false;
+                                    continue;
+                                };
+                                let Some(id) = Uuid::parse_str(name_str).ok().filter(|id| !id.is_nil() && *id != transaction)
+                                else {
+                                    continue;
+                                };
+                                if !entry.file_type().map_err(to_file_error)?.is_dir() {
+                                    return Err(DiskError::InvalidPath);
+                                }
+                                let dir_path = entry.path();
+                                let prepared = regular_bytes(&dir_path.join(PREPARED))?;
+                                let receipt = regular_bytes(&dir_path.join(DELETE_QUORUM_COMMITTED_FILE))?;
+                                if sole_transaction && name_str == id.hyphenated().to_string() && old_dir != Some(id) {
+                                    let is_surviving_data_dir = original_versions.iter().any(|v| v.data_dir == Some(id))
+                                        && prepared.is_none()
+                                        && receipt.is_none()
+                                        && !dir_path.join(STORAGE_FORMAT_FILE_BACKUP).exists();
+                                    if !is_surviving_data_dir {
+                                        sole_transaction = false;
+                                    }
+                                }
+                                let mut records = Vec::new();
+                                if let Some(bytes) = prepared {
+                                    records.push(
+                                        serde_json::from_slice::<DeleteRecoveryJournal>(&bytes)
+                                            .map_err(|_| DiskError::FileCorrupt)?,
+                                    );
+                                }
+                                if let Some(bytes) = receipt.filter(|bytes| !bytes.is_empty()) {
+                                    records.push(
+                                        serde_json::from_slice::<CommitReceipt>(&bytes)
+                                            .map_err(|_| DiskError::FileCorrupt)?
+                                            .journal,
+                                    );
+                                }
+                                for record in records {
+                                    if record.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION
+                                        || record.volume != volume_str
+                                        || record.object != object_str
+                                        || record.transaction != id
+                                        || incarnation.zip(record.bucket_incarnation).is_some_and(|(a, b)| a != b)
+                                        || record.predecessors.iter().any(|before| before.is_nil() || *before == id)
+                                    {
+                                        return Err(DiskError::FileCorrupt);
+                                    }
+                                    predecessors.insert(id);
                                 }
-                                predecessors.insert(id);
                             }
                         }
-                    }
-                    let mut sparse = fi_clone;
-                    sparse.data = None;
-                    let journal = DeleteRecoveryJournal {
-                        protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
-                        volume: volume_str,
-                        object: object_str,
-                        transaction,
-                        bucket_incarnation: incarnation,
-                        original_digest: Some(digest(&buf)),
-                        target_digest: None,
-                        data_dirs: old_dir.into_iter().collect(),
-                        deleted_versions,
-                        requests: vec![sparse],
-                        original_versions,
-                        predecessors: predecessors.into_iter().collect(),
+                        sole_transaction &= predecessors.is_empty();
+                        let mut sparse = fi_clone;
+                        sparse.data = None;
+                        let journal = DeleteRecoveryJournal {
+                            protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
+                            volume: volume_str,
+                            object: object_str,
+                            transaction,
+                            bucket_incarnation: incarnation,
+                            original_digest: Some(digest(&buf)),
+                            target_digest: target.as_deref().map(digest),
+                            data_dirs: old_dir.into_iter().collect(),
+                            deleted_versions,
+                            requests: vec![sparse],
+                            original_versions,
+                            predecessors: predecessors.into_iter().collect(),
+                        };
+                        let raw = journal_bytes(&journal)?;
+                        Ok(Some((old_dir, target, raw, journal, sole_transaction)))
+                    })() {
+                        Ok(Some(tuple)) => tuple,
+                        Ok(None) => return Ok(Ok(None)),
+                        Err(err) => return Ok(Err(err)),
                     };
-                    Ok(Some((old_dir, journal_bytes(&journal)?)))
-                })() {
-                    Ok(Some(pair)) => pair,
-                    Ok(None) => return Ok(Ok(None)),
-                    Err(err) => return Ok(Err(err)),
-                };
                 std::fs::create_dir_all(&staging_root)?;
-                let staged = tempfile::Builder::new()
-                    .prefix("delete-preparation-")
-                    .tempdir_in(&staging_root)?;
-                for (name, bytes) in [
-                    (STORAGE_FORMAT_FILE_BACKUP, buf.as_slice()),
-                    (PREPARED, journal_raw.as_slice()),
-                ] {
-                    let mut file = std::fs::File::create(staged.path().join(name))?;
-                    std::io::Write::write_all(&mut file, bytes)?;
-                    if sync && !bytes.is_empty() {
+                let staged = tempfile::Builder::new().prefix("delete-prep-").tempdir_in(&staging_root)?;
+                let staged_dir = staged.path().to_path_buf();
+                let backup_path = staged_dir.join(STORAGE_FORMAT_FILE_BACKUP);
+                if std::fs::hard_link(&xl_path, &backup_path).is_err() {
+                    let mut file = std::fs::File::create(&backup_path)?;
+                    std::io::Write::write_all(&mut file, &buf)?;
+                    if sync && !buf.is_empty() {
                         file.sync_data()?;
                     }
                 }
+                let mut written_files = Vec::with_capacity(2);
+                for (name, bytes) in [(TARGET, target.as_deref()), (PREPARED, Some(journal_raw.as_slice()))] {
+                    let Some(bytes) = bytes else {
+                        continue;
+                    };
+                    let mut file = std::fs::File::create(staged_dir.join(name))?;
+                    std::io::Write::write_all(&mut file, bytes)?;
+                    if sync && !bytes.is_empty() {
+                        written_files.push(file);
+                    }
+                }
+                for file in written_files {
+                    file.sync_data()?;
+                }
                 if sync {
-                    os::fsync_dir_std(staged.path())?;
+                    os::fsync_dir_std(&staged_dir)?;
                 }
                 match std::fs::symlink_metadata(&destination) {
                     Ok(metadata) if metadata.is_dir() => match std::fs::remove_dir(&destination) {
@@ -2257,27 +2322,62 @@ impl LocalDisk {
                     Err(err) if err.kind() == ErrorKind::NotFound => {}
                     Err(err) => return Err(err),
                 }
-                if let Some(parent) = destination.parent() {
-                    std::fs::create_dir_all(parent)?;
+                let staged_dir = staged.keep();
+                if let Err(err) = std::fs::rename(&staged_dir, &destination) {
+                    let _ = std::fs::remove_dir_all(&staged_dir);
+                    return Err(err);
                 }
-                std::fs::rename(staged.path(), &destination)?;
-                if sync {
-                    os::fsync_dir_std(&file_path)?;
+                let old_data_path = old_dir.map(|uuid| file_path.join(uuid.to_string())).filter(|p| p.is_dir());
+                if sync && let Err(err) = os::fsync_dir_std(&file_path) {
+                    let _ = std::fs::remove_dir_all(&destination);
+                    return Err(err);
                 }
                 let mut reserved_data_path = None;
-                if let Some(uuid) = old_dir {
-                    let old_path = file_path.join(uuid.to_string());
-                    if old_path.is_dir() {
-                        let reserved_marker = old_path.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"));
+                if let Some(old_path) = old_data_path {
+                    let reserved_marker = old_path.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"));
+                    let reserve_res = (|| -> std::io::Result<()> {
                         let marker = std::fs::File::create(&reserved_marker)?;
                         if sync {
                             marker.sync_all()?;
                             os::fsync_dir_std(&old_path)?;
                         }
-                        reserved_data_path = Some((old_path, reserved_marker));
+                        Ok(())
+                    })();
+                    if let Err(err) = reserve_res {
+                        let _ = std::fs::remove_file(&reserved_marker);
+                        let _ = std::fs::remove_dir_all(&destination);
+                        return Err(err);
                     }
+                    reserved_data_path = Some((old_path, reserved_marker));
                 }
-                if let Err(err) = std::fs::remove_file(&xl_path)
+                if let Some(ref target_bytes) = target {
+                    let target_in_rollback = destination.join(TARGET);
+                    let linked_xl = staging_root.join(format!("delete-xl-{transaction}"));
+                    let _ = std::fs::remove_file(&linked_xl);
+                    let write_res = if std::fs::hard_link(&target_in_rollback, &linked_xl).is_ok() {
+                        let rename_res = std::fs::rename(&linked_xl, &xl_path);
+                        if rename_res.is_err() {
+                            let _ = std::fs::remove_file(&linked_xl);
+                        }
+                        rename_res
+                    } else {
+                        (|| -> std::io::Result<()> {
+                            let mut staged_xl = tempfile::NamedTempFile::new_in(&staging_root)?;
+                            std::io::Write::write_all(staged_xl.as_file_mut(), target_bytes)?;
+                            if sync && !target_bytes.is_empty() {
+                                staged_xl.as_file().sync_data()?;
+                            }
+                            std::fs::rename(staged_xl.path(), &xl_path)
+                        })()
+                    };
+                    if let Err(err) = write_res {
+                        if let Some((_, reserved_marker)) = &reserved_data_path {
+                            let _ = std::fs::remove_file(reserved_marker);
+                        }
+                        let _ = std::fs::remove_dir_all(&destination);
+                        return Err(err);
+                    }
+                } else if let Err(err) = std::fs::remove_file(&xl_path)
                     && err.kind() != ErrorKind::NotFound
                 {
                     if let Some((_, reserved_marker)) = &reserved_data_path {
@@ -2298,6 +2398,7 @@ impl LocalDisk {
                         Err(err) => return Err(err),
                     }
                 }
+                put_prepared_delete_receipt(destination, (journal, sole_transaction));
                 Ok(Ok(Some(old_dir)))
             },
         )
@@ -2343,6 +2444,26 @@ impl LocalDisk {
             lease,
             sync,
             move || -> std::io::Result<Result<Option<(DeleteRecoveryJournal, bool, bool)>>> {
+                if let Some((journal, sole_transaction)) = take_prepared_delete_receipt(&rollback)
+                    && rollback.join(PREPARED).is_file()
+                {
+                    let receipt = CommitReceipt {
+                        journal: journal.clone(),
+                        finished: false,
+                    };
+                    let receipt_bytes = serde_json::to_vec(&receipt).map_err(std::io::Error::other)?;
+                    std::fs::create_dir_all(&staging_root)?;
+                    let mut staged = tempfile::NamedTempFile::new_in(&staging_root)?;
+                    std::io::Write::write_all(staged.as_file_mut(), &receipt_bytes)?;
+                    if sync && !receipt_bytes.is_empty() {
+                        staged.as_file().sync_data()?;
+                    }
+                    std::fs::rename(staged.path(), &receipt_path)?;
+                    if sync {
+                        os::fsync_dir_std(&rollback)?;
+                    }
+                    return Ok(Ok(Some((journal, sole_transaction, true))));
+                }
                 let prepared = match regular_bytes(&rollback.join(PREPARED)) {
                     Ok(Some(bytes)) => bytes,
                     Ok(None) => return Ok(Ok(None)),
@@ -2360,12 +2481,18 @@ impl LocalDisk {
                 {
                     return Ok(Err(DiskError::FileCorrupt));
                 }
-                let current_exists = match regular_file_exists(&root.join(STORAGE_FORMAT_FILE)) {
-                    Ok(exists) => exists,
-                    Err(e) => return Ok(Err(e)),
+                let target_satisfied = match journal.target_digest {
+                    None => match regular_file_exists(&root.join(STORAGE_FORMAT_FILE)) {
+                        Ok(exists) => !exists,
+                        Err(e) => return Ok(Err(e)),
+                    },
+                    Some(expected_target) => match regular_bytes(&root.join(STORAGE_FORMAT_FILE)) {
+                        Ok(Some(bytes)) => digest(&bytes) == expected_target,
+                        Ok(None) => false,
+                        Err(e) => return Ok(Err(e)),
+                    },
                 };
-                let unversioned_target_absent = journal.target_digest.is_none() && !current_exists;
-                if !unversioned_target_absent {
+                if !target_satisfied {
                     return Ok(Ok(None));
                 }
                 let mut sole_transaction = journal.predecessors.is_empty();
@@ -2382,8 +2509,15 @@ impl LocalDisk {
                                     && id != transaction
                                     && !journal.data_dirs.contains(&id)
                                 {
-                                    sole_transaction = false;
-                                    break;
+                                    let dir_path = entry.path();
+                                    let is_surviving_data_dir = journal.original_versions.iter().any(|v| v.data_dir == Some(id))
+                                        && !dir_path.join(PREPARED).exists()
+                                        && !dir_path.join(DELETE_QUORUM_COMMITTED_FILE).exists()
+                                        && !dir_path.join(STORAGE_FORMAT_FILE_BACKUP).exists();
+                                    if !is_surviving_data_dir {
+                                        sole_transaction = false;
+                                        break;
+                                    }
                                 }
                             }
                         }
@@ -2406,7 +2540,7 @@ impl LocalDisk {
                 if sync {
                     os::fsync_dir_std(&rollback)?;
                 }
-                Ok(Ok(Some((journal, sole_transaction, unversioned_target_absent))))
+                Ok(Ok(Some((journal, sole_transaction, target_satisfied))))
             },
         )
         .await
@@ -2458,12 +2592,15 @@ impl LocalDisk {
         }
 
         let bucket_path = self.io_get_bucket_path(volume)?;
+        let staging_root = self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?;
         let root = self.io_get_object_path(volume, object)?;
         let rollback = root.join(transaction.to_string());
         let receipt_path = rollback.join(DELETE_QUORUM_COMMITTED_FILE);
         let sync = effective_durability(volume).syncs_commit_metadata();
-        let lease = os::acquire_namespace_mutation_lease_with_owner(&receipt_path, owner).await;
         let data_dirs = journal.data_dirs.clone();
+        let has_target = journal.target_digest.is_some();
+        let journal_for_receipt = journal.clone();
+        let lease = os::acquire_namespace_mutation_lease_with_owner(&receipt_path, owner).await;
         let prune_paths: Vec<_> = rollback
             .ancestors()
             .take_while(|dir| *dir != bucket_path.as_path() && dir.starts_with(&bucket_path))
@@ -2474,11 +2611,13 @@ impl LocalDisk {
             let committed = format!("{DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
             let recovery_committed = format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
             let reserved = format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
+            let mut had_physical_data_dirs = false;
             for data_dir in &data_dirs {
                 for base in [&root, &rollback] {
                     let dir_path = base.join(data_dir.to_string());
                     match std::fs::read_dir(&dir_path) {
                         Ok(entries) => {
+                            had_physical_data_dirs = true;
                             let mut removable = Vec::new();
                             let mut preserve_dir = false;
                             for entry in entries {
@@ -2519,23 +2658,45 @@ impl LocalDisk {
                     }
                 }
             }
-            if !data_dirs.is_empty() && sync && root.is_dir() {
+            if had_physical_data_dirs && sync && root.is_dir() {
                 os::fsync_dir_std(&root)?;
             }
-            for name in [
-                STORAGE_FORMAT_FILE_BACKUP,
-                DELETE_MARKER_ROLLBACK_FILE,
-                TARGET,
-                PREPARED,
-                FINISHED,
-            ] {
+            for name in [STORAGE_FORMAT_FILE_BACKUP, DELETE_MARKER_ROLLBACK_FILE] {
                 ignore_not_found(std::fs::remove_file(rollback.join(name)))?;
             }
+            if has_target || had_physical_data_dirs {
+                let finished_receipt = CommitReceipt {
+                    journal: journal_for_receipt,
+                    finished: true,
+                };
+                let receipt_bytes = serde_json::to_vec(&finished_receipt).map_err(std::io::Error::other)?;
+                std::fs::create_dir_all(&staging_root)?;
+                let mut staged_receipt = tempfile::NamedTempFile::new_in(&staging_root)?;
+                std::io::Write::write_all(staged_receipt.as_file_mut(), &receipt_bytes)?;
+                if sync && !receipt_bytes.is_empty() {
+                    staged_receipt.as_file().sync_data()?;
+                }
+                std::fs::rename(staged_receipt.path(), &receipt_path)?;
+                let _ = std::fs::File::create(rollback.join(FINISHED))?;
+                if sync {
+                    os::fsync_dir_std(&rollback)?;
+                }
+            }
+            for name in [TARGET, PREPARED, FINISHED] {
+                let path = rollback.join(name);
+                #[cfg(all(any(test, feature = "test-util"), not(windows)))]
+                os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &path);
+                ignore_not_found(std::fs::remove_file(path))?;
+            }
             if sync {
                 os::fsync_dir_std(&rollback)?;
             }
+            #[cfg(all(any(test, feature = "test-util"), not(windows)))]
+            os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &receipt_path);
             ignore_not_found(std::fs::remove_file(&receipt_path))?;
             for dir_path in &prune_paths {
+                #[cfg(all(any(test, feature = "test-util"), not(windows)))]
+                os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, dir_path);
                 if let Err(err) = std::fs::remove_dir(dir_path) {
                     if is_dir_not_empty_error(&err) {
                         break;
@@ -6282,6 +6443,214 @@ mod tests {
             b"outside bytes"
         );
     }
+
+    #[tokio::test]
+    async fn delete_recovery_fused_versioned_delete_marker_and_publish_fast_path() {
+        let (_temp, disk, info, _) = fixture().await;
+        let root = disk.io_get_object_path("recovery", "object").expect("object root");
+        let transaction = Uuid::new_v4();
+        let mut marker = FileInfo {
+            name: "object".to_owned(),
+            version_id: Some(Uuid::new_v4()),
+            deleted: true,
+            mark_deleted: true,
+            mod_time: info.mod_time.map(|t| t + time::Duration::seconds(1)),
+            ..Default::default()
+        };
+        let opts = DeleteOptions {
+            old_data_dir: Some(transaction),
+            recoverable_delete: true,
+            ..Default::default()
+        };
+        let fused = disk
+            .try_fused_recoverable_delete_version("recovery", "object", &marker, &opts, transaction, None)
+            .await
+            .expect("fused delete marker preparation must succeed");
+        assert_eq!(fused, Some(()), "versioned delete marker must use fused path");
+        assert!(root.join(transaction.to_string()).join(TARGET).is_file());
+        assert!(root.join(STORAGE_FORMAT_FILE).is_file());
+
+        let published = disk
+            .try_publish_prepared_delete_local("recovery", "object", transaction, None)
+            .await
+            .expect("local publish must succeed")
+            .expect("versioned target must satisfy local publish");
+        assert!(published.1, "sole transaction must be true");
+        assert!(published.2, "target_satisfied must be true");
+
+        let finalized = disk
+            .try_finish_and_finalize_prepared_delete_local("recovery", "object", transaction, &published.0, None)
+            .await
+            .expect("local finalize must succeed");
+        assert!(finalized);
+        assert!(!root.join(transaction.to_string()).exists());
+        let final_meta = FileMeta::load(&fs::read(root.join(STORAGE_FORMAT_FILE)).await.expect("read final xl.meta"))
+            .expect("decode final xl.meta");
+        assert_eq!(final_meta.versions.len(), 2);
+        assert_eq!(final_meta.versions[0].header.version_id, marker.version_id);
+        marker.deleted = false;
+    }
+
+    #[tokio::test]
+    async fn delete_recovery_fused_versioned_delete_with_surviving_version_and_interrupted_target_cleanup() {
+        let (_temp, disk, mut v1, _) = fixture().await;
+        let root = disk.io_get_object_path("recovery", "object").expect("object path");
+
+        // Configure v1 and v2 with distinct physical data directories so deleting v2
+        // exercises both `old_dir` reservation and `is_surviving_data_dir`.
+        v1.data = None;
+        rustfs_utils::http::metadata_compat::remove_str(
+            &mut v1.metadata,
+            rustfs_utils::http::metadata_compat::SUFFIX_INLINE_DATA,
+        );
+        let v1_dir = Uuid::new_v4();
+        v1.data_dir = Some(v1_dir);
+        fs::create_dir_all(root.join(v1_dir.to_string()))
+            .await
+            .expect("create v1 dir");
+        fs::write(root.join(v1_dir.to_string()).join("part.1"), b"v1-data")
+            .await
+            .expect("write v1 part");
+
+        let mut v2 = v1.clone();
+        v2.version_id = Some(Uuid::new_v4());
+        let v2_dir = Uuid::new_v4();
+        v2.data_dir = Some(v2_dir);
+        fs::create_dir_all(root.join(v2_dir.to_string()))
+            .await
+            .expect("create v2 dir");
+        fs::write(root.join(v2_dir.to_string()).join("part.1"), b"v2-data")
+            .await
+            .expect("write v2 part");
+
+        let mut meta = FileMeta::new();
+        meta.add_version(v1.clone()).expect("add v1");
+        meta.add_version(v2.clone()).expect("add v2");
+        fs::write(root.join(STORAGE_FORMAT_FILE), meta.marshal_msg().expect("marshal meta"))
+            .await
+            .expect("write xl.meta");
+
+        let transaction = Uuid::new_v4();
+        let opts = DeleteOptions {
+            recoverable_delete: true,
+            ..Default::default()
+        };
+        let fused = disk
+            .try_fused_recoverable_delete_version("recovery", "object", &v2, &opts, transaction, None)
+            .await
+            .expect("fused versioned delete preparation must succeed");
+        assert_eq!(fused, Some(()));
+        assert!(
+            root.join(v2_dir.to_string())
+                .join(format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"))
+                .is_file(),
+            "deleted version data_dir must have recovery marker"
+        );
+
+        let (journal, sole_transaction, target_satisfied) = disk
+            .try_publish_prepared_delete_local("recovery", "object", transaction, None)
+            .await
+            .expect("local publish must succeed")
+            .expect("versioned target must satisfy local publish");
+        assert!(sole_transaction, "surviving v1 data_dir must not clear sole_transaction");
+        assert!(target_satisfied);
+        assert!(journal.target_digest.is_some());
+
+        // Simulate interruption during finish-before-finalize right after TARGET is removed
+        // (after finished receipt + FINISHED marker were persisted, before receipt_path is removed).
+        let rollback = root.join(transaction.to_string());
+        let finished_receipt = CommitReceipt {
+            journal: journal.clone(),
+            finished: true,
+        };
+        fs::write(
+            rollback.join(DELETE_QUORUM_COMMITTED_FILE),
+            serde_json::to_vec(&finished_receipt).expect("serialize finished receipt"),
+        )
+        .await
+        .expect("write finished receipt");
+        fs::write(rollback.join(FINISHED), []).await.expect("write FINISHED");
+        fs::remove_file(rollback.join(STORAGE_FORMAT_FILE_BACKUP))
+            .await
+            .expect("remove backup");
+        fs::remove_file(rollback.join(TARGET)).await.expect("remove TARGET");
+
+        let inspected = disk
+            .inspect_delete_transaction("recovery", "object", transaction)
+            .await
+            .expect("interrupted post-TARGET state must remain inspectable without FileCorrupt");
+        assert_eq!(inspected.state, DeleteRecoveryState::CommitFinished);
+
+        // Complete cleanup via try_finish_and_finalize_prepared_delete_local.
+        let finalized = disk
+            .try_finish_and_finalize_prepared_delete_local("recovery", "object", transaction, &journal, None)
+            .await
+            .expect("local finalize must succeed after interruption");
+        assert!(finalized);
+        assert!(!rollback.exists(), "rollback directory must be pruned");
+        assert!(!root.join(v2_dir.to_string()).exists(), "deleted v2 data_dir must be removed");
+        assert!(
+            root.join(v1_dir.to_string()).join("part.1").is_file(),
+            "surviving v1 data_dir must remain intact"
+        );
+    }
+
+    #[tokio::test]
+    async fn delete_recovery_fused_unversioned_physical_delete_retains_receipt_until_finalize() {
+        let (_temp, disk, mut info, _) = fixture().await;
+        let root = disk.io_get_object_path("recovery", "object").expect("object path");
+
+        info.data = None;
+        rustfs_utils::http::metadata_compat::remove_str(
+            &mut info.metadata,
+            rustfs_utils::http::metadata_compat::SUFFIX_INLINE_DATA,
+        );
+        let data_dir = Uuid::new_v4();
+        info.data_dir = Some(data_dir);
+        fs::create_dir_all(root.join(data_dir.to_string()))
+            .await
+            .expect("create data_dir");
+        fs::write(root.join(data_dir.to_string()).join("part.1"), b"physical-payload")
+            .await
+            .expect("write part.1");
+        let mut meta = FileMeta::new();
+        meta.add_version(info.clone()).expect("add version");
+        fs::write(root.join(STORAGE_FORMAT_FILE), meta.marshal_msg().expect("marshal meta"))
+            .await
+            .expect("write xl.meta");
+
+        let transaction = Uuid::new_v4();
+        let opts = DeleteOptions {
+            recoverable_delete: true,
+            ..Default::default()
+        };
+        let fused = disk
+            .try_fused_recoverable_delete_version("recovery", "object", &info, &opts, transaction, None)
+            .await
+            .expect("fused unversioned physical delete must succeed");
+        assert_eq!(fused, Some(()));
+
+        let (journal, sole_transaction, target_satisfied) = disk
+            .try_publish_prepared_delete_local("recovery", "object", transaction, None)
+            .await
+            .expect("local publish must succeed")
+            .expect("absent target must satisfy local publish");
+        assert!(sole_transaction);
+        assert!(target_satisfied);
+        assert!(journal.target_digest.is_none());
+
+        let rollback = root.join(transaction.to_string());
+        assert!(rollback.join(DELETE_QUORUM_COMMITTED_FILE).is_file());
+
+        let finalized = disk
+            .try_finish_and_finalize_prepared_delete_local("recovery", "object", transaction, &journal, None)
+            .await
+            .expect("local finalize must succeed");
+        assert!(finalized);
+        assert!(!rollback.exists(), "rollback directory must be pruned");
+        assert!(!root.join(data_dir.to_string()).exists(), "physical data_dir must be removed");
+        assert!(!root.exists(), "empty object directory must be pruned");
+    }
 }
 
 #[cfg(test)]

crates/ecstore/src/disk/local/multipart_source_abort.rs

@@ -421,6 +421,7 @@ impl LocalDisk {
                     return Err(DiskError::ConditionalDeleteConflict);
                 }
                 let permit = AbortDeletePermit::new(&store, &expected, certificate.clone())?;
+                super::multipart_source_state::invalidate_active_multipart_source_head(&store.source);
                 let destination = self.io_get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, &Uuid::new_v4().to_string())?;
                 let source = store.source.clone();
                 let base = store.base.clone();
@@ -618,6 +619,9 @@ pub(super) struct OrdinaryMultipartDeleteCheck {
     root: os::PublicationRoot,
 }
 impl OrdinaryMultipartDeleteCheck {
+    pub(super) fn source(&self) -> &Path {
+        &self.source
+    }
     pub(super) fn validate(&self) -> Result<()> {
         let pinned = match os::pin_multipart_source_directory(&self.source, &self.base, &self.root) {
             Ok(pinned) => pinned,

crates/ecstore/src/disk/local/multipart_source_state.rs

@@ -238,9 +238,177 @@ pub(super) fn read_optional_in_pinned(
         Err(error) => Err(error.into()),
     }
 }
+#[derive(Clone, Copy, PartialEq, Eq)]
+struct FileStamp {
+    len: u64,
+    modified: Option<std::time::SystemTime>,
+    #[cfg(unix)]
+    ino: u64,
+}
+
+impl FileStamp {
+    fn from_path(path: &Path) -> Option<Self> {
+        let meta = std::fs::symlink_metadata(path).ok()?;
+        Some(Self {
+            len: meta.len(),
+            modified: meta.modified().ok(),
+            #[cfg(unix)]
+            ino: {
+                use std::os::unix::fs::MetadataExt;
+                meta.ino()
+            },
+        })
+    }
+}
+
+#[derive(Clone)]
+struct ActiveMultipartSourceEntry {
+    raw: Bytes,
+    fi: FileInfo,
+    head_stamp: FileStamp,
+    unsettled_parts: std::collections::HashSet<usize>,
+    parts: std::collections::HashMap<usize, (FileStamp, rustfs_filemeta::ObjectPartInfo)>,
+}
+
+const ACTIVE_MULTIPART_SOURCE_HEADS_PER_SHARD: usize = 64;
+
+static ACTIVE_MULTIPART_SOURCE_HEADS: std::sync::LazyLock<os::ShardedPathCache<ActiveMultipartSourceEntry>> =
+    std::sync::LazyLock::new(|| os::ShardedPathCache::new(ACTIVE_MULTIPART_SOURCE_HEADS_PER_SHARD));
+
+pub(super) fn put_active_multipart_source_head(source: impl AsRef<Path>, raw: Bytes, fi: FileInfo) {
+    let source = source.as_ref();
+    let Some(head_stamp) = FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) else {
+        return;
+    };
+    ACTIVE_MULTIPART_SOURCE_HEADS.insert(
+        source,
+        ActiveMultipartSourceEntry {
+            raw,
+            fi,
+            head_stamp,
+            unsettled_parts: std::collections::HashSet::new(),
+            parts: std::collections::HashMap::new(),
+        },
+    );
+}
+
+fn stamp_matches_active_disk(source: &Path, expected_head_stamp: FileStamp) -> bool {
+    FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) == Some(expected_head_stamp)
+        && !source.join(STATE).exists()
+        && !source.join(crate::disk::MULTIPART_RECOVERY_DIR).exists()
+        && !source.join(".mpu-completions").exists()
+}
+
+pub(super) fn get_active_multipart_source_head(source: &Path) -> Option<(Bytes, FileInfo)> {
+    let (raw, fi, head_stamp) =
+        ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| (entry.raw.clone(), entry.fi.clone(), entry.head_stamp))?;
+    if !stamp_matches_active_disk(source, head_stamp) {
+        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
+        return None;
+    }
+    Some((raw, fi))
+}
+
+pub(super) fn has_active_multipart_source_head(source: &Path) -> bool {
+    let Some(head_stamp) = ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| entry.head_stamp) else {
+        return false;
+    };
+    if !stamp_matches_active_disk(source, head_stamp) {
+        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
+        return false;
+    }
+    true
+}
+
+pub(super) fn has_clean_active_multipart_source(source: &Path) -> bool {
+    let Some((head_stamp, clean)) =
+        ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| (entry.head_stamp, entry.unsettled_parts.is_empty()))
+    else {
+        return false;
+    };
+    if !stamp_matches_active_disk(source, head_stamp) {
+        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
+        return false;
+    }
+    clean
+}
+
+pub(super) fn begin_active_multipart_part_transaction(source: &Path, part_number: usize) {
+    ACTIVE_MULTIPART_SOURCE_HEADS.with_get_mut(source, |entry| {
+        entry.unsettled_parts.insert(part_number);
+        entry.parts.remove(&part_number);
+    });
+}
+
+pub(super) fn record_active_multipart_part_committed(source: &Path, part: rustfs_filemeta::ObjectPartInfo) {
+    let Some(data_dir) = ACTIVE_MULTIPART_SOURCE_HEADS
+        .with_get(source, |entry| entry.fi.data_dir)
+        .flatten()
+        .filter(|dir| !dir.is_nil())
+    else {
+        return;
+    };
+    let part_meta_path = source.join(data_dir.to_string()).join(format!("part.{}.meta", part.number));
+    let Some(part_stamp) = FileStamp::from_path(&part_meta_path) else {
+        return;
+    };
+    ACTIVE_MULTIPART_SOURCE_HEADS.with_get_mut(source, |entry| {
+        entry.parts.insert(part.number, (part_stamp, part));
+    });
+}
+
+pub(super) fn finish_active_multipart_part_transaction(
+    source: &Path,
+    part_number: Option<usize>,
+    committed: bool,
+    retired_existing_txn: bool,
+) {
+    let Some(part_number) = part_number.filter(|_| committed) else {
+        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
+        return;
+    };
+    if !retired_existing_txn {
+        return;
+    }
+    ACTIVE_MULTIPART_SOURCE_HEADS.remove_if_mut(source, |entry| !entry.unsettled_parts.remove(&part_number));
+}
+
+pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize) -> Option<rustfs_filemeta::ObjectPartInfo> {
+    let (head_stamp, data_dir, expected_stamp, part) = ACTIVE_MULTIPART_SOURCE_HEADS
+        .with_get(source, |entry| {
+            if !entry.unsettled_parts.is_empty() {
+                return None;
+            }
+            let data_dir = entry.fi.data_dir.filter(|dir| !dir.is_nil())?;
+            let (expected_stamp, part) = entry.parts.get(&part_number)?;
+            Some((entry.head_stamp, data_dir, *expected_stamp, part.clone()))
+        })
+        .flatten()?;
+    if !stamp_matches_active_disk(source, head_stamp) {
+        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
+        return None;
+    }
+    let part_meta_path = source.join(data_dir.to_string()).join(format!("part.{part_number}.meta"));
+    if FileStamp::from_path(&part_meta_path) != Some(expected_stamp) {
+        return None;
+    }
+    Some(part)
+}
+
+pub(super) fn invalidate_active_multipart_source_head(source: &Path) {
+    ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
+}
+
+pub(super) fn invalidate_active_multipart_sources_under(prefix: &Path) {
+    ACTIVE_MULTIPART_SOURCE_HEADS.retain_outside_prefix(prefix);
+}
+
 /// Canonical source mutations are admitted under SourceRW at the syscall,
 /// not on the earlier async caller that may have timed out.
 pub(super) fn validate_source_mutation(source: &Path, base: &Path, root: &os::PublicationRoot, initial: bool) -> Result<()> {
+    if !initial && has_active_multipart_source_head(source) {
+        return Ok(());
+    }
     let pinned = match os::pin_multipart_source_directory(source, base, root) {
         Ok(pinned) => pinned,
         Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
@@ -440,6 +608,7 @@ impl SourceStore {
         terminal: Option<&Path>,
         sync_source_before: bool,
     ) -> std::io::Result<()> {
+        invalidate_active_multipart_source_head(source);
         #[cfg(unix)]
         let mut pinned = os::pin_multipart_source_directory(source, base, root)?;
         #[cfg(not(unix))]
@@ -470,7 +639,7 @@ impl SourceStore {
         let terminal = terminal_record_path(&source, state);
         let bytes = bytes.to_vec();
         os::run_blocking_namespace_operation_with_sync(self.lease.clone(), true, move || {
-            Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes)?;
+            Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes, false)?;
             Self::persist_state_std(&source, &base, &root, &raw, terminal.as_deref(), false)
         })
         .await
@@ -539,7 +708,7 @@ impl SourceStore {
         let root = self.publication_root.clone();
         let bytes = bytes.to_vec();
         os::run_blocking_namespace_operation(self.lease.clone(), move || {
-            Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes)
+            Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes, true)
         })
         .await
         .map_err(DiskError::from)
@@ -551,6 +720,7 @@ impl SourceStore {
         owner: Uuid,
         target: bool,
         bytes: &[u8],
+        sync_source_root: bool,
     ) -> std::io::Result<()> {
         let path = if target {
             target_backup_path(source, owner)
@@ -582,9 +752,10 @@ impl SourceStore {
         if !existed {
             os::replace_multipart_source_metadata(&path, bytes, base, root)?;
         }
-        os::sync_multipart_owned_record_parents(source, parent, existed, base, root)
+        os::sync_multipart_owned_record_parents_options(source, parent, existed, sync_source_root, base, root)
     }
     pub async fn replace_head(&self, bytes: &[u8]) -> Result<()> {
+        invalidate_active_multipart_source_head(&self.source);
         let path = self.source.join(STORAGE_FORMAT_FILE);
         let base = self.base.clone();
         let root = self.publication_root.clone();
@@ -1097,6 +1268,82 @@ pub(super) fn load_source(
     })
 }
 
+fn load_uninitialized_source(
+    raw: Bytes,
+    path: &str,
+    disk_id: Uuid,
+    physical_slot: usize,
+    disk_ids: &[Uuid],
+) -> Result<LoadedSource> {
+    let sha = raw_sha(&raw);
+    let length = raw.len() as u64;
+    let current = get_file_info(
+        &raw,
+        RUSTFS_META_MULTIPART_BUCKET,
+        path,
+        "",
+        FileInfoOpts {
+            data: false,
+            include_free_versions: false,
+            include_part_checksums: true,
+        },
+    )
+    .map_err(Error::other)?;
+    native_source_protocol(&current)?;
+    if source_status(&current)?.is_some()
+        || publication_identity(&current)?.is_some()
+        || SourceConditionV1::take(&mut current.clone())?.is_some()
+    {
+        return Err(DiskError::FileCorrupt);
+    }
+    let bucket = rustfs_utils::http::metadata_compat::get_consistent_str(&current.metadata, "multipart-bucket")
+        .ok_or(DiskError::FileCorrupt)?;
+    let object = rustfs_utils::http::metadata_compat::get_consistent_str(&current.metadata, "multipart-object")
+        .ok_or(DiskError::FileCorrupt)?;
+    let condition = capture_source_condition(bucket, object, path, &current, current.data_dir.ok_or(DiskError::FileCorrupt)?)?;
+    if condition.generation.epoch != 0 {
+        return Err(DiskError::FileCorrupt);
+    }
+    let slot = condition
+        .lineage
+        .geometry
+        .distribution
+        .iter()
+        .position(|index| *index == current.erasure.index)
+        .ok_or(DiskError::FileCorrupt)?;
+    if slot != physical_slot || condition.lineage.geometry.disks() != disk_ids.len() {
+        return Err(DiskError::ConditionalDeleteConflict);
+    }
+    let inventory = raw_sha(&encode(&(None::<[u8; 32]>, Some(sha), sha, None::<[u8; 32]>, None::<[u8; 32]>))?);
+    let snapshot = Some(MultipartSourceSnapshot {
+        disk_id,
+        physical_slot: slot,
+        lineage: condition.lineage,
+        generation: condition.generation,
+        revision: 0,
+        phase: MultipartSourcePhase::Active,
+        state_sha256: [0; 32],
+        head_sha256: Some(sha),
+        inventory_sha256: inventory,
+        capability_sha256: None,
+        publication: None,
+    });
+    Ok(LoadedSource {
+        root_missing: false,
+        state: None,
+        head: Some(SourceHead {
+            sha256: sha,
+            length,
+            raw: Some(raw.clone()),
+        }),
+        original: Some(raw),
+        info: Some(current),
+        snapshot,
+        head_matches_state: true,
+        head_matches_activation: false,
+    })
+}
+
 pub(super) const RETIREMENT_ANCHOR: &str = ".mpu-source-retirement";
 
 #[derive(Serialize, Deserialize)]
@@ -1323,41 +1570,6 @@ impl LocalDisk {
 }
 
 impl SourceStore {
-    /// Only native source validation may create provenance. Ordinary TMP/heal
-    /// publication can retain a valid old local anchor, never issue/rebind one.
-    pub async fn write_retirement_anchor(
-        &self,
-        state: &SourceState,
-        publication: &PublicationIdentity,
-        directory: &Path,
-        lease: Arc<os::NamespaceMutationLease>,
-    ) -> Result<()> {
-        if state.lineage != publication.lineage
-            || state.generation != publication.source_generation
-            || state.disk_id != self.disk_id
-            || state.physical_slot != self.physical_slot
-        {
-            return Err(DiskError::ConditionalDeleteConflict);
-        }
-        let record = NativeRetirementAnchor {
-            protocol_version: MULTIPART_SOURCE_PROTOCOL_VERSION,
-            disk_id: self.disk_id,
-            physical_slot: self.physical_slot,
-            publication: publication.clone(),
-            source_original_sha256: state.original_sha256,
-            source_validation_receipt_id: state.backup_owner,
-        };
-        let bytes = encode(&record)?;
-        let directory = directory.to_path_buf();
-        let base = self.base.clone();
-        let root = self.publication_root.clone();
-        os::run_blocking_namespace_operation(lease, move || {
-            let _parent = os::mkdir_all_below_existing_base_std(&directory, &base, &root)?;
-            os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &bytes, &base, &root)
-        })
-        .await
-        .map_err(DiskError::from)
-    }
     /// This caller already owns SourceRW and destination metadata ownership.
     /// An Aborting survivor can confirm a historical target only with its
     /// independent, same-disk publication anchor. Closure never creates one.
@@ -1456,6 +1668,8 @@ struct ValidatedPreparedPublication {
     length: u64,
 }
 
+type DeferredInitialPublication = (Vec<u8>, PathBuf, Vec<u8>);
+
 pub(super) struct NativeMultipartPublication {
     pub store: Arc<SourceStore>,
     pub publication: PublicationIdentity,
@@ -1463,22 +1677,49 @@ pub(super) struct NativeMultipartPublication {
     state: parking_lot::Mutex<SourceState>,
     capability: parking_lot::Mutex<Option<MultipartUndoCapability>>,
     validated_prepared: parking_lot::Mutex<Option<ValidatedPreparedPublication>>,
+    deferred_initial: parking_lot::Mutex<Option<DeferredInitialPublication>>,
     pub legacy: bool,
 }
 
 impl LocalDisk {
+    #[cfg(test)]
     pub(super) async fn prepare_native_multipart_publication(
         &self,
         path: &str,
         incoming: &mut FileInfo,
         staging: &MultipartCommitStaging,
+    ) -> Result<NativeMultipartPublication> {
+        self.prepare_native_multipart_publication_inner(path, incoming, staging, false)
+            .await
+    }
+
+    pub(super) async fn prepare_native_multipart_publication_deferred(
+        &self,
+        path: &str,
+        incoming: &mut FileInfo,
+        staging: &MultipartCommitStaging,
+    ) -> Result<NativeMultipartPublication> {
+        self.prepare_native_multipart_publication_inner(path, incoming, staging, true)
+            .await
+    }
+
+    async fn prepare_native_multipart_publication_inner(
+        &self,
+        path: &str,
+        incoming: &mut FileInfo,
+        staging: &MultipartCommitStaging,
+        defer_persist: bool,
     ) -> Result<NativeMultipartPublication> {
         // Real Complete is parts-backed, including a zero-byte last part.
         if incoming.data.is_some() || incoming.inline_data() {
             return Err(DiskError::UnsupportedDisk);
         }
         let store = Arc::new(self.multipart_store_from_staging(path, staging).await?);
-        let loaded = store.load().await?;
+        let loaded = if defer_persist && let Some(raw) = staging.fresh_source_head() {
+            load_uninitialized_source(raw, &store.path, store.disk_id, store.physical_slot, &store.disk_ids)?
+        } else {
+            store.load().await?
+        };
         let snapshot = loaded.snapshot.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
         let original = loaded
             .head
@@ -1569,7 +1810,44 @@ impl LocalDisk {
         };
         // Copy the small source FI to an independent immutable inode. Its
         // canonical inode is never truncated to build the destination proxy.
-        store.write_owned_backup_and_persist(owner, false, original, &state).await?;
+        let directory = staging.private_directory(publication.final_data_dir);
+        let anchor_record = NativeRetirementAnchor {
+            protocol_version: MULTIPART_SOURCE_PROTOCOL_VERSION,
+            disk_id: store.disk_id,
+            physical_slot: store.physical_slot,
+            publication: publication.clone(),
+            source_original_sha256: state.original_sha256,
+            source_validation_receipt_id: state.backup_owner,
+        };
+        let anchor_bytes = encode(&anchor_record)?;
+        state.validate(store.disk_id, &store.path)?;
+        let original_bytes = original.to_vec();
+        let deferred_initial = if defer_persist {
+            Some((original_bytes, directory, anchor_bytes))
+        } else {
+            let state_raw = encode(&state)?;
+            let source_dir = store.source.clone();
+            let base_dir = store.base.clone();
+            let pub_root = store.publication_root.clone();
+            let terminal = terminal_record_path(&source_dir, &state);
+            #[cfg(target_os = "linux")]
+            let can_mark_presynced = has_active_multipart_source_head(&source_dir)
+                && super::effective_durability(RUSTFS_META_MULTIPART_BUCKET).syncs_data_shards();
+            os::run_blocking_namespace_operation_with_sync(staging.lease.clone(), true, move || {
+                SourceStore::write_owned_backup_std(&source_dir, &base_dir, &pub_root, owner, false, &original_bytes, false)?;
+                SourceStore::persist_state_std(&source_dir, &base_dir, &pub_root, &state_raw, terminal.as_deref(), false)?;
+                let _parent = os::mkdir_all_below_existing_base_std(&directory, &base_dir, &pub_root)?;
+                os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, &base_dir, &pub_root)?;
+                #[cfg(target_os = "linux")]
+                if can_mark_presynced {
+                    super::mark_presynced_tmp_data_dir(directory);
+                }
+                Ok(())
+            })
+            .await
+            .map_err(DiskError::from)?;
+            None
+        };
         let prepared_path = store.source.join(format!(".mpu-prepared-{owner}"));
         let context = NativeMultipartPublication {
             store,
@@ -1578,14 +1856,9 @@ impl LocalDisk {
             state: parking_lot::Mutex::new(state),
             capability: parking_lot::Mutex::new(Some(capability)),
             validated_prepared: parking_lot::Mutex::new(None),
+            deferred_initial: parking_lot::Mutex::new(deferred_initial),
             legacy,
         };
-        let directory = staging.private_directory(context.publication.final_data_dir);
-        let reserved = context.state.lock().clone();
-        context
-            .store
-            .write_retirement_anchor(&reserved, &context.publication, &directory, staging.lease.clone())
-            .await?;
         Ok(context)
     }
 }
@@ -1594,18 +1867,15 @@ impl NativeMultipartPublication {
     pub fn canonical_head(&self) -> PathBuf {
         self.store.source.join(STORAGE_FORMAT_FILE)
     }
+    #[cfg(test)]
     pub async fn prepare_metadata(&self, new_metadata: &[u8], old_metadata: Option<&[u8]>) -> Result<()> {
         let mut state = self.state.lock().clone();
-        let current = self.store.load().await?;
-        let current_state = current.state.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
-        if encode(current_state)? != encode(&state)? || !current.head_matches_state {
-            return Err(DiskError::ConditionalDeleteConflict);
-        }
         // Validate the actual persisted representation once, after FileMeta's
         // canonicalization. No unvalidated input can install a source proxy.
         if !self.store.target_matches(&self.publication, new_metadata)? {
             return Err(DiskError::ConditionalDeleteConflict);
         }
+        let expected_state_raw = encode(&state)?;
         let validated = ValidatedPreparedPublication {
             publication: self.publication.clone(),
             sha256: raw_sha(new_metadata),
@@ -1618,17 +1888,298 @@ impl NativeMultipartPublication {
         state.prepared_sha256 = Some(validated.sha256);
         state.prepared_length = Some(validated.length);
         state.advance(MultipartSourcePhase::Prepared)?;
-        if let Some(previous) = old_metadata {
-            self.store
-                .write_owned_backup_and_persist(state.backup_owner, true, previous, &state)
-                .await?;
-        } else {
-            self.store.persist(&state).await?;
-        }
+        state.validate(self.store.disk_id, &self.store.path)?;
+        let next_state_raw = encode(&state)?;
+        let source = self.store.source.clone();
+        let base = self.store.base.clone();
+        let root = self.store.publication_root.clone();
+        let path = self.store.path.clone();
+        let disk_id = self.store.disk_id;
+        let physical_slot = self.store.physical_slot;
+        let disk_ids = self.store.disk_ids.clone();
+        let backup_owner = state.backup_owner;
+        let old_bytes = old_metadata.map(|b| b.to_vec());
+        let terminal = terminal_record_path(&source, &state);
+        os::run_blocking_namespace_operation_with_sync(self.store.lease.clone(), true, move || {
+            let current = load_source(&source, &base, &root, &path, disk_id, physical_slot, &disk_ids).map_err(IoError::other)?;
+            let current_state = current
+                .state
+                .as_ref()
+                .ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
+            let current_raw = encode(current_state).map_err(IoError::other)?;
+            if current_raw != expected_state_raw || !current.head_matches_state {
+                return Err(IoError::other(DiskError::ConditionalDeleteConflict));
+            }
+            if let Some(previous) = old_bytes.as_deref() {
+                SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
+            }
+            SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)
+        })
+        .await
+        .map_err(DiskError::from)?;
         *self.state.lock() = state;
         *self.validated_prepared.lock() = Some(validated);
         Ok(())
     }
+    #[cfg(target_os = "linux")]
+    pub(super) fn prepare_and_install_head_on_thread(
+        &self,
+        new_metadata: &[u8],
+        old_metadata: Option<&[u8]>,
+        dst_file_path: &Path,
+        directory_guard: &os::RenameCommitGuard,
+    ) -> Result<os::PreparedRenameSource> {
+        let mut state = self.state.lock().clone();
+        if !self.store.target_matches(&self.publication, new_metadata)? {
+            return Err(DiskError::ConditionalDeleteConflict);
+        }
+        let deferred_initial = self.deferred_initial.lock().take();
+        let expected_state_raw = encode(&state)?;
+        let expected_lineage = state.lineage.clone();
+        let expected_generation = state.generation;
+        let expected_revision = state.revision;
+        let validated = ValidatedPreparedPublication {
+            publication: self.publication.clone(),
+            sha256: raw_sha(new_metadata),
+            length: new_metadata.len() as u64,
+        };
+        if let Some(previous) = old_metadata {
+            state.previous_target_sha256 = Some(raw_sha(previous));
+            state.previous_target_length = Some(previous.len() as u64);
+        }
+        state.prepared_sha256 = Some(validated.sha256);
+        state.prepared_length = Some(validated.length);
+        state.advance(MultipartSourcePhase::Prepared)?;
+        state.validate(self.store.disk_id, &self.store.path)?;
+        let next_state_raw = encode(&state)?;
+        let source = &self.store.source;
+        let base = &self.store.base;
+        let root = &self.store.publication_root;
+        let path = &self.store.path;
+        let disk_id = self.store.disk_id;
+        let physical_slot = self.store.physical_slot;
+        let disk_ids = &self.store.disk_ids;
+        let backup_owner = state.backup_owner;
+        #[cfg(target_os = "linux")]
+        let can_mark_presynced = has_active_multipart_source_head(source)
+            && super::effective_durability(RUSTFS_META_MULTIPART_BUCKET).syncs_data_shards();
+        let terminal = terminal_record_path(source, &state);
+        let prepared_path = &self.prepared_path;
+        let canonical = self.canonical_head();
+        let prepared = (|| -> std::io::Result<os::PreparedRenameSource> {
+            if let Some((ref original_bytes, _, _)) = deferred_initial {
+                let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
+                let snapshot = current
+                    .snapshot
+                    .as_ref()
+                    .ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
+                let current_head = current
+                    .head
+                    .as_ref()
+                    .and_then(SourceHead::raw)
+                    .ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
+                if !current.head_matches_state
+                    || snapshot.phase != MultipartSourcePhase::Active
+                    || snapshot.capability_sha256.is_some()
+                    || snapshot.lineage != expected_lineage
+                    || snapshot.generation != expected_generation
+                    || snapshot.revision.checked_add(1) != Some(expected_revision)
+                    || current_head != original_bytes.as_slice()
+                {
+                    return Err(IoError::other(DiskError::ConditionalDeleteConflict));
+                }
+            } else {
+                let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
+                let current_state = current
+                    .state
+                    .as_ref()
+                    .ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
+                let current_raw = encode(current_state).map_err(IoError::other)?;
+                if current_raw != expected_state_raw || !current.head_matches_state {
+                    return Err(IoError::other(DiskError::ConditionalDeleteConflict));
+                }
+            }
+            #[cfg(test)]
+            super::run_owned_file_before_open(prepared_path);
+            let mut prepared_metadata_source =
+                os::create_prepared_rename_source_with_commit_guard(prepared_path, dst_file_path, directory_guard)?;
+            if let Err(err) = prepared_metadata_source.write_all(new_metadata, false) {
+                drop(prepared_metadata_source);
+                let _ = std::fs::remove_file(prepared_path);
+                return Err(err);
+            }
+            let setup_res: std::io::Result<()> = (|| {
+                if let Some((original_bytes, directory, anchor_bytes)) = deferred_initial {
+                    SourceStore::write_owned_backup_std(source, base, root, backup_owner, false, &original_bytes, false)?;
+                    if let Some(previous) = old_metadata {
+                        SourceStore::write_owned_backup_std(source, base, root, backup_owner, true, previous, false)?;
+                    }
+                    SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), false)?;
+                    let _parent = os::mkdir_all_below_existing_base_std(&directory, base, root)?;
+                    os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, base, root)?;
+                    #[cfg(target_os = "linux")]
+                    if can_mark_presynced {
+                        super::mark_presynced_tmp_data_dir(directory);
+                    }
+                } else {
+                    if let Some(previous) = old_metadata {
+                        SourceStore::write_owned_backup_std(source, base, root, backup_owner, true, previous, false)?;
+                    }
+                    SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), false)?;
+                }
+                Ok(())
+            })();
+            if let Err(err) = setup_res {
+                drop(prepared_metadata_source);
+                let _ = std::fs::remove_file(prepared_path);
+                return Err(err);
+            }
+            if let Err(err) = prepared_metadata_source.sync_data() {
+                drop(prepared_metadata_source);
+                let _ = std::fs::remove_file(prepared_path);
+                return Err(err);
+            }
+            os::install_multipart_prepared_head(prepared_metadata_source, &canonical, base, root)
+        })()
+        .map_err(DiskError::from)?;
+        *self.state.lock() = state;
+        *self.validated_prepared.lock() = Some(validated);
+        Ok(prepared)
+    }
+
+    #[cfg(not(target_os = "linux"))]
+    pub async fn prepare_and_install_head(
+        &self,
+        new_metadata: &[u8],
+        old_metadata: Option<&[u8]>,
+        dst_file_path: &Path,
+        directory_guard: &os::RenameCommitGuard,
+        mutation_lease: Arc<os::NamespaceMutationLease>,
+    ) -> Result<os::PreparedRenameSource> {
+        let mut state = self.state.lock().clone();
+        if !self.store.target_matches(&self.publication, new_metadata)? {
+            return Err(DiskError::ConditionalDeleteConflict);
+        }
+        let deferred_initial = self.deferred_initial.lock().take();
+        let expected_state_raw = encode(&state)?;
+        let expected_lineage = state.lineage.clone();
+        let expected_generation = state.generation;
+        let expected_revision = state.revision;
+        let validated = ValidatedPreparedPublication {
+            publication: self.publication.clone(),
+            sha256: raw_sha(new_metadata),
+            length: new_metadata.len() as u64,
+        };
+        if let Some(previous) = old_metadata {
+            state.previous_target_sha256 = Some(raw_sha(previous));
+            state.previous_target_length = Some(previous.len() as u64);
+        }
+        state.prepared_sha256 = Some(validated.sha256);
+        state.prepared_length = Some(validated.length);
+        state.advance(MultipartSourcePhase::Prepared)?;
+        state.validate(self.store.disk_id, &self.store.path)?;
+        let next_state_raw = encode(&state)?;
+        let source = self.store.source.clone();
+        let base = self.store.base.clone();
+        let root = self.store.publication_root.clone();
+        let path = self.store.path.clone();
+        let disk_id = self.store.disk_id;
+        let physical_slot = self.store.physical_slot;
+        let disk_ids = self.store.disk_ids.clone();
+        let backup_owner = state.backup_owner;
+        #[cfg(target_os = "linux")]
+        let can_mark_presynced = has_active_multipart_source_head(&source)
+            && super::effective_durability(RUSTFS_META_MULTIPART_BUCKET).syncs_data_shards();
+        let old_bytes = old_metadata.map(|b| b.to_vec());
+        let new_bytes = new_metadata.to_vec();
+        let terminal = terminal_record_path(&source, &state);
+        let prepared_path = self.prepared_path.clone();
+        let canonical = self.canonical_head();
+        let dst_file_path = dst_file_path.to_path_buf();
+        let owned_directory_guard = directory_guard.clone();
+        let prepared = os::run_blocking_namespace_operation_with_sync(mutation_lease, true, move || {
+            let current = load_source(&source, &base, &root, &path, disk_id, physical_slot, &disk_ids).map_err(IoError::other)?;
+            if let Some((original_bytes, directory, anchor_bytes)) = deferred_initial {
+                let snapshot = current
+                    .snapshot
+                    .as_ref()
+                    .ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
+                let current_head = current
+                    .head
+                    .as_ref()
+                    .and_then(SourceHead::raw)
+                    .ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
+                if !current.head_matches_state
+                    || snapshot.phase != MultipartSourcePhase::Active
+                    || snapshot.capability_sha256.is_some()
+                    || snapshot.lineage != expected_lineage
+                    || snapshot.generation != expected_generation
+                    || snapshot.revision.checked_add(1) != Some(expected_revision)
+                    || current_head != original_bytes.as_slice()
+                {
+                    return Err(IoError::other(DiskError::ConditionalDeleteConflict));
+                }
+                SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, false, &original_bytes, false)?;
+                if let Some(previous) = old_bytes.as_deref() {
+                    SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
+                }
+                SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)?;
+                let _parent = os::mkdir_all_below_existing_base_std(&directory, &base, &root)?;
+                os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, &base, &root)?;
+                #[cfg(target_os = "linux")]
+                if can_mark_presynced {
+                    super::mark_presynced_tmp_data_dir(directory);
+                }
+            } else {
+                let current_state = current
+                    .state
+                    .as_ref()
+                    .ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
+                let current_raw = encode(current_state).map_err(IoError::other)?;
+                if current_raw != expected_state_raw || !current.head_matches_state {
+                    return Err(IoError::other(DiskError::ConditionalDeleteConflict));
+                }
+                if let Some(previous) = old_bytes.as_deref() {
+                    SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
+                }
+                SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)?;
+            }
+            #[cfg(test)]
+            super::run_owned_file_before_open(&prepared_path);
+            let mut prepared_metadata_source =
+                os::create_prepared_rename_source_with_commit_guard(&prepared_path, &dst_file_path, &owned_directory_guard)?;
+            prepared_metadata_source.write_all(&new_bytes, true)?;
+            os::install_multipart_prepared_head(prepared_metadata_source, &canonical, &base, &root)
+        })
+        .await
+        .map_err(DiskError::from)?;
+        *self.state.lock() = state;
+        *self.validated_prepared.lock() = Some(validated);
+        Ok(prepared)
+    }
+
+    #[cfg(target_os = "linux")]
+    pub async fn prepare_and_install_head(
+        self: &Arc<Self>,
+        new_metadata: &[u8],
+        old_metadata: Option<&[u8]>,
+        dst_file_path: &Path,
+        directory_guard: &os::RenameCommitGuard,
+        mutation_lease: Arc<os::NamespaceMutationLease>,
+    ) -> Result<os::PreparedRenameSource> {
+        let this = Arc::clone(self);
+        let new_bytes = new_metadata.to_vec();
+        let old_bytes = old_metadata.map(|b| b.to_vec());
+        let dst_file_path = dst_file_path.to_path_buf();
+        let owned_directory_guard = directory_guard.clone();
+        os::run_blocking_namespace_operation_with_sync(mutation_lease, true, move || {
+            this.prepare_and_install_head_on_thread(&new_bytes, old_bytes.as_deref(), &dst_file_path, &owned_directory_guard)
+                .map_err(IoError::other)
+        })
+        .await
+        .map_err(DiskError::from)
+    }
+    #[cfg(test)]
     pub async fn install_head(&self, prepared: os::PreparedRenameSource) -> Result<os::PreparedRenameSource> {
         {
             let guard = self.validated_prepared.lock();
@@ -1699,13 +2250,13 @@ impl NativeMultipartPublication {
         // return an error after consuming the last claimable source heads.
         self.store.restore_exact_target_with_lease(&state, lease).await
     }
-    pub async fn finish(&self, target_lease: Arc<os::NamespaceMutationLease>) -> Result<MultipartPublicationResult> {
+    pub(super) fn finish_on_thread(
+        &self,
+        target_already_synced: bool,
+        staging_cleanup: Option<(PathBuf, Arc<std::sync::atomic::AtomicBool>)>,
+    ) -> Result<MultipartPublicationResult> {
         let mut state = self.state.lock().clone();
-        let current = self.store.load().await?;
-        let current_state = current.state.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
-        if current.head.is_some() || encode(current_state)? != encode(&state)? {
-            return Err(DiskError::ConditionalDeleteConflict);
-        }
+        let expected_state_raw = encode(&state)?;
         let (expected_sha256, expected_length) = {
             let guard = self.validated_prepared.lock();
             let validated = guard.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
@@ -1717,16 +2268,88 @@ impl NativeMultipartPublication {
             }
             (validated.sha256, validated.length)
         };
-        self.store
-            .read_and_sync_target_with_lease(&self.publication, target_lease, move |target| {
-                if target.len() as u64 != expected_length || raw_sha(target) != expected_sha256 {
-                    return Err(DiskError::ConditionalDeleteConflict);
-                }
-                Ok(())
-            })
-            .await?;
+        let target_dir = self.store.target_path(&self.publication)?;
         state.advance(MultipartSourcePhase::ConsumedUnsealed)?;
-        let snapshot = self.store.persist_with_source_sync(&state, true).await?;
+        state.validate(self.store.disk_id, &self.store.path)?;
+        let next_state_raw = encode(&state)?;
+        let snapshot = snapshot_for_state(self.store.disk_id, &state, &next_state_raw, None)?;
+        let source = &self.store.source;
+        let base = &self.store.base;
+        let root = &self.store.publication_root;
+        let path = &self.store.path;
+        let disk_id = self.store.disk_id;
+        let physical_slot = self.store.physical_slot;
+        let disk_ids = &self.store.disk_ids;
+        let terminal = terminal_record_path(source, &state);
+        (|| -> std::io::Result<()> {
+            let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
+            let current_state = current
+                .state
+                .as_ref()
+                .ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
+            let current_raw = encode(current_state).map_err(IoError::other)?;
+            if current.head.is_some() || current_raw != expected_state_raw {
+                return Err(IoError::other(DiskError::ConditionalDeleteConflict));
+            }
+            if target_already_synced {
+                let pinned = os::pin_multipart_source_directory(&target_dir, base, root).map_err(|error| {
+                    if error.kind() == std::io::ErrorKind::NotFound {
+                        IoError::other(DiskError::ConditionalDeleteConflict)
+                    } else {
+                        error
+                    }
+                })?;
+                let target = os::read_multipart_source_metadata_in_pinned(
+                    &pinned,
+                    &target_dir,
+                    std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
+                    base,
+                    root,
+                )
+                .map_err(|error| {
+                    if error.kind() == std::io::ErrorKind::NotFound {
+                        IoError::other(DiskError::ConditionalDeleteConflict)
+                    } else {
+                        error
+                    }
+                })?;
+                if target.len() as u64 != expected_length || raw_sha(&target) != expected_sha256 {
+                    return Err(IoError::other(DiskError::ConditionalDeleteConflict));
+                }
+            } else {
+                os::read_and_sync_multipart_target_metadata(
+                    &target_dir,
+                    std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
+                    base,
+                    root,
+                    |target| {
+                        if target.len() as u64 != expected_length || raw_sha(target) != expected_sha256 {
+                            return Err(IoError::other(DiskError::ConditionalDeleteConflict));
+                        }
+                        Ok(())
+                    },
+                )
+                .map_err(|error| {
+                    if error.kind() == std::io::ErrorKind::NotFound {
+                        IoError::other(DiskError::ConditionalDeleteConflict)
+                    } else {
+                        error
+                    }
+                })?;
+            }
+            SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), true)?;
+            if let Some((io_path, cleaned)) = staging_cleanup {
+                match std::fs::remove_dir(&io_path) {
+                    Ok(()) => cleaned.store(true, std::sync::atomic::Ordering::Release),
+                    Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
+                        cleaned.store(true, std::sync::atomic::Ordering::Release);
+                    }
+                    Err(_) => {}
+                }
+            }
+            Ok(())
+        })()
+        .map_err(DiskError::from)?;
         *self.state.lock() = state;
         let proof = self
             .store
@@ -1738,6 +2361,21 @@ impl NativeMultipartPublication {
             undo_capability: capability,
         })
     }
+
+    pub async fn finish(
+        self: &Arc<Self>,
+        target_lease: Arc<os::NamespaceMutationLease>,
+        target_already_synced: bool,
+        staging_cleanup: Option<(PathBuf, Arc<std::sync::atomic::AtomicBool>)>,
+    ) -> Result<MultipartPublicationResult> {
+        let this = Arc::clone(self);
+        os::run_blocking_namespace_operation_with_sync(target_lease, true, move || {
+            this.finish_on_thread(target_already_synced, staging_cleanup)
+                .map_err(IoError::other)
+        })
+        .await
+        .map_err(DiskError::from)
+    }
 }
 
 impl SourceStore {

crates/ecstore/src/disk/local/multipart_staging.rs

@@ -207,6 +207,12 @@ pub(super) struct MultipartSourceAdmission {
     initial: bool,
 }
 impl MultipartSourceAdmission {
+    pub(super) fn source(&self) -> &Path {
+        &self.source
+    }
+    pub(super) fn is_initial(&self) -> bool {
+        self.initial
+    }
     pub(super) fn precondition(&self) -> Arc<dyn Fn() -> std::io::Result<()> + Send + Sync> {
         let admission = self.clone();
         Arc::new(move || admission.validate())
@@ -250,6 +256,8 @@ pub(super) struct MultipartCommitStaging {
     logical_path: PathBuf,
     io_path: PathBuf,
     owner: Option<Arc<dyn Send + Sync>>,
+    cleaned: Arc<std::sync::atomic::AtomicBool>,
+    fresh_source_head: Option<bytes::Bytes>,
 }
 
 impl MultipartCommitStaging {
@@ -259,10 +267,19 @@ impl MultipartCommitStaging {
     pub(super) fn private_directory(&self, directory: Uuid) -> PathBuf {
         self.io_path.join(directory.to_string())
     }
+    pub(super) fn cleanup_handle(&self) -> (PathBuf, Arc<std::sync::atomic::AtomicBool>) {
+        (self.io_path.clone(), Arc::clone(&self.cleaned))
+    }
+    pub(super) fn fresh_source_head(&self) -> Option<bytes::Bytes> {
+        self.fresh_source_head.clone()
+    }
 }
 
 impl Drop for MultipartCommitStaging {
     fn drop(&mut self) {
+        if self.cleaned.load(std::sync::atomic::Ordering::Acquire) {
+            return;
+        }
         let logical_path = self.logical_path.clone();
         let io_path = self.io_path.clone();
         let owner = self.owner.clone();
@@ -285,6 +302,14 @@ impl Drop for MultipartCommitStaging {
 }
 
 impl LocalDisk {
+    pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
+        let root = upload_scope_root(upload_or_part_path, part_path);
+        let Ok(source) = self.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, root) else {
+            return false;
+        };
+        super::multipart_source_state::has_clean_active_multipart_source(&source)
+    }
+
     pub(super) fn multipart_source_admission(
         &self,
         volume: &str,
@@ -356,12 +381,14 @@ impl LocalDisk {
         let io_path = self.io_get_object_path(RUSTFS_META_TMP_BUCKET, &attempt)?;
         // The attempt root is exactly the namespace used by the tmp sweeper.
         let lease = os::acquire_namespace_mutation_lease_with_owner(&logical_path, owner.clone()).await;
-        let staging = MultipartCommitStaging {
+        let mut staging = MultipartCommitStaging {
             path: attempt,
             lease,
             logical_path,
             io_path,
             owner,
+            cleaned: Arc::new(std::sync::atomic::AtomicBool::new(false)),
+            fresh_source_head: None,
         };
         let canonical_metadata = self.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, &format!("{source_path}/xl.meta"))?;
         let source_object = self.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, source_path)?;
@@ -371,31 +398,37 @@ impl LocalDisk {
         let source_path = source_path.to_owned();
         let incoming = incoming.clone();
         let sync_directory = effective_durability(destination_volume).syncs_data_shards();
-        os::run_cancellable_blocking_namespace_operation(Arc::clone(&staging.lease), move |cancelled| {
+        let fresh_head = os::run_cancellable_blocking_namespace_operation(Arc::clone(&staging.lease), move |cancelled| {
             if cancelled() {
                 return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "multipart staging cancelled"));
             }
-            let raw = os::read_multipart_source_metadata(&canonical_metadata, &base, &publication_root).map_err(|error| {
-                // A retired source has no authority for this delayed completion.
-                // Preserve confinement and other I/O failures as their real errors.
-                if error.kind() == std::io::ErrorKind::NotFound {
-                    DiskError::FileNotFound.into()
-                } else {
-                    error
-                }
-            })?;
-            let source = get_file_info(
-                &raw,
-                RUSTFS_META_MULTIPART_BUCKET,
-                &source_path,
-                "",
-                FileInfoOpts {
-                    data: false,
-                    include_free_versions: false,
-                    include_part_checksums: true,
-                },
-            )
-            .map_err(std::io::Error::other)?;
+            let cached_head = super::multipart_source_state::get_active_multipart_source_head(&source_object);
+            let (raw, source, from_active_cache) = if let Some((raw, source)) = cached_head {
+                (raw, source, true)
+            } else {
+                let raw = os::read_multipart_source_metadata(&canonical_metadata, &base, &publication_root).map_err(|error| {
+                    // A retired source has no authority for this delayed completion.
+                    // Preserve confinement and other I/O failures as their real errors.
+                    if error.kind() == std::io::ErrorKind::NotFound {
+                        DiskError::FileNotFound.into()
+                    } else {
+                        error
+                    }
+                })?;
+                let source = get_file_info(
+                    &raw,
+                    RUSTFS_META_MULTIPART_BUCKET,
+                    &source_path,
+                    "",
+                    FileInfoOpts {
+                        data: false,
+                        include_free_versions: false,
+                        include_part_checksums: true,
+                    },
+                )
+                .map_err(std::io::Error::other)?;
+                (bytes::Bytes::from(raw), source, false)
+            };
             let source_dir = source
                 .data_dir
                 .filter(|dir| !dir.is_nil())
@@ -412,23 +445,31 @@ impl LocalDisk {
             let _attempt = os::mkdir_all_below_existing_base_std(&private_object, &base, &publication_root)?;
             if incoming.data.is_none() && !incoming.is_remote() {
                 let source_data_dir = source_object.join(source_dir.to_string());
-                let pinned_source_data = (!incoming.parts.is_empty())
-                    .then(|| os::pin_multipart_source_directory(&source_data_dir, &base, &publication_root))
-                    .transpose()?;
+                let mut pinned_source_data = None;
                 let mut names = Vec::with_capacity(incoming.parts.len().saturating_mul(3));
                 for part in &incoming.parts {
                     if cancelled() {
                         return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "multipart staging cancelled"));
                     }
                     let meta_name = format!("part.{}.meta", part.number);
-                    let bytes = os::read_multipart_source_metadata_in_pinned(
-                        pinned_source_data.as_ref().expect("pinned source data directory"),
-                        &source_data_dir,
-                        std::ffi::OsStr::new(&meta_name),
-                        &base,
-                        &publication_root,
-                    )?;
-                    let actual = rustfs_filemeta::ObjectPartInfo::unmarshal(&bytes).map_err(std::io::Error::other)?;
+                    let actual = if let Some(cached_part) =
+                        super::multipart_source_state::get_active_multipart_source_part(&source_object, part.number)
+                    {
+                        cached_part
+                    } else {
+                        if pinned_source_data.is_none() {
+                            pinned_source_data =
+                                Some(os::pin_multipart_source_directory(&source_data_dir, &base, &publication_root)?);
+                        }
+                        let bytes = os::read_multipart_source_metadata_in_pinned(
+                            pinned_source_data.as_ref().expect("pinned source data directory"),
+                            &source_data_dir,
+                            std::ffi::OsStr::new(&meta_name),
+                            &base,
+                            &publication_root,
+                        )?;
+                        rustfs_filemeta::ObjectPartInfo::unmarshal(&bytes).map_err(std::io::Error::other)?
+                    };
                     let normalized = i64::try_from(actual.size)
                         .map_err(std::io::Error::other)?
                         .max(actual.actual_size);
@@ -464,10 +505,15 @@ impl LocalDisk {
                     cancelled,
                 )?;
             }
-            Ok(())
+            let has_state = !from_active_cache
+                && (source_object.join(".mpu-source-state").exists()
+                    || source_object.join(".mpu-recovery").exists()
+                    || source_object.join(".mpu-completions").exists());
+            Ok((!has_state).then_some(raw))
         })
         .await
         .map_err(DiskError::from)?;
+        staging.fresh_source_head = fresh_head;
         Ok(staging)
     }
 }
@@ -1219,4 +1265,211 @@ mod tests {
             .expect("non-part empty hash cleanup must remain supported");
         assert!(!directory.exists(), "a per-upload safety guard must not strand empty hash directories");
     }
+
+    #[tokio::test]
+    async fn multipart_source_active_head_cache_populates_and_invalidates_across_part_and_delete_paths() {
+        use crate::disk::{DiskAPI as _, PartTransactionAction, ReadOptions};
+        let (_root, disk, completed, _) = fixture().await;
+        std::fs::create_dir_all(disk.io_get_bucket_path(RUSTFS_META_TMP_BUCKET).expect("TMP path")).expect("TMP bucket");
+        let upload = format!("{}/{}", "c".repeat(64), Uuid::new_v4());
+        let data_dir = Uuid::new_v4();
+        let mut init_fi = FileInfo::new(&upload, 2, 2);
+        init_fi.erasure.index = 1;
+        init_fi.data_dir = Some(data_dir);
+        init_fi.fresh = true;
+        init_fi.mod_time = Some(time::OffsetDateTime::now_utc());
+
+        disk.write_metadata("", RUSTFS_META_MULTIPART_BUCKET, &upload, init_fi.clone())
+            .await
+            .expect("fresh multipart upload metadata");
+        assert!(disk.has_clean_active_multipart_source(&upload, false));
+
+        let read_fi = disk
+            .read_version("", RUSTFS_META_MULTIPART_BUCKET, &upload, "", &ReadOptions::default())
+            .await
+            .expect("cached read_version");
+        assert_eq!(read_fi.data_dir, Some(data_dir));
+
+        let destination = format!("{upload}/{data_dir}/part.1");
+        let part_meta = bytes::Bytes::from(completed.parts[0].marshal_msg().expect("part meta"));
+        disk.write_all(
+            RUSTFS_META_TMP_BUCKET,
+            "cache-test-part/part.1",
+            bytes::Bytes::from_static(b"immutable shard"),
+        )
+        .await
+        .expect("stage tmp part");
+        disk.prepare_part_transaction(
+            RUSTFS_META_TMP_BUCKET,
+            "cache-test-part/part.1",
+            RUSTFS_META_MULTIPART_BUCKET,
+            &destination,
+            part_meta.clone(),
+        )
+        .await
+        .expect("prepare part transaction");
+        assert!(
+            !disk.has_clean_active_multipart_source(&upload, false),
+            "in-flight part transaction must mark active source as having unsettled part txn"
+        );
+        disk.rename_part(
+            RUSTFS_META_TMP_BUCKET,
+            "cache-test-part/part.1",
+            RUSTFS_META_MULTIPART_BUCKET,
+            &destination,
+            part_meta,
+        )
+        .await
+        .expect("rename part");
+        disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &destination, PartTransactionAction::Commit)
+            .await
+            .expect("settle part transaction");
+        assert!(
+            disk.has_clean_active_multipart_source(&upload, false),
+            "settled commit must restore clean active multipart source status"
+        );
+        let read_parts = disk
+            .read_parts(RUSTFS_META_MULTIPART_BUCKET, &[format!("{destination}.meta")])
+            .await
+            .expect("cached read_parts");
+        assert_eq!(read_parts.len(), 1);
+        assert_eq!(read_parts[0].number, 1);
+
+        // Prepare part 2, then replay Commit for already-settled part 1: part 2's
+        // pending transaction must keep the upload marked as having unsettled part txns.
+        let destination2 = format!("{upload}/{data_dir}/part.2");
+        let mut part2_info = completed.parts[0].clone();
+        part2_info.number = 2;
+        let part2_meta = bytes::Bytes::from(part2_info.marshal_msg().expect("part 2 meta"));
+        disk.write_all(
+            RUSTFS_META_TMP_BUCKET,
+            "cache-test-part/part.2",
+            bytes::Bytes::from_static(b"immutable shard"),
+        )
+        .await
+        .expect("stage tmp part 2");
+        disk.prepare_part_transaction(
+            RUSTFS_META_TMP_BUCKET,
+            "cache-test-part/part.2",
+            RUSTFS_META_MULTIPART_BUCKET,
+            &destination2,
+            part2_meta.clone(),
+        )
+        .await
+        .expect("prepare part 2 transaction");
+        assert!(!disk.has_clean_active_multipart_source(&upload, false));
+        disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &destination, PartTransactionAction::Commit)
+            .await
+            .expect("replayed part 1 commit settlement");
+        assert!(
+            !disk.has_clean_active_multipart_source(&upload, false),
+            "replayed settlement for part 1 must not clear part 2's pending transaction"
+        );
+        disk.rename_part(
+            RUSTFS_META_TMP_BUCKET,
+            "cache-test-part/part.2",
+            RUSTFS_META_MULTIPART_BUCKET,
+            &destination2,
+            part2_meta,
+        )
+        .await
+        .expect("rename part 2");
+        disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &destination2, PartTransactionAction::Commit)
+            .await
+            .expect("settle part 2 transaction");
+        assert!(disk.has_clean_active_multipart_source(&upload, false));
+
+        // Deleting {upload}/xl.meta through delete_paths must invalidate the canonical {upload} cache key.
+        disk.delete_paths(RUSTFS_META_MULTIPART_BUCKET, &[format!("{upload}/xl.meta")])
+            .await
+            .expect("delete xl.meta via delete_paths");
+        assert!(
+            !disk.has_clean_active_multipart_source(&upload, false),
+            "per-file head deletion must invalidate the canonical upload cache key"
+        );
+        assert!(matches!(
+            disk.read_version("", RUSTFS_META_MULTIPART_BUCKET, &upload, "", &ReadOptions::default())
+                .await,
+            Err(DiskError::FileNotFound)
+        ));
+    }
+
+    #[cfg(not(windows))]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn multipart_source_cache_publication_holds_source_lock_against_concurrent_retirement() {
+        use crate::disk::{DiskAPI as _, disk_store::LocalDiskWrapper, os::prepared_publication_test_hooks as hooks};
+
+        let (_root, disk, _, _) = fixture().await;
+        std::fs::create_dir_all(disk.io_get_bucket_path(RUSTFS_META_TMP_BUCKET).expect("TMP path")).expect("TMP bucket");
+        let disk = Arc::new(disk);
+        let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false);
+        let upload = format!("{}/{}", "d".repeat(64), Uuid::new_v4());
+        let data_dir = Uuid::new_v4();
+        let mut init_fi = FileInfo::new(&upload, 2, 2);
+        init_fi.erasure.index = 1;
+        init_fi.data_dir = Some(data_dir);
+        init_fi.fresh = true;
+        init_fi.mod_time = Some(time::OffsetDateTime::now_utc());
+
+        let source_dir = disk
+            .io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, &upload)
+            .expect("source dir");
+        let base_dir = disk
+            .io_get_bucket_path(RUSTFS_META_MULTIPART_BUCKET)
+            .expect("multipart bucket path");
+        let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
+        let (release_tx, release_rx) = std::sync::mpsc::channel();
+        let _hook = hooks::install_at(hooks::Stage::DirFsync, &source_dir, move || {
+            let _ = entered_tx.send(());
+            let _ = release_rx.recv();
+        });
+
+        let init_wrapper = wrapper.clone();
+        let init_upload = upload.clone();
+        let init_task = tokio::spawn(async move {
+            init_wrapper
+                .write_metadata("", RUSTFS_META_MULTIPART_BUCKET, &init_upload, init_fi)
+                .await
+        });
+
+        tokio::time::timeout(std::time::Duration::from_secs(5), entered_rx)
+            .await
+            .expect("initial write_metadata reaches parent dir fsync")
+            .expect("entered channel");
+
+        let retire_wrapper = wrapper.clone();
+        let retire_path = format!("{upload}/xl.meta");
+        let mut retire_fut = Box::pin(async move {
+            retire_wrapper
+                .delete_paths(RUSTFS_META_MULTIPART_BUCKET, &[retire_path])
+                .await
+        });
+        assert!(
+            futures::poll!(tokio::task::unconstrained(retire_fut.as_mut())).is_pending(),
+            "concurrent retirement must block on SourceRW while initial head publication is in flight"
+        );
+
+        let _ = release_tx.send(());
+        init_task.await.expect("init join").expect("initial write_metadata succeeds");
+        retire_fut
+            .await
+            .expect("concurrent retirement completes after SourceRW release");
+
+        assert!(
+            !super::super::multipart_source_state::has_active_multipart_source_head(&source_dir),
+            "retired source must not remain active in cache after interleaved initial publication"
+        );
+        assert!(
+            matches!(
+                super::super::multipart_source_state::validate_source_mutation(
+                    &source_dir,
+                    &base_dir,
+                    &disk.publication_root,
+                    false
+                ),
+                Err(DiskError::FileNotFound)
+            ),
+            "validate_source_mutation must reject non-initial mutation on retired source"
+        );
+    }
 }

crates/ecstore/src/disk/mod.rs

@@ -401,6 +401,13 @@ impl Disk {
         }
     }
 
+    pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
+        match self {
+            Self::Local(disk) => disk.has_clean_active_multipart_source(upload_or_part_path, part_path),
+            Self::Remote(_) => false,
+        }
+    }
+
     pub(crate) async fn try_publish_prepared_delete_local(
         &self,
         volume: &str,

crates/ecstore/src/io_support/bitrot.rs

@@ -828,13 +828,27 @@ pub async fn create_bitrot_writer(
     length: i64,
     shard_size: usize,
     checksum_algo: HashAlgorithm,
+) -> disk::error::Result<BitrotWriterWrapper> {
+    create_bitrot_writer_with_orig_volume(is_inline_buffer, disk, "", volume, path, length, shard_size, checksum_algo).await
+}
+
+#[allow(clippy::too_many_arguments)]
+pub async fn create_bitrot_writer_with_orig_volume(
+    is_inline_buffer: bool,
+    disk: Option<&DiskStore>,
+    origvolume: &str,
+    volume: &str,
+    path: &str,
+    length: i64,
+    shard_size: usize,
+    checksum_algo: HashAlgorithm,
 ) -> disk::error::Result<BitrotWriterWrapper> {
     let writer = if is_inline_buffer {
         CustomWriter::new_inline_buffer()
     } else if let Some(disk) = disk {
         let length = bitrot_create_file_size(length, shard_size, &checksum_algo);
 
-        let file = disk.create_file("", volume, path, length).await?;
+        let file = disk.create_file(origvolume, volume, path, length).await?;
         #[cfg(feature = "hotpath")]
         let file = instrument_raw_shard_writer(file, disk.is_local());
         CustomWriter::new_tokio_writer(file)

crates/ecstore/src/io_support/rio.rs

@@ -429,6 +429,7 @@ impl WritePlan {
         let checksum = reader.content_hash().clone();
         let trailer = reader.get_trailer().cloned();
 
+        let delegate_inner_etag = self.compression.is_none() && reader.is_etag_reader();
         let encrypted = self.encryption.is_some();
         if let Some(algorithm) = self.compression {
             reader = HashReader::from_reader(
@@ -448,7 +449,8 @@ impl WritePlan {
                 }
                 WriteEncryptionMode::SinglepartObjectKey | WriteEncryptionMode::MultipartObjectKey { .. } => [0u8; 12],
             };
-            let protected_reader = rustfs_rio::EtagReader::new_encrypted(reader, encryption.key_bytes, etag_nonce)?;
+            let protected_reader = rustfs_rio::EtagReader::new_encrypted(reader, encryption.key_bytes, etag_nonce)?
+                .with_inner_etag_delegation(delegate_inner_etag);
             reader = match encryption.mode {
                 WriteEncryptionMode::SinglepartObjectKey => HashReader::from_reader(
                     #[cfg(feature = "rio-v2")]

crates/ecstore/src/io_support/shard_integrity.rs

@@ -333,6 +333,34 @@ struct ProofCache {
     preferred: usize,
 }
 
+type VerifiedStripeKey = (uuid::Uuid, [u8; 32], u32);
+
+static VERIFIED_STRIPE_RECORDS: std::sync::LazyLock<parking_lot::Mutex<std::collections::HashMap<VerifiedStripeKey, Bytes>>> =
+    std::sync::LazyLock::new(|| parking_lot::Mutex::new(std::collections::HashMap::new()));
+
+fn get_verified_stripe_record(part: &PartIntegrity, stripe: u32) -> Option<Bytes> {
+    VERIFIED_STRIPE_RECORDS
+        .lock()
+        .get(&(part.generation, part.root, stripe))
+        .cloned()
+}
+
+fn put_verified_stripe_record(part: &PartIntegrity, stripe: u32, record: Bytes) {
+    let mut map = VERIFIED_STRIPE_RECORDS.lock();
+    if map.len() >= 4096
+        && let Some(evict_key) = map.keys().next().copied()
+    {
+        map.remove(&evict_key);
+    }
+    map.insert((part.generation, part.root, stripe), record);
+}
+
+pub(crate) fn invalidate_verified_stripe_records(generation: uuid::Uuid) {
+    VERIFIED_STRIPE_RECORDS
+        .lock()
+        .retain(|(stored_gen, _, _), _| *stored_gen != generation);
+}
+
 impl PartProofReader {
     pub(crate) async fn reconstruction_proof(
         &self,
@@ -410,6 +438,9 @@ impl PartProofReader {
 
     pub(crate) async fn record(&self, stripe: u32) -> io::Result<Bytes> {
         let offset = self.part.record_offset(stripe).map_err(|_| corrupt())?;
+        if let Some(bytes) = get_verified_stripe_record(&self.part, stripe) {
+            return Ok(bytes);
+        }
         let mut cached = self.cached.lock().await;
         if let Some((_, bytes)) = cached.records.iter().find(|(number, _)| *number == stripe) {
             return Ok(bytes.clone());
@@ -465,6 +496,9 @@ impl PartProofReader {
             bytes
         };
         self.part.verify_record(stripe, &record).map_err(|_| corrupt())?;
+        if self.part.stripes <= 4 {
+            put_verified_stripe_record(&self.part, stripe, record.clone());
+        }
         if cached.records.len() == 2 {
             cached.records.pop_front();
         }
@@ -863,4 +897,41 @@ mod tests {
             prepared.part.verify_shard(stripe, 15, b"x", &record).expect("spilled proof");
         }
     }
+
+    #[tokio::test]
+    async fn verified_stripe_record_cache_serves_verified_record_and_still_verifies_payload() {
+        let layout = IntegrityLayout::new(2, 2, 8, false).expect("layout");
+        let mut builder = IntegrityBuilder::new(layout, 1).expect("builder");
+        let shards = [b"ABCD".as_slice(), b"EFGH", b"IJKL", b"MNOP"];
+        builder.push(shards.into_iter()).await.expect("stripe");
+        let prepared = builder.finish(8).await.expect("index");
+        let inline_bytes = prepared.inline_bytes().expect("inline bytes");
+        let part = prepared.part;
+
+        let warm_reader = PartProofReader {
+            part: part.clone(),
+            sources: Vec::new(),
+            volume: "bucket".to_owned(),
+            inline: Some(inline_bytes),
+            cached: Mutex::new(ProofCache::default()),
+        };
+        warm_reader.verify(0, 0, shards[0]).await.expect("warm verify");
+        assert!(
+            get_verified_stripe_record(&part, 0).is_some(),
+            "small verified stripe record should populate the verified stripe cache"
+        );
+
+        let cached_reader = Arc::new(PartProofReader {
+            part,
+            sources: Vec::new(),
+            volume: "bucket".to_owned(),
+            inline: None,
+            cached: Mutex::new(ProofCache::default()),
+        });
+        cached_reader.verify(0, 0, shards[0]).await.expect("cached verify");
+        assert!(
+            cached_reader.verify(0, 0, shards[1]).await.is_err(),
+            "cached record must still reject corrupt or mismatched shard payloads"
+        );
+    }
 }

crates/ecstore/src/object_api/types.rs

@@ -1543,6 +1543,8 @@ pub struct ObjectInfo {
     /// published by the same upload. Callers must not repeat quota, replication,
     /// lifecycle, or object-created side effects for that response.
     pub multipart_completion_replayed: bool,
+    /// Previous current version's size observed during `rename_data` commit.
+    pub backfilled_old_current_size: Option<crate::disk::OldCurrentSize>,
 }
 
 impl Default for ObjectInfo {
@@ -1585,6 +1587,7 @@ impl Default for ObjectInfo {
             replication_decision: String::new(),
             checksum: None,
             multipart_completion_replayed: false,
+            backfilled_old_current_size: None,
         }
     }
 }
@@ -1628,6 +1631,7 @@ impl Clone for ObjectInfo {
             replication_decision: self.replication_decision.clone(),
             checksum: self.checksum.clone(),
             multipart_completion_replayed: self.multipart_completion_replayed,
+            backfilled_old_current_size: self.backfilled_old_current_size,
             expires: self.expires,
         }
     }

crates/ecstore/src/runtime/instance.rs

@@ -941,6 +941,32 @@ impl InstanceContext {
             .clone()
     }
 
+    /// Remove the `BucketMetadataSys` when its backing `ECStore` is gone.
+    ///
+    /// Called from `ECStore::drop`. When test fixtures share the bootstrap
+    /// context, the `BucketMetadataSys` stores a `Weak<ECStore>` to prevent a
+    /// reference cycle; once the store drops the weak is dangling and any
+    /// `object_store()` call panics. Clearing the slot here lets the next
+    /// fixture install a fresh `BucketMetadataSys` without hitting the stale one.
+    pub fn clear_bucket_metadata_sys_if_stale(&self) {
+        let mut guard = self.bucket_metadata_sys.lock().expect("bucket_metadata_sys lock poisoned");
+        if let Some(sys) = guard.as_ref() {
+            // `read()` may block briefly; the async RwLock is acceptable here
+            // because Drop is sync and the lock is held only long enough to
+            // check the weak reference.
+            let is_stale = {
+                let s = sys.try_read();
+                match s {
+                    Ok(s) => s.is_store_stale(),
+                    Err(_) => true, // contended: treat as stale-safe to clear
+                }
+            };
+            if is_stale {
+                *guard = None;
+            }
+        }
+    }
+
     /// Set this instance's background-services cancellation token (once).
     pub fn init_background_cancel_token(&self, token: CancellationToken) -> Result<(), CancellationToken> {
         self.background_cancel_token.set(token)

crates/ecstore/src/services/rebalance/entry.rs

@@ -283,6 +283,17 @@ impl ECStore {
             arrived.notify_one();
             release.notified().await;
         }
+        #[cfg(feature = "e2e-test-hooks")]
+        tokio::select! {
+            // Stop must drain this pause before taking the activation writer.
+            _ = cancel.cancelled() => return Err(Error::OperationCanceled),
+            result = data_movement::e2e_barrier::wait(
+                "rebalance",
+                bucket.as_str(),
+                entry.name.as_str(),
+                data_movement::e2e_barrier::STAGE_AFTER_ENTRY_ADMISSION,
+            ) => result.map_err(Error::from)?,
+        }
         let lock_lost_signal = run_guard.lock_lost_signal();
         #[cfg(test)]
         let _run_signal_test_fence = lock_lost_signal

crates/ecstore/src/services/rebalance/worker.rs

@@ -318,6 +318,21 @@ fn is_rebalance_transient_lock_error(err: &rustfs_lock::LockError) -> bool {
     }
 }
 
+fn is_rebalance_metadata_admission_retryable(err: &Error) -> bool {
+    match rebalance_error_source(err) {
+        Error::Lock(err) => is_rebalance_transient_lock_error(err),
+        // No metadata was written when lock admission failed. Each attempt
+        // must acquire its quorum again before loading or saving this run.
+        Error::NamespaceLockQuorumUnavailable {
+            mode: "write",
+            bucket,
+            object,
+            ..
+        } => bucket == crate::disk::RUSTFS_META_BUCKET && object == REBAL_META_NAME,
+        _ => false,
+    }
+}
+
 fn is_rebalance_transient_io_error(err: &std::io::Error) -> bool {
     if err.kind() == std::io::ErrorKind::TimedOut {
         return true;
@@ -385,9 +400,7 @@ where
         match result {
             Ok(value) => return Ok(value),
             Err(err) => {
-                if attempt.saturating_add(1) >= max_attempts.max(1)
-                    || !matches!(rebalance_error_source(&err), Error::Lock(lock_err) if is_rebalance_transient_lock_error(lock_err))
-                {
+                if attempt.saturating_add(1) >= max_attempts.max(1) || !is_rebalance_metadata_admission_retryable(&err) {
                     return Err(err);
                 }
                 #[cfg(test)]
@@ -719,6 +732,82 @@ mod error_source_tests {
         }
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn rebalance_metadata_retry_recovers_after_write_quorum_failure() {
+        let mut attempts = 0;
+        retry_rebalance_metadata_access(None, 3, || {
+            attempts += 1;
+            std::future::ready(if attempts == 1 {
+                resolve_rebalance_meta_save_result(
+                    Err(rebalance_meta_lock_error(
+                        rustfs_lock::LockError::QuorumNotReached {
+                            required: 1,
+                            achieved: 0,
+                        },
+                        "write",
+                    )),
+                    "stop_rebalance",
+                )
+            } else {
+                Ok(())
+            })
+        })
+        .await
+        .expect("a later quorum-protected admission must be allowed to succeed");
+        assert_eq!(attempts, 2);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn rebalance_metadata_retry_retains_persistent_write_quorum_failure() {
+        for max_attempts in [0, 1, 3] {
+            let mut attempts = 0;
+            let result = retry_rebalance_metadata_access(None, max_attempts, || {
+                attempts += 1;
+                std::future::ready(Err::<(), _>(rebalance_meta_lock_error(
+                    rustfs_lock::LockError::QuorumNotReached {
+                        required: 1,
+                        achieved: 0,
+                    },
+                    "write",
+                )))
+            })
+            .await;
+            let error = result.expect_err("exhausted admission attempts must never acknowledge a write");
+            assert_eq!(attempts, max_attempts.max(1));
+            assert!(matches!(
+                rebalance_error_source(&error),
+                Error::NamespaceLockQuorumUnavailable {
+                    mode: "write",
+                    required: 1,
+                    achieved: 0,
+                    ..
+                }
+            ));
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn rebalance_metadata_retry_stops_after_admission_is_cancelled() {
+        let mut attempts = 0;
+        let result = retry_rebalance_metadata_access(None, 3, || {
+            attempts += 1;
+            std::future::ready(Err::<(), _>(if attempts == 1 {
+                rebalance_meta_lock_error(
+                    rustfs_lock::LockError::QuorumNotReached {
+                        required: 1,
+                        achieved: 0,
+                    },
+                    "write",
+                )
+            } else {
+                Error::OperationCanceled
+            }))
+        })
+        .await;
+        assert_eq!(attempts, 2, "a stale or cancelled run cannot enter another admission attempt");
+        assert!(matches!(result, Err(Error::OperationCanceled)));
+    }
+
     #[tokio::test]
     async fn rebalance_metadata_retry_does_not_retry_permanent_or_untyped_errors() {
         for err in [
@@ -731,6 +820,20 @@ mod error_source_tests {
                 required: 3,
                 achieved: 2,
             },
+            Error::NamespaceLockQuorumUnavailable {
+                mode: "write",
+                bucket: "data-bucket".to_string(),
+                object: REBAL_META_NAME.to_string(),
+                required: 1,
+                achieved: 0,
+            },
+            Error::NamespaceLockQuorumUnavailable {
+                mode: "write",
+                bucket: crate::disk::RUSTFS_META_BUCKET.to_string(),
+                object: "other-metadata.bin".to_string(),
+                required: 1,
+                achieved: 0,
+            },
             Error::other("stale rebalance run rejected: lock acquisition timed out"),
             Error::other("rebalance distributed run fence lost"),
             Error::Io(std::io::Error::from(std::io::ErrorKind::TimedOut)),

crates/ecstore/src/set_disk/core/io_primitives.rs

@@ -5340,6 +5340,14 @@ impl SetDisks {
         }
 
         let disks = self.get_disks_internal().await;
+        if disks.len() == self.set_drive_count
+            && disks.iter().all(|disk| {
+                disk.as_ref()
+                    .is_some_and(|disk| disk.has_clean_active_multipart_source(dst_object, true))
+            })
+        {
+            return Ok(false);
+        }
         let transaction_path = part_transaction_path(dst_object);
         let transaction_meta_path = format!("{transaction_path}/{PART_TRANSACTION_NEW_META}");
 
@@ -5539,6 +5547,14 @@ impl SetDisks {
         write_quorum: usize,
     ) -> disk::error::Result<()> {
         let disks = self.get_disks_internal().await;
+        if disks.len() == self.set_drive_count
+            && disks.iter().all(|disk| {
+                disk.as_ref()
+                    .is_some_and(|disk| disk.has_clean_active_multipart_source(part_path, false))
+            })
+        {
+            return Ok(());
+        }
         let listings = join_all(disks.iter().map(|disk| {
             let disk = disk.clone();
             async move {
@@ -5598,10 +5614,11 @@ impl SetDisks {
         write_quorum: usize,
         quorum_context: Option<MultipartWriteQuorumContext<'_>>,
     ) -> disk::error::Result<Vec<Option<DiskStore>>> {
-        self.recover_part_transaction(dst_object, write_quorum).await?;
         let part = ObjectPartInfo::unmarshal(&meta)?;
         let integrity = part.integrity.map(Arc::new);
 
+        self.recover_part_transaction(dst_object, write_quorum).await?;
+
         let src_bucket = Arc::new(src_bucket.to_string());
         let src_object = Arc::new(src_object.to_string());
         let dst_bucket = Arc::new(dst_bucket.to_string());
@@ -5623,7 +5640,9 @@ impl SetDisks {
                 if let Err(error) = prepared {
                     return Some(Err(error));
                 }
-                if let Some(integrity) = integrity {
+                if let Some(integrity) = integrity
+                    && !disk.is_local()
+                {
                     let Some((directory, _)) = dst_object.rsplit_once('/') else {
                         return Some(Err(DiskError::FileCorrupt));
                     };
@@ -5740,7 +5759,36 @@ impl SetDisks {
             return Err(err);
         }
 
-        let committed = self.recover_part_transaction(&dst_object, write_quorum).await?;
+        let all_prepared_succeeded = errs
+            .iter()
+            .zip(prepared_disks.iter())
+            .all(|(err, disk)| err.is_none() && disk.as_ref().is_some_and(|d| d.is_local()));
+        let committed = if all_prepared_succeeded {
+            let settles = prepared_disks.iter().map(|disk| {
+                let disk = disk.clone();
+                let dst_object = dst_object.clone();
+                async move {
+                    let Some(disk) = disk else {
+                        return Err(DiskError::DiskNotFound);
+                    };
+                    disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &dst_object, PartTransactionAction::Commit)
+                        .await
+                }
+            });
+            let settle_results = join_all(settles).await;
+            if settle_results.iter().filter(|res| res.is_ok()).count() >= write_quorum
+                && settle_results
+                    .iter()
+                    .zip(prepared_disks.iter())
+                    .all(|(res, disk)| disk.is_none() || res.is_ok())
+            {
+                true
+            } else {
+                self.recover_part_transaction(&dst_object, write_quorum).await?
+            }
+        } else {
+            self.recover_part_transaction(&dst_object, write_quorum).await?
+        };
         if !committed {
             let err = DiskError::ErasureWriteQuorum;
             if let Some(context) = quorum_context {
@@ -6737,9 +6785,7 @@ impl SetDisks {
                         object,
                     );
                 }
-                let if_none_match = http_preconditions.if_none_match_value().map(str::to_owned);
-                let if_match = http_preconditions.if_match_value().map(str::to_owned);
-                if should_prevent_write(&oi, if_none_match, if_match)
+                if should_prevent_write(&oi, http_preconditions.if_none_match_value(), http_preconditions.if_match_value())
                     && !crate::data_movement::can_replace_stale_data_movement_target(&oi, opts)
                 {
                     return Some(StorageError::PreconditionFailed);

crates/ecstore/src/set_disk/metadata.rs

@@ -161,7 +161,7 @@ impl SetDisks {
             )
         });
         let mut metadata = FileMeta::new();
-        for parts in versions.into_values() {
+        for (version_id, parts) in versions {
             let errors = parts
                 .iter()
                 .zip(&missing)
@@ -173,15 +173,25 @@ impl SetDisks {
                     }
                 })
                 .collect::<Vec<_>>();
-            let read_quorum = match Self::object_quorum_from_meta(&parts, &errors, default_parity) {
-                Ok((quorum, _)) => usize::try_from(quorum).map_err(|_| DiskError::FileCorrupt)?,
+            let (read_quorum, write_quorum) = match Self::object_quorum_from_meta(&parts, &errors, default_parity) {
+                Ok((read, write)) => (
+                    usize::try_from(read).map_err(|_| DiskError::FileCorrupt)?,
+                    usize::try_from(write).map_err(|_| DiskError::FileCorrupt)?,
+                ),
                 Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) if complete => continue,
                 Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) => return Err(DiskError::ErasureReadQuorum),
                 Err(DiskError::ErasureReadQuorum) if complete => continue,
                 Err(error) => return Err(error),
             };
-            let mod_time = Self::common_time(&Self::list_object_modtimes(&parts, &errors), read_quorum);
-            let selected = Self::pick_valid_fileinfo(&parts, mod_time, None, read_quorum)?;
+            let selected = if crate::bucket::utils::is_meta_bucketname(bucket) && version_id.is_none() {
+                // A null-version system record must carry the same commit proof
+                // as GET. Listing only needs the selected metadata, so no disk
+                // handles are retained from the shared read selector.
+                Self::select_committed_fileinfo(&vec![None; disk_count], &parts, &errors, "", read_quorum, write_quorum)?.1
+            } else {
+                let mod_time = Self::common_time(&Self::list_object_modtimes(&parts, &errors), read_quorum);
+                Self::pick_valid_fileinfo(&parts, mod_time, None, read_quorum)?
+            };
             metadata.add_version(selected)?;
         }
         if metadata.versions.is_empty() {

crates/ecstore/src/set_disk/mod.rs

@@ -80,7 +80,7 @@ use crate::erasure::codec::bridge::{
 use crate::erasure::coding;
 use crate::error::{Error, Result, is_err_version_not_found};
 use crate::error::{GenericError, ObjectApiError, is_err_object_not_found};
-use crate::io_support::bitrot::{create_bitrot_reader_from_bytes, create_bitrot_writer};
+use crate::io_support::bitrot::{create_bitrot_reader_from_bytes, create_bitrot_writer, create_bitrot_writer_with_orig_volume};
 use crate::object_api::ObjectOptions;
 use crate::object_api::get_object_body_cache_hook;
 use crate::object_api::object_api_utils::get_raw_etag;
@@ -120,7 +120,6 @@ use bytesize::ByteSize;
 use futures::future::join_all;
 use http::HeaderMap;
 use md5::{Digest as Md5Digest, Md5};
-use regex::Regex;
 use rustfs_config::MI_B;
 use rustfs_filemeta::metadata_keys;
 use rustfs_filemeta::{
@@ -4697,6 +4696,17 @@ pub(crate) fn same_distributed_lock_domain(left: &[Arc<dyn LockClient>], right:
             .all(|right_client| left.iter().any(|left_client| Arc::ptr_eq(left_client, right_client)))
 }
 
+// Reusing a held writer additionally requires the same quorum, so repeated
+// clients must not give different weight to an otherwise identical domain.
+fn same_distributed_lock_quorum(left: &[Arc<dyn LockClient>], right: &[Arc<dyn LockClient>]) -> bool {
+    !left.is_empty()
+        && left.len() == right.len()
+        && left.iter().all(|client| {
+            left.iter().filter(|other| Arc::ptr_eq(client, other)).count() == 1
+                && right.iter().filter(|other| Arc::ptr_eq(client, other)).count() == 1
+        })
+}
+
 const ERASURE_CACHE_MAX_ENTRIES: usize = 32;
 
 #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
@@ -5009,6 +5019,11 @@ impl DiskHealthEntry {
 #[error("committed system metadata write requires repair by its owning writer")]
 pub(crate) struct SystemMetadataRepairRequired;
 
+/// An inline metadata write was rejected before staging or commit began.
+#[derive(Debug, thiserror::Error)]
+#[error("system metadata write rejected before staging: {0}")]
+pub(crate) struct SystemMetadataWriteNotStarted(#[source] pub(crate) Error);
+
 impl SetDisks {
     pub(in crate::set_disk) async fn persist_delete_marker_purge(
         &self,
@@ -5646,9 +5661,9 @@ impl SetDisks {
     pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool {
         match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) {
             (false, false) => Arc::ptr_eq(&self.local_lock_manager, &other.local_lock_manager),
-            (true, true) => {
-                self.set_lock_namespace == other.set_lock_namespace && same_distributed_lock_domain(&self.lockers, &other.lockers)
-            }
+            // Distributed lock clients receive the raw ObjectKey; the pool
+            // namespace is not part of the key stored by their backends.
+            (true, true) => same_distributed_lock_quorum(&self.lockers, &other.lockers),
             _ => false,
         }
     }
@@ -8036,16 +8051,29 @@ fn parts_after_marker(part_numbers: &[usize], part_number_marker: usize) -> Opti
     Some(&part_numbers[index..])
 }
 
-pub fn canonicalize_etag(etag: &str) -> String {
-    let re = Regex::new("\"*?([^\"]*?)\"*?$").unwrap();
-    re.replace_all(etag, "$1").to_string()
+fn canonicalized_etag(etag: &str) -> Cow<'_, str> {
+    // Preserve the legacy suffix-matching behavior for embedded quotes.
+    let trimmed = etag.trim_end_matches('"');
+    let Some(last_quote) = trimmed.rfind('"') else {
+        return Cow::Borrowed(trimmed);
+    };
+    let prefix = trimmed[..last_quote].trim_end_matches('"');
+    let suffix = &trimmed[last_quote + 1..];
+    if prefix.is_empty() {
+        Cow::Borrowed(suffix)
+    } else {
+        let mut result = String::with_capacity(prefix.len() + suffix.len());
+        result.push_str(prefix);
+        result.push_str(suffix);
+        Cow::Owned(result)
+    }
 }
 
 pub fn e_tag_matches(etag: &str, condition: &str) -> bool {
     if condition.trim() == "*" {
         return true;
     }
-    canonicalize_etag(etag) == canonicalize_etag(condition)
+    canonicalized_etag(etag) == canonicalized_etag(condition)
 }
 
 fn weak_etag_matches(etag: &str, condition: &str) -> bool {
@@ -8060,15 +8088,12 @@ fn weak_etag_matches(etag: &str, condition: &str) -> bool {
             .or_else(|| trimmed.strip_prefix("w/"))
             .unwrap_or(trimmed)
     }
-    canonicalize_etag(strip_weak(etag)) == canonicalize_etag(strip_weak(condition))
+    canonicalized_etag(strip_weak(etag)) == canonicalized_etag(strip_weak(condition))
 }
 
-pub fn should_prevent_write(oi: &ObjectInfo, if_none_match: Option<String>, if_match: Option<String>) -> bool {
-    let if_none_match = if_none_match
-        .as_deref()
-        .map(str::trim)
-        .filter(|condition| !condition.is_empty());
-    let if_match = if_match.as_deref().map(str::trim).filter(|condition| !condition.is_empty());
+pub fn should_prevent_write(oi: &ObjectInfo, if_none_match: Option<&str>, if_match: Option<&str>) -> bool {
+    let if_none_match = if_none_match.map(str::trim).filter(|condition| !condition.is_empty());
+    let if_match = if_match.map(str::trim).filter(|condition| !condition.is_empty());
 
     match &oi.etag {
         Some(etag) => {
@@ -9029,6 +9054,13 @@ mod tests {
             "constructing an object lock must not clone each client Arc"
         );
 
+        // Check shared ownership before acquisition can spawn pending cleanup.
+        let read_lock = set
+            .new_ns_lock("bucket", "read-object")
+            .await
+            .expect("second namespace lock should be created");
+        assert_eq!(Arc::strong_count(&set.shared_lockers), clients_before + 2);
+
         let write_error = write_lock
             .get_write_lock(Duration::from_millis(500))
             .await
@@ -9043,11 +9075,6 @@ mod tests {
             ),
             "the shared client representation must preserve the exact write quorum result: {write_error}"
         );
-        let read_lock = set
-            .new_ns_lock("bucket", "read-object")
-            .await
-            .expect("second namespace lock should be created");
-        assert_eq!(Arc::strong_count(&set.shared_lockers), clients_before + 2);
         let read_guard = read_lock
             .get_read_lock(Duration::from_millis(500))
             .await
@@ -9871,7 +9898,7 @@ mod tests {
 
     #[tokio::test(flavor = "multi_thread")]
     #[serial]
-    async fn namespace_lock_domain_includes_pool_namespace() {
+    async fn distributed_namespace_lock_domain_follows_backend_not_pool_name() {
         let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
 
         let first: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
@@ -9910,11 +9937,25 @@ mod tests {
             "sets in the same pool share the object namespace lock domain"
         );
         assert!(
-            !same_pool_first_set.shares_namespace_lock_domain(&other_pool_set).await,
-            "different pool namespaces must not be deduplicated solely by identical clients"
+            same_pool_first_set.shares_namespace_lock_domain(&other_pool_set).await,
+            "different pools using identical backends lock the same raw object key"
         );
     }
 
+    #[test]
+    fn distributed_lock_domain_rejects_partial_or_duplicate_backends() {
+        let a: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
+        let b: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
+        let c: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
+        assert!(same_distributed_lock_quorum(&[a.clone(), b.clone()], &[b.clone(), a.clone()]));
+        assert!(!same_distributed_lock_quorum(&[a.clone(), b.clone()], &[a.clone(), c]));
+        assert!(!same_distributed_lock_quorum(
+            &[a.clone(), a.clone(), b.clone()],
+            &[a.clone(), b.clone(), b]
+        ));
+        assert!(!same_distributed_lock_quorum(&[], &[]));
+    }
+
     #[tokio::test(flavor = "multi_thread")]
     #[serial]
     async fn streaming_reader_holds_read_lock_until_eof() {
@@ -13772,6 +13813,22 @@ mod tests {
         assert!(result2.iter().all(|d| d.is_none()));
     }
 
+    #[test]
+    fn canonicalized_etag_preserves_legacy_regex_semantics() {
+        let regex = regex::Regex::new("\"*?([^\"]*?)\"*?$").expect("valid legacy ETag pattern");
+        let alphabet = ['a', '"', '\n', 'é', '\0'];
+        for length in 0..=6_u32 {
+            for mut number in 0..alphabet.len().pow(length) {
+                let mut input = String::new();
+                for _ in 0..length {
+                    input.push(alphabet[number % alphabet.len()]);
+                    number /= alphabet.len();
+                }
+                assert_eq!(canonicalized_etag(&input), regex.replace_all(&input, "$1"), "input: {input:?}");
+            }
+        }
+    }
+
     #[test]
     fn test_etag_matches() {
         assert!(e_tag_matches("abc", "abc"));
@@ -14347,61 +14404,61 @@ mod tests {
             etag: Some("abc".to_string()),
             ..Default::default()
         };
-        let if_none_match = Some("abc".to_string());
+        let if_none_match = Some("abc");
         let if_match = None;
         assert!(should_prevent_write(&oi, if_none_match, if_match));
 
-        let if_none_match = Some("*".to_string());
+        let if_none_match = Some("*");
         let if_match = None;
         assert!(should_prevent_write(&oi, if_none_match, if_match));
 
         let if_none_match = None;
-        let if_match = Some("def".to_string());
+        let if_match = Some("def");
         assert!(should_prevent_write(&oi, if_none_match, if_match));
 
         let if_none_match = None;
-        let if_match = Some("*".to_string());
+        let if_match = Some("*");
         assert!(!should_prevent_write(&oi, if_none_match, if_match));
 
-        let if_none_match = Some("def".to_string());
+        let if_none_match = Some("def");
         let if_match = None;
         assert!(!should_prevent_write(&oi, if_none_match, if_match));
 
-        let if_none_match = Some("def".to_string());
-        let if_match = Some("*".to_string());
+        let if_none_match = Some("def");
+        let if_match = Some("*");
         assert!(!should_prevent_write(&oi, if_none_match, if_match));
 
-        let if_none_match = Some("def".to_string());
-        let if_match = Some("\"abc\"".to_string());
+        let if_none_match = Some("def");
+        let if_match = Some("\"abc\"");
         assert!(!should_prevent_write(&oi, if_none_match, if_match));
 
-        let if_none_match = Some("*".to_string());
-        let if_match = Some("\"abc\"".to_string());
+        let if_none_match = Some("*");
+        let if_match = Some("\"abc\"");
         assert!(should_prevent_write(&oi, if_none_match, if_match));
 
         let oi = ObjectInfo {
             etag: None,
             ..Default::default()
         };
-        let if_none_match = Some("*".to_string());
-        let if_match = Some("\"abc\"".to_string());
+        let if_none_match = Some("*");
+        let if_match = Some("\"abc\"");
         assert!(should_prevent_write(&oi, if_none_match, if_match));
 
         let if_none_match = None;
         let if_match = None;
         assert!(!should_prevent_write(&oi, if_none_match, if_match));
 
-        let if_none_match = Some(String::new());
-        let if_match = Some(" ".to_string());
+        let if_none_match = Some("");
+        let if_match = Some(" ");
         assert!(!should_prevent_write(&oi, if_none_match, if_match));
 
         let oi = ObjectInfo {
             etag: Some("abc".to_string()),
             ..Default::default()
         };
-        assert!(should_prevent_write(&oi, Some("W/\"abc\"".to_string()), None));
-        assert!(!should_prevent_write(&oi, Some("W/\"def\"".to_string()), None));
-        assert!(should_prevent_write(&oi, None, Some("W/\"abc\"".to_string())));
+        assert!(should_prevent_write(&oi, Some("W/\"abc\""), None));
+        assert!(!should_prevent_write(&oi, Some("W/\"def\""), None));
+        assert!(should_prevent_write(&oi, None, Some("W/\"abc\"")));
     }
 
     #[test]

crates/ecstore/src/set_disk/ops/locking.rs

@@ -95,13 +95,8 @@ impl SetDisks {
         bucket: &str,
         object: &str,
     ) -> Result<Arc<rustfs_lock::NamespaceLockGuard>> {
-        let resource = ObjectKey::with_version(
-            bucket,
-            object_name_disk_identity(object).as_ref(),
-            format!("rustfs:delete-transaction:{}:{}", self.pool_index, self.set_index),
-        );
         let guard = self
-            .namespace_lock_for_key(resource)
+            .namespace_lock_for_key(self.delete_transaction_resource(bucket, object))
             .await
             .get_write_lock(super::super::get_lock_acquire_timeout())
             .await
@@ -111,6 +106,94 @@ impl SetDisks {
         Ok(Arc::new(guard))
     }
 
+    fn delete_transaction_resource(&self, bucket: &str, object: &str) -> ObjectKey {
+        ObjectKey::with_version(
+            bucket,
+            object_name_disk_identity(object).as_ref(),
+            format!("rustfs:delete-transaction:{}:{}", self.pool_index, self.set_index),
+        )
+    }
+
+    /// Batched write-lock acquisition for the whole key set of one bulk
+    /// mutation: one lock round instead of one quorum round-trip per key.
+    ///
+    /// Bulk delete is the only multi-key write-lock holder. The batched round
+    /// is deadlock-free by construction (a single `lock_batch` RPC per lock
+    /// client, and every lock server orders concurrent batches by the same
+    /// global (shard, key) order), so unlike an optimistic probe wave it
+    /// needs no canonical serial fallback. Outcomes are per key and use the
+    /// same error shape as the single-key paths.
+    async fn namespace_write_locks_batch(
+        &self,
+        bucket: &str,
+        resources: &[ObjectKey],
+        timeout: std::time::Duration,
+        mode: &'static str,
+    ) -> Vec<Result<rustfs_lock::NamespaceLockGuard>> {
+        if resources.is_empty() {
+            return Vec::new();
+        }
+        let lock = self.namespace_lock_for_key(resources[0].clone()).await;
+        lock.get_write_locks_batch(resources, timeout)
+            .await
+            .into_iter()
+            .zip(resources)
+            .map(|(result, resource)| {
+                result.map_err(|error| self.map_namespace_lock_error(bucket, resource.object.as_ref(), mode, error))
+            })
+            .collect()
+    }
+
+    /// Batched raw namespace write locks for a bulk delete's key set. The
+    /// resource keys use the same disk-identity folding as `new_ns_lock`, so
+    /// aliases (`a\b` vs `a/b`) still share one lock.
+    pub(crate) async fn acquire_object_write_locks_batch(
+        &self,
+        bucket: &str,
+        objects: &[&str],
+        timeout: std::time::Duration,
+    ) -> Vec<Result<rustfs_lock::NamespaceLockGuard>> {
+        let resources: Vec<ObjectKey> = objects
+            .iter()
+            .map(|object| ObjectKey {
+                bucket: Arc::from(bucket),
+                object: Arc::from(object_name_disk_identity(object).as_ref()),
+                version: None,
+            })
+            .collect();
+        self.namespace_write_locks_batch(bucket, &resources, timeout, "write").await
+    }
+
+    /// Batched delete-transaction locks for a bulk delete's key set; see
+    /// [`Self::acquire_delete_transaction_lock`] for the resource shape.
+    pub(in crate::set_disk) async fn acquire_delete_transaction_locks_batch(
+        &self,
+        bucket: &str,
+        objects: &[&str],
+        timeout: std::time::Duration,
+    ) -> Vec<Result<Arc<rustfs_lock::NamespaceLockGuard>>> {
+        let resources: Vec<ObjectKey> = objects
+            .iter()
+            .map(|object| self.delete_transaction_resource(bucket, object))
+            .collect();
+        let mut guards = Vec::with_capacity(resources.len());
+        for (object, result) in objects.iter().zip(
+            self.namespace_write_locks_batch(bucket, &resources, timeout, "delete_transaction")
+                .await,
+        ) {
+            match result {
+                Ok(guard) => {
+                    #[cfg(test)]
+                    super::delete_recovery::test_hooks::run_after_transaction_lock_acquired(bucket, object, &guard);
+                    let _ = object;
+                    guards.push(Ok(Arc::new(guard)));
+                }
+                Err(error) => guards.push(Err(error)),
+            }
+        }
+        guards
+    }
+
     #[allow(dead_code, reason = "lock diagnostics formatter with no caller in this port (backlog#1823)")]
     pub(in crate::set_disk) fn format_lock_error(&self, bucket: &str, object: &str, mode: &str, err: &LockResult) -> String {
         match err {
@@ -375,7 +458,10 @@ impl SetDisks {
                         return None;
                     }
 
-                    let _ = disk.close().await;
+                    // A failed remote reconnect must retain its existing recovery monitor.
+                    if disk.is_local() {
+                        let _ = disk.close().await;
+                    }
                 }
 
                 let endpoint = self.set_endpoints.get(i)?;
@@ -497,19 +583,26 @@ impl SetDisks {
         let _ = new_disk.set_disk_id(Some(fm.erasure.this)).await;
 
         if !new_disk.is_local() {
-            let mut disks = self.disks.write().await;
-            let slot = &mut disks[disk_idx];
-            let unchanged = match (slot.as_ref(), previous_disk.as_ref()) {
-                (None, None) => true,
-                (Some(current), Some(previous)) => Arc::ptr_eq(current, previous),
-                _ => false,
+            let displaced = {
+                let mut disks = self.disks.write().await;
+                let slot = &mut disks[disk_idx];
+                let unchanged = match (slot.as_ref(), previous_disk.as_ref()) {
+                    (None, None) => true,
+                    (Some(current), Some(previous)) => Arc::ptr_eq(current, previous),
+                    _ => false,
+                };
+                // A GET probe or another reconnect may have filled this slot while
+                // format I/O was pending. Only the successful publisher owns the
+                // displaced monitor; a failed or superseded probe must retain it.
+                if unchanged {
+                    new_disk.enable_health_check();
+                    slot.replace(new_disk)
+                } else {
+                    None
+                }
             };
-            // A GET probe or another reconnect may have filled this slot while
-            // format I/O was pending. Do not replace its handle or start a
-            // second health monitor for an unpublished handle.
-            if unchanged {
-                new_disk.enable_health_check();
-                *slot = Some(new_disk);
+            if let Some(disk) = displaced {
+                let _ = disk.close().await;
             }
             return false;
         }

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -41,8 +41,8 @@ use super::super::{
     SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_RESTORE_OPERATION_ID, SUFFIX_RESTORE_WORKER_LOCK, SetDisks, SmallWritePath, StorageError,
     Uuid, WriteLayout, check_object_lock_for_deletion_with_state, classify_multipart_part_write_path, coding,
     complete_multipart_part_error, complete_multipart_part_error_result, complete_part_checksum, contains_key_str,
-    create_bitrot_writer, debug, disk, error, file_info_is_valid_for_metadata, get_complete_multipart_md5, get_header_map,
-    get_str, insert_str, is_err_object_not_found, is_err_version_not_found, is_min_allowed_part_size,
+    create_bitrot_writer_with_orig_volume, debug, disk, error, file_info_is_valid_for_metadata, get_complete_multipart_md5,
+    get_header_map, get_str, insert_str, is_err_object_not_found, is_err_version_not_found, is_min_allowed_part_size,
     log_multipart_write_quorum_failure, multipart_part_logical_size, parts_after_marker, path_join_buf,
     prepare_complete_multipart_parts, record_compression_total_memory, record_multipart_write_quorum_failure,
     reduce_read_quorum_errs, reduce_write_quorum_errs, remove_header_map, resolve_write_layout,
@@ -57,7 +57,6 @@ use super::object::{
     object_transaction_fencing_requested, read_object_transaction_epoch_fence, verify_object_transaction_epoch_fence,
 };
 use crate::api::config::storageclass;
-#[cfg(test)]
 use crate::bucket::metadata_sys::ObjectLockConfigState;
 use crate::bucket::quota::reservation;
 use crate::bucket::replication::ReplicationStatusType;
@@ -124,9 +123,10 @@ async fn create_part_writers(
         let Some(disk) = disk else {
             return (None, Some(DiskError::DiskNotFound));
         };
-        match create_bitrot_writer(
+        match create_bitrot_writer_with_orig_volume(
             false,
             Some(disk),
+            RUSTFS_META_MULTIPART_BUCKET,
             RUSTFS_META_TMP_BUCKET,
             path,
             length,
@@ -977,7 +977,7 @@ async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str, root_pref
         return Err(DiskError::DiskNotFound);
     }
 
-    if !root_prefix.is_empty() {
+    if !root_prefix.is_empty() && root_prefix != DATA_MOVEMENT_MULTIPART_PREFIX {
         let upload_dirs = match disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, root_prefix, -1).await {
             Ok(entries) => entries,
             Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => return Ok(Vec::new()),
@@ -1002,6 +1002,13 @@ async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str, root_pref
         .into_iter()
         .map(|sha_dir| sha_dir.trim_end_matches('/').to_owned())
         .filter(|sha_dir| sha_dir.len() == 64 && sha_dir.bytes().all(|byte| byte.is_ascii_hexdigit()))
+        .map(|sha_dir| {
+            if root_prefix.is_empty() {
+                sha_dir
+            } else {
+                format!("{root_prefix}/{sha_dir}")
+            }
+        })
         .collect::<Vec<_>>();
     let listed_upload_dirs = stream::iter(sha_dirs)
         .map(|sha_dir| {
@@ -1124,6 +1131,7 @@ impl SetDisks {
         orig_bucket: &str,
         error_path: &str,
         root_prefix: &str,
+        require_all_disks: bool,
     ) -> Result<(Vec<Option<DiskStore>>, Vec<String>, usize, bool)> {
         let disks = self.disks.read().await.clone();
         if disks.is_empty() {
@@ -1165,7 +1173,8 @@ impl SetDisks {
             }
         }
 
-        if let Some(err) = reduce_read_quorum_errs(&discovery_errors, OBJECT_OP_IGNORED_ERRS, discovery_quorum) {
+        let required_discoveries = if require_all_disks { disks.len() } else { discovery_quorum };
+        if let Some(err) = reduce_read_quorum_errs(&discovery_errors, OBJECT_OP_IGNORED_ERRS, required_discoveries) {
             return Err(to_object_err(err.into(), vec![orig_bucket, error_path]));
         }
 
@@ -1187,7 +1196,7 @@ impl SetDisks {
 
     pub(crate) async fn first_multipart_upload_path_for_decommission(&self, bucket: &str) -> Result<Option<String>> {
         let (_, paths, _, _) = self
-            .discover_multipart_upload_paths(bucket, RUSTFS_META_MULTIPART_BUCKET, "")
+            .discover_multipart_upload_paths(bucket, RUSTFS_META_MULTIPART_BUCKET, "", false)
             .await?;
         Ok(paths.into_iter().next())
     }
@@ -1200,8 +1209,9 @@ impl SetDisks {
         upload_identity: &str,
     ) -> Result<Vec<String>> {
         let expected_parent = format!("{DATA_MOVEMENT_MULTIPART_PREFIX}/{}", Self::get_multipart_sha_dir(bucket, object));
-        let (_, candidate_paths, _, has_minority_candidate) =
-            self.discover_multipart_upload_paths(bucket, object, &expected_parent).await?;
+        let (_, candidate_paths, _, has_minority_candidate) = self
+            .discover_multipart_upload_paths(bucket, object, &expected_parent, false)
+            .await?;
         if has_minority_candidate {
             return Err(Error::DecommissionCapacityBlocked {
                 message: "data movement multipart cleanup found an upload path on fewer than the discovery quorum of disks"
@@ -1750,6 +1760,54 @@ impl SetDisks {
         })
     }
 
+    pub(crate) async fn ensure_no_multipart_uploads(&self, bucket: &str) -> Result<()> {
+        // Every disk must confirm absence in both namespaces before cleanup can
+        // remove catalog protection from the bucket.
+        for root_prefix in ["", DATA_MOVEMENT_MULTIPART_PREFIX] {
+            let (disks, paths, _, has_minority_candidate) =
+                self.discover_multipart_upload_paths(bucket, "", root_prefix, true).await?;
+            if has_minority_candidate {
+                return Err(Error::ErasureReadQuorum);
+            }
+            for upload_path in paths {
+                let (sha_dir, _) = upload_path
+                    .rsplit_once('/')
+                    .filter(|(directory, upload_id)| !directory.is_empty() && !upload_id.is_empty())
+                    .ok_or(DiskError::CorruptedFormat)?;
+                let (metadata, errors) =
+                    Self::read_all_fileinfo(&disks, bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_path, "", false, false, false)
+                        .await?;
+                let (read_quorum, _) = Self::object_quorum_from_meta(&metadata, &errors, self.default_parity_count)?;
+                let read_quorum = usize::try_from(read_quorum).map_err(|_| DiskError::ErasureReadQuorum)?;
+                if let Some(error) = reduce_read_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, read_quorum) {
+                    return Err(to_object_err(error.into(), vec![bucket, &upload_path]));
+                }
+                let (_, modified, etag) = Self::list_online_disks(&disks, &metadata, &errors, read_quorum);
+                let file = Self::pick_valid_fileinfo(&metadata, modified, etag, read_quorum)?;
+                let (Some(owner_bucket), Some(owner_object)) = (
+                    file.metadata.get(RUSTFS_MULTIPART_BUCKET_KEY),
+                    file.metadata.get(RUSTFS_MULTIPART_OBJECT_KEY),
+                ) else {
+                    return Err(to_object_err(DiskError::CorruptedFormat.into(), vec![bucket, &upload_path]));
+                };
+                if owner_bucket.is_empty()
+                    || owner_object.is_empty()
+                    || Self::get_multipart_sha_dir(owner_bucket, owner_object)
+                        != sha_dir
+                            .strip_prefix(DATA_MOVEMENT_MULTIPART_PREFIX)
+                            .and_then(|suffix| suffix.strip_prefix('/'))
+                            .unwrap_or(sha_dir)
+                {
+                    return Err(to_object_err(DiskError::CorruptedFormat.into(), vec![bucket, &upload_path]));
+                }
+                if owner_bucket == bucket {
+                    return Err(Error::BucketNotEmpty(bucket.to_string()));
+                }
+            }
+        }
+        Ok(())
+    }
+
     #[allow(clippy::too_many_arguments)]
     pub(crate) async fn list_multipart_uploads_for_incarnation(
         &self,
@@ -1761,7 +1819,8 @@ impl SetDisks {
         max_uploads: usize,
         expected_incarnation_id: Option<Uuid>,
     ) -> Result<ListMultipartsInfo> {
-        let (disks, candidate_paths, discovery_quorum, _) = self.discover_multipart_upload_paths(bucket, prefix, "").await?;
+        let (disks, candidate_paths, discovery_quorum, _) =
+            self.discover_multipart_upload_paths(bucket, prefix, "", false).await?;
         let listed_uploads = stream::iter(candidate_paths)
             .map(|upload_path| {
                 let disks = &disks;
@@ -2372,7 +2431,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
             let multipart_part_size = data.size();
             let stream = mem::replace(
                 &mut data.stream,
-                HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, false)?,
+                HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, true)?,
             );
 
             let write_path = classify_multipart_part_write_path(multipart_part_size, fi.erasure.block_size);
@@ -2436,7 +2495,16 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
             }
 
             let part_integrity = if let Some(integrity) = integrity {
-                Some(integrity.write(&mut shuffle_disks, bucket, RUSTFS_META_TMP_BUCKET, &tmp_part).await?)
+                Some(
+                    integrity
+                        .write(
+                            &mut shuffle_disks,
+                            RUSTFS_META_MULTIPART_BUCKET,
+                            RUSTFS_META_TMP_BUCKET,
+                            &tmp_part,
+                        )
+                        .await?,
+                )
             } else { None };
             if shuffle_disks.iter().filter(|disk| disk.is_some()).count() < write_quorum {
                 return Err(Error::ErasureWriteQuorum);
@@ -3471,10 +3539,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
         bind_multipart_proof_snapshot(&disks, &self.multipart_proof_verification_context())?;
 
         let part_path = format!("{}/{}/", upload_id_path, fi.data_dir.unwrap_or(Uuid::nil()));
-        self.recover_part_transactions(&part_path, read_quorum, write_quorum)
-            .await
-            .map_err(|err| to_object_err(err.into(), vec![bucket, object]))?;
-
         let part_meta_paths = uploaded_parts
             .iter()
             .map(|v| format!("{part_path}part.{0}.meta", v.part_num))
@@ -3482,6 +3546,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
 
         let part_numbers = uploaded_parts.iter().map(|v| v.part_num).collect::<Vec<usize>>();
 
+        self.recover_part_transactions(&part_path, read_quorum, write_quorum)
+            .await
+            .map_err(|err| to_object_err(err.into(), vec![bucket, object]))?;
+
         let object_parts = Self::read_parts(&disks, RUSTFS_META_MULTIPART_BUCKET, &part_meta_paths, &part_numbers, read_quorum)
             .await
             .map_err(|err| to_object_err(err.into(), vec![bucket, object]))?;
@@ -4045,6 +4113,40 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
         )
         .await?;
 
+        if !opts.versioned
+            && !crate::bucket::utils::is_meta_bucketname(bucket)
+            && let Some(snapshot) = opts.object_lock_config_snapshot.as_deref()
+            && !matches!(snapshot.state(), ObjectLockConfigState::ConfirmedAbsent)
+        {
+            // Completion replaces the null version, not necessarily the latest
+            // version inspected by the request-level Object Lock check.
+            let current = self
+                .get_object_info(
+                    bucket,
+                    object,
+                    &ObjectOptions {
+                        version_id: Some(Uuid::nil().to_string()),
+                        no_lock: true,
+                        metadata_cache_safe: false,
+                        versioned: opts.versioned,
+                        version_suspended: opts.version_suspended,
+                        ..Default::default()
+                    },
+                )
+                .await;
+            match current {
+                Ok(existing) => {
+                    if check_object_lock_for_deletion_with_state(snapshot.state(), &existing, false)?.is_some() {
+                        return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object.to_string()));
+                    }
+                }
+                // A null delete marker has no protected payload to replace.
+                Err(StorageError::MethodNotAllowed) => {}
+                Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
+                Err(err) => return Err(err),
+            }
+        }
+
         if opts.data_movement
             && opts.http_preconditions.is_some()
             && let Some(err) = self.check_write_precondition(bucket, object, opts).await
@@ -4389,6 +4491,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
             let needs_immediate_heal = rename_commit.needs_immediate_heal();
             let op_old_dir = rename_commit.data_dir;
             let cleanup_disks = rename_commit.cleanup_disks;
+            let old_current_size = rename_commit.old_current_size;
             let committed_file_info = rename_commit.committed_file_info;
 
             if needs_immediate_heal {
@@ -4457,7 +4560,13 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
             // parts; deleting them before the commit would strand the upload
             // permanently. This mirrors the "clean up only after commit" pattern
             // already used for the old data-dir GC and the upload-dir delete_all below.
-            commit_set.cleanup_multipart_path(&parts).await;
+            let can_skip_part_cleanup = !needs_upgrade
+                && commit_disks
+                    .iter()
+                    .all(|disk| disk.as_ref().is_some_and(|disk| disk.is_local()));
+            if !can_skip_part_cleanup {
+                commit_set.cleanup_multipart_path(&parts).await;
+            }
 
             if let Some(old_dir) = op_old_dir {
                 // backlog#898: best-effort reclaim of the dereferenced old data dir.
@@ -4503,7 +4612,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
 
             drop(_upload_guard.take());
 
-            Ok(ObjectInfo::from_file_info(&fi, &commit_bucket, &commit_object, commit_is_versioned))
+            let mut obj_info = ObjectInfo::from_file_info(&fi, &commit_bucket, &commit_object, commit_is_versioned);
+            obj_info.backfilled_old_current_size = old_current_size;
+            Ok(obj_info)
         };
 
         let result = if detach_commit_owner {
@@ -9982,7 +10093,7 @@ mod tests {
 
         // Real disk callbacks and health probes must share wall-clock deadlines.
         // Advancing Tokio time can expire unrelated healthy-drive probes.
-        temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("2"))], async {
+        temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, None::<&str>)], async {
             let (dirs, store) = crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks().await;
             crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
             let bucket = "mpu-quorum-success-physical";
@@ -10009,6 +10120,8 @@ mod tests {
                 local.observe_multipart_cleanup_for_test(&upload_path)
             }).collect::<Vec<_>>();
             let disk::Disk::Local(paused_local) = disks[3].as_ref() else { panic!("actual local physical tail") };
+            // Fault only the held leg; healthy disks keep their normal I/O budgets.
+            let _paused_timeout = paused_local.override_operation_timeout_for_test(Duration::from_secs(2));
             let io_target = paused_local.get_disk().get_object_path_for_io(bucket, object).expect("real IO target");
             let mutation_key = paused_local.get_disk().get_object_path(bucket, object).expect("logical native mutation key");
             let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
@@ -10165,7 +10278,15 @@ mod tests {
                 let size = copy_info.get_actual_size().expect("actual source size");
                 copy_info.put_object_reader = Some(PutObjReader::new(HashReader::from_stream(reader.stream, size, size, None, None, false).expect("actual Copy body stream")));
                 tokio::time::timeout(Duration::from_secs(5), store.copy_object(bucket, object, bucket, copy_object, &mut copy_info, &ObjectOptions::default(), &ObjectOptions::default()))
-                    .await.expect("cross-key Copy cannot wait on the old successful logical owner").expect("actual CopyObject");
+                    .await.expect("cross-key Copy cannot wait on the old successful logical owner")
+                    .unwrap_or_else(|error| {
+                        let health = disks.iter().enumerate().map(|(slot, disk)| {
+                            let disk::Disk::Local(local) = disk.as_ref() else { panic!("actual local Copy backend") };
+                            let metrics = local.metrics_snapshot();
+                            (slot, disk.runtime_state(), metrics.total_errors_timeout, metrics.total_errors_availability, metrics.total_waiting, metrics.api_calls)
+                        }).collect::<Vec<_>>();
+                        panic!("actual CopyObject: {error:?}; drive health (slot, state, timeouts, unavailable, waiting, calls): {health:?}");
+                    });
                 drop(copy_info);
                 let mut copied = store.get_object_reader(bucket, copy_object, None, HeaderMap::new(), &ObjectOptions::default()).await.expect("actual copied body");
                 let mut actual = Vec::new();
@@ -11194,6 +11315,163 @@ mod tests {
         assert_eq!(upload_uuid_suffix(&recovered[0]), upload_uuid_suffix(&upload.upload_id));
     }
 
+    #[tokio::test]
+    async fn multipart_absence_inspection_distinguishes_bucket_ownership() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "multipart-cleanup-target";
+        let other = "multipart-cleanup-other";
+        for disk in &disk_stores {
+            disk.make_volume(bucket).await.expect("create target bucket");
+            disk.make_volume(other).await.expect("create other bucket");
+        }
+        set_disks
+            .ensure_no_multipart_uploads(bucket)
+            .await
+            .expect("an empty upload namespace permits cleanup");
+        set_disks
+            .new_multipart_upload(other, "object.bin", &ObjectOptions::default())
+            .await
+            .expect("create upload in other bucket");
+        set_disks
+            .ensure_no_multipart_uploads(bucket)
+            .await
+            .expect("an upload owned by another bucket permits cleanup");
+        set_disks
+            .new_multipart_upload(bucket, "object.bin", &ObjectOptions::default())
+            .await
+            .expect("create upload in target bucket");
+        assert!(
+            matches!(set_disks.ensure_no_multipart_uploads(bucket).await, Err(Error::BucketNotEmpty(name)) if name == bucket)
+        );
+    }
+
+    #[tokio::test]
+    async fn multipart_absence_inspection_requires_every_disk_online() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "multipart-cleanup-offline";
+        for disk in &disk_stores {
+            disk.make_volume(bucket).await.expect("create bucket");
+        }
+        set_disks.disks.write().await[3] = None;
+        let listed = set_disks
+            .list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
+            .await
+            .expect("ordinary listing preserves its discovery quorum");
+        assert!(listed.uploads.is_empty());
+        let error = set_disks
+            .ensure_no_multipart_uploads(bucket)
+            .await
+            .expect_err("an offline disk must prevent confirmation of an empty multipart namespace");
+        assert!(
+            matches!(&error, Error::InsufficientReadQuorum(error_bucket, error_path) if error_bucket == bucket && error_path.is_empty()),
+            "the offline disk must produce a bucket-scoped read quorum error: {error:?}"
+        );
+
+        set_disks.disks.write().await[3] = Some(disk_stores[3].clone());
+        set_disks
+            .ensure_no_multipart_uploads(bucket)
+            .await
+            .expect("all disks can confirm the empty namespace after recovery");
+    }
+
+    #[tokio::test]
+    async fn multipart_absence_inspection_rejects_a_failed_disk_enumeration() {
+        let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "multipart-cleanup-list-error";
+        for disk in &disk_stores {
+            disk.make_volume(bucket).await.expect("create bucket");
+        }
+        let sha_dir = SetDisks::get_multipart_sha_dir(bucket, "object.bin");
+        let multipart_dir = temp_dirs[0].path().join(RUSTFS_META_MULTIPART_BUCKET);
+        tokio::fs::create_dir_all(&multipart_dir)
+            .await
+            .expect("create multipart staging directory");
+        tokio::fs::write(multipart_dir.join(&sha_dir), b"not a directory")
+            .await
+            .expect("replace one upload parent directory with a file");
+        multipart_upload_paths_on_disk(disk_stores[0].clone(), bucket, "")
+            .await
+            .expect_err("the affected disk must fail directory enumeration");
+        let listed = set_disks
+            .list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
+            .await
+            .expect("ordinary listing still accepts the remaining discovery quorum");
+        assert!(listed.uploads.is_empty());
+        let error = set_disks
+            .ensure_no_multipart_uploads(bucket)
+            .await
+            .expect_err("a failed disk enumeration must prevent confirmation of an empty multipart namespace");
+        assert!(
+            matches!(&error, Error::InsufficientReadQuorum(error_bucket, error_path) if error_bucket == bucket && error_path.is_empty()),
+            "the failed enumeration must produce a bucket-scoped read quorum error: {error:?}"
+        );
+
+        tokio::fs::remove_file(temp_dirs[0].path().join(RUSTFS_META_MULTIPART_BUCKET).join(sha_dir))
+            .await
+            .expect("remove the invalid directory entry");
+        set_disks
+            .ensure_no_multipart_uploads(bucket)
+            .await
+            .expect("all disks can confirm the empty namespace after repair");
+    }
+
+    #[tokio::test]
+    async fn multipart_absence_inspection_rejects_data_movement_uploads() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "multipart-cleanup-data-movement";
+        for disk in &disk_stores {
+            disk.make_volume(bucket).await.expect("create bucket");
+        }
+        let mut metadata = HashMap::new();
+        rustfs_utils::http::insert_str(
+            &mut metadata,
+            rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
+            format!("v1:{}:{}", Uuid::new_v4(), OffsetDateTime::now_utc().unix_timestamp_nanos()),
+        );
+        set_disks
+            .new_multipart_upload(
+                bucket,
+                "object.bin",
+                &ObjectOptions {
+                    data_movement: true,
+                    user_defined: metadata,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("create data movement upload");
+        assert!(
+            matches!(set_disks.ensure_no_multipart_uploads(bucket).await, Err(Error::BucketNotEmpty(name)) if name == bucket)
+        );
+    }
+
+    #[tokio::test]
+    async fn multipart_absence_inspection_rejects_unreadable_ownership() {
+        let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "multipart-cleanup-corrupt";
+        for disk in &disk_stores {
+            disk.make_volume(bucket).await.expect("create bucket");
+        }
+        set_disks
+            .new_multipart_upload(bucket, "object.bin", &ObjectOptions::default())
+            .await
+            .expect("create multipart upload");
+        let mut corrupted = 0;
+        for temp_dir in &temp_dirs {
+            for path in multipart_meta_files_on_disk(temp_dir, "xl.meta").await {
+                tokio::fs::write(path, b"invalid upload metadata")
+                    .await
+                    .expect("corrupt upload metadata");
+                corrupted += 1;
+            }
+        }
+        assert!(corrupted > 0, "upload metadata must exist before corruption");
+        set_disks
+            .ensure_no_multipart_uploads(bucket)
+            .await
+            .expect_err("unreadable upload ownership must block cleanup");
+    }
+
     /// Regression (issue #5716): a single upload directory whose `xl.meta` was
     /// destroyed (crash mid-write, torn disk state) must degrade to that upload
     /// alone. Failing the whole ListMultipartUploads turns one piece of stale

crates/ecstore/src/set_disk/ops/object.rs

@@ -46,9 +46,9 @@ use super::super::{
     build_get_object_info, build_inline_bitrot_readers, build_inline_bitrot_readers_from_refs, can_try_inline_data_shards_direct,
     check_object_lock_delete, check_object_lock_for_deletion_with_state, check_object_lock_retention_update,
     classify_get_codec_streaming_object_class, classify_put_write_path, classify_storage_error,
-    collect_inline_data_shard_fileinfos_by_index, contains_key_str, create_bitrot_writer, debug, delete_file_info_version_id,
-    disk, ensure_delete_commit_locks_held, error, explicit_delete_removed_marker, finish_set_disk_read_lock,
-    get_codec_streaming_reader_gate_with_plan, get_object_body_cache_hook, get_raw_etag,
+    collect_inline_data_shard_fileinfos_by_index, contains_key_str, create_bitrot_writer_with_orig_volume, debug,
+    delete_file_info_version_id, disk, ensure_delete_commit_locks_held, error, explicit_delete_removed_marker,
+    finish_set_disk_read_lock, get_codec_streaming_reader_gate_with_plan, get_object_body_cache_hook, get_raw_etag,
     get_small_object_direct_memory_decision_with_threshold_and_plan, get_small_object_direct_memory_threshold,
     get_stage_timer_if_enabled, get_str, get_transitioned_object_reader_with_tier_manager, inline_admission_shard_size,
     inline_erasure_shard_file_offset, inline_erasure_shard_size, insert_str, is_deadlock_detection_enabled,
@@ -4459,9 +4459,10 @@ impl SetDisks {
                             if let Some(disk) = disk_op
                                 && disk.is_online().await
                             {
-                                match create_bitrot_writer(
+                                match create_bitrot_writer_with_orig_volume(
                                     is_inline_buffer,
                                     Some(disk),
+                                    bucket,
                                     RUSTFS_META_TMP_BUCKET,
                                     &tmp_obj,
                                     shard_file_size,
@@ -4525,7 +4526,16 @@ impl SetDisks {
                     "Set disk write quorum unavailable"
                 );
                 if let Some(write_err) = reduce_write_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, write_quorum) {
-                    return Err(to_object_err(write_err.into(), vec![bucket, object]));
+                    let error = to_object_err(write_err.into(), vec![bucket, object]);
+                    if direct_inline_commit
+                        && is_meta_bucketname(bucket)
+                        && opts.write_completion == crate::object_api::WriteCompletion::TailDrainedWithOwnerRepair
+                    {
+                        // Inline writer setup only checks disk availability;
+                        // no shard or metadata write has been dispatched yet.
+                        return Err(Error::other(crate::set_disk::SystemMetadataWriteNotStarted(error)));
+                    }
+                    return Err(error);
                 }
 
                 return Err(Error::other(format!("not enough disks to write: {errors:?}")));
@@ -4533,7 +4543,7 @@ impl SetDisks {
 
             let stream = mem::replace(
                 &mut data.stream,
-                HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, false)?,
+                HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, true)?,
             );
 
             let small_size_hint = if matches!(write_path, SmallWritePath::Inline | SmallWritePath::SingleBlockNonInline) {
@@ -4957,7 +4967,20 @@ impl SetDisks {
                 );
             }
 
-            if let Some(version_id) = opts.version_id.as_deref()
+            // A write without a version id to a key that is not versioning-Enabled
+            // replaces the key's null version in place, so the WORM gate must judge
+            // that version too. The request-layer pre-check reads the latest
+            // version, which is not the one replaced when a UUID version or a
+            // delete marker sits above a retained null version.
+            let worm_gate_version_id = opts.version_id.clone().or_else(|| {
+                (!opts.versioned
+                    && opts
+                        .object_lock_config_snapshot
+                        .as_ref()
+                        .is_some_and(|snapshot| !matches!(snapshot.state(), ObjectLockConfigState::ConfirmedAbsent)))
+                .then(|| Uuid::nil().to_string())
+            });
+            if let Some(version_id) = worm_gate_version_id.as_deref()
                 && !is_meta_bucketname(bucket)
             {
                 let explicit_version_lookup_started = rustfs_io_metrics::put_stage_timer();
@@ -4977,9 +5000,10 @@ impl SetDisks {
                     .await;
                 match current {
                     Ok(existing) => {
-                        let object_lock_config = opts.object_lock_config_snapshot.as_deref().ok_or_else(|| {
-                            Error::other("explicit-version PUT is missing its Object Lock configuration snapshot")
-                        })?;
+                        let object_lock_config = opts
+                            .object_lock_config_snapshot
+                            .as_deref()
+                            .ok_or_else(|| Error::other("destructive PUT is missing its Object Lock configuration snapshot"))?;
                         // The WORM gate protects the locked version from local
                         // overwrites; an authorized replication write passes it
                         // only when the LWW merge below will judge every
@@ -5010,6 +5034,8 @@ impl SetDisks {
                             }
                         }
                     }
+                    // A null delete marker has no protected payload to replace.
+                    Err(StorageError::MethodNotAllowed) if version_id == Uuid::nil().to_string() => {}
                     Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
                     Err(err) => return Err(err),
                 }
@@ -6128,7 +6154,11 @@ impl SetDisks {
                     );
                 }
 
-                if !tail_owns_tmp_cleanup {
+                let all_local_disks_cleaned_tmp = all_commit_disks_local
+                    && online_disks.len() == commit_set.set_drive_count
+                    && online_disks.iter().all(Option::is_some)
+                    && !issue3031_diag_enabled();
+                if !tail_owns_tmp_cleanup && !all_local_disks_cleaned_tmp {
                     let cleanup_set = commit_set.clone();
                     let cleanup_tmp_dir = commit_tmp_dir.clone();
                     cleanup_runtime::spawn(commit_cleanup_reservation.clone(), async move {
@@ -9299,6 +9329,60 @@ impl SetDisks {
 }
 
 impl SetDisks {
+    /// Judge the version a versionless destructive delete removes.
+    ///
+    /// For a key that is not versioning-`Enabled`, a delete without a version
+    /// id removes the `null` version, while the preceding Object Lock check
+    /// judged the key's latest entry. When a UUID version or delete marker
+    /// sits above a retained `null` version, the two differ and the retained
+    /// version must be judged on its own. The caller holds the object's write
+    /// lock. Plain unversioned keys and versioning-`Enabled` keys keep a single
+    /// lookup: their latest entry is the version removed.
+    ///
+    /// The lookup future is boxed from a synchronous wrapper so it leaves the
+    /// delete futures' frames: `delete_object` already nests the boxed batch
+    /// path, and inlining a second `get_object_info` overflowed the 4 MiB test
+    /// stack on Linux.
+    fn check_null_version_object_lock_delete<'a>(
+        &'a self,
+        bucket: &'a str,
+        object: &'a str,
+        latest: &'a ObjectInfo,
+        opts: &'a ObjectOptions,
+    ) -> impl Future<Output = Result<()>> + Send + 'a {
+        Box::pin(self.check_null_version_object_lock_delete_inner(bucket, object, latest, opts))
+    }
+
+    async fn check_null_version_object_lock_delete_inner(
+        &self,
+        bucket: &str,
+        object: &str,
+        latest: &ObjectInfo,
+        opts: &ObjectOptions,
+    ) -> Result<()> {
+        if is_meta_bucketname(bucket)
+            || opts.version_id.is_some()
+            || set_disk_delete_creates_delete_marker(opts)
+            || opts.transition.expire_restored
+            || latest.version_id.is_none_or(|version| version.is_nil())
+        {
+            return Ok(());
+        }
+        let null_opts = ObjectOptions {
+            version_id: Some(Uuid::nil().to_string()),
+            no_lock: true,
+            metadata_cache_safe: false,
+            ..opts.clone()
+        };
+        match self.get_object_info(bucket, object, &null_opts).await {
+            Ok(existing) => check_object_lock_delete(&self.ctx, bucket, object, &existing, &null_opts).await,
+            // A null delete marker has no protected payload to remove.
+            Err(StorageError::MethodNotAllowed) => Ok(()),
+            Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(()),
+            Err(err) => Err(err),
+        }
+    }
+
     /// Read current metadata without cache admission or read-side repair. The
     /// outer batch owner already holds the sorted namespace write-lock set.
     pub(crate) async fn conditional_delete_current_snapshot(
@@ -9333,6 +9417,76 @@ where
     futures::future::try_join_all(probes).await.map(|_| ()).map_err(Error::from)
 }
 
+/// Per-key source-read inputs for the bulk-delete admission loop. The bounded
+/// metadata pre-fetch and the loop itself must resolve these identically, so
+/// both sides derive them here.
+struct BatchDeleteSourceInputs {
+    replication_object_name: String,
+    explicit_null_version: bool,
+    version_id: Option<Uuid>,
+    versioned: bool,
+    version_suspended: bool,
+    check_opts: ObjectOptions,
+    object_lock_check_required: bool,
+    replicate_delete: bool,
+    needs_source_read: bool,
+}
+
+fn batch_delete_source_inputs(
+    bucket: &str,
+    dobj: &ObjectToDelete,
+    opts: &ObjectOptions,
+    delete_config_snapshot: &DeleteReplicationConfigSnapshot,
+    object_lock_config: &Arc<ObjectLockConfigSnapshot>,
+    guarded: bool,
+) -> BatchDeleteSourceInputs {
+    let replication_object_name = decode_dir_object(&dobj.object_name);
+    let explicit_null_version = is_explicit_null_version(dobj.version_id);
+    let version_id = delete_file_info_version_id(dobj.version_id);
+    let (versioned, version_suspended) = delete_config_snapshot
+        .versioning_config()
+        .delete_state(replication_object_name.as_str());
+    let check_opts = ObjectOptions {
+        version_id: dobj.version_id.map(|version_id| version_id.to_string()),
+        versioned,
+        version_suspended,
+        object_lock_delete: opts.object_lock_delete.clone(),
+        object_lock_config_snapshot: Some(Arc::clone(object_lock_config)),
+        no_lock: true,
+        suppress_read_repair: guarded,
+        ..Default::default()
+    };
+    // A missing bucket Object Lock configuration does not prove that
+    // persisted object metadata is clean: migrated or corrupt data may
+    // still carry an explicit retention or legal hold. Only a real
+    // delete-marker creation can skip the object-level WORM check.
+    let object_lock_check_required = !is_meta_bucketname(bucket) && !set_disk_delete_creates_delete_marker(&check_opts);
+    let replicate_delete = delete_config_snapshot.has_active_rule(&replication_object_name);
+    let marker_delete = dobj.version_id.is_none() || dobj.synthetic_version_id;
+    let replication_needs_source = replicate_delete
+        && (!marker_delete || delete_config_snapshot.active_delete_marker_rules_require_tags(&replication_object_name));
+    // Guarded explicit-version deletes are rejected before any read and must
+    // not join the pre-fetch wave.
+    let admission_error = guarded && dobj.version_id.is_some();
+    let needs_source_read = !admission_error
+        && (object_lock_check_required
+            || replication_needs_source
+            || opts.tier_delete_journal_api.is_some()
+            || dobj.expected_identity.is_some()
+            || dobj.version_id.is_some());
+    BatchDeleteSourceInputs {
+        replication_object_name,
+        explicit_null_version,
+        version_id,
+        versioned,
+        version_suspended,
+        check_opts,
+        object_lock_check_required,
+        replicate_delete,
+        needs_source_read,
+    }
+}
+
 #[async_trait::async_trait]
 impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
     type Error = Error;
@@ -9731,7 +9885,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         let unique_lock_count = batch.requests.len();
 
         let mut failed_map = HashMap::new();
-        let mut _local_batch_guards: Vec<FastLockGuard> = Vec::with_capacity(batch.requests.len());
+        let mut local_batch_guards: Vec<FastLockGuard> = Vec::with_capacity(batch.requests.len());
         let mut locked_objects = HashSet::new();
 
         // Instance-scoped, not the ambient facade (backlog#1052) — see
@@ -9748,7 +9902,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             (failed_map, locked_objects, dist_batch_lock_ids) = self.acquire_dist_delete_object_locks_batch(&batch).await;
         } else {
             let batch_result = self.local_lock_manager.acquire_locks_batch(batch).await;
-            _local_batch_guards = batch_result.guards;
+            local_batch_guards = batch_result.guards;
 
             for key in batch_result.successful_locks {
                 locked_objects.insert(key.object.as_ref().to_string());
@@ -9790,8 +9944,17 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             // Every raw batch lock is acquired first. Canonical sorting also
             // gives aliases of one disk key exactly one transaction owner.
             let keys: std::collections::BTreeSet<_> = locked_objects.iter().cloned().collect();
-            for key in keys {
-                match self.acquire_delete_transaction_lock(bucket, &key).await {
+            // Bulk delete is the only multi-key transaction-lock holder; one
+            // batched lock round replaces one quorum round-trip per key and is
+            // deadlock-free by construction (a single `lock_batch` RPC per lock
+            // client, global (shard, key) order at each server). Per-key
+            // failures keep the set layer's per-key error reporting.
+            let key_names: Vec<&str> = keys.iter().map(|key| key.as_str()).collect();
+            let acquired = self
+                .acquire_delete_transaction_locks_batch(bucket, &key_names, super::super::get_lock_acquire_timeout())
+                .await;
+            for (key, result) in keys.into_iter().zip(acquired) {
+                match result {
                     Ok(guard) => {
                         opts.add_namespace_lock_guard(&guard);
                         transaction_guards.insert(key, guard);
@@ -9827,6 +9990,54 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 },
             }
         };
+        // Per-key read inputs are resolved exactly once; the pre-fetch and the
+        // admission loop below consume this table so the two cannot disagree
+        // on eligibility or read options.
+        let mut source_inputs: Vec<Option<BatchDeleteSourceInputs>> = Vec::with_capacity(objects.len());
+        for dobj in &objects {
+            let guarded = current_generations.is_some_and(|generations| generations.contains_key(&dobj.object_name));
+            source_inputs.push(Some(batch_delete_source_inputs(
+                bucket,
+                dobj,
+                &opts,
+                &delete_config_snapshot,
+                &object_lock_config,
+                guarded,
+            )));
+        }
+        // Bounded-parallel pre-fetch of the source metadata each admitted key
+        // needs below. These reads are pure quorum lookups under the locks
+        // already held; sequencing one metadata fan-out per key made a
+        // 1000-key batch pay a full round of drive reads at a time.
+        const DELETE_OBJECTS_SOURCE_READ_CONCURRENCY: usize = 16;
+        let mut source_reads: HashMap<usize, (ObjectInfo, FileInfo, Option<Error>)> = HashMap::with_capacity(objects.len());
+        {
+            let mut pending = Vec::new();
+            for (i, source) in source_inputs.iter().enumerate() {
+                if del_errs[i].is_some() {
+                    continue;
+                }
+                let Some(source) = source else {
+                    continue;
+                };
+                if !source.needs_source_read {
+                    continue;
+                }
+                let object_name = objects[i].object_name.clone();
+                let check_opts = &source.check_opts;
+                pending.push(async move {
+                    let (goi, file_info, _write_quorum, gerr) = self
+                        .get_object_info_fileinfo_and_quorum(bucket, &object_name, check_opts)
+                        .await;
+                    (i, (goi, file_info, gerr))
+                });
+            }
+            let reads = futures::stream::iter(pending)
+                .buffer_unordered(DELETE_OBJECTS_SOURCE_READ_CONCURRENCY)
+                .collect::<Vec<_>>()
+                .await;
+            source_reads.extend(reads);
+        }
         let mut vers_map: HashMap<&String, FileInfoVersions> = HashMap::new();
         let mut tier_reference_leases: Vec<(usize, String, Option<TierDestinationId>)> = Vec::new();
         let mut tier_free_version_receipt_candidates: HashMap<usize, TierFreeVersionReceiptCandidate> = HashMap::new();
@@ -9848,41 +10059,30 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 ));
                 continue;
             }
-            let replication_object_name = decode_dir_object(&dobj.object_name);
-            let explicit_null_version = is_explicit_null_version(dobj.version_id);
-            let version_id = delete_file_info_version_id(dobj.version_id);
-            let (versioned, version_suspended) = delete_config_snapshot
-                .versioning_config()
-                .delete_state(replication_object_name.as_str());
-            let check_opts = ObjectOptions {
-                version_id: dobj.version_id.map(|version_id| version_id.to_string()),
+            let Some(source) = source_inputs[i].take() else {
+                continue;
+            };
+            let BatchDeleteSourceInputs {
+                replication_object_name,
+                explicit_null_version,
+                version_id,
                 versioned,
                 version_suspended,
-                object_lock_delete: opts.object_lock_delete.clone(),
-                object_lock_config_snapshot: Some(Arc::clone(&object_lock_config)),
-                no_lock: true,
-                suppress_read_repair: guarded,
-                ..Default::default()
-            };
-            // A missing bucket Object Lock configuration does not prove that
-            // persisted object metadata is clean: migrated or corrupt data may
-            // still carry an explicit retention or legal hold. Only a real
-            // delete-marker creation can skip the object-level WORM check.
-            let object_lock_check_required = !is_meta_bucketname(bucket) && !set_disk_delete_creates_delete_marker(&check_opts);
-            let replicate_delete = delete_config_snapshot.has_active_rule(&replication_object_name);
-            let marker_delete = dobj.version_id.is_none() || dobj.synthetic_version_id;
-            let replication_needs_source = replicate_delete
-                && (!marker_delete || delete_config_snapshot.active_delete_marker_rules_require_tags(&replication_object_name));
-            let (mut goi, mut authoritative_file_info, mut gerr) = if object_lock_check_required
-                || replication_needs_source
-                || opts.tier_delete_journal_api.is_some()
-                || dobj.expected_identity.is_some()
-                || dobj.version_id.is_some()
-            {
-                let (goi, file_info, _write_quorum, gerr) = self
-                    .get_object_info_fileinfo_and_quorum(bucket, &dobj.object_name, &check_opts)
-                    .await;
-                (goi, file_info, gerr)
+                check_opts,
+                object_lock_check_required,
+                replicate_delete,
+                needs_source_read,
+            } = source;
+            let (mut goi, mut authoritative_file_info, mut gerr) = if needs_source_read {
+                match source_reads.remove(&i) {
+                    Some(read) => read,
+                    None => {
+                        let (goi, file_info, _write_quorum, gerr) = self
+                            .get_object_info_fileinfo_and_quorum(bucket, &dobj.object_name, &check_opts)
+                            .await;
+                        (goi, file_info, gerr)
+                    }
+                }
             } else {
                 (ObjectInfo::default(), FileInfo::default(), None)
             };
@@ -9899,6 +10099,17 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 && !guarded
                 && goi.name.is_empty()
             {
+                // Recovery can restore or retire versions of this key
+                // (rolling back an interrupted prepared delete brings a
+                // retained version back). A pre-fetched read for a later
+                // duplicate of the same name would still report the stale
+                // miss and skip its object-lock check — bypassing retention —
+                // so evict those entries and let the loop re-read them live.
+                for (other_index, other) in objects.iter().enumerate() {
+                    if other_index != i && other.object_name == dobj.object_name {
+                        source_reads.remove(&other_index);
+                    }
+                }
                 let recovery = self
                     .recover_delete_key(
                         bucket,
@@ -10017,6 +10228,17 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 del_errs[i] = Some(err);
                 continue;
             }
+            // Not gated on `source_missing`: with a delete marker on top the
+            // latest-version lookup reports the key as missing while the null
+            // version below it may still be retained.
+            if object_lock_check_required
+                && let Err(err) = self
+                    .check_null_version_object_lock_delete(bucket, &dobj.object_name, &goi, &check_opts)
+                    .await
+            {
+                del_errs[i] = Some(err);
+                continue;
+            }
 
             if opts.tier_delete_journal_api.is_some()
                 && !source_missing
@@ -10472,6 +10694,10 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             // response can hide a persisted decision and must never roll back.
             let groups = groups.collect::<Vec<_>>();
             published_groups.extend(groups.iter().map(|index| vers[*index].name.clone()));
+            // Journal publish fans out per object per drive; keep enough
+            // publications in flight to feed the drives' durable-mutation
+            // lanes (8 workers each) across a 16-drive set.
+            const DELETE_JOURNAL_PUBLISH_CONCURRENCY: usize = 32;
             let publications = futures::stream::iter(groups.into_iter().map(|index: usize| {
                 let group = &vers[index];
                 let recovery_ticket = recovery_tickets[index].clone();
@@ -10507,7 +10733,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                     (index, result)
                 }
             }))
-            .buffer_unordered(16)
+            .buffer_unordered(DELETE_JOURNAL_PUBLISH_CONCURRENCY)
             .collect::<Vec<_>>()
             .await;
             for (index, result) in publications {
@@ -10647,9 +10873,14 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         // Deferred recovery takes fresh raw and transaction locks. Discard the
         // completed leases' deadlines while retaining every caller-owned fence.
         opts.namespace_lock_fence = outer_namespace_lock_fence;
-        // Release inner owners before reacquiring any raw key lock.
+        // Release inner owners before reacquiring any raw key lock. The raw
+        // batch guards must go too: deferred recovery takes a fresh write lock
+        // on the same keys below, and a still-held local batch guard would
+        // deadlock it against itself (the distributed path releases its batch
+        // ids in the same region).
         drop(namespace_owner);
         drop(transaction_guards);
+        drop(local_batch_guards);
 
         // TODO(backlog): support partial object deletion for multi-part objects
 
@@ -11064,7 +11295,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         if let Some(err) = &gerr
             && goi.name.is_empty()
         {
-            if should_force_delete_marker_for_missing_version(&opts) {
+            if (is_err_object_not_found(err) || is_err_version_not_found(err))
+                && should_force_delete_marker_for_missing_version(&opts)
+            {
                 version_found = false;
             } else {
                 if is_err_object_not_found(err) {
@@ -11087,6 +11320,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             opts.precondition_check(&goi)?;
             check_object_lock_delete(&self.ctx, bucket, object, &goi, &opts).await?;
         }
+        self.check_null_version_object_lock_delete(bucket, object, &goi, &opts)
+            .await?;
         let delete_marker_purge_candidate = opts
             .version_id
             .as_deref()
@@ -24985,6 +25220,131 @@ mod delete_objects_lock_gating_tests {
             .expect("plain object should be written");
     }
 
+    /// Regression for the PR9155 review finding: a pre-fetched source-missing
+    /// read of a later duplicate must not skip the object-lock check after an
+    /// earlier entry's recovery rolls back an interrupted prepared delete and
+    /// restores a GOVERNANCE-retained version. Without the eviction the later
+    /// duplicate deletes the restored version, bypassing retention.
+    #[tokio::test]
+    async fn duplicate_entries_recheck_object_lock_after_recovery_restores_retained_version() {
+        let (_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "recovery-retention-dup";
+        let object = "retained.bin";
+        for disk in &disk_stores {
+            disk.make_volume(bucket).await.expect("create bucket");
+        }
+
+        // A versioned put, so the batch can address the version explicitly.
+        let mut reader = PutObjReader::from_vec(vec![7u8; 256]);
+        set_disks
+            .put_object(
+                bucket,
+                object,
+                &mut reader,
+                &ObjectOptions {
+                    versioned: true,
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("versioned object should be written");
+        let version = set_disks
+            .get_object_info(bucket, object, &ObjectOptions::default())
+            .await
+            .expect("read back the version identity")
+            .version_id
+            .expect("versioned put assigns a version id");
+
+        // GOVERNANCE retention on that version, persisted the way
+        // PutObjectRetention persists it.
+        let retain_until = OffsetDateTime::now_utc() + time::Duration::days(60);
+        set_disks
+            .put_object_metadata(
+                bucket,
+                object,
+                &ObjectOptions {
+                    no_lock: true,
+                    version_id: Some(version.to_string()),
+                    eval_metadata: Some(HashMap::from([
+                        (metadata_keys::OBJECT_LOCK_MODE.to_string(), "GOVERNANCE".to_string()),
+                        (
+                            metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
+                            retain_until
+                                .format(&time::format_description::well_known::Rfc3339)
+                                .expect("retention date formats"),
+                        ),
+                    ])),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("governance retention should persist");
+
+        // Interrupted authorized delete: the recoverable per-disk delete
+        // applies the version removal and leaves the prepared journal
+        // undecided — exactly a crash between apply and publish.
+        let transaction = Uuid::new_v4();
+        for disk in &disk_stores {
+            let fi = disk
+                .read_version("", bucket, object, &version.to_string(), &crate::disk::ReadOptions::default())
+                .await
+                .expect("version exists before the interrupted delete");
+            let errors = disk
+                .delete_versions_recoverable(
+                    bucket,
+                    vec![FileInfoVersions {
+                        name: object.to_string(),
+                        versions: vec![fi],
+                        ..Default::default()
+                    }],
+                    DeleteOptions {
+                        recoverable_delete: true,
+                        old_data_dir: Some(transaction),
+                        ..Default::default()
+                    },
+                )
+                .await;
+            assert!(errors.iter().all(Option::is_none), "the plant must apply: {errors:?}");
+        }
+
+        // A no-bypass batch naming the retained version twice: the first
+        // entry rolls the prepared delete back (restoring the retained
+        // version) and must be denied; the second must re-read and stay
+        // denied too instead of reusing the stale pre-fetched miss.
+        let victim = ObjectToDelete {
+            object_name: object.to_string(),
+            version_id: Some(version),
+            ..Default::default()
+        };
+        let (_deleted, errors) = set_disks
+            .delete_objects(
+                bucket,
+                vec![victim.clone(), victim],
+                ObjectOptions {
+                    versioned: true,
+                    object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
+                        ObjectLockConfigState::ConfirmedAbsent,
+                    ))),
+                    ..Default::default()
+                },
+            )
+            .await;
+        assert!(
+            errors.iter().all(|error| error
+                .as_ref()
+                .is_some_and(|error| matches!(error, StorageError::PrefixAccessDenied(_, _)))),
+            "both duplicates must be denied by the restored retention: {errors:?}"
+        );
+
+        // The retained version survives the batch.
+        for disk in &disk_stores {
+            disk.read_version("", bucket, object, &version.to_string(), &crate::disk::ReadOptions::default())
+                .await
+                .expect("the retained version must survive the batch");
+        }
+    }
+
     #[tokio::test]
     #[serial_test::serial(shard_integrity_rollout)]
     async fn conditional_delete_snapshot_preserves_multipart_etag() {

crates/ecstore/src/storage_api_contracts/mod.rs

@@ -44,15 +44,16 @@ pub(crate) mod lifecycle {
 
 pub(crate) mod list {
     pub(crate) use rustfs_storage_api::{
-        ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
-        ListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, VersionMarker, WalkOptions as StorageWalkOptions,
-        WalkVersionsSortOrder,
+        AuthoritativeListOperations, ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsInfo,
+        ListObjectsV2Info as StorageListObjectsV2Info, ListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, VersionMarker,
+        WalkOptions as StorageWalkOptions, WalkVersionsSortOrder,
     };
 }
 
 pub(crate) mod multipart {
     pub(crate) use rustfs_storage_api::{
-        CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations, MultipartUploadResult, PartInfo,
+        CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations, MultipartUploadInspection,
+        MultipartUploadResult, PartInfo,
     };
 }
 

crates/ecstore/src/store/bucket.rs

@@ -523,7 +523,22 @@ impl ECStore {
     where
         F: FnOnce() -> Result<()> + Send,
     {
-        metadata_sys::update_validated_in(&self.ctx, bucket, config_file, data, validate).await
+        metadata_sys::update_validated_in(&self.ctx, bucket, config_file, data, None, validate).await
+    }
+
+    /// Validate an external fence and the bucket incarnation in one metadata transaction.
+    pub async fn update_bucket_metadata_config_for_incarnation_validated<F>(
+        &self,
+        bucket: &str,
+        config_file: &str,
+        data: Vec<u8>,
+        expected_incarnation_id: Uuid,
+        validate: F,
+    ) -> Result<OffsetDateTime>
+    where
+        F: FnOnce() -> Result<()> + Send,
+    {
+        metadata_sys::update_validated_in(&self.ctx, bucket, config_file, data, Some(expected_incarnation_id), validate).await
     }
 
     pub async fn bucket_incarnation_id(&self, bucket: &str) -> Result<Uuid> {

crates/ecstore/src/store/heal.rs

@@ -3692,6 +3692,177 @@ mod tests {
         );
     }
 
+    // The RPC lock service uses the raw ObjectKey and one backend per node,
+    // even when the requesting sets carry different pool namespace strings.
+    async fn pool_meta_heal_store_with_lock_domains(source: &Arc<ECStore>, shared: bool) -> Arc<ECStore> {
+        use rustfs_lock::client::{LockClient, local::LocalClient};
+
+        let clients = || {
+            (0..2)
+                .map(|_| {
+                    Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new()))) as Arc<dyn LockClient>
+                })
+                .collect::<Vec<_>>()
+        };
+        let first = clients();
+        let second = if shared { first.clone() } else { clients() };
+        let mut pools = source.pools.clone();
+        for (pool, lockers) in pools.iter_mut().zip([first, second]) {
+            for set in &mut Arc::make_mut(pool).disk_set {
+                Arc::make_mut(set).lockers = lockers.clone();
+            }
+        }
+        let endpoints: EndpointServerPools = pools.iter().map(|pool| pool.endpoints.clone()).collect::<Vec<_>>().into();
+        let state = source.pool_meta_save_gate.lock().await.independent_clone_for_test();
+        Arc::new(ECStore {
+            id: Uuid::new_v4(),
+            disk_map: Default::default(),
+            pools,
+            peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoints, Arc::clone(&source.ctx)),
+            pool_meta: tokio::sync::RwLock::new(source.pool_meta.read().await.clone()),
+            rebalance_meta: tokio::sync::RwLock::new(None),
+            decommission_cancelers: tokio::sync::RwLock::new(vec![None; 2]),
+            start_gate: tokio::sync::Mutex::new(()),
+            pool_meta_save_gate: tokio::sync::Mutex::new(state),
+            ctx: Arc::clone(&source.ctx),
+            bucket_fence_registry: Arc::default(),
+        })
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn pool_meta_same_distributed_backend_has_one_raw_lock_across_pools() {
+        let (_dirs, source, _peer) = test_two_pool_stores(None).await;
+        let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
+        let first = store.pools[0].get_disks_by_key(POOL_META_NAME);
+        let second = store.pools[1].get_disks_by_key(POOL_META_NAME);
+        let first_lock = first
+            .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
+            .await
+            .expect("first pool lock");
+        let second_lock = second
+            .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
+            .await
+            .expect("second pool lock");
+        let guard = first_lock
+            .get_write_lock(std::time::Duration::from_secs(2))
+            .await
+            .expect("first writer");
+        assert!(
+            second_lock
+                .get_write_lock(std::time::Duration::from_millis(150))
+                .await
+                .is_err(),
+            "different pool namespaces still conflict on the same backend and object key"
+        );
+        drop(guard);
+        second_lock
+            .get_write_lock(std::time::Duration::from_secs(3))
+            .await
+            .expect("second writer should acquire after the first releases");
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn pool_meta_heal_reuses_same_distributed_backend_across_pools() {
+        let (_dirs, source, _peer) = test_two_pool_stores(None).await;
+        let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
+        let (item, error) = tokio::time::timeout(
+            std::time::Duration::from_secs(3),
+            store.handle_heal_object(
+                RUSTFS_META_BUCKET,
+                POOL_META_NAME,
+                "",
+                &HealOpts {
+                    pool: Some(1),
+                    dry_run: true,
+                    ..Default::default()
+                },
+            ),
+        )
+        .await
+        .expect("inspection must reuse the fixed pool's writer when the lock backend is shared")
+        .expect("pool metadata heal inspection should return a result");
+        assert!(error.is_none(), "inspection should succeed: {error:?}");
+        assert_eq!(item.object, POOL_META_NAME);
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn pool_meta_heal_repairs_other_pool_with_shared_distributed_backend() {
+        let (_dirs, source, _peer) = test_two_pool_stores(None).await;
+        let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
+        let missing = remove_pool_meta_shard(&store, 1).await;
+        let (item, error) = tokio::time::timeout(
+            std::time::Duration::from_secs(10),
+            store.handle_heal_object(
+                RUSTFS_META_BUCKET,
+                POOL_META_NAME,
+                "",
+                &HealOpts {
+                    pool: Some(1),
+                    ..Default::default()
+                },
+            ),
+        )
+        .await
+        .expect("cross-pool repair must not reenter the shared writer")
+        .expect("pool metadata repair should return a result");
+        assert!(error.is_none(), "cross-pool repair should complete: {error:?}");
+        assert_eq!(item.object, POOL_META_NAME);
+        assert!(
+            missing.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_ok(),
+            "the selected missing shard must be restored"
+        );
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn pool_meta_heal_keeps_target_lock_for_independent_distributed_backend() {
+        let (_dirs, source, _peer) = test_two_pool_stores(None).await;
+        let store = pool_meta_heal_store_with_lock_domains(&source, false).await;
+        let target = store.pools[1].get_disks_by_key(POOL_META_NAME);
+        let lock = target
+            .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
+            .await
+            .expect("target lock");
+        let guard = lock
+            .get_write_lock(std::time::Duration::from_secs(2))
+            .await
+            .expect("hold target lock");
+        let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
+        let heal_store = Arc::clone(&store);
+        let mut heal = tokio::spawn(async move {
+            heal_store
+                .handle_heal_object(
+                    RUSTFS_META_BUCKET,
+                    POOL_META_NAME,
+                    "",
+                    &HealOpts {
+                        pool: Some(1),
+                        dry_run: true,
+                        ..Default::default()
+                    },
+                )
+                .await
+        });
+        tokio::time::timeout(
+            std::time::Duration::from_secs(3),
+            barrier.wait_until_external_heal_target_lock_attempted(),
+        )
+        .await
+        .expect("inspection must attempt the independent target lock");
+        assert!(!heal.is_finished(), "inspection must wait while the target writer is held");
+        drop(guard);
+        let (item, error) = tokio::time::timeout(std::time::Duration::from_secs(3), &mut heal)
+            .await
+            .expect("inspection should progress after target lock release")
+            .expect("heal task should not panic")
+            .expect("pool metadata heal inspection should return a result");
+        assert!(error.is_none(), "inspection should succeed: {error:?}");
+        assert_eq!(item.object, POOL_META_NAME);
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn pool_meta_heal_cas_repairs_only_the_requested_pool() {

crates/ecstore/src/store/init.rs

@@ -540,6 +540,7 @@ impl ECStore {
             // failures can still spawn recovery probes for peers that come up
             // after this node.
             let (mut disks, errs) = init_format::init_disks(
+                &instance_ctx,
                 &pool_eps.endpoints,
                 &DiskOption {
                     cleanup: true,
@@ -5340,6 +5341,203 @@ mod tests {
         shutdown.cancel();
     }
 
+    #[cfg(feature = "test-util")]
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn conditional_multipart_must_not_commit_against_an_older_readable_pool() {
+        use crate::storage_api_contracts::multipart::CompletePart;
+        use crate::storage_api_contracts::object::HTTPPreconditions;
+
+        let temp_dir = tempfile::tempdir().expect("create conditional multipart test directory");
+        let (_ctx, store, shutdown) =
+            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "conditional-mpu-unreadable", &[4, 4])).await;
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let bucket = format!("conditional-mpu-{}", uuid::Uuid::new_v4().simple());
+        let object = "object";
+        store
+            .make_bucket(&bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create conditional multipart test bucket");
+        let mut old_body = PutObjReader::from_vec(b"older-pool".to_vec());
+        let old = store.pools[0]
+            .put_object(
+                &bucket,
+                object,
+                &mut old_body,
+                &ObjectOptions {
+                    mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1)),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("write old object to staging pool");
+        let upload = store.pools[0]
+            .new_multipart_upload(&bucket, object, &ObjectOptions::default())
+            .await
+            .expect("create multipart upload in pool 0");
+        let mut part_body = PutObjReader::from_vec(b"stale-writer".to_vec());
+        let part = store.pools[0]
+            .put_object_part(&bucket, object, &upload.upload_id, 1, &mut part_body, &ObjectOptions::default())
+            .await
+            .expect("stage multipart part in pool 0");
+        let mut new_body = PutObjReader::from_vec(b"newer-pool".to_vec());
+        let new = store.pools[1]
+            .put_object(
+                &bucket,
+                object,
+                &mut new_body,
+                &ObjectOptions {
+                    mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(2)),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("write newer object to pool 1");
+        let old_etag = old.etag.clone().expect("old object must have an ETag");
+        let before = store.get_object_info(&bucket, object, &ObjectOptions::default()).await;
+
+        let mut metadata = Vec::new();
+        for disk_index in 0..4 {
+            let path = temp_dir
+                .path()
+                .join(format!("pool1/set0/disk{disk_index}/{bucket}/{object}/xl.meta"));
+            let original = tokio::fs::read(&path)
+                .await
+                .expect("save original object metadata before fault injection");
+            metadata.push((path, original));
+        }
+        let mut fault_error = None;
+        for (path, _) in &metadata {
+            if let Err(err) = tokio::fs::write(path, b"invalid object metadata").await {
+                fault_error = Some(err);
+                break;
+            }
+        }
+        let mut fault_contents = Vec::new();
+        if fault_error.is_none() {
+            for (path, _) in &metadata {
+                fault_contents.push(tokio::fs::read(path).await);
+            }
+            store.pools[1]
+                .get_disks_by_key(object)
+                .invalidate_get_object_metadata_cache(&bucket, object)
+                .await;
+        }
+        let unreadable = if fault_error.is_none() {
+            Some(
+                store.pools[1]
+                    .get_object_info(
+                        &bucket,
+                        object,
+                        &ObjectOptions {
+                            no_lock: true,
+                            metadata_cache_safe: false,
+                            suppress_read_repair: true,
+                            ..Default::default()
+                        },
+                    )
+                    .await,
+            )
+        } else {
+            None
+        };
+        let completed = if fault_error.is_none() {
+            let completion_store = Arc::clone(&store);
+            let completion_bucket = bucket.clone();
+            let upload_id = upload.upload_id.clone();
+            let part_etag = part.etag.clone();
+            Some(
+                tokio::spawn(async move {
+                    completion_store
+                        .complete_multipart_upload(
+                            &completion_bucket,
+                            object,
+                            &upload_id,
+                            vec![CompletePart {
+                                part_num: 1,
+                                etag: part_etag,
+                                ..Default::default()
+                            }],
+                            &ObjectOptions {
+                                http_preconditions: Some(HTTPPreconditions {
+                                    if_match: Some(old_etag),
+                                    ..Default::default()
+                                }),
+                                ..Default::default()
+                            },
+                        )
+                        .await
+                })
+                .await,
+            )
+        } else {
+            None
+        };
+        let mut restore_error = None;
+        for (path, original) in &metadata {
+            if let Err(err) = tokio::fs::write(path, original).await {
+                restore_error = Some(err);
+            }
+        }
+        store.pools[1]
+            .get_disks_by_key(object)
+            .invalidate_get_object_metadata_cache(&bucket, object)
+            .await;
+        let after = store.get_object_info(&bucket, object, &ObjectOptions::default()).await;
+        let recovered_body = match store
+            .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
+            .await
+        {
+            Ok(mut reader) => {
+                let mut body = Vec::new();
+                let result = reader
+                    .read_to_end(&mut body)
+                    .await
+                    .map(|_| body)
+                    .map_err(|err| err.to_string());
+                drop(reader);
+                result
+            }
+            Err(err) => Err(err.to_string()),
+        };
+        let parts = store
+            .list_object_parts(&bucket, object, &upload.upload_id, None, 100, &ObjectOptions::default())
+            .await;
+        shutdown.cancel();
+
+        assert!(fault_error.is_none(), "failed to inject metadata fault: {fault_error:?}");
+        assert!(restore_error.is_none(), "failed to restore metadata: {restore_error:?}");
+        assert_eq!(fault_contents.len(), 4, "all four pool 1 metadata files must be inspected");
+        for observed in fault_contents {
+            assert_eq!(
+                observed.expect("read injected metadata"),
+                b"invalid object metadata",
+                "each pool 1 metadata file must contain the injected corruption"
+            );
+        }
+        assert_eq!(before.expect("latest object before fault").etag, new.etag);
+        assert_ne!(old.etag, new.etag, "fixture bodies must produce different ETags");
+        let read_error = unreadable
+            .expect("metadata fault should have been injected")
+            .expect_err("the newer pool must fail its object metadata read during the fault");
+        assert!(
+            !crate::error::is_err_object_not_found(&read_error) && !crate::error::is_err_version_not_found(&read_error),
+            "corrupt metadata must produce a non-not-found read error: {read_error:?}"
+        );
+        let completed = completed
+            .expect("completion should have been attempted")
+            .expect("completion task should not panic");
+        let Err(completion_error) = completed else {
+            panic!("stale If-Match must not commit while the newer object's metadata is unreadable");
+        };
+        assert_eq!(completion_error, read_error, "completion must propagate the unreadable pool's error");
+        assert_eq!(after.expect("newer object after recovery").etag, new.etag);
+        assert_eq!(recovered_body.expect("newer body after recovery"), b"newer-pool");
+        let parts = parts.expect("failed completion must leave the upload retryable");
+        assert_eq!(parts.parts.len(), 1);
+        assert_eq!(parts.parts[0].etag, part.etag);
+    }
+
     async fn migrate_versioned_decommission_test_object(
         store: &Arc<crate::store::ECStore>,
         bucket: &str,
@@ -27155,4 +27353,282 @@ mod tests {
             .expect("recovery must preserve the local source object");
         assert_eq!(source_after.size, i64::try_from(payload.len()).expect("payload length should fit i64"));
     }
+
+    #[cfg(feature = "test-util")]
+    async fn seed_retained_null_version(
+        store: &Arc<crate::store::ECStore>,
+        bucket: &str,
+        object: &str,
+        protection: &str,
+        latest_is_marker: bool,
+    ) {
+        let metadata = if protection == "legal-hold" {
+            HashMap::from([("x-amz-object-lock-legal-hold".to_string(), "ON".to_string())])
+        } else {
+            HashMap::from([
+                ("x-amz-object-lock-mode".to_string(), protection.to_string()),
+                ("x-amz-object-lock-retain-until-date".to_string(), "2099-01-01T00:00:00Z".to_string()),
+            ])
+        };
+        let mut original = PutObjReader::from_vec(b"protected null version".to_vec());
+        store
+            .put_object(
+                bucket,
+                object,
+                &mut original,
+                &ObjectOptions {
+                    user_defined: metadata,
+                    mod_time: Some(OffsetDateTime::now_utc() - time::Duration::days(1)),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("seed retained null version");
+        let mut latest = PutObjReader::from_vec(b"unprotected latest version".to_vec());
+        store
+            .put_object(
+                bucket,
+                object,
+                &mut latest,
+                &ObjectOptions {
+                    versioned: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("seed newer unprotected UUID version");
+        if latest_is_marker {
+            store
+                .delete_object(
+                    bucket,
+                    object,
+                    ObjectOptions {
+                        versioned: true,
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("create a UUID delete marker above the retained null version");
+        }
+    }
+
+    #[cfg(feature = "test-util")]
+    async fn assert_retained_null_version_body(store: &Arc<crate::store::ECStore>, bucket: &str, object: &str) {
+        let mut reader = store
+            .get_object_reader(
+                bucket,
+                object,
+                None,
+                HeaderMap::new(),
+                &ObjectOptions {
+                    version_id: Some(Uuid::nil().to_string()),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("retained null version must remain readable");
+        let mut body = Vec::new();
+        reader
+            .stream
+            .read_to_end(&mut body)
+            .await
+            .expect("read retained null version");
+        assert_eq!(body, b"protected null version");
+    }
+
+    /// A versionless delete removes the null version, so the Object Lock check
+    /// must judge that version even when a UUID version or delete marker is
+    /// the key's latest entry (rustfs/backlog#3017).
+    #[cfg(feature = "test-util")]
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn null_version_object_lock_delete_checks_actual_target() {
+        use crate::bucket::replication::DeleteReplicationConfigSnapshot;
+        use s3s::dto::{BucketVersioningStatus, ExcludedPrefix, VersioningConfiguration};
+
+        let temp = tempfile::tempdir().expect("create null-version delete store");
+        let (_ctx, store, _shutdown) =
+            without_storage_class_env(build_isolated_test_store(temp.path(), "null-version-delete-lock", &[4])).await;
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let bucket = format!("null-delete-lock-{}", Uuid::new_v4());
+        store
+            .make_bucket(
+                &bucket,
+                &MakeBucketOptions {
+                    lock_enabled: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("create Object Lock bucket");
+        let configs = [
+            VersioningConfiguration::default(),
+            VersioningConfiguration {
+                status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)),
+                ..Default::default()
+            },
+            VersioningConfiguration {
+                status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
+                excluded_prefixes: Some(vec![ExcludedPrefix {
+                    prefix: Some("archive/".to_string()),
+                }]),
+                ..Default::default()
+            },
+        ];
+        for (case, config) in configs.into_iter().enumerate() {
+            let snapshot = Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(config, None));
+            for protection in ["COMPLIANCE", "GOVERNANCE", "legal-hold"] {
+                for latest_is_marker in [false, true] {
+                    let object = format!("archive/{case}-{protection}-{latest_is_marker}");
+                    seed_retained_null_version(&store, &bucket, &object, protection, latest_is_marker).await;
+                    let (_, errors) = store
+                        .delete_objects(
+                            &bucket,
+                            vec![ObjectToDelete {
+                                object_name: object.clone(),
+                                ..Default::default()
+                            }],
+                            ObjectOptions {
+                                delete_replication_config_snapshot: Some(Arc::clone(&snapshot)),
+                                ..Default::default()
+                            },
+                        )
+                        .await;
+                    assert!(
+                        matches!(errors.as_slice(), [Some(StorageError::PrefixAccessDenied(_, _))]),
+                        "batch delete must judge the retained null version: {object}: {errors:?}"
+                    );
+                    let error = store
+                        .delete_object(
+                            &bucket,
+                            &object,
+                            ObjectOptions {
+                                version_suspended: true,
+                                ..Default::default()
+                            },
+                        )
+                        .await
+                        .expect_err("suspended single delete must also judge the retained null version");
+                    assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
+                    assert_retained_null_version_body(&store, &bucket, &object).await;
+                    if protection == "GOVERNANCE" {
+                        let (_, errors) = store
+                            .delete_objects(
+                                &bucket,
+                                vec![ObjectToDelete {
+                                    object_name: object.clone(),
+                                    ..Default::default()
+                                }],
+                                ObjectOptions {
+                                    delete_replication_config_snapshot: Some(Arc::clone(&snapshot)),
+                                    object_lock_delete: Some(crate::storage_api_contracts::object::ObjectLockDeleteOptions {
+                                        bypass_governance: true,
+                                    }),
+                                    ..Default::default()
+                                },
+                            )
+                            .await;
+                        assert!(errors.iter().all(Option::is_none), "authorized governance bypass: {errors:?}");
+                        assert!(
+                            store
+                                .get_object_reader(
+                                    &bucket,
+                                    &object,
+                                    None,
+                                    HeaderMap::new(),
+                                    &ObjectOptions {
+                                        version_id: Some(Uuid::nil().to_string()),
+                                        ..Default::default()
+                                    }
+                                )
+                                .await
+                                .is_err(),
+                            "authorized deletion must make the retained null payload unreadable"
+                        );
+                    }
+                }
+            }
+        }
+    }
+
+    /// A versionless PUT or multipart completion replaces the null version in
+    /// place, so the commit-time WORM gate must judge that version even when a
+    /// UUID version or delete marker is the key's latest entry
+    /// (rustfs/backlog#3019).
+    #[cfg(feature = "test-util")]
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn null_version_object_lock_put_and_multipart_preserve_retained_target() {
+        let temp = tempfile::tempdir().expect("create null-version overwrite store");
+        let (_ctx, store, _shutdown) =
+            without_storage_class_env(build_isolated_test_store(temp.path(), "null-version-overwrite-lock", &[4])).await;
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let bucket = format!("null-overwrite-lock-{}", Uuid::new_v4());
+        store
+            .make_bucket(
+                &bucket,
+                &MakeBucketOptions {
+                    lock_enabled: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("create Object Lock bucket");
+        for protection in ["COMPLIANCE", "GOVERNANCE", "legal-hold"] {
+            for latest_is_marker in [false, true] {
+                let object = format!("{protection}-{latest_is_marker}");
+                seed_retained_null_version(&store, &bucket, &object, protection, latest_is_marker).await;
+                let mut replacement = PutObjReader::from_vec(b"replacement".to_vec());
+                let error = store
+                    .put_object(&bucket, &object, &mut replacement, &ObjectOptions::default())
+                    .await
+                    .expect_err("PUT must reject replacement of a retained null version");
+                assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
+
+                let upload = store
+                    .clone()
+                    .new_multipart_upload(&bucket, &object, &ObjectOptions::default())
+                    .await
+                    .expect("initiate multipart upload");
+                let mut part_reader = PutObjReader::from_vec(b"multipart replacement".to_vec());
+                let part = store
+                    .clone()
+                    .put_object_part(&bucket, &object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
+                    .await
+                    .expect("stage multipart replacement");
+                let error = store
+                    .clone()
+                    .complete_multipart_upload(
+                        &bucket,
+                        &object,
+                        &upload.upload_id,
+                        vec![crate::storage_api_contracts::multipart::CompletePart {
+                            part_num: part.part_num,
+                            etag: part.etag,
+                            ..Default::default()
+                        }],
+                        &ObjectOptions::default(),
+                    )
+                    .await
+                    .expect_err("multipart completion must reject replacement of a retained null version");
+                assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
+                assert_retained_null_version_body(&store, &bucket, &object).await;
+
+                let mut next_version = PutObjReader::from_vec(b"new UUID version".to_vec());
+                store
+                    .put_object(
+                        &bucket,
+                        &object,
+                        &mut next_version,
+                        &ObjectOptions {
+                            versioned: true,
+                            ..Default::default()
+                        },
+                    )
+                    .await
+                    .expect("a new UUID version must remain allowed");
+                assert_retained_null_version_body(&store, &bucket, &object).await;
+            }
+        }
+    }
 }

crates/ecstore/src/store/init_format.rs

@@ -44,12 +44,32 @@ const LEGACY_FORMAT_READ_CONCURRENCY: usize = 16;
 const LEGACY_FORMAT_BASE_MAX_BYTES: usize = 16 * 1024;
 const LEGACY_FORMAT_MAX_BYTES_PER_DISK: usize = 64;
 
-pub async fn init_disks(eps: &Endpoints, opt: &DiskOption) -> (Vec<Option<DiskStore>>, Vec<Option<DiskError>>) {
+pub async fn init_disks(
+    instance_ctx: &InstanceContext,
+    eps: &Endpoints,
+    opt: &DiskOption,
+) -> (Vec<Option<DiskStore>>, Vec<Option<DiskError>>) {
     let mut futures = Vec::with_capacity(eps.as_ref().len());
+    let local_disk_map = instance_ctx.local_disk_map();
+    let local_disks = local_disk_map.read().await;
 
     for ep in eps.as_ref().iter() {
-        futures.push(new_disk(ep, opt));
+        // Foundation disks may already serve RPC writes. Opening them again
+        // with startup cleanup can move live staging out from under a writer.
+        let registered = if ep.is_local {
+            local_disks.get(&ep.to_string()).cloned().flatten()
+        } else {
+            None
+        };
+        futures.push(async move {
+            match registered {
+                Some(disk) if disk.endpoint() == *ep => Ok(disk),
+                Some(_) => Err(DiskError::InconsistentDisk),
+                None => new_disk(ep, opt).await,
+            }
+        });
     }
+    drop(local_disks);
 
     let mut res = Vec::with_capacity(eps.as_ref().len());
     let mut errors = Vec::with_capacity(eps.as_ref().len());
@@ -1030,6 +1050,7 @@ mod tests {
         }
 
         let (disks, errors) = init_disks(
+            &InstanceContext::new(),
             &Endpoints::from(endpoints),
             &DiskOption {
                 cleanup: false,
@@ -1049,6 +1070,91 @@ mod tests {
         (temp_dir, disks)
     }
 
+    #[tokio::test]
+    async fn init_disks_preserves_live_staging_on_registered_local_disks() {
+        let (temp_dir, disks) = local_disks(1).await;
+        let registered = disks[0].as_ref().expect("registered disk should exist");
+        let instance_ctx = Arc::new(InstanceContext::new());
+        record_local_disks(&instance_ctx, vec![registered.clone()]).await;
+        let staging = temp_dir.path().join("disk-0/.rustfs.sys/tmp/in-flight/xl.meta");
+        tokio::fs::create_dir_all(staging.parent().expect("staging has a parent"))
+            .await
+            .expect("create staging directory");
+        tokio::fs::write(&staging, b"in-flight pool metadata")
+            .await
+            .expect("stage metadata through the registered disk");
+
+        let (initialized, errors) = init_disks(
+            &instance_ctx,
+            &Endpoints::from(vec![registered.endpoint()]),
+            &DiskOption {
+                cleanup: true,
+                health_check: false,
+            },
+        )
+        .await;
+
+        assert!(errors.iter().all(Option::is_none), "disk initialization failed: {errors:?}");
+        assert_eq!(
+            tokio::fs::read(&staging)
+                .await
+                .expect("startup must preserve live RPC staging"),
+            b"in-flight pool metadata"
+        );
+        assert!(Arc::ptr_eq(registered, initialized[0].as_ref().expect("initialized disk should exist")));
+    }
+
+    #[tokio::test]
+    async fn init_disks_cleans_stale_staging_without_a_registered_disk() {
+        let temp_dir = tempfile::tempdir().expect("create disk root");
+        let endpoint =
+            Endpoint::try_from(temp_dir.path().to_str().expect("disk root should be UTF-8")).expect("endpoint should parse");
+        let staging = temp_dir.path().join(".rustfs.sys/tmp/stale/xl.meta");
+        tokio::fs::create_dir_all(staging.parent().expect("staging has a parent"))
+            .await
+            .expect("create stale staging directory");
+        tokio::fs::write(&staging, b"stale pool metadata")
+            .await
+            .expect("write stale staging");
+
+        let (disks, errors) = init_disks(
+            &InstanceContext::new(),
+            &Endpoints::from(vec![endpoint]),
+            &DiskOption {
+                cleanup: true,
+                health_check: false,
+            },
+        )
+        .await;
+
+        assert!(errors.iter().all(Option::is_none), "disk initialization failed: {errors:?}");
+        assert!(disks[0].is_some());
+        assert!(!staging.exists(), "a fresh disk open must still clear stale staging");
+    }
+
+    #[tokio::test]
+    async fn init_disks_rejects_registered_disk_topology_mismatch() {
+        let (_temp_dir, disks) = local_disks(1).await;
+        let registered = disks[0].as_ref().expect("registered disk should exist");
+        let instance_ctx = Arc::new(InstanceContext::new());
+        record_local_disks(&instance_ctx, vec![registered.clone()]).await;
+        let mut endpoint = registered.endpoint();
+        endpoint.set_pool_index(1);
+
+        let (initialized, errors) = init_disks(
+            &instance_ctx,
+            &Endpoints::from(vec![endpoint]),
+            &DiskOption {
+                cleanup: true,
+                health_check: false,
+            },
+        )
+        .await;
+
+        assert!(initialized[0].is_none(), "a registered disk must not be reopened in a different slot");
+        assert_eq!(errors[0], Some(DiskError::InconsistentDisk));
+    }
+
     async fn format_bytes(disks: &[Option<DiskStore>]) -> Vec<Option<Vec<u8>>> {
         let mut snapshots = Vec::with_capacity(disks.len());
         for disk in disks {
@@ -1758,6 +1864,7 @@ mod tests {
             disk.set_disk_id(None).await.expect("test disk ID should be reset");
         }
         let (prewarm_disks, prewarm_errors) = init_disks(
+            &InstanceContext::new(),
             &prewarm_endpoints,
             &DiskOption {
                 cleanup: false,

crates/ecstore/src/store/list_objects.rs

@@ -37,8 +37,8 @@ use crate::error::{
 use crate::object_api::{ObjectInfo, ObjectOptions};
 use crate::set_disk::SetDisks;
 use crate::storage_api_contracts::list::{
-    ListObjectsInfo as StorageListObjectsInfo, StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr,
-    StorageWalkOptions, VersionMarker, WalkVersionsSortOrder,
+    AuthoritativeListOperations, ListObjectsInfo as StorageListObjectsInfo, StorageListObjectVersionsInfo,
+    StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions, VersionMarker, WalkVersionsSortOrder,
 };
 use crate::store::ECStore;
 use crate::store::utils::is_reserved_or_invalid_bucket;
@@ -80,6 +80,21 @@ type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
 type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
 type WalkOptions = StorageWalkOptions<fn(&rustfs_filemeta::FileInfo) -> bool>;
 
+#[async_trait::async_trait]
+impl AuthoritativeListOperations for ECStore {
+    async fn walk_authoritative(
+        self: Arc<Self>,
+        rx: CancellationToken,
+        bucket: &str,
+        prefix: &str,
+        result: Sender<ObjectInfoOrErr>,
+        opts: WalkOptions,
+    ) -> Result<()> {
+        check_list_objs_args(bucket, prefix, &None)?;
+        self.walk_authoritatively(rx, bucket, prefix, result, opts).await
+    }
+}
+
 /// Build the versioned listing options used by recursive object walks.
 ///
 /// `list_path_result` normally derives these scan bounds before dispatching to
@@ -1292,6 +1307,37 @@ impl ECStore {
         );
 
         let rx = rx.child_token();
+        if self.pools.len() == 1 && self.pools[0].disk_set.len() == 1 {
+            rustfs_io_metrics::record_list_objects_merge(LIST_OBJECTS_SOURCE_WALKER, 1, 1);
+            let res = self.pools[0].disk_set[0].list_path(rx, opts, sender).await;
+            let mut outcome = ListPathRawOutcome::default();
+            let err = match res {
+                Ok(set_outcome) => {
+                    outcome.limit_reached = set_outcome.limit_reached;
+                    None
+                }
+                Err(err) => Some(err),
+            };
+            let errs = [err];
+            if is_all_not_found(&errs) {
+                return Ok(outcome);
+            }
+            if let Some(err) = errs.into_iter().flatten().next() {
+                return Err(err);
+            }
+            rustfs_io_metrics::record_stage_duration(
+                "store_list_objects_list_merged",
+                merge_started.elapsed().as_secs_f64() * 1000.0,
+            );
+            debug!(
+                set_pair_count = 1,
+                limit_reached = outcome.limit_reached,
+                error_count = 0,
+                "store list_merged finished"
+            );
+            return Ok(outcome);
+        }
+
         let mut futures = Vec::new();
 
         let mut inputs = Vec::new();
@@ -1519,6 +1565,57 @@ impl ECStore {
 
         result
     }
+
+    async fn walk_authoritatively(
+        self: Arc<Self>,
+        cancellation: CancellationToken,
+        bucket: &str,
+        prefix: &str,
+        result: Sender<ObjectInfoOrErr>,
+        opts: WalkOptions,
+    ) -> Result<()> {
+        let cancel = cancellation.child_token();
+        let _cancel_on_drop = cancel.clone().drop_guard();
+        let mut scans = Vec::new();
+        let mut inputs = Vec::new();
+        for pool in &self.pools {
+            for set in &pool.disk_set {
+                let (sender, receiver) = mpsc::channel(1);
+                inputs.push(receiver);
+                let mut scan = recursive_walk_list_path_options(bucket.to_owned(), prefix, &opts);
+                // Internal absence checks must include reserved object prefixes.
+                scan.skip_hidden_prefix_check = true;
+                scans.push(set.list_versions_authoritatively(cancel.clone(), scan, sender));
+            }
+        }
+        if scans.is_empty() {
+            return Err(StorageError::ErasureReadQuorum);
+        }
+
+        let (sender, receiver) = mpsc::channel(100);
+        let forwarder = spawn_walk_entry_forwarder(cancel.clone(), receiver, result, opts, bucket.to_owned());
+        let merge = spawn_listing_merge(cancel.clone(), inputs, sender);
+        let scan_results = join_all(scans).await;
+        let merge_result = merge.await.map_err(Error::from)?;
+        let forward_result = forwarder.await.map_err(Error::from);
+
+        // An unavailable set cannot establish absence, even when another set
+        // completed. Only an explicitly missing directory or volume is empty.
+        for scan in scan_results {
+            match scan {
+                Ok(outcome) if !outcome.limit_reached => {}
+                Ok(_) => return Err(StorageError::ErasureReadQuorum),
+                Err(StorageError::FileNotFound | StorageError::VolumeNotFound) => {}
+                Err(err) => return Err(err),
+            }
+        }
+        merge_result?;
+        forward_result?;
+        if cancellation.is_cancelled() {
+            return Err(Error::other("authoritative object walk was cancelled"));
+        }
+        Ok(())
+    }
 }
 
 /// Forwards merged raw entries to a walk consumer as `ObjectInfoOrErr`,
@@ -1811,7 +1908,7 @@ async fn gather_results(
         if opts.versioned
             && is_object
             && entry
-                .xl_meta()
+                .xl_meta_ref()
                 .is_ok_and(|meta| !meta.versions.is_empty() && meta.versions.iter().all(|version| version.header.free_version()))
         {
             continue;
@@ -2489,7 +2586,8 @@ impl SetDisks {
     /// Version scan for set-scoped callers that act on absence: set walks
     /// (tier reference proofs, tier-delete journals, pool maintenance) and the
     /// recursive prefix-delete scan. Every online drive is read and each
-    /// version is resolved at the read quorum of its own erasure layout; a
+    /// version uses the read quorum of its own erasure layout. Null versions
+    /// in the system bucket also retain GET's commit-proof selection. A
     /// version that cannot be ruled out fails the scan instead of being
     /// omitted. Merged `ListObjectVersions` pages go through `list_path`.
     async fn list_versions_authoritatively(
@@ -2828,6 +2926,7 @@ mod test {
     use crate::error::{Result, StorageError};
     use crate::object_api::{ObjectInfo, ObjectOptions, PutObjReader};
     use crate::set_disk::{SetDisks, hermetic_set_disks_isolated};
+    use crate::storage_api_contracts::list::AuthoritativeListOperations as _;
     use crate::storage_api_contracts::namespace::NamespaceLocking as _;
     use crate::storage_api_contracts::object::ObjectIO as _;
     use bytes::Bytes;
@@ -5688,6 +5787,126 @@ mod test {
         ));
     }
 
+    #[test]
+    fn authoritative_system_bucket_scan_rejects_uncommitted_null_version_split() {
+        let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
+        let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
+        for new_etag in ["new", "old"] {
+            let new =
+                test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), new_etag, Uuid::from_u128(2));
+            let entries = MetaCacheEntries(vec![Some(old.clone()), Some(old.clone()), Some(new.clone()), Some(new)]);
+
+            assert!(matches!(
+                SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &[None, None, None, None], 4, 2),
+                Err(DiskError::ErasureReadQuorum)
+            ));
+        }
+    }
+
+    #[test]
+    fn authoritative_system_bucket_scan_selects_committed_null_version() {
+        let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
+        let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
+        let new = test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), "new", Uuid::from_u128(2));
+        for (new_copies, expected) in [(1, "old"), (3, "new"), (4, "new")] {
+            let entries = MetaCacheEntries(
+                (0..4)
+                    .map(|slot| Some(if slot < new_copies { new.clone() } else { old.clone() }))
+                    .collect(),
+            );
+            let resolved = SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &[None, None, None, None], 4, 2)
+                .expect("committed generation should resolve")
+                .expect("committed record should be present");
+            let info = resolved
+                .to_fileinfo(RUSTFS_META_BUCKET)
+                .expect("resolved metadata should decode");
+
+            assert_eq!(info.metadata.get("etag").map(String::as_str), Some(expected));
+        }
+    }
+
+    #[test]
+    fn authoritative_system_bucket_scan_matches_degraded_null_version_reads() {
+        let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
+        let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
+        let new = test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), "new", Uuid::from_u128(2));
+        for (old_copies, new_copies, expected) in [
+            (2, 1, Some("old")),
+            (1, 2, None),
+            (0, 3, Some("new")),
+            (0, 2, Some("new")),
+            (0, 1, None),
+        ] {
+            let entries = MetaCacheEntries(
+                (0..4)
+                    .map(|slot| {
+                        if slot < old_copies {
+                            Some(old.clone())
+                        } else if slot < old_copies + new_copies {
+                            Some(new.clone())
+                        } else {
+                            None
+                        }
+                    })
+                    .collect(),
+            );
+            let errors = (0..4)
+                .map(|slot| (slot >= old_copies + new_copies).then_some(DiskError::DiskNotFound))
+                .collect::<Vec<_>>();
+            let resolved = SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &errors, 4, 2);
+
+            if let Some(expected) = expected {
+                let info = resolved
+                    .expect("degraded committed record should resolve")
+                    .expect("degraded committed record should be present")
+                    .to_fileinfo(RUSTFS_META_BUCKET)
+                    .expect("resolved metadata should decode");
+                assert_eq!(info.metadata.get("etag").map(String::as_str), Some(expected));
+            } else {
+                assert!(matches!(resolved, Err(DiskError::ErasureReadQuorum)));
+            }
+        }
+    }
+
+    #[test]
+    fn authoritative_user_bucket_scan_keeps_null_version_read_quorum_selection() {
+        let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
+        let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
+        let new = test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), "new", Uuid::from_u128(2));
+        for old_copies in [1, 2] {
+            let entries = MetaCacheEntries(vec![
+                Some(new.clone()),
+                Some(new.clone()),
+                Some(old.clone()),
+                (old_copies == 2).then(|| old.clone()),
+            ]);
+            let errors = [None, None, None, (old_copies == 1).then_some(DiskError::DiskNotFound)];
+            let resolved = SetDisks::resolve_listed_versions("bucket", entries, &errors, 4, 2)
+                .expect("user bucket read quorum should resolve")
+                .expect("user bucket record should be present");
+            let info = resolved.to_fileinfo("bucket").expect("resolved metadata should decode");
+
+            assert_eq!(info.metadata.get("etag").map(String::as_str), Some("new"));
+        }
+    }
+
+    #[test]
+    fn authoritative_system_bucket_scan_keeps_explicit_version_read_quorum() {
+        let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
+        let entry = test_object_meta_entry_with_erasure_versions("entry.json", &[(time, "version", 2, 2)]);
+        let entries = MetaCacheEntries(vec![Some(entry.clone()), Some(entry), None, None]);
+        let errors = [None, None, Some(DiskError::DiskNotFound), Some(DiskError::DiskNotFound)];
+        let resolved = SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &errors, 4, 2)
+            .expect("explicit version read quorum should resolve")
+            .expect("explicit version should be present");
+        let info = resolved
+            .to_fileinfo(RUSTFS_META_BUCKET)
+            .expect("resolved metadata should decode");
+
+        assert_eq!(info.version_id, Some(Uuid::from_u128(1)));
+        assert_eq!(info.metadata.get("etag").map(String::as_str), Some("version"));
+    }
+
     #[test]
     fn authoritative_version_scan_tolerates_corrupt_replicas_only_with_version_quorum() {
         let entry = test_object_meta_entry_with_erasure_versions(
@@ -6029,6 +6248,49 @@ mod test {
         Bytes::from(meta.marshal_msg().expect("encode fixture xl.meta"))
     }
 
+    #[tokio::test]
+    async fn authoritative_store_walk_includes_reserved_prefix_versions() {
+        let bucket = "catalog-cleanup-versions";
+        let name = ".rustfs-table/namespaces/ns/tables/table/metadata/metadata.json";
+        let (_dirs, store) = single_set_store(16).await;
+        let set = &store.pools[0].disk_set[0];
+        for (slot, disk) in set.disks.read().await.iter().enumerate() {
+            let disk = disk.as_ref().expect("fixture drive is online");
+            disk.make_volume(bucket).await.expect("create bucket volume");
+            disk.write_all(
+                bucket,
+                &format!("{name}/{STORAGE_FORMAT_FILE}"),
+                versioned_object_metadata(bucket, name, &[(2, 1_700_000_002), (1, 1_700_000_001)], slot),
+            )
+            .await
+            .expect("persist reserved metadata versions");
+        }
+        let (sender, mut receiver) = mpsc::channel(1);
+        let walk = store
+            .clone()
+            .walk_authoritative(CancellationToken::new(), bucket, "", sender, super::WalkOptions::default());
+        let collect = async {
+            let mut versions = Vec::new();
+            while let Some(item) = receiver.recv().await {
+                assert!(item.err.is_none(), "reserved metadata must decode: {:?}", item.err);
+                let item = item.item.expect("walk must return an object");
+                versions.push((item.name, item.version_id));
+            }
+            versions
+        };
+        let (walk, versions) = timeout(Duration::from_secs(10), async { tokio::join!(walk, collect) })
+            .await
+            .expect("authoritative reserved-prefix walk must terminate");
+        walk.expect("authoritative walk must succeed");
+        assert_eq!(
+            versions,
+            [
+                (name.to_string(), Some(Uuid::from_u128(1))),
+                (name.to_string(), Some(Uuid::from_u128(2)))
+            ]
+        );
+    }
+
     // A versioned DeleteObject(versionId) can reach write quorum while three
     // drives are offline; they keep the deleted version after they return. A
     // different drive going offline later must not turn that sub-quorum
@@ -6099,6 +6361,16 @@ mod test {
                 .await
                 .expect("set walk must terminate");
             assert!(matches!(walk, Err(StorageError::ErasureReadQuorum)), "{walk:?}");
+            let (sender, mut receiver) = mpsc::channel(16);
+            let walk =
+                store
+                    .clone()
+                    .walk_authoritative(CancellationToken::new(), bucket, "", sender, super::WalkOptions::default());
+            let drain = async { while receiver.recv().await.is_some() {} };
+            let (walk, ()) = timeout(Duration::from_secs(10), async { tokio::join!(walk, drain) })
+                .await
+                .expect("authoritative store walk must terminate");
+            assert!(matches!(walk, Err(StorageError::ErasureReadQuorum)), "{walk:?}");
             let recursive_delete_scan = set
                 .clone()
                 .inner_list_object_versions_for_recursive_delete(bucket, "", None, None, 1000)

crates/ecstore/src/store/mod.rs

@@ -498,6 +498,17 @@ pub struct ECStore {
     pub(crate) bucket_fence_registry: Arc<bucket_fence::BucketFenceRegistry>,
 }
 
+impl Drop for ECStore {
+    /// Clear the instance context's `BucketMetadataSys` when its backing store
+    /// is dropped. Without this, the shared bootstrap context retains a
+    /// `BucketMetadataSys` whose `Weak<ECStore>` is dangling after the store
+    /// goes away (e.g. between sequential test fixtures that share the
+    /// bootstrap), and the next `object_store()` call panics.
+    fn drop(&mut self) {
+        self.ctx.clear_bucket_metadata_sys_if_stale();
+    }
+}
+
 const METRIC_SCANNER_DATA_MOVEMENT_PAUSED: &str = "rustfs_scanner_data_movement_paused";
 const METRIC_SCANNER_DATA_MOVEMENT_PAUSE_DURATION_SECONDS: &str = "rustfs_scanner_data_movement_pause_duration_seconds";
 const METRIC_SCANNER_DATA_MOVEMENT_BACKLOG_WORK_ITEMS: &str = "rustfs_scanner_data_movement_backlog_work_items";
@@ -3284,6 +3295,7 @@ mod tests {
         init_local_disks(endpoint_pools.clone()).await.expect("init local disks");
 
         let (mut disks, errs) = init_disks(
+            &crate::runtime::global::current_ctx(),
             &endpoint_pools.as_ref().first().expect("pool endpoints").endpoints,
             &DiskOption {
                 cleanup: true,

crates/ecstore/src/store/multipart.rs

@@ -19,10 +19,28 @@ use crate::set_disk::get_lock_acquire_timeout;
 use crate::storage_api_contracts::multipart::MultipartOperations as _;
 use crate::storage_api_contracts::object::ObjectOperations as _;
 use futures::{StreamExt, stream};
+use rustfs_utils::path::encode_dir_object_ref;
 use std::collections::HashSet;
 
 const MULTIPART_LIST_SET_CONCURRENCY: usize = 4;
 
+#[async_trait::async_trait]
+impl crate::storage_api_contracts::multipart::MultipartUploadInspection for ECStore {
+    type Error = Error;
+
+    async fn ensure_no_multipart_uploads(&self, bucket: &str) -> Result<()> {
+        let (_, lifecycle_guard) = self
+            .guard_multipart_bucket_incarnation(bucket, &ObjectOptions::default())
+            .await?;
+        for pool in &self.pools {
+            for set in &pool.disk_set {
+                set.ensure_no_multipart_uploads(bucket).await?;
+            }
+        }
+        ensure_multipart_bucket_lifecycle_guard_held(lifecycle_guard.as_ref(), bucket, "")
+    }
+}
+
 #[cfg(test)]
 static DATA_MOVEMENT_MULTIPART_DISCOVERY_COUNTS: std::sync::OnceLock<std::sync::Mutex<std::collections::HashMap<Uuid, usize>>> =
     std::sync::OnceLock::new();
@@ -612,7 +630,7 @@ impl ECStore {
         }
         if opts.http_preconditions.is_some()
             && !opts.data_movement
-            && let Some(err) = self.check_write_precondition(bucket, object, &opts).await
+            && let Some(err) = self.check_write_precondition(bucket, object, &opts, false).await
         {
             return Err(err);
         }
@@ -1154,12 +1172,31 @@ impl ECStore {
         Ok(())
     }
 
-    async fn check_write_precondition(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Option<StorageError> {
+    async fn check_write_precondition(
+        &self,
+        bucket: &str,
+        object: &str,
+        opts: &ObjectOptions,
+        require_all_pools: bool,
+    ) -> Option<StorageError> {
         let mut lookup_opts = opts.clone();
         let http_preconditions = lookup_opts.http_preconditions?;
         lookup_opts.http_preconditions = None;
         lookup_opts.no_lock = true;
-        let oi = self.get_object_info_for_delete(bucket, object, &lookup_opts).await;
+        let oi = if require_all_pools {
+            lookup_opts.metadata_cache_safe = false;
+            lookup_opts.suppress_read_repair = true;
+            // Only completion uses this branch, after validating bucket and object.
+            let encoded_object = encode_dir_object_ref(object);
+            self.get_latest_object_info_for_conditional_write(bucket, encoded_object.as_ref(), &lookup_opts)
+                .await
+                .and_then(|(info, _)| {
+                    lookup_opts.precondition_check(&info)?;
+                    Ok(info)
+                })
+        } else {
+            self.get_object_info_for_delete(bucket, object, &lookup_opts).await
+        };
         crate::set_disk::SetDisks::evaluate_write_precondition(oi, &http_preconditions, bucket, object, opts)
     }
 
@@ -1173,7 +1210,34 @@ impl ECStore {
         opts: &ObjectOptions,
     ) -> Result<ObjectInfo> {
         check_complete_multipart_args(bucket, object, upload_id)?;
+        let encoded_object = encode_dir_object_ref(object);
+        let object = encoded_object.as_ref();
         let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
+        if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) {
+            // The set-level commit judges the null version this completion
+            // replaces; give it the same authoritative snapshot a PUT carries.
+            let expected_incarnation_id = opts
+                .expected_bucket_incarnation_id
+                .ok_or_else(|| Error::other("multipart completion is missing its bucket incarnation"))?;
+            let lifecycle_fence = opts
+                .bucket_lifecycle_lock_fence
+                .as_ref()
+                .ok_or_else(|| Error::other("multipart completion is missing its bucket lifecycle fence"))?;
+            let snapshot = match opts.object_lock_config_snapshot.as_ref() {
+                Some(snapshot) => Arc::clone(snapshot),
+                None => {
+                    self.object_lock_config_snapshot_under_lifecycle_fence(bucket, lifecycle_fence)
+                        .await?
+                }
+            };
+            if !snapshot.is_valid_for_destructive_put(self.id, bucket, expected_incarnation_id) {
+                return Err(Error::other(
+                    "Object Lock snapshot does not hold valid target bucket generation and configuration fences",
+                ));
+            }
+            snapshot.add_lock_fences(&mut opts);
+            opts.object_lock_config_snapshot = Some(snapshot);
+        }
         if !self.single_pool() {
             opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
         }
@@ -1186,11 +1250,13 @@ impl ECStore {
                 .await;
             return enqueue_transition_after_write(self.as_ref(), result, LcEventSrc::S3CompleteMultipartUpload, opts).await;
         }
-        if opts.http_preconditions.is_some()
-            && !opts.data_movement
-            && let Some(err) = self.check_write_precondition(bucket, object, opts).await
-        {
-            return Err(err);
+        if opts.http_preconditions.is_some() && !opts.data_movement {
+            // Preserve explicitly scoped internal reads and borrowed-lock callers.
+            let require_all_pools =
+                !opts.replication_request && !opts.no_lock && !opts.skip_decommissioned && !opts.skip_rebalancing;
+            if let Some(err) = self.check_write_precondition(bucket, object, opts, require_all_pools).await {
+                return Err(err);
+            }
         }
 
         let result = match self.multipart_upload_pool_idx(bucket, object, upload_id, opts).await {

crates/ecstore/src/store/object.rs

@@ -1352,8 +1352,8 @@ impl RestoreWorkerGuard {
     }
 }
 
-impl Drop for ObjectLockDiagGuard {
-    fn drop(&mut self) {
+impl ObjectLockDiagGuard {
+    fn record_hold_diag(&self) {
         if !self.enabled || self.guard.as_ref().is_released() {
             return;
         }
@@ -1378,6 +1378,12 @@ impl Drop for ObjectLockDiagGuard {
     }
 }
 
+impl Drop for ObjectLockDiagGuard {
+    fn drop(&mut self) {
+        self.record_hold_diag();
+    }
+}
+
 /// A failure to preserve one object generation for a SelectObjectContent read.
 #[derive(Debug, thiserror::Error)]
 #[non_exhaustive]
@@ -3467,22 +3473,53 @@ impl ECStore {
         }
     }
 
+    /// Acquire the per-object write locks a bulk delete needs before any
+    /// mutation, one batched lock round for the whole key set. Bulk delete is
+    /// the only multi-key write-lock holder, and the batched round is
+    /// deadlock-free by construction (one `lock_batch` RPC per lock client,
+    /// global key order at each server), so no optimistic wave or serial
+    /// fallback is needed. Outcomes are per key: a locked-out key fails alone
+    /// and the rest of the batch proceeds.
     async fn acquire_delete_objects_write_locks(
         &self,
         bucket: &str,
         objects: &[ObjectToDelete],
         opts: &mut ObjectOptions,
-    ) -> Result<Vec<ObjectLockDiagGuard>> {
+    ) -> Result<(Vec<ObjectLockDiagGuard>, HashMap<String, StorageError>)> {
         if opts.no_lock || objects.is_empty() {
-            return Ok(Vec::new());
+            return Ok((Vec::new(), HashMap::new()));
         }
 
         let object_names = sorted_unique_delete_object_names(objects);
-        // Lock order: encoded object names are acquired in ascending order, then
-        // the set-layer calls receive no_lock so they do not reacquire them.
+        let acquired = self.pools[0].disk_set[0]
+            .acquire_object_write_locks_batch(bucket, &object_names, get_lock_acquire_timeout())
+            .await;
+
+        let diag_enabled = is_object_lock_diag_enabled();
         let mut guards = Vec::with_capacity(object_names.len());
-        for object in object_names {
-            guards.push(self.acquire_object_write_lock("delete_objects", bucket, object).await?);
+        let mut failures = HashMap::new();
+        for (object, result) in object_names.iter().zip(acquired) {
+            match result {
+                Ok(guard) => {
+                    #[cfg(any(test, feature = "test-util"))]
+                    notify_delete_namespace_pending(bucket);
+                    guards.push(
+                        ObjectLockDiagGuard::new(
+                            guard,
+                            diag_enabled,
+                            "delete_objects",
+                            diag_enabled.then(|| bucket.to_string()),
+                            diag_enabled.then(|| (*object).to_string()),
+                            diag_enabled.then(|| self.pools[0].disk_set[0].locker_owner.clone()),
+                            ObjectLockDiagMode::Write,
+                        )
+                        .with_write_key(bucket, object),
+                    );
+                }
+                Err(error) => {
+                    failures.insert((*object).to_owned(), error);
+                }
+            }
         }
         opts.no_lock = true;
         for signal in guards.iter().filter_map(ObjectLockDiagGuard::lock_lost_signal) {
@@ -3490,7 +3527,7 @@ impl ECStore {
         }
         opts.ensure_namespace_lock_fence();
 
-        Ok(guards)
+        Ok((guards, failures))
     }
 
     pub(super) async fn acquire_object_read_lock_if_needed(
@@ -5637,13 +5674,11 @@ impl ECStore {
         }
 
         if self.single_pool()
-            && !creates_latest_marker
             && !opts.data_movement
             && opts.http_preconditions.is_none()
             && !opts.replication_request
-            && !opts.versioned
             && !opts.version_suspended
-            && opts.version_id.is_none()
+            && !opts.delete_marker
             && !self.is_suspended(0).await
             && !self.is_pool_rebalancing(0).await
         {
@@ -5655,6 +5690,10 @@ impl ECStore {
                 .await
             {
                 Ok(mut res) => {
+                    #[cfg(test)]
+                    if creates_latest_marker {
+                        pause_versioned_delete_marker_after_commit(bucket, object).await;
+                    }
                     res.name = decode_dir_object(res.name.as_str());
                     return Ok(res);
                 }
@@ -6163,10 +6202,21 @@ impl ECStore {
         if current_bucket_incarnation_id.is_some() {
             pause_delete_after_object_lock_snapshot(bucket).await;
         }
-        let _object_lock_guards = Arc::new(match self.acquire_delete_objects_write_locks(bucket, &objects, &mut opts).await {
-            Ok(guards) => guards,
-            Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
-        });
+        let (object_lock_guards, object_lock_failures) =
+            match self.acquire_delete_objects_write_locks(bucket, &objects, &mut opts).await {
+                Ok(result) => result,
+                Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
+            };
+        // Lock outcomes are per key: a locked-out key fails alone and the rest
+        // of the batch proceeds, matching the set layer's per-key lock errors.
+        if !object_lock_failures.is_empty() {
+            for (index, object) in objects.iter().enumerate() {
+                if let Some(error) = object_lock_failures.get(&object.object_name) {
+                    del_errs[index] = Some(error.clone());
+                }
+            }
+        }
+        let _object_lock_guards = Arc::new(object_lock_guards);
         opts.delete_mutation_owner = Some(Arc::new((opts.delete_mutation_owner.take(), _object_lock_guards.clone())));
         #[cfg(any(test, feature = "test-util"))]
         if !_object_lock_guards.is_empty() {
@@ -6286,8 +6336,11 @@ impl ECStore {
         inject_conditional_delete_post_admission_mutations(self, bucket).await;
 
         if !has_conditions {
-            let indices = (0..objects.len()).collect::<Vec<_>>();
-            (del_objects, del_errs, accounting) = self
+            // Keys whose write lock failed are already reported in `del_errs`;
+            // the phase only runs for the keys this batch actually holds, and
+            // only its outcomes replace the outer per-key state.
+            let indices = (0..objects.len()).filter(|idx| del_errs[*idx].is_none()).collect::<Vec<_>>();
+            let (phase_deleted, mut phase_errors, mut phase_accounting) = self
                 .run_batch_delete_phase(
                     bucket,
                     &objects,
@@ -6299,6 +6352,11 @@ impl ECStore {
                     },
                 )
                 .await;
+            for idx in indices {
+                del_objects[idx] = phase_deleted[idx].clone();
+                del_errs[idx] = phase_errors[idx].take();
+                accounting[idx] = phase_accounting[idx].take();
+            }
         } else {
             let mut conditional_actions = HashMap::with_capacity(objects.len());
             let mut conditional_indices = Vec::with_capacity(objects.len());
@@ -12942,11 +13000,12 @@ mod tests {
         ];
         let mut opts = ObjectOptions::default();
 
-        let guards = store
+        let (guards, failures) = store
             .acquire_delete_objects_write_locks("bucket", &objects, &mut opts)
             .await
             .expect("delete object locks should be acquired");
 
+        assert!(failures.is_empty(), "uncontended keys must all lock: {failures:?}");
         assert_eq!(guards.len(), 2, "duplicate object names should share one namespace lock");
         assert!(opts.no_lock, "set layer should not reacquire locks already held by ECStore");
         assert!(
@@ -13578,3 +13637,129 @@ mod missing_key_delete_quorum_tests {
         .await;
     }
 }
+
+/// The bulk-delete lock rounds are batched and per key: one `lock_batch` wave
+/// per lock layer replaces one quorum round-trip per key, a locked-out key
+/// fails alone, and the rest of the batch proceeds (rustfs/backlog#3146).
+#[cfg(test)]
+mod batch_delete_lock_tests {
+    use super::*;
+    use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
+    use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions};
+
+    async fn seeded_store(bucket: &str) -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
+        let (dirs, store) = isolated_store_over_temp_disks().await;
+        init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("fixture bucket");
+        for key in ["a-object", "b-object"] {
+            let mut reader = PutObjReader::from_vec(vec![0x53; 4096]);
+            store
+                .put_object(
+                    bucket,
+                    key,
+                    &mut reader,
+                    &ObjectOptions {
+                        write_completion: crate::object_api::WriteCompletion::TailDrained,
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("seed object");
+        }
+        (dirs, store)
+    }
+
+    fn two_key_batch() -> Vec<ObjectToDelete> {
+        ["b-object", "a-object"]
+            .into_iter()
+            .map(|object_name| ObjectToDelete {
+                object_name: object_name.to_owned(),
+                ..Default::default()
+            })
+            .collect()
+    }
+
+    #[tokio::test]
+    async fn uncontended_batch_deletes_every_key() {
+        let (_dirs, store) = seeded_store("lock-batch-success").await;
+
+        let (deleted, errors, _) = store
+            .delete_objects_with_tier_delete_journal_and_accounting(
+                "lock-batch-success",
+                two_key_batch(),
+                ObjectOptions::default(),
+            )
+            .await;
+        assert!(errors.iter().all(Option::is_none), "batch outcome: {errors:?}");
+        assert_eq!(deleted.len(), 2);
+    }
+
+    #[tokio::test]
+    async fn contended_key_fails_alone_and_the_rest_of_the_batch_proceeds() {
+        let (_dirs, store) = seeded_store("lock-batch-partial").await;
+
+        // Hold one key's write lock: the batched round must fail that key
+        // alone (per-key outcome) while the other key still deletes.
+        let held = store
+            .acquire_object_write_lock("delete_objects", "lock-batch-partial", "a-object")
+            .await
+            .expect("hold contended key");
+        let (deleted, errors, _) = tokio::time::timeout(
+            Duration::from_secs(30),
+            store.delete_objects_with_tier_delete_journal_and_accounting(
+                "lock-batch-partial",
+                two_key_batch(),
+                ObjectOptions::default(),
+            ),
+        )
+        .await
+        .expect("per-key lock outcomes must not block the whole batch");
+
+        assert!(errors[1].is_some(), "contended a-object must fail alone: {errors:?}");
+        assert!(errors[0].is_none(), "b-object must proceed: {errors:?}");
+        assert!(deleted[1].object_name.is_empty(), "failed key must not report success");
+        assert_eq!(deleted[0].object_name, "b-object");
+        drop(held);
+    }
+
+    #[tokio::test]
+    async fn overlapping_batches_reach_per_key_order_without_deadlock() {
+        let (_dirs, store) = seeded_store("lock-batch-overlap").await;
+
+        let first = store.clone();
+        let second = store.clone();
+        let run = |store: Arc<ECStore>| {
+            tokio::spawn(async move {
+                store
+                    .delete_objects_with_tier_delete_journal_and_accounting(
+                        "lock-batch-overlap",
+                        two_key_batch(),
+                        ObjectOptions::default(),
+                    )
+                    .await
+            })
+        };
+        let first = run(first);
+        let second = run(second);
+
+        // Whatever mix of per-key lock outcomes the overlap produces, both
+        // batches must finish: the batched round is deadlock-free by
+        // construction (global key order at each lock server).
+        for batch in [first, second] {
+            let (_, errors, _) = tokio::time::timeout(Duration::from_secs(30), batch)
+                .await
+                .expect("overlapping batches must not deadlock")
+                .expect("delete task");
+            for error in errors.iter().flatten() {
+                let message = error.to_string();
+                assert!(
+                    message.contains("lock") || message.contains("Lock") || message.contains("namespace"),
+                    "overlap may only surface lock outcomes: {message}"
+                );
+            }
+        }
+    }
+}

crates/ecstore/src/store/rebalance.rs

@@ -837,7 +837,29 @@ impl ECStore {
         object: &str,
         opts: &ObjectOptions,
     ) -> Result<(ObjectInfo, usize)> {
-        let suspended_pools = if opts.skip_decommissioned {
+        self.get_latest_object_info_with_pool_policy(bucket, object, opts, false)
+            .await
+    }
+
+    /// Observe every pool for a conditional-write preflight. This does not
+    /// hold the namespace locks needed to prove a condition at publication.
+    pub(super) async fn get_latest_object_info_for_conditional_write(
+        &self,
+        bucket: &str,
+        object: &str,
+        opts: &ObjectOptions,
+    ) -> Result<(ObjectInfo, usize)> {
+        self.get_latest_object_info_with_pool_policy(bucket, object, opts, true).await
+    }
+
+    async fn get_latest_object_info_with_pool_policy(
+        &self,
+        bucket: &str,
+        object: &str,
+        opts: &ObjectOptions,
+        require_all_pools: bool,
+    ) -> Result<(ObjectInfo, usize)> {
+        let suspended_pools = if opts.skip_decommissioned && !require_all_pools {
             let pool_meta = self.pool_meta.read().await;
             Some(
                 (0..self.pools.len())
@@ -853,7 +875,7 @@ impl ECStore {
                 continue;
             }
 
-            if opts.skip_rebalancing && self.is_pool_rebalancing(idx).await {
+            if opts.skip_rebalancing && !require_all_pools && self.is_pool_rebalancing(idx).await {
                 continue;
             }
 
@@ -873,6 +895,9 @@ impl ECStore {
                     });
                 }
                 Err(e) => {
+                    if require_all_pools && !is_err_object_not_found(&e) && !is_err_version_not_found(&e) {
+                        return Err(e);
+                    }
                     candidates.push(LatestObjectInfoCandidate {
                         info: None,
                         idx,

crates/filemeta/src/filemeta.rs

@@ -2070,6 +2070,53 @@ mod test {
         assert!(meta.versions.iter().all(|version| !version.header.free_version()));
     }
 
+    #[test]
+    fn bytes_count_toward_listing_limit_matches_fileinfo_visibility() {
+        let mut meta = FileMeta::new();
+        let mut obj = FileInfo::new("object", 2, 2);
+        obj.version_id = Some(Uuid::new_v4());
+        obj.mod_time = Some(OffsetDateTime::now_utc());
+        meta.add_version(obj.clone()).expect("add object version");
+        let obj_bytes = meta.marshal_msg().expect("marshal object");
+        assert!(FileMeta::bytes_count_toward_listing_limit(&obj_bytes));
+
+        let dm = FileInfo {
+            name: "object".to_string(),
+            version_id: Some(Uuid::new_v4()),
+            deleted: true,
+            mark_deleted: true,
+            mod_time: obj.mod_time.map(|t| t + time::Duration::seconds(1)),
+            ..Default::default()
+        };
+        meta.delete_version(&dm).expect("add delete marker");
+        let dm_bytes = meta.marshal_msg().expect("marshal delete marker");
+        assert!(!FileMeta::bytes_count_toward_listing_limit(&dm_bytes));
+
+        let (_, mut source) = tier_overwrite_fixture(crate::TransitionVersionState::Exact);
+        source.version_id = Some(Uuid::new_v4());
+        let mut transitioned = FileMeta::new();
+        transitioned.add_version(source.clone()).expect("seed transitioned version");
+        let mut del_req = FileInfo {
+            name: "object".to_string(),
+            version_id: source.version_id,
+            mod_time: source.mod_time,
+            ..Default::default()
+        };
+        del_req.set_tier_free_version_id(&Uuid::new_v4().to_string());
+        transitioned
+            .delete_version(&del_req)
+            .expect("delete transitioned version leaving free version");
+        assert!(!FileMeta::bytes_count_toward_listing_limit(
+            &transitioned.marshal_msg().expect("marshal free-only")
+        ));
+
+        transitioned.add_version(obj).expect("add live object version");
+        assert!(FileMeta::bytes_count_toward_listing_limit(
+            &transitioned.marshal_msg().expect("marshal free + live object")
+        ));
+        assert!(FileMeta::bytes_count_toward_listing_limit(b"corrupt-xl-meta"));
+    }
+
     #[test]
     fn add_version_filemata_uses_canonical_equal_time_order() {
         let mod_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid test timestamp");

crates/filemeta/src/filemeta/codec.rs

@@ -324,6 +324,63 @@ impl FileMeta {
         true
     }
 
+    pub(crate) fn buffers_match_strict(self_buf: &[u8], other_buf: &[u8]) -> bool {
+        let Ok((s_meta, _)) = Self::parse_validated_xl_envelope(self_buf) else {
+            return false;
+        };
+        let Ok((o_meta, _)) = Self::parse_validated_xl_envelope(other_buf) else {
+            return false;
+        };
+
+        if s_meta.is_empty() || o_meta.is_empty() {
+            return s_meta.is_empty() && o_meta.is_empty();
+        }
+
+        let Ok((s_len, s_header_ver, _s_meta_ver, s_meta)) = Self::decode_validated_xl_headers(s_meta) else {
+            return false;
+        };
+        let Ok((o_len, o_header_ver, _o_meta_ver, o_meta)) = Self::decode_validated_xl_headers(o_meta) else {
+            return false;
+        };
+
+        if s_len != o_len {
+            return false;
+        }
+
+        let mut s_cur: Cursor<&[u8]> = Cursor::new(s_meta);
+        let mut o_cur: Cursor<&[u8]> = Cursor::new(o_meta);
+        for _ in 0..s_len {
+            let Ok((s_header_raw, _)) = Self::read_next_raw_slices(s_meta, &mut s_cur) else {
+                return false;
+            };
+            let Ok((o_header_raw, _)) = Self::read_next_raw_slices(o_meta, &mut o_cur) else {
+                return false;
+            };
+            if s_header_ver == XL_HEADER_VERSION
+                && o_header_ver == XL_HEADER_VERSION
+                && s_header_raw.len() == 38
+                && o_header_raw.len() == 38
+                && s_header_raw == o_header_raw
+                && parse_canonical_v3_header_fields(s_header_raw).is_some()
+            {
+                continue;
+            }
+            let mut s_header = FileMetaVersionHeader::default();
+            if s_header.unmarshal_v(s_header_ver, s_header_raw).is_err() {
+                return false;
+            }
+            let mut o_header = FileMetaVersionHeader::default();
+            if o_header.unmarshal_v(o_header_ver, o_header_raw).is_err() {
+                return false;
+            }
+            if !Self::headers_match_strict(&s_header, &o_header) {
+                return false;
+            }
+        }
+
+        true
+    }
+
     #[inline]
     fn header_matches_canonical_v3_raw_strict(s_header: &FileMetaVersionHeader, buf: &[u8]) -> Option<bool> {
         if buf.len() != 38 {
@@ -625,6 +682,49 @@ impl FileMeta {
         }
     }
 
+    pub fn bytes_count_toward_listing_limit(buf: &[u8]) -> bool {
+        let Ok((meta, _inline_buf)) = Self::parse_validated_xl_envelope(buf) else {
+            return true;
+        };
+        if meta.is_empty() {
+            return false;
+        }
+        let Ok((versions_len, header_ver, _meta_ver, meta)) = Self::decode_validated_xl_headers(meta) else {
+            return true;
+        };
+        if versions_len == 0 {
+            return false;
+        }
+        let mut cur = Cursor::new(meta);
+        let mut selected: Option<(FileMetaVersionHeader, &[u8])> = None;
+        let mut any_non_hidden = false;
+        for _ in 0..versions_len {
+            let Ok((header, ver_meta)) = Self::read_next_shallow_slice(meta, &mut cur, header_ver) else {
+                return true;
+            };
+            if header.version_type != VersionType::Delete && !header.free_version() {
+                any_non_hidden = true;
+            }
+            if selected.is_some() {
+                continue;
+            }
+            if header.free_version() && header.version_id.is_some_and(|id| !id.is_nil()) {
+                continue;
+            }
+            selected = Some((header, ver_meta));
+        }
+        let Some((header, ver_meta)) = selected else {
+            return false;
+        };
+        if !header.free_version() && header.version_type != VersionType::Delete {
+            return true;
+        }
+        if !any_non_hidden {
+            return false;
+        }
+        FileMetaVersion::decode_fileinfo_from_slice(ver_meta, "", "", true, false).is_err()
+    }
+
     #[hotpath::measure(impl_type = "FileMeta")]
     pub fn marshal_msg(&self) -> Result<Vec<u8>> {
         let mut wr =

crates/filemeta/src/metacache.rs

@@ -352,6 +352,22 @@ impl MetaCacheEntry {
         (prefer, true)
     }
 
+    pub fn matches_strict_borrowed(&mut self, other: &MetaCacheEntry) -> bool {
+        if self.name != other.name {
+            return false;
+        }
+
+        if other.is_dir() || self.is_dir() {
+            return other.is_dir() == self.is_dir();
+        }
+
+        if self.cached.is_none() && other.cached.is_none() {
+            return FileMeta::buffers_match_strict(&self.metadata, &other.metadata);
+        }
+
+        self.matches_strict(other)
+    }
+
     pub fn matches_strict(&mut self, other: &MetaCacheEntry) -> bool {
         if self.name != other.name {
             return false;
@@ -909,6 +925,13 @@ impl MetaCacheEntriesSorted {
 
 const METACACHE_STREAM_VERSION: u8 = 2;
 const METACACHE_READ_CHUNK_SIZE: usize = 64 << 10;
+/// Upper bound on one incremental reservation while accumulating a single
+/// metacache element. Read rounds double from [`METACACHE_READ_CHUNK_SIZE`]
+/// up to this bound so a legitimately large xl.meta does not pay one small
+/// read per chunk, while a corrupt length prefix still cannot force a
+/// reservation larger than the payload that has already arrived (see
+/// rustfs/rustfs#2715 and rustfs/rustfs#9120).
+const METACACHE_READ_MAX_CHUNK_SIZE: usize = 4 << 20;
 /// msgpack `false`: the entry-stream terminator, and the only byte a legacy
 /// reader accepts after it (see `MetacacheWriter::close_with_limit_reached`).
 const METACACHE_TERMINATOR: u8 = 0xc2;
@@ -989,6 +1012,17 @@ impl<W: AsyncWrite + Unpin> MetacacheWriter<W> {
         Ok(())
     }
 
+    pub async fn write_obj_buffered(&mut self, obj: &MetaCacheEntry) -> Result<()> {
+        self.init().await?;
+
+        Self::encode_obj(&mut self.buf, obj)?;
+        if self.buf.len() >= METACACHE_READ_CHUNK_SIZE {
+            self.flush().await?;
+        }
+
+        Ok(())
+    }
+
     pub async fn close(&mut self) -> Result<()> {
         self.close_with_limit_reached(false).await
     }
@@ -1130,13 +1164,14 @@ impl<R: AsyncRead + Unpin> MetacacheReader<R> {
         // A whole xl.meta can legitimately exceed the limit on its individual
         // msgpack values. Grow only as payload arrives, so a corrupt length
         // prefix cannot cause a speculative allocation of the advertised size.
+        let mut chunk = METACACHE_READ_CHUNK_SIZE;
         while self.buf.len() < read_size {
             let start = self.buf.len();
             let extra = if lookahead {
                 let min_chunk = METACACHE_READ_CHUNK_SIZE.saturating_sub(start);
-                (read_size - start).clamp(min_chunk, METACACHE_READ_CHUNK_SIZE)
+                (read_size - start).clamp(min_chunk, chunk)
             } else {
-                (read_size - start).min(METACACHE_READ_CHUNK_SIZE)
+                (read_size - start).min(chunk)
             };
             if let Err(e) = self.buf.try_reserve(extra) {
                 let err = Error::other(format!("metacache stream: buffer allocation of {extra} bytes failed: {e}"));
@@ -1167,6 +1202,7 @@ impl<R: AsyncRead + Unpin> MetacacheReader<R> {
                     return Err(err);
                 }
             }
+            chunk = chunk.saturating_mul(2).min(METACACHE_READ_MAX_CHUNK_SIZE);
         }
 
         self.pos = read_size;
@@ -1282,6 +1318,85 @@ impl<R: AsyncRead + Unpin> MetacacheReader<R> {
         Ok(u32::from_be_bytes(buf.try_into().expect("Slice with incorrect length")))
     }
 
+    pub fn skip_current(&mut self) -> bool {
+        if self.has_current && self.init && self.err.is_none() {
+            self.has_current = false;
+            true
+        } else {
+            false
+        }
+    }
+
+    pub fn try_peek_buffered(&mut self) -> Option<Result<Option<MetaCacheEntry>>> {
+        if !self.init || self.err.is_some() || self.terminated {
+            return None;
+        }
+
+        let avail = &self.buf[self.pos..];
+        let &first = avail.first()?;
+        if first != 0xc3 {
+            return None;
+        }
+
+        let str_marker = *avail.get(1)?;
+        let (str_hdr, str_len) = match str_marker {
+            0xa0..=0xbf => (1usize, usize::from(str_marker & 0x1f)),
+            0xd9 => (2usize, usize::from(*avail.get(2)?)),
+            0xda => (3usize, usize::from(u16::from_be_bytes([*avail.get(2)?, *avail.get(3)?]))),
+            0xdb => (
+                5usize,
+                usize::try_from(u32::from_be_bytes([*avail.get(2)?, *avail.get(3)?, *avail.get(4)?, *avail.get(5)?])).ok()?,
+            ),
+            _ => return None,
+        };
+
+        let bin_offset = 1usize.checked_add(str_hdr)?.checked_add(str_len)?;
+        let bin_marker = *avail.get(bin_offset)?;
+        let (bin_hdr, bin_len) = match bin_marker {
+            0xc4 => (2usize, usize::from(*avail.get(bin_offset + 1)?)),
+            0xc5 => (
+                3usize,
+                usize::from(u16::from_be_bytes([*avail.get(bin_offset + 1)?, *avail.get(bin_offset + 2)?])),
+            ),
+            0xc6 => (
+                5usize,
+                usize::try_from(u32::from_be_bytes([
+                    *avail.get(bin_offset + 1)?,
+                    *avail.get(bin_offset + 2)?,
+                    *avail.get(bin_offset + 3)?,
+                    *avail.get(bin_offset + 4)?,
+                ]))
+                .ok()?,
+            ),
+            _ => return None,
+        };
+
+        let total_len = bin_offset.checked_add(bin_hdr)?.checked_add(bin_len)?;
+        if avail.len() < total_len {
+            return None;
+        }
+
+        let Ok(decoded) = from_utf8(&avail[1 + str_hdr..bin_offset]) else {
+            return None;
+        };
+        let name = decoded.to_owned();
+        let metadata = if bin_len == 0 {
+            Vec::new()
+        } else {
+            avail[bin_offset + bin_hdr..total_len].to_vec()
+        };
+
+        self.pos += total_len;
+        self.reset();
+        self.has_current = true;
+        Some(Ok(Some(MetaCacheEntry {
+            name,
+            metadata,
+            cached: None,
+            reusable: false,
+        })))
+    }
+
     pub async fn skip(&mut self, size: usize) -> Result<()> {
         self.check_init().await?;
 
@@ -2049,6 +2164,76 @@ mod tests {
         }
     }
 
+    /// rustfs/rustfs#9120: a listing stream made of many ordinary entries must
+    /// decode completely even when the aggregate stream exceeds the per-value
+    /// `MAX_MSGP_ELEMENT_SIZE` bound that release 1.0.4 applied to every
+    /// element.
+    #[tokio::test]
+    async fn test_reader_wide_listing_aggregate_exceeds_per_value_limit() {
+        // ~2.9 KiB of metadata per entry matches the per-entry cost observed in
+        // the #9120 production report; 6,000 entries push the stream past 16 MiB.
+        const ENTRIES: usize = 6_000;
+        let metadata = vec![0xab; 2_880];
+        let entries: Vec<MetaCacheEntry> = (0..ENTRIES)
+            .map(|i| MetaCacheEntry {
+                name: format!("bigdir/object-{i:05}"),
+                metadata: metadata.clone(),
+                ..Default::default()
+            })
+            .collect();
+        let mut output = Cursor::new(Vec::new());
+        let mut writer = MetacacheWriter::new(&mut output);
+        writer.write(&entries).await.expect("wide listing must encode");
+        writer.close().await.expect("stream must close");
+        let bytes = output.into_inner();
+        assert!(
+            bytes.len() > 16 << 20,
+            "fixture must exceed the per-value limit, got {} bytes",
+            bytes.len()
+        );
+        let mut reader = MetacacheReader::new(Cursor::new(bytes));
+        let decoded = reader.read_all().await.expect("wide listing must decode");
+        assert_eq!(decoded, entries);
+    }
+
+    /// The #9120 failure shape seen in production: one entry whose metadata is
+    /// an xl.meta that grew past 16 MiB with its version history, listed among
+    /// ordinary neighbors and skipped past without losing the framing.
+    #[tokio::test]
+    async fn test_reader_oversized_entry_among_ordinary_neighbors_stays_listable() {
+        let small = |i: u8| MetaCacheEntry {
+            name: format!("bigdir/object-{i}"),
+            metadata: vec![i; 64],
+            ..Default::default()
+        };
+        let oversized = MetaCacheEntry {
+            name: "bigdir/huge-versions".to_string(),
+            metadata: vec![0xcd; (16 << 20) + 12_345],
+            ..Default::default()
+        };
+        let entries = vec![small(0), small(1), oversized.clone(), small(2), small(3)];
+        let mut output = Cursor::new(Vec::new());
+        let mut writer = MetacacheWriter::new(&mut output);
+        writer.write(&entries).await.expect("mixed listing must encode");
+        writer.close().await.expect("stream must close");
+
+        let mut reader = MetacacheReader::new(Cursor::new(output.into_inner()));
+        assert_eq!(
+            reader.peek().await.expect("first entry").expect("present"),
+            entries[0],
+            "an ordinary entry before the oversized one must decode"
+        );
+        reader.skip(1).await.expect("skip the first entry");
+        assert_eq!(reader.peek().await.expect("second entry").expect("present"), entries[1]);
+        assert_eq!(
+            reader.peek().await.expect("oversized entry").expect("present"),
+            oversized,
+            "an entry past the per-value limit must round-trip whole"
+        );
+        reader.skip(1).await.expect("skipping the oversized entry must stay aligned");
+        assert_eq!(reader.read_all().await.expect("tail must decode"), entries[3..]);
+    }
+
     #[test]
     fn file_info_versions_with_free_versions_includes_persisted_tier_cleanup_records() {
         let version_id = Uuid::new_v4();
@@ -3669,6 +3854,74 @@ mod tests {
         assert!(corrupt_crc_entry.is_latest_delete_marker());
     }
 
+    #[test]
+    fn scanner_and_list_select_same_non_free_version_after_cleanup_record() {
+        let mut fm = FileMeta::load(&crate::test_data::create_realistic_object_xlmeta().expect("realistic xl.meta"))
+            .expect("load realistic xl.meta");
+        let object = fm.versions[0].into_fileinfo("bucket", "object", true).expect("decode object");
+        let mut free_meta_sys = HashMap::new();
+        rustfs_utils::http::insert_bytes(&mut free_meta_sys, rustfs_utils::http::SUFFIX_FREE_VERSION, Vec::new());
+        let free = FileMetaShallowVersion::try_from(FileMetaVersion {
+            version_type: VersionType::Delete,
+            delete_marker: Some(MetaDeleteMarker {
+                version_id: Some(Uuid::from_u128(42)),
+                mod_time: Some(OffsetDateTime::from_unix_timestamp(2_000_000_000).expect("free timestamp")),
+                meta_sys: free_meta_sys,
+            }),
+            write_version: 1,
+            ..Default::default()
+        })
+        .expect("encode cleanup record");
+        fm.versions.insert(0, free);
+        let entry = MetaCacheEntry {
+            name: "object".to_string(),
+            metadata: fm.marshal_msg().expect("encode metadata"),
+            cached: None,
+            reusable: false,
+        };
+        let scanned = fm
+            .get_file_info_versions("bucket", "object", false)
+            .expect("scanner version decode");
+        assert_eq!(scanned.free_versions.len(), 1);
+        assert_eq!(scanned.versions.len(), 1);
+        assert!(!scanned.versions[0].is_latest, "free record occupies the original first position");
+        assert!(!entry.clone().is_latest_delete_marker(), "cleanup record is not a visible delete marker");
+        let listed = entry.to_fileinfo("bucket").expect("LIST version decode");
+        assert_eq!(listed.version_id, object.version_id);
+        assert_eq!(listed.version_id, scanned.versions[0].version_id);
+        assert_eq!(listed.mod_time, scanned.versions[0].mod_time);
+        assert_eq!(listed.size, scanned.versions[0].size);
+        assert!(!listed.deleted);
+
+        let marker_id = Uuid::from_u128(43);
+        fm.add_version(FileInfo {
+            volume: "bucket".to_string(),
+            name: "object".to_string(),
+            version_id: Some(marker_id),
+            deleted: true,
+            mod_time: Some(OffsetDateTime::from_unix_timestamp(2_100_000_000).expect("marker timestamp")),
+            ..Default::default()
+        })
+        .expect("add current delete marker");
+        let mut marker_entry = MetaCacheEntry {
+            name: "object".to_string(),
+            metadata: fm.marshal_msg().expect("encode marker metadata"),
+            cached: None,
+            reusable: false,
+        };
+        let scanned = fm
+            .get_file_info_versions("bucket", "object", false)
+            .expect("scan marker stack");
+        assert_eq!(scanned.versions.len(), 2);
+        assert_eq!(scanned.versions[0].version_id, Some(marker_id));
+        assert!(scanned.versions[0].deleted);
+        assert!(!scanned.versions[1].deleted, "older object must remain hidden by the current marker");
+        assert!(marker_entry.is_latest_delete_marker());
+        let listed = marker_entry.to_fileinfo("bucket").expect("LIST marker version");
+        assert_eq!(listed.version_id, scanned.versions[0].version_id);
+        assert!(listed.deleted);
+    }
+
     #[test]
     fn file_info_versions_and_borrowed_slice_decode_match_cached_and_full_decoders() {
         let mut fm = FileMeta::load(&crate::test_data::create_realistic_object_xlmeta().expect("realistic xl.meta"))
@@ -4257,4 +4510,95 @@ mod tests {
             .expect("resolve mixed empty");
         assert_eq!(res_mixed_empty.cached.expect("cached").versions.len(), 1);
     }
+
+    #[test]
+    fn matches_strict_borrowed_matches_shallow_headers_and_rejects_corrupt_or_divergent_buffers() {
+        let base_bytes = create_real_xlmeta().expect("create real xlmeta");
+        let mut peer_fm = FileMeta::load(&base_bytes).expect("load base xlmeta");
+        // Mutate inner version payload (ver.meta) without changing shallow header.
+        let mut ver = FileMetaVersion::try_from(peer_fm.versions[1].meta.as_slice()).expect("decode version");
+        ver.object.as_mut().expect("object version").erasure_index += 1;
+        peer_fm.versions[1].meta = ver.marshal_msg().expect("marshal peer version");
+        let peer_bytes = peer_fm.marshal_msg().expect("marshal peer xlmeta");
+        assert_ne!(base_bytes, peer_bytes, "peer shard has distinct erasure index payload");
+
+        let mut e1 = MetaCacheEntry {
+            name: "obj-a".to_owned(),
+            metadata: base_bytes.clone(),
+            cached: None,
+            reusable: false,
+        };
+        let e1_clone = e1.clone();
+        let e2 = MetaCacheEntry {
+            name: "obj-a".to_owned(),
+            metadata: peer_bytes,
+            cached: None,
+            reusable: false,
+        };
+        assert!(e1.matches_strict_borrowed(&e2));
+        assert!(e1.matches_strict_borrowed(&e1_clone));
+
+        // Different shallow header must not match.
+        let mut newer_fm = FileMeta::load(&base_bytes).expect("load base xlmeta");
+        newer_fm.versions[0].header.version_id = Some(Uuid::new_v4());
+        let newer_entry = MetaCacheEntry {
+            name: "obj-a".to_owned(),
+            metadata: newer_fm.marshal_msg().expect("marshal newer xlmeta"),
+            cached: None,
+            reusable: false,
+        };
+        assert!(!e1.matches_strict_borrowed(&newer_entry));
+
+        // Identical corrupt buffers must return false, never true via byte-equality shortcut.
+        let mut corrupt = MetaCacheEntry {
+            name: "obj-a".to_owned(),
+            metadata: b"XL2 corrupt-payload".to_vec(),
+            cached: None,
+            reusable: false,
+        };
+        let corrupt_peer = corrupt.clone();
+        assert!(!corrupt.matches_strict_borrowed(&corrupt_peer));
+    }
+
+    #[tokio::test]
+    async fn try_peek_buffered_and_skip_current_match_async_peek_and_skip() {
+        let entries: Vec<_> = (0..5)
+            .map(|i| MetaCacheEntry {
+                name: format!("buffered-key-{i}"),
+                metadata: vec![i as u8; 16],
+                cached: None,
+                reusable: false,
+            })
+            .collect();
+        let mut buf = Cursor::new(Vec::new());
+        let mut writer = MetacacheWriter::new(&mut buf);
+        writer.write(&entries).await.expect("write entries");
+        writer.close_with_limit_reached(false).await.expect("close stream");
+
+        let mut reader = MetacacheReader::new(Cursor::new(buf.into_inner()));
+        assert!(reader.try_peek_buffered().is_none(), "uninitialized reader must defer to async peek");
+        assert!(!reader.skip_current(), "no current entry before first peek");
+
+        let first = reader.peek().await.expect("first peek").expect("entry 0");
+        assert_eq!(first, entries[0]);
+        assert!(reader.skip_current(), "skip_current must consume the peeked entry");
+        assert!(!reader.skip_current(), "second skip_current without peek must return false");
+
+        for expected in &entries[1..] {
+            let next = reader
+                .try_peek_buffered()
+                .expect("remaining entries are already buffered")
+                .expect("valid entry")
+                .expect("present entry");
+            assert_eq!(&next, expected);
+            assert!(reader.skip_current());
+        }
+
+        assert!(
+            reader.try_peek_buffered().is_none(),
+            "stream terminator must defer to async peek for trailer handling"
+        );
+        assert!(reader.peek().await.expect("eof peek").is_none());
+        assert_eq!(reader.limit_reached(), Some(false));
+    }
 }