refactor(ecstore): share the flushable file open for Windows sync sites

Windows flushes only a handle with write access. Three sync sites each carried their own cfg pair for that; route them through one helper that opens read-write on Windows and read-only elsewhere.

.config/e2e-distributed-selection.txt

@@ -1,2 +1,2 @@
-sha256-linux=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
+sha256-linux=348fa5eed9ae58962c06b33283ff4f15122cff97bce9b4eb1e3ffcb227264b0b
-sha256-darwin=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
+sha256-darwin=348fa5eed9ae58962c06b33283ff4f15122cff97bce9b4eb1e3ffcb227264b0b

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=67251cb7d9978d38e899268cac4d7898297ff7e5b5c48c9c1546dc760b93568d
+sha256-darwin=219ac11e720b3f54dfee4d5d5e95171785ba159b2fe01db3037a8f5061c1ad32
-sha256-linux=673da76c2e5710c4622fe2fa32d2b297730895769487fcf1152ee1c2366f32f4
+sha256-linux=2d34698cc8ec52d5b6c37ac612919a1d21f36cd56a1af539a7522a4eea518e26

.config/e2e-nightly-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=3814cdde8cacf4187e18188d342ed70931cc32e268e42e5bb438d565bb4d0f81
+sha256-darwin=e84e04208f19dc0596b10d4d1b799ce43d6cc431415902f9c5458aa8232bc2d8
-sha256-linux=663025a80273579f95ae3f49af02c0afcbfce94df09701c360a13e607b7637db
+sha256-linux=1f9c7add72c44d1c12ef4ac050bb7917431be657c0c90b29f70c3df56f8bbae3

.config/e2e-smoke-selection.txt

@@ -1 +1 @@
-sha256=e9dc896774122a304e2b8b90f28c82d4d4e0af8ded36a82b147b30945ec994e3
+sha256=5bdd4c2eadbc41625ae3184fff34062e000d328108436259b2b7c4fe2531ecaf

.config/nextest.toml

@@ -234,7 +234,7 @@ test-group = 'embedded-test-ports'
 # processes cannot exhaust those budgets. Keep deadlines and assertions intact.
 # The short HTTP fixtures are tracked in rustfs/backlog#2714.
 [[profile.default.overrides]]
-filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
+filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
 threads-required = "num-test-threads"
 
 # This cancellation probe hashes the serving executable before acquiring its
@@ -548,7 +548,7 @@ filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnost
 test-group = 'embedded-test-ports'
 
 [[profile.ci.overrides]]
-filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
+filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
 threads-required = "num-test-threads"
 
 # Preserve the cancellation probe's startup budget and assertions in CI too.
@@ -701,7 +701,7 @@ test-group = 'ecstore-serial-flaky'
 [profile.e2e-smoke]
 default-filter = """
     package(e2e_test) & (
-        test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
+        test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|multipart_upload_id|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
         | test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
         | test(/^reliant::lifecycle::/)
         | test(/^conditional_delete_objects_test::/)
@@ -827,7 +827,7 @@ slow-timeout = { period = "120s", terminate-after = 12, grace-period = "10s" }
 # ---------------------------------------------------------------------------
 # e2e-distributed profile — 4-node 4-disk Actions suite
 # ---------------------------------------------------------------------------
-# Storage-sensitive PR / nightly / dispatch lane owned by
+# Nightly and manual-dispatch lane owned by
 # .github/workflows/e2e-distributed.yml.
 # Each case starts four rustfs processes (and for site replication, two
 # clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
@@ -976,6 +976,12 @@ test-group = 'e2e-vault'
 filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
 threads-required = "num-test-threads"
 
+# The mc fixture uploads hundreds of objects concurrently before measuring listing.
+# Reserve the runner so other server fixtures do not starve its disk operations.
+[[profile.e2e-full.overrides]]
+filter = 'package(e2e_test) & test(=mc_mirror_small_bucket_test::test_mc_mirror_small_bucket_completes_without_list_timeout)'
+threads-required = "num-test-threads"
+
 # Fixed, small contract for every executable server change. These existing
 # scenarios exercise write/read, anonymous denial and retained bytes on restart.
 [profile.core-contract]

.config/release-validation-plan.json

@@ -0,0 +1,77 @@
+{
+  "schema_version": 1,
+  "plan_id": "rustfs-release-validation",
+  "description": "Versioned certification plan for every public RustFS release exit (rustfs/backlog#2662, R01). preview, prerelease and stable all require this explicit plan. Scripts/release/release_gate.py fail-closes publication on any missing, duplicate, stale, substituted or non-PASS required item.",
+  "release_types": ["preview", "prerelease", "stable"],
+  "targets": [
+    {"os": "linux", "arch": "x86_64", "abi": "musl"},
+    {"os": "linux", "arch": "aarch64", "abi": "musl"},
+    {"os": "linux", "arch": "x86_64", "abi": "gnu"},
+    {"os": "linux", "arch": "aarch64", "abi": "gnu"},
+    {"os": "macos", "arch": "aarch64", "abi": "none"},
+    {"os": "windows", "arch": "x86_64", "abi": "msvc"}
+  ],
+  "targets_basis": "Platform matrix of .github/workflows/build.yml (prepare-platform-matrix). gnu and macOS targets build with the pyroscope feature; musl and Windows targets build with no extra features.",
+  "features": ["pyroscope"],
+  "trusted_workflows": [
+    {"repository": "rustfs/s3chaos", "workflow_id": "release-gate.yml"}
+  ],
+  "trusted_workflows_basis": "The shared evidence envelope producer lives in rustfs/s3chaos (G01, fault/release_gate.rs) and its plan/live split workflow is .github/workflows/release-gate.yml there. Only results from these identities may enter a release certification; PR-uploaded JSON never gains release authority. R02 must extend this list when additional verifier workflows land, never widen it to untrusted sources.",
+  "lanes": [
+    {
+      "lane": "identity",
+      "harness_repository": "rustfs/s3chaos",
+      "harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
+      "harness_basis": "Fixed G01 evidence-envelope producer revision (rustfs/s3chaos PR #102 head recorded in rustfs/backlog#2658). R02 re-pins this when the live verifier ships.",
+      "timeout_seconds": 1800,
+      "environment_capabilities": ["process-exec"],
+      "cases": ["artifact-identity", "startup"],
+      "applies_to": {
+        "release_types": ["preview", "prerelease", "stable"],
+        "platforms": "all"
+      },
+      "applies_basis": "Artifact identity and startup checks are mandatory for every release type, preview included, on every shipped target (rustfs/backlog#2662 frozen design item 5)."
+    },
+    {
+      "lane": "upgrade",
+      "harness_repository": "rustfs/s3chaos",
+      "harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
+      "harness_basis": "Same fixed envelope producer as the identity lane; the workflow SHA and the product build SHA are bound separately and need not be equal.",
+      "timeout_seconds": 5400,
+      "environment_capabilities": ["process-exec", "previous-release-artifact"],
+      "cases": [
+        "direct-upgrade",
+        "mixed-version-upgrade",
+        "bucket-config-upgrade",
+        "bucket-config-rollback",
+        "multipart-layout-upgrade",
+        "multipart-layout-baseline"
+      ],
+      "cases_basis": "Case IDs of the pinned previous-release matrix in .github/workflows/e2e-upgrade.yml (source.version/asset/sha256 and the case list mapped there).",
+      "applies_to": {
+        "release_types": ["prerelease", "stable"],
+        "platforms": [{"os": "linux", "arch": "x86_64", "abi": "gnu"}]
+      },
+      "applies_basis": "preview releases use the pre-declared smaller set (identity lane only). The upgrade lane is only exercised on linux-x86_64-gnu, the platform of the pinned source assets."
+    }
+  ],
+  "upgrade_sources": {
+    "basis": "Exactly the two previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
+    "sources": [
+      {
+        "version": "1.0.1",
+        "role": "N-1",
+        "artifact_name": "rustfs-linux-x86_64-gnu-v1.0.1.zip",
+        "artifact_sha256": "9742e183cb0dfb0d48fa909797af572fe47b7f827569036baa4156e0d3958ee5"
+      },
+      {
+        "version": "1.0.0-rc.5",
+        "role": "N-2",
+        "artifact_name": "rustfs-linux-x86_64-gnu-v1.0.0-rc.5.zip",
+        "artifact_sha256": "3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d"
+      }
+    ],
+    "rollback_supported": false,
+    "rollback_basis": "Only the bucket-config-rollback case (rollback_to_previous_release_reads_current_bucket_metadata) is exercised. Full data-plane rollback is not certified and is never a default claim."
+  }
+}

.docker/README.md

@@ -22,9 +22,9 @@ The following files in the project root are essential for Docker operations:
 | File | Description | Usage |
 | :--- | :--- | :--- |
 | **`docker-buildx.sh`** | **Multi-Arch Build Script**<br>Automates building and pushing Docker images for `amd64` and `arm64`. Supports release and dev channels. | `./docker-buildx.sh --push` |
-| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads pre-built binaries from GitHub Releases. | `docker build -t rustfs:latest .` |
+| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads verified RustFS and `rc` binaries from GitHub Releases and bundles `/etc/rustfs/bootstrap`. | `docker build -t rustfs:latest .` |
-| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Useful if you need specific dynamic libraries. | `docker build -f Dockerfile.glibc .` |
+| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Includes the bundled bootstrap configuration. | `docker build -f Dockerfile.glibc .` |
-| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code. Includes build tools. Ideal for local development and CI. | `docker build -f Dockerfile.source .` |
+| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code and bundles `rc` and the default bootstrap configuration. | `docker build -f Dockerfile.source .` |
 
 ### Docker Compose Configurations
 

.github/dependabot.yml

@@ -32,6 +32,8 @@ updates:
       - "overtrue"
       - "majinghe"
     ignore:
+      - dependency-name: "pulsar"
+        versions: [ "6.9.x" ]
       - dependency-name: "object_store"
         versions: [ "0.13.x" ]
       - dependency-name: "libunftp"

.github/scheduled-validations.json

@@ -296,6 +296,32 @@
       "origin": "baseline",
       "alert_contract": "scheduled-validation"
     },
+    {
+      "workflow": ".github/workflows/rustfs-validation-sweep.yml",
+      "name": "RustFS Validation Sweep",
+      "roles": [
+        "periodic",
+        "manual"
+      ],
+      "events": [
+        "schedule",
+        "workflow_dispatch"
+      ],
+      "cadence": "nightly",
+      "timezone": "UTC",
+      "schedule": [
+        "30 19 * * *"
+      ],
+      "owner": "@overtrue",
+      "policy": "active",
+      "monitor": "workflow",
+      "max_age_hours": 60,
+      "result_owner": "rustfs-validation-sweep.yml",
+      "disposition": "canonical validation cases executed in rotating batches",
+      "failure_owner": "workflow",
+      "origin": "baseline",
+      "alert_contract": "scheduled-validation"
+    },
     {
       "workflow": ".github/workflows/btf-prototype-reference.yml",
       "name": "BTF prototype reference",
@@ -743,12 +769,10 @@
       "workflow": ".github/workflows/e2e-distributed.yml",
       "name": "e2e-distributed",
       "roles": [
-        "pr",
         "periodic",
         "manual"
       ],
       "events": [
-        "pull_request",
         "workflow_dispatch",
         "schedule"
       ],
@@ -823,12 +847,11 @@
       "workflow": ".github/workflows/e2e-upgrade.yml",
       "name": "Upgrade Compatibility",
       "roles": [
-        "pr",
         "periodic",
-        "manual"
+        "manual",
+        "release"
       ],
       "events": [
-        "pull_request",
         "push",
         "schedule",
         "workflow_dispatch"

.github/workflows/build.yml

@@ -814,13 +814,18 @@ jobs:
           export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
           export AWS_DEFAULT_REGION="auto"
 
-          # Determine upload path based on build type
+          # Determine upload path based on build type. Release and prerelease
+          # builds only stage their bytes here: the official
+          # artifacts/rustfs/release/ path is written solely by the
+          # certification-gated promote-release-artifacts job, from these same
+          # staged objects. Development builds keep the uncertified dev channel
+          # and never touch official or latest channels.
           if [[ "$BUILD_TYPE" == "development" ]]; then
             R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/dev/"
             echo "📤 Uploading development build to R2 dev directory"
           else
-            R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
+            R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
-            echo "📤 Uploading release build to R2 release directory"
+            echo "📤 Staging release build for certification-gated promotion"
           fi
 
           # Upload all rustfs zip files to R2 using S3-compatible API
@@ -869,6 +874,221 @@ jobs:
             echo "This run does not trigger Docker publishing."
           fi
 
+  # Aggregate the parent candidate batch. This job always collects results and
+  # retains its evidence, including on failure, and every publication job
+  # below requires its success. Missing, stale, substituted or non-PASS
+  # evidence fails closed here: only the trusted verification sources
+  # (rustfs/s3chaos release-gate envelopes, rustfs/backlog#2658) can satisfy
+  # it, so a PR-uploaded JSON set never gains release authority.
+  release-certification:
+    name: Release Certification
+    needs: [ build-check, build-rustfs ]
+    if: ${{ !cancelled() && needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') }}
+    runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      actions: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Download all build artifacts
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          path: ./artifacts
+          pattern: rustfs-*
+          merge-multiple: true
+
+      - name: Write candidate and collect trusted validation results
+        env:
+          # Cross-repo read of the trusted verification run. Falls back to the
+          # workflow token, which cannot read rustfs/s3chaos; a missing token
+          # therefore fail-closes instead of certifying.
+          GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
+          PRODUCT_COMMIT: ${{ github.sha }}
+          VERSION: ${{ needs.build-check.outputs.version }}
+          BUILD_TYPE: ${{ needs.build-check.outputs.build_type }}
+          FEATURES: ${{ needs.build-check.outputs.build_type == 'development' && '' || 'pyroscope' }}
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          mkdir -p results release-gate-output
+          PLAN_SHA256=$(python3 -c "import hashlib,json,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('.config/release-validation-plan.json'))))")
+
+          python3 - "$PLAN_SHA256" > candidate.json <<'PY'
+          import hashlib, json, os, sys, zipfile
+
+          plan_sha256 = sys.argv[1]
+          profile = {"release": "stable", "prerelease": "prerelease", "preview": "preview"}.get(os.environ["BUILD_TYPE"])
+          if profile is None:
+              raise SystemExit("release certification only covers preview, prerelease and stable builds")
+
+          def sha256(path):
+              digest = hashlib.sha256()
+              with open(path, "rb") as handle:
+                  for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+                      digest.update(chunk)
+              return digest.hexdigest()
+
+          deliverables = []
+          for name in sorted(os.listdir("artifacts")):
+              if not name.startswith("rustfs-") or not name.endswith(".zip"):
+                  continue
+              path = os.path.join("artifacts", name)
+              members = zipfile.ZipFile(path).namelist()
+              binary_name = "rustfs.exe" if "rustfs.exe" in members else "rustfs"
+              with zipfile.ZipFile(path) as archive:
+                  binary_bytes = archive.read(binary_name)
+              import tempfile
+              with tempfile.NamedTemporaryFile() as handle:
+                  handle.write(binary_bytes)
+                  handle.flush()
+                  binary_sha = sha256(handle.name)
+              deliverables.append({
+                  "kind": "zip",
+                  "name": name,
+                  "location": "https://github.com/rustfs/rustfs/releases/download/{}/{}".format(os.environ["VERSION"], name),
+                  "sha256": sha256(path),
+                  "size": os.path.getsize(path),
+                  "binary_sha256": binary_sha,
+              })
+          if not deliverables:
+              raise SystemExit("no built deliverables found to certify")
+          candidate = {
+              "schema_version": 1,
+              "product_commit": os.environ["PRODUCT_COMMIT"],
+              "version": os.environ["VERSION"],
+              "release_profile": profile,
+              "features": [f for f in os.environ.get("FEATURES", "").split(",") if f],
+              "plan_sha256": plan_sha256,
+              "validation_batch": {"run_id": os.environ["GITHUB_RUN_ID"], "run_attempt": int(os.environ["GITHUB_RUN_ATTEMPT"])},
+              "deliverables": deliverables,
+          }
+          json.dump(candidate, sys.stdout, indent=2, sort_keys=True)
+          print()
+          PY
+
+          # Trusted verification evidence only: the release-gate envelopes of
+          # rustfs/s3chaos for this exact product commit. Absence blocks below.
+          TRUSTED_REPO="rustfs/s3chaos"
+          TRUSTED_WORKFLOW="release-gate.yml"
+          TRUSTED_RUN=$(gh api "repos/${TRUSTED_REPO}/actions/workflows/${TRUSTED_WORKFLOW}/runs?head_sha=${PRODUCT_COMMIT}&status=success&per_page=1" --jq '.workflow_runs[0].id // empty' || true)
+          if [[ -z "$TRUSTED_RUN" ]]; then
+            echo "❌ No trusted verification run of ${TRUSTED_REPO}/${TRUSTED_WORKFLOW} for ${PRODUCT_COMMIT}; refusing to certify" >&2
+          else
+            gh run download "$TRUSTED_RUN" -R "$TRUSTED_REPO" -n release-validation-evidence -D results || \
+              echo "❌ Trusted verification run ${TRUSTED_RUN} published no release-validation-evidence artifact" >&2
+          fi
+
+          python3 scripts/release/release_gate.py validate \
+            --plan .config/release-validation-plan.json \
+            --candidate candidate.json \
+            --results results \
+            --output release-gate-output
+
+      - name: Upload certification evidence
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: release-certification
+          path: |
+            candidate.json
+            results/
+            release-gate-output/
+          if-no-files-found: warn
+          retention-days: 30
+
+  # The only writer of the official R2 release path. Promotion re-checks the
+  # exact bytes about to be published against the passed certification list
+  # and copies the staged objects byte-for-byte: no rebuild-after-verify.
+  promote-release-artifacts:
+    name: Promote Release Artifacts
+    needs: [ build-check, build-rustfs, release-certification ]
+    # Target-channel lock for the official R2 release path.
+    concurrency:
+      group: release-channel-r2-release
+      cancel-in-progress: false
+    if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      actions: read
+    env:
+      R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
+      R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
+      R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
+      R2_BUCKET: ${{ secrets.R2_BUCKET }}
+      AWS_EC2_METADATA_DISABLED: true
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Download build and certification artifacts
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          path: ./artifacts
+          pattern: rustfs-*
+          merge-multiple: true
+
+      - name: Download certification evidence
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: release-certification
+          path: ./certification
+
+      - name: Verify and promote staged release artifacts
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
+            echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
+            exit 1
+          fi
+
+          for zip_file in artifacts/rustfs-*.zip; do
+            [[ -f "$zip_file" ]] || continue
+            name="$(basename "$zip_file")"
+            python3 scripts/release/release_gate.py verify-promotion \
+              --plan .config/release-validation-plan.json \
+              --candidate certification/candidate.json \
+              --summary certification/release-gate-output \
+              --artifact "$zip_file" \
+              --artifact-name "$name"
+          done
+
+          if ! command -v aws >/dev/null 2>&1; then
+            sudo apt-get update && sudo apt-get install -y awscli
+          fi
+          export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
+          export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
+          export AWS_DEFAULT_REGION="auto"
+
+          STAGING_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
+          RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
+          for zip_file in artifacts/rustfs-*.zip; do
+            [[ -f "$zip_file" ]] || continue
+            name="$(basename "$zip_file")"
+            staged="$(mktemp)"
+            aws s3 cp "${STAGING_PATH}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+            staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
+            built_sha="$(sha256sum "$zip_file" | awk '{print $1}')"
+            if [[ "$staged_sha" != "$built_sha" ]]; then
+              echo "❌ Staged object ${name} does not match the verified bytes" >&2
+              exit 1
+            fi
+            aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+            rm -f "$staged"
+            echo "✅ Promoted ${name} (${built_sha})"
+          done
+
   # Create GitHub Release for every valid release tag, including previews
   create-release:
     name: Create GitHub Release
@@ -1028,6 +1248,10 @@ jobs:
   update-latest-version:
     name: Update Latest Version
     needs: [ build-check, publish-release ]
+    # Target-channel lock for the latest.json version pointer.
+    concurrency:
+      group: release-channel-oss-latest
+      cancel-in-progress: false
     if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
     runs-on: ubuntu-latest
     timeout-minutes: 30
@@ -1068,6 +1292,45 @@ jobs:
           rm -rf "$OSSUTIL_DIR" "$OSSUTIL_ZIP"
           chmod +x "$OSSUTIL_BIN"
 
+          # Candidate/pointer ordering: an older candidate must never overwrite
+          # a newer pointer, even if its promotion job finishes later.
+          CURRENT_LATEST="${RUNNER_TEMP}/current-latest.json"
+          if "$OSSUTIL_BIN" cp oss://rustfs-version/latest.json "$CURRENT_LATEST" 2>/dev/null; then
+            python3 - "$CURRENT_LATEST" "$VERSION" <<'PY'
+          import json
+          import re
+          import sys
+
+          def key(version):
+              match = re.match(r"^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$", version)
+              if not match:
+                  raise SystemExit(3)
+              core = tuple(int(part) for part in match.groups()[:3])
+              prerelease = match.group(4)
+              if prerelease is None:
+                  return core + (1,)
+              return core + (0, tuple(int(p) if p.isdigit() else p for p in re.split(r"[.-]", prerelease)))
+
+          try:
+              current = json.load(open(sys.argv[1], encoding="utf-8")).get("version", "")
+              proposed = sys.argv[2]
+              if current and key(current) > key(proposed):
+                  raise SystemExit(1)
+          except SystemExit:
+              raise
+          except (OSError, ValueError, TypeError):
+              raise SystemExit(3)
+          PY
+            guard_status=$?
+            if [[ "$guard_status" -eq 1 ]]; then
+              echo "Current latest.json points at a newer version; refusing to downgrade" >&2
+              exit 0
+            elif [[ "$guard_status" -ne 0 ]]; then
+              echo "Existing latest.json is unparsable; refusing to overwrite it blindly" >&2
+              exit 1
+            fi
+          fi
+
           # Create latest.json
           cat > latest.json << EOF
           {
@@ -1084,16 +1347,59 @@ jobs:
 
           echo "✅ Updated latest.json for ${RELEASE_TYPE} release $VERSION"
 
-  # Publish release (remove draft status)
+  # Publish release (remove draft status). Requires the release-certification
+  # aggregate: without a complete legal success there is no public write.
   publish-release:
     name: Publish Release
-    needs: [ build-check, create-release, upload-release-assets ]
+    needs: [ build-check, create-release, upload-release-assets, release-certification ]
+    # Target-channel lock: concurrent releases serialize on this channel so an
+    # older candidate cannot interleave its publish with a newer one.
+    concurrency:
+      group: release-channel-github-release
+      cancel-in-progress: false
     if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
     runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
     timeout-minutes: 30
     permissions:
       contents: write
+      actions: read
     steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Download certification evidence
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: release-certification
+          path: ./certification
+
+      - name: Verify release assets against certification
+        env:
+          GH_TOKEN: ${{ github.token }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          TAG="${{ needs.build-check.outputs.version }}"
+          mkdir -p "$RUNNER_TEMP/promotion-assets"
+          gh release download "$TAG" --dir "$RUNNER_TEMP/promotion-assets" --pattern '*.zip'
+          found=0
+          for zip_file in "$RUNNER_TEMP"/promotion-assets/*.zip; do
+            [[ -f "$zip_file" ]] || continue
+            found=1
+            python3 scripts/release/release_gate.py verify-promotion \
+              --plan .config/release-validation-plan.json \
+              --candidate certification/candidate.json \
+              --summary certification/release-gate-output \
+              --artifact "$zip_file" \
+              --artifact-name "$(basename "$zip_file")"
+          done
+          if [[ "$found" -ne 1 ]]; then
+            echo "No release zip assets to verify against the certification" >&2
+            exit 1
+          fi
+
       - name: Publish release
         env:
           GH_TOKEN: ${{ github.token }}

.github/workflows/cache-warm.yml

@@ -96,7 +96,7 @@ env:
   RUST_BACKTRACE: 1
 
 jobs:
-  # Readers: test-and-lint (workspace and ILM), build-rustfs-debug-binary,
+  # Readers: workspace-lint-and-build, workspace-tests (workspace and ILM), build-rustfs-debug-binary,
   # e2e-tests, e2e-full.
   warm-ci-dev:
     name: Warm ci-dev
@@ -146,7 +146,7 @@ jobs:
           if-no-files-found: error
 
       # --all-targets covers the test binaries nextest builds, including
-      # e2e_test, which test-and-lint's own run excludes. The second build adds
+      # e2e_test, which the workspace run excludes. The second build adds
       # the e2e-test-hooks feature resolution that build-rustfs-debug-binary uses
       # and that no lint lane enables.
       - name: Build ci-dev superset

.github/workflows/ci.yml

@@ -113,7 +113,7 @@ jobs:
             printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
             echo "Base CI policy unavailable; running the full matrix."
           fi
-          if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^rehearse_shards=' "$GITHUB_OUTPUT"; then
+          if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=3$' "$GITHUB_OUTPUT"; then
             printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
             echo "Base policy does not cover this workflow contract; retaining the full matrix."
           fi
@@ -160,7 +160,7 @@ jobs:
       - name: Run shared quick checks
         uses: ./.github/actions/quick-checks
 
-  test-and-lint:
+  workspace-lint-and-build:
     name: Workspace Lint and Build
     if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
     needs: [ quick-checks, classify-changes ]
@@ -193,13 +193,13 @@ jobs:
 
       - name: Prepare test evidence
         run: |
-          mkdir -p artifacts/test-and-lint
+          mkdir -p artifacts/workspace
           {
             echo "run_id=${GITHUB_RUN_ID}"
             echo "job=${GITHUB_JOB}"
             echo "runner=${RUNNER_NAME}"
             echo "started_at=$(date --utc --iso-8601=seconds)"
-          } > artifacts/test-and-lint/run-metadata.txt
+          } > artifacts/workspace/run-metadata.txt
 
       # Clippy gates every event, pull requests included. When it ran only after
       # merge, a lint regression merged freely and then failed every main run
@@ -233,11 +233,11 @@ jobs:
 
       - name: Run documentation tests
         run: |
-          mkdir -p artifacts/test-and-lint
+          mkdir -p artifacts/workspace
           set +e
           timeout --verbose --signal=TERM --kill-after=30s 15m \
             python3 scripts/ci_test_artifact.py cargo doc --configuration workspace \
-            2>&1 | tee artifacts/test-and-lint/doctest.log
+            2>&1 | tee artifacts/workspace/doctest.log
           status=${PIPESTATUS[0]}
           {
             echo "command=cargo test --all --exclude e2e_test --doc"
@@ -246,7 +246,7 @@ jobs:
             echo
             echo "Remaining test-related processes:"
             pgrep -af 'cargo|rustdoc|target/.*/deps/' || true
-          } > artifacts/test-and-lint/doctest-diagnostics.txt
+          } > artifacts/workspace/doctest-diagnostics.txt
           exit "${status}"
 
       - name: Upload test reports and diagnostics
@@ -255,7 +255,7 @@ jobs:
         with:
           name: junit-test-and-lint-${{ github.run_number }}-${{ github.run_attempt }}
           path: |
-            artifacts/test-and-lint
+            artifacts/workspace
           retention-days: 3
           if-no-files-found: error
 
@@ -268,7 +268,7 @@ jobs:
 
   workspace-tests:
     name: Workspace tests
-    needs: [ test-and-lint, classify-changes ]
+    needs: [ workspace-lint-and-build, classify-changes ]
     # The archive retains compile-time fixture paths and the large test set
     # requires the same runner image until the hosted-runtime pilot passes.
     runs-on: sm-standard-4
@@ -293,14 +293,14 @@ jobs:
 
       - name: Extract and list workspace tests
         run: |
-          mkdir -p artifacts/test-and-lint
+          mkdir -p artifacts/workspace
           cargo nextest list --profile ci --archive-file "$RUNNER_TEMP/workspace-tests.tar.zst" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json \
-            > artifacts/test-and-lint/core-test-listing.json
+            > artifacts/workspace/core-test-listing.json
-          python3 scripts/check_test_wiring.py --check-core artifacts/test-and-lint/core-test-listing.json
+          python3 scripts/check_test_wiring.py --check-core artifacts/workspace/core-test-listing.json
 
       - name: Run nextest tests
         run: |
-          mkdir -p artifacts/test-and-lint
+          mkdir -p artifacts/workspace
           rm -f target/nextest/ci/junit.xml
           ./scripts/ci/resource_sampler.sh start nextest
           trap './scripts/ci/resource_sampler.sh stop' EXIT
@@ -309,7 +309,7 @@ jobs:
             cargo nextest run --profile ci --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json \
             --workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" \
             --status-level all --final-status-level all \
-            2>&1 | tee artifacts/test-and-lint/nextest.log
+            2>&1 | tee artifacts/workspace/nextest.log
           status=${PIPESTATUS[0]}
           {
             echo "command=workspace archive execution"
@@ -321,13 +321,13 @@ jobs:
             echo
             echo "Kernel OOM / kill events:"
             dmesg -T 2>/dev/null | grep -iE 'oom|out of memory|killed process' | tail -20 || true
-          } > artifacts/test-and-lint/nextest-diagnostics.txt
+          } > artifacts/workspace/nextest-diagnostics.txt
           exit "${status}"
 
       - name: Verify workspace execution completeness
         run: |
-          python3 scripts/ci_test_artifact.py evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
+          python3 scripts/ci_test_artifact.py evidence artifacts/workspace/core-test-listing.json target/nextest/ci/junit.xml
-          ./scripts/check_migration_gate_count.sh evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
+          ./scripts/check_migration_gate_count.sh evidence artifacts/workspace/core-test-listing.json target/nextest/ci/junit.xml
 
       - name: Run ignored ILM integration tests serially
         if: needs.classify-changes.outputs.mode == 'full'
@@ -358,7 +358,7 @@ jobs:
           name: workspace-test-evidence-${{ github.run_attempt }}
           path: |
             target/nextest/ci/junit.xml
-            artifacts/test-and-lint
+            artifacts/workspace
             artifacts/ilm-integration
           retention-days: 3
           if-no-files-found: error
@@ -368,7 +368,7 @@ jobs:
         if: needs.classify-changes.outputs.rehearse_shards == 'true'
         run: |
           python3 scripts/ci_test_shards.py prepare \
-            --listing artifacts/test-and-lint/core-test-listing.json \
+            --listing artifacts/workspace/core-test-listing.json \
             --junit target/nextest/ci/junit.xml --output artifacts/workspace-shard-plan
       - name: Upload same-run shard plan
         if: needs.classify-changes.outputs.rehearse_shards == 'true'
@@ -449,7 +449,7 @@ jobs:
 
   connect-feature-boundaries:
     name: Connect Feature Boundaries
-    if: needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect != 'false' && (github.event_name != 'pull_request' || github.event.action != 'closed')
+    if: needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect != 'false' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
     needs: [ quick-checks, classify-changes ]
     runs-on: sm-standard-4
     timeout-minutes: 90
@@ -497,13 +497,72 @@ jobs:
       - name: Check offline enrollment E2E root boundary
         run: ./scripts/check_offline_enrollment_e2e.sh
 
+  fips-provider:
+    name: FIPS provider boundaries
+    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
+    needs: [ quick-checks, classify-changes ]
+    # This builds the RustFS library test target, which exceeds hosted-runner disk capacity.
+    runs-on: sm-standard-4
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        mode: [default, fips]
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+      CARGO_BUILD_JOBS: "2"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-fips-provider
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'
+
+      - name: Install FIPS build tools
+        if: matrix.mode == 'fips'
+        run: sudo apt-get install -y cmake golang clang libclang-dev
+
+      - name: Verify provider installation
+        shell: bash
+        run: |
+          args=()
+          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
+            args+=(--features fips)
+          fi
+          cargo nextest run --locked -p rustfs-tls-runtime --lib "${args[@]}" --no-tests fail -E 'test(provider_policy_isolated)'
+          cargo nextest run --locked -p rustfs-s3-client --lib "${args[@]}" --no-tests fail -E 'test(overflowing_closed_range_is_rejected_before_network_io)'
+          cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(validate_pulsar)'
+          cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(webhook_provider_policy_isolated)'
+          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
+            cargo nextest run --locked -p rustfs-protocols --lib --features ftps,webdav,fips --no-tests fail -E 'test(protocols_reject_incompatible_host_provider)'
+            cargo nextest run --locked -p rustfs-targets --lib --features fips --no-tests fail -E 'test(fips_postgres_rejects_incompatible_host_provider)'
+            cargo nextest run --locked -p rustfs-s3-client --lib --features fips --no-tests fail -E 'test(fips_transition_rejects_incompatible_host_provider)'
+          fi
+          tests=(provider_installation_isolated prepared_generation_shares_crypto_and_serves_sni_with_mtls)
+          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
+            tests+=(fips_provider_builds_fips_tls_configs fips_provider_tls_handshake_accepts_and_rejects_algorithms)
+          fi
+          for test_name in "${tests[@]}"; do
+            cargo nextest run --locked -p rustfs --lib "${args[@]}" --no-tests fail -E "test($test_name)"
+          done
+          cargo clippy --locked -p rustfs --lib "${args[@]}" -- -D warnings
+
   test-and-lint-rio-v2:
     name: Test and Lint (rio-v2)
-    # Alternate feature configurations run on the nightly schedule, merge
+    # Specialized boundary, hardware and alternate-feature lanes run only on
-    # groups and manual dispatch. The default workspace remains the PR gate and
+    # the nightly schedule and manual dispatch. They do not gate a merge: the
-    # the per-push main baseline: repeating four feature builds on every merge
+    # nightly run reports a regression within a day, and repeating these builds
-    # bought nothing the nightly run does not report within a day.
+    # on every PR revision and merge group bought no earlier signal worth the
-    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["merge_group", "schedule", "workflow_dispatch"]'), github.event_name)
+    # runner occupancy. Workspace lint and tests, the core contract, the debug
+    # build, E2E smoke and the S3 lanes remain the pull-request gate.
+    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
     needs: [ quick-checks, classify-changes ]
     runs-on: sm-standard-4
     timeout-minutes: 90
@@ -540,12 +599,12 @@ jobs:
           # --profile ci so the quarantine list (and its junit flaky markers)
           # covers this leg too; the default profile is the local no-retry
           # profile and silently ignored quarantined flakes here (rustfs#6703).
-          mkdir -p artifacts/test-and-lint
+          mkdir -p artifacts/workspace
           rm -f target/nextest/ci/junit.xml
           ./scripts/ci/resource_sampler.sh start nextest-rio-v2
           trap './scripts/ci/resource_sampler.sh stop' EXIT
           cargo nextest run --profile ci -p rustfs -p rustfs-ecstore --features rio-v2 \
-            2>&1 | tee artifacts/test-and-lint/nextest.log
+            2>&1 | tee artifacts/workspace/nextest.log
           cargo test -p rustfs --doc --features rio-v2
 
       - name: Upload rio-v2 test reports and diagnostics
@@ -555,14 +614,14 @@ jobs:
           name: junit-test-and-lint-rio-v2-${{ github.run_number }}-${{ github.run_attempt }}
           path: |
             target/nextest/ci/junit.xml
-            artifacts/test-and-lint
+            artifacts/workspace
           retention-days: 3
           if-no-files-found: error
 
   test-and-lint-protocols:
     name: "Test and Lint (${{ matrix.features.name }})"
     # Same cadence as test-and-lint-rio-v2 above.
-    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["merge_group", "schedule", "workflow_dispatch"]'), github.event_name)
+    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
     needs: [ quick-checks, classify-changes ]
     runs-on: sm-standard-4
     timeout-minutes: 90
@@ -631,7 +690,7 @@ jobs:
           path: |
             target/nextest/ci/junit.xml
             artifacts/protocol-tests
-            artifacts/test-and-lint
+            artifacts/workspace
           retention-days: 3
           if-no-files-found: warn
 
@@ -783,7 +842,7 @@ jobs:
     # job had neither, so each closed/merged PR really ran the whole io_uring
     # suite (measured 4m17s / 7m19s / 7m31s on runs 30678272341 / 30678117601 /
     # 30662728539) and kept the cancellation run in progress for minutes.
-    if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
+    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
     needs: [ quick-checks, classify-changes ]
     # GitHub-hosted ubuntu-latest runs a recent kernel with io_uring and, unlike
     # a container, applies no seccomp filter that would block io_uring_setup — so
@@ -978,7 +1037,7 @@ jobs:
           path: |
             ${{ runner.temp }}/rustfs-e2e-smoke-logs/
             ${{ runner.temp }}/rustfs-e2e-smoke-list.json
-            artifacts/test-and-lint
+            artifacts/workspace
           if-no-files-found: warn
 
       - name: Upload e2e smoke JUnit report
@@ -1389,12 +1448,13 @@ jobs:
       - classify-changes
       - typos
       - quick-checks
-      - test-and-lint
+      - workspace-lint-and-build
       - workspace-tests
       - workspace-shards
       - workspace-shard-evidence
       - core-contract
       - connect-feature-boundaries
+      - fips-provider
       - test-and-lint-rio-v2
       - test-and-lint-protocols
       - build-rustfs-debug-binary
@@ -1422,10 +1482,11 @@ jobs:
     needs:
       - classify-changes
       - connect-feature-boundaries
+      - fips-provider
       - required-checks
       - typos
       - quick-checks
-      - test-and-lint
+      - workspace-lint-and-build
       - workspace-tests
       - core-contract
       - test-and-lint-rio-v2

.github/workflows/cla.yml

@@ -95,7 +95,7 @@ jobs:
 
       - name: Run CLA Bot
         if: github.event_name != 'merge_group'
-        uses: overtrue/cla-bot@5e5929846a60cfb42b16d0d38e135ef67d94fc88 # v0.0.9
+        uses: overtrue/cla-bot@f054da99d5777e47e1af9933737b9c0c5584be8d # v0.1.0
         with:
           github-token: ${{ github.token }}
           registry-token: ${{ steps.registry-token.outputs.token }}

.github/workflows/docker.yml

@@ -329,13 +329,75 @@ jobs:
           echo "  - Create latest: $create_latest"
           echo "  - Source ref: $source_ref"
 
+  # Consume the trusted release certification for the exact version this
+  # workflow wraps and verify the certified input digests. A manual dispatch
+  # without that certification fails here and pushes no official tag; only
+  # results from the trusted verification sources carry release authority.
+  release-certification:
+    name: Release Certification
+    needs: build-check
+    if: needs.build-check.outputs.should_build == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      actions: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+          ref: ${{ needs.build-check.outputs.source_ref }}
+
+      - name: Download and verify release certification
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
+          TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
+          VERSION: ${{ needs.build-check.outputs.version }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p certification
+          run_id="$TRIGGERING_RUN_ID"
+          if [[ -z "$run_id" ]]; then
+            run_id="$(gh api --method GET \
+              "repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
+              -f branch="$VERSION" -f status=success -F per_page=1 \
+              --jq '.workflow_runs[0].id // empty' || true)"
+          fi
+          if [[ -z "$run_id" ]]; then
+            echo "No certified Build and Release run found for ${VERSION}; failing closed" >&2
+            exit 1
+          fi
+          if ! gh run download "$run_id" -n release-certification -D certification; then
+            echo "Build run ${run_id} published no release-certification artifact; failing closed" >&2
+            exit 1
+          fi
+          python3 scripts/release/release_gate.py verify-promotion \
+            --plan .config/release-validation-plan.json \
+            --candidate certification/candidate.json \
+            --summary certification/release-gate-output \
+            --artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
+            --artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
+
+          # Input digest binding: every zip this image wraps must be the
+          # certified byte-for-byte content, never a mutable latest pointer.
+          while IFS=$'\t' read -r name location expected; do
+            echo "Verifying certified input ${name}"
+            actual="$(curl -fsSL "$location" | sha256sum | awk '{print $1}')"
+            if [[ "$actual" != "$expected" ]]; then
+              echo "Certified input ${name} does not match its recorded digest" >&2
+              exit 1
+            fi
+          done < <(jq -r '.deliverables[] | select(.name | contains("linux")) | [.name, .location, .sha256] | @tsv' certification/candidate.json)
+
   # Build multi-arch Docker images
   # Strategy: Build images using pre-built binaries from dl.rustfs.com
   # Supports both release and dev channel binaries based on build context
   # Only runs when should_build is true (which includes workflow success check)
   build-docker:
     name: Build Docker Images
-    needs: build-check
+    needs: [ build-check, release-certification ]
     if: needs.build-check.outputs.should_build == 'true'
     runs-on: ubuntu-latest
     timeout-minutes: 60
@@ -443,6 +505,14 @@ jobs:
           # Output tags
           echo "tags=$TAGS" >> "$GITHUB_OUTPUT"
 
+          # Staging identity for this run's promotion: official tags are only
+          # created after the staged image verifies against the certification,
+          # and they point at this same digest (no rebuild-after-verify).
+          STAGING_TAG="staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${VARIANT_SUFFIX}"
+          STAGING_TAGS="${{ env.REGISTRY_DOCKERHUB }}:${STAGING_TAG},${{ env.REGISTRY_GHCR }}:${STAGING_TAG},${{ env.REGISTRY_QUAY }}:${STAGING_TAG}"
+          echo "staging_tags=$STAGING_TAGS" >> "$GITHUB_OUTPUT"
+          echo "staging_tag=$STAGING_TAG" >> "$GITHUB_OUTPUT"
+
           # Generate labels
           LABELS="org.opencontainers.image.title=RustFS"
           LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system"
@@ -467,7 +537,9 @@ jobs:
           file: ${{ matrix.file }}
           platforms: ${{ env.DOCKER_PLATFORMS }}
           push: ${{ needs.build-check.outputs.should_push == 'true' }}
-          tags: ${{ steps.meta.outputs.tags }}
+          # Push to staging tags only: official tags are created by the
+          # certification-gated promotion step below from this same digest.
+          tags: ${{ steps.meta.outputs.staging_tags }}
           labels: ${{ steps.meta.outputs.labels }}
           # No layer cache. This build compiles nothing — it downloads a
           # release zip and runs apk/apt — so the cache could only save the
@@ -493,6 +565,114 @@ jobs:
           no-cache: false
           pull: true
 
+      # A later-built wrapper: the staged image must complete this platform's
+      # own binary identity verification before any official tag is written.
+      # It may not inherit an unverified package's conclusion.
+      - name: Verify staged image against certification
+        if: needs.build-check.outputs.should_push == 'true'
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
+          TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
+          VERSION: ${{ needs.build-check.outputs.version }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p certification
+          run_id="$TRIGGERING_RUN_ID"
+          if [[ -z "$run_id" ]]; then
+            run_id="$(gh api --method GET \
+              "repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
+              -f branch="$VERSION" -f status=success -F per_page=1 \
+              --jq '.workflow_runs[0].id // empty' || true)"
+          fi
+          gh run download "$run_id" -n release-certification -D certification
+
+          STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
+          case "${{ matrix.variant }}" in
+            musl) libc="musl" ;;
+            glibc) libc="gnu" ;;
+          esac
+          for platform in linux/amd64 linux/arm64; do
+            case "$platform" in
+              linux/amd64) arch_sub="x86_64" ;;
+              linux/arm64) arch_sub="aarch64" ;;
+            esac
+            docker pull --platform "$platform" "$STAGING_REF"
+            container="$(docker create --platform "$platform" "$STAGING_REF")"
+            docker cp "$container:/usr/bin/rustfs" "$RUNNER_TEMP/staged-rustfs"
+            docker rm "$container" >/dev/null
+            actual="$(sha256sum "$RUNNER_TEMP/staged-rustfs" | awk '{print $1}')"
+            expected="$(jq -r --arg needle "linux-${arch_sub}-${libc}" \
+              '[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
+              certification/candidate.json)"
+            if [[ -z "$expected" || "$actual" != "$expected" ]]; then
+              echo "Staged image binary for ${platform} does not match the certified binary digest" >&2
+              exit 1
+            fi
+            python3 scripts/release/release_gate.py verify-promotion \
+              --plan .config/release-validation-plan.json \
+              --candidate certification/candidate.json \
+              --summary certification/release-gate-output \
+              --artifact "$RUNNER_TEMP/staged-rustfs" \
+              --artifact-role binary
+          done
+
+      - name: Promote staged image to official tags
+        if: needs.build-check.outputs.should_push == 'true'
+        shell: bash
+        run: |
+          set -euo pipefail
+          STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
+          IFS=',' read -r -a official_tags <<< "${{ steps.meta.outputs.tags }}"
+          for tag in "${official_tags[@]}"; do
+            docker buildx imagetools create -t "$tag" "$STAGING_REF"
+            echo "Promoted $tag -> $STAGING_REF"
+          done
+
+      # Downstream consumers (Helm) verify this record instead of inheriting
+      # an unverified wrapper's conclusion.
+      - name: Record image promotion
+        if: needs.build-check.outputs.should_push == 'true'
+        shell: bash
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
+          TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
+          VERSION: ${{ needs.build-check.outputs.version }}
+        run: |
+          set -euo pipefail
+          mkdir -p certification
+          run_id="$TRIGGERING_RUN_ID"
+          if [[ -z "$run_id" ]]; then
+            run_id="$(gh api --method GET \
+              "repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
+              -f branch="$VERSION" -f status=success -F per_page=1 \
+              --jq '.workflow_runs[0].id // empty' || true)"
+          fi
+          gh run download "$run_id" -n release-certification -D certification
+          STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
+          digest="$(docker buildx imagetools inspect "$STAGING_REF" --format '{{json .Manifest}}' | jq -r '.digest')"
+          candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
+          jq -n \
+            --arg version "$VERSION" \
+            --arg digest "$digest" \
+            --arg staging_ref "$STAGING_REF" \
+            --arg candidate_sha256 "$candidate_sha" \
+            --arg tags "${{ steps.meta.outputs.tags }}" \
+            --slurpfile candidate certification/candidate.json \
+            '{schema_version: 1, version: $version, image_digest: $digest, staging_ref: $staging_ref,
+              promoted_tags: ($tags | split(",")), candidate_sha256: $candidate_sha256,
+              binary_sha256s: ($candidate[0].deliverables | map(.binary_sha256) | unique)}' \
+            > image-promotion-record.json
+
+      - name: Upload image promotion record
+        if: needs.build-check.outputs.should_push == 'true'
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: image-promotion-record${{ matrix.suffix }}
+          path: image-promotion-record.json
+          if-no-files-found: error
+          retention-days: 30
+
   # Note: Manifest creation is no longer needed as we only build one variant
   # Multi-arch manifests are automatically created by docker/build-push-action
 

.github/workflows/e2e-distributed.yml

@@ -17,8 +17,8 @@
 # Each selected test starts a real localhost cluster via
 # `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
 # case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
-# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
+# `[profile.e2e-distributed]` in `.config/nextest.toml`. The nightly schedule
-# nightly runs, and manual dispatches all execute the same fail-closed suite.
+# and manual dispatches execute the same fail-closed suite.
 # Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
 #
 # Isolated pool filesystems: expand/decommission/rebalance cases require
@@ -33,32 +33,6 @@
 name: e2e-distributed
 
 on:
-  pull_request:
-    paths:
-      - "Cargo.lock"
-      - "Cargo.toml"
-      - ".config/nextest.toml"
-      - ".config/e2e-distributed-selection.txt"
-      - ".github/workflows/e2e-distributed.yml"
-      - "crates/audit/**"
-      - "crates/common/**"
-      - "crates/config/**"
-      - "crates/e2e_test/**"
-      - "crates/ecstore/**"
-      - "crates/filemeta/**"
-      - "crates/heal/**"
-      - "crates/iam/**"
-      - "crates/lock/**"
-      - "crates/madmin/**"
-      - "crates/notify/**"
-      - "crates/replication/**"
-      - "crates/s3-client/**"
-      - "crates/s3-ops/**"
-      - "crates/s3-types/**"
-      - "crates/scanner/**"
-      - "crates/storage-api/**"
-      - "crates/utils/**"
-      - "rustfs/**"
   workflow_dispatch:
     inputs:
       filter:

.github/workflows/e2e-upgrade.yml

@@ -12,30 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Single-node upgrade and rollback lane.
+#
+# The weekly Monday schedule, version tags, and workflow_dispatch run this
+# suite. Distributed upgrade cases stay on the nightly e2e-distributed lane.
+
 name: Upgrade Compatibility
 
 on:
-  pull_request:
-    paths:
-      - ".github/workflows/e2e-upgrade.yml"
-      - "crates/e2e_test/src/common.rs"
-      - "crates/e2e_test/src/distributed/**"
-      - "crates/e2e_test/src/fake_s3_target/**"
-      - "crates/e2e_test/src/lib.rs"
-      - "crates/e2e_test/src/replication_extension_test.rs"
-      - "crates/e2e_test/src/rpc_fault_proxy.rs"
-      - "crates/e2e_test/src/upgrade_compatibility_test.rs"
-      - "crates/ecstore/**"
-      - "crates/filemeta/**"
-      - "crates/heal/**"
-      - "crates/kms/**"
-      - "crates/storage-api/**"
-      - "rustfs/**"
-      - "Cargo.lock"
   push:
     tags:
       - "[0-9]*.[0-9]*.[0-9]*"
   schedule:
+    # 03:17 UTC every Monday.
     - cron: "17 3 * * 1"
   workflow_dispatch:
 
@@ -44,7 +33,7 @@ permissions:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+  cancel-in-progress: false
 
 env:
   CARGO_TERM_COLOR: always

.github/workflows/helm-package.yml

@@ -30,9 +30,102 @@ permissions:
   contents: read
 
 jobs:
+  # Consume the same trusted release certification as every other publication
+  # exit and verify the image promotion record of the Docker workflow. The
+  # chart wraps no binary of its own, so it may only reference an image whose
+  # platform verification already passed; it must not inherit an unverified
+  # package's conclusion. A run without that evidence fails closed here and
+  # never reaches the chart repository.
+  release-certification:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      actions: read
+    outputs:
+      version: ${{ steps.version.outputs.raw_tag }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Normalize release version
+        id: version
+        env:
+          RAW_INPUT: ${{ github.event.inputs.version }}
+          RAW_BRANCH: ${{ github.event.workflow_run.head_branch }}
+        run: |
+          set -euo pipefail
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            RAW="$RAW_INPUT"
+          else
+            RAW="$RAW_BRANCH"
+          fi
+          case "$RAW" in
+            refs/tags/*) RAW_TAG="${RAW#refs/tags/}" ;;
+            *) RAW_TAG="$RAW" ;;
+          esac
+          echo "raw_tag=$RAW_TAG" >> "$GITHUB_OUTPUT"
+
+      - name: Download and verify release certification and image record
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
+          TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
+          RAW_TAG: ${{ steps.version.outputs.raw_tag }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p certification image-record
+
+          build_run="$TRIGGERING_RUN_ID"
+          if [[ -z "$build_run" ]]; then
+            build_run="$(gh api --method GET \
+              "repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
+              -f branch="$RAW_TAG" -f status=success -F per_page=1 \
+              --jq '.workflow_runs[0].id // empty' || true)"
+          fi
+          if [[ -z "$build_run" ]]; then
+            echo "No certified Build and Release run found for ${RAW_TAG}; failing closed" >&2
+            exit 1
+          fi
+          if ! gh run download "$build_run" -n release-certification -D certification; then
+            echo "Build run ${build_run} published no release-certification artifact; failing closed" >&2
+            exit 1
+          fi
+          python3 scripts/release/release_gate.py verify-promotion \
+            --plan .config/release-validation-plan.json \
+            --candidate certification/candidate.json \
+            --summary certification/release-gate-output \
+            --artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
+            --artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
+
+          docker_run="$(gh api --method GET \
+            "repos/${GITHUB_REPOSITORY}/actions/workflows/docker.yml/runs" \
+            -f branch="$RAW_TAG" -f status=success -F per_page=1 \
+            --jq '.workflow_runs[0].id // empty' || true)"
+          if [[ -z "$docker_run" ]] || ! gh run download "$docker_run" -n image-promotion-record -D image-record; then
+            echo "No image promotion record for ${RAW_TAG}; a chart may not wrap an unverified image" >&2
+            exit 1
+          fi
+          candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
+          record_sha="$(jq -r '.candidate_sha256 // empty' image-record/image-promotion-record.json)"
+          if [[ "$record_sha" != "$candidate_sha" ]]; then
+            echo "Image promotion record does not belong to this certification" >&2
+            exit 1
+          fi
+          overlap="$(jq -r --slurpfile candidate certification/candidate.json \
+            '[.binary_sha256s[]] - ($candidate[0].deliverables | map(.binary_sha256)) | length' \
+            image-record/image-promotion-record.json)"
+          if [[ "$overlap" -ne 0 ]]; then
+            echo "Image promotion record carries binary digests outside the certification" >&2
+            exit 1
+          fi
+
   build-helm-package:
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    needs: [ release-certification ]
     if: |
       (github.event_name == 'workflow_dispatch' && !contains(github.event.inputs.version, '-preview')) ||
       (

.github/workflows/nightly-gnu.yml

@@ -87,18 +87,206 @@ jobs:
           install-build-packaging-tools: 'false'
           install-test-tools: 'false'
 
+      - name: Download static console assets
+        shell: bash
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          # Packaging tools are off on this lane, so setup does not install unzip.
+          if ! command -v unzip >/dev/null 2>&1; then
+            sudo apt-get update -qq
+            sudo apt-get install -y -qq unzip
+          fi
+          if ! command -v unzip >/dev/null 2>&1; then
+            echo "unzip is required to extract console assets" >&2
+            exit 1
+          fi
+
+          mkdir -p ./rustfs/static
+
+          verify_sha256() {
+            local expected="$1"
+            local file="$2"
+            local actual=""
+            local actual_line=""
+
+            if command -v sha256sum >/dev/null 2>&1; then
+              actual_line=$(sha256sum -- "$file") || return 1
+            elif command -v shasum >/dev/null 2>&1; then
+              actual_line=$(shasum -a 256 -- "$file") || return 1
+            else
+              echo "No SHA-256 verification tool found" >&2
+              return 1
+            fi
+            actual="${actual_line%%[[:space:]]*}"
+
+            if [ "$actual" = "$expected" ]; then
+              echo "SHA256 verified OK: $actual"
+              return 0
+            else
+              echo "SHA256 mismatch: expected=$expected actual=$actual" >&2
+              return 1
+            fi
+          }
+
+          download_console_assets() {
+            local curl_bin="curl"
+            local python_bin="python3"
+            local console_api="https://api.github.com/repos/rustfs/console/releases/latest"
+            local console_json="console-release.json"
+            local console_url
+            local console_sha256
+            local curl_auth_args=()
+            console_tag=""
+
+            chmod +w ./rustfs/static/LICENSE || true
+
+            if ! command -v "$python_bin" >/dev/null 2>&1; then
+              python_bin="python"
+            fi
+            if ! command -v "$python_bin" >/dev/null 2>&1; then
+              echo "No Python interpreter found for release metadata parsing" >&2
+              return 1
+            fi
+
+            if [[ -n "${GITHUB_TOKEN:-}" ]]; then
+              curl_auth_args=(-H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28")
+            fi
+
+            "$curl_bin" "${curl_auth_args[@]}" --fail -L "$console_api" \
+              -o "$console_json" --retry 3 --retry-delay 5 --max-time 300 || return 1
+
+            read -r console_tag console_url console_sha256 < <("$python_bin" - "$console_json" <<'PY'
+          import json
+          import re
+          import sys
+
+          with open(sys.argv[1], encoding="utf-8") as handle:
+              release = json.load(handle)
+
+          tag = release.get("tag_name", "") or "unknown"
+
+          for asset in release.get("assets", []):
+              name = asset.get("name", "")
+              digest = asset.get("digest", "")
+              url = asset.get("browser_download_url", "")
+              if name.startswith("rustfs-console-") and name.endswith(".zip") and digest.startswith("sha256:") and url:
+                  sha256 = digest.split(":", 1)[1]
+                  if not re.fullmatch(r"[0-9a-fA-F]{64}", sha256):
+                      raise SystemExit(f"console zip asset has invalid sha256 digest: {sha256}")
+                  sys.stdout.buffer.write(f"{tag} {url} {sha256}\n".encode("utf-8"))
+                  break
+          else:
+              raise SystemExit("no console zip asset with sha256 digest found")
+          PY
+            ) || return 1
+
+            if [[ -z "$console_url" || -z "$console_sha256" ]]; then
+              echo "Console release metadata is missing URL or SHA-256 digest" >&2
+              return 1
+            fi
+
+            echo "Console release: ${console_tag}"
+            echo "Downloading console asset: ${console_url}"
+            "$curl_bin" --fail -L "$console_url" -o console.zip --retry 3 --retry-delay 5 --max-time 300 || return 1
+            verify_sha256 "$console_sha256" console.zip || return 2
+            unzip -o console.zip -d ./rustfs/static || return 2
+          }
+
+          if download_console_assets; then
+            rm -f console.zip console-release.json
+          else
+            status=$?
+            rm -f console.zip console-release.json
+            if [[ "$status" -eq 2 ]]; then
+              echo "Console asset integrity verification failed" >&2
+            fi
+            echo "Failed to download verified console assets" >&2
+            exit 1
+          fi
+
+          if [[ ! -s ./rustfs/static/index.html ]]; then
+            echo "Console asset archive is missing static/index.html" >&2
+            exit 1
+          fi
+
+          asset_count=$(find ./rustfs/static -type f | wc -l | tr -d '[:space:]')
+          echo "Console assets ready: version=${console_tag:-unknown}, ${asset_count} files extracted to ./rustfs/static"
+
       - name: Build RustFS
         # Pin the glibc baseline (`<target>.<glibc>` for cargo-zigbuild) so
         # nightlies run on older distros such as RHEL/Rocky 9 (glibc 2.34); #8062.
-        run: cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bins
+        shell: bash
+        run: |
+          # Force rebuild by touching build.rs so a restored cache cannot
+          # reuse a rustfs artifact compiled before these console assets existed.
+          touch rustfs/build.rs
+          cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bin rustfs
 
       - name: Verify glibc symbol baseline
         run: |
           python3 scripts/release/check_glibc.py 2.31 \
-            target/x86_64-unknown-linux-gnu/release/rustfs \
+            target/x86_64-unknown-linux-gnu/release/rustfs
-            target/x86_64-unknown-linux-gnu/release/rustfs-cli
           target/x86_64-unknown-linux-gnu/release/rustfs --version | grep -Fx 'build target : x86_64-unknown-linux-gnu'
 
+      - name: Verify console
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          find_free_port() {
+            python3 - <<'PY'
+          import socket
+
+          with socket.socket() as sock:
+              sock.bind(("127.0.0.1", 0))
+              print(sock.getsockname()[1])
+          PY
+          }
+
+          binary="target/x86_64-unknown-linux-gnu/release/rustfs"
+          data_dir="$(mktemp -d)"
+          log_file="${RUNNER_TEMP}/rustfs-console-smoke.log"
+          server_pid=""
+          trap '
+            if [[ -n "${server_pid:-}" ]]; then
+              kill "$server_pid" 2>/dev/null || true
+              wait "$server_pid" 2>/dev/null || true
+            fi
+            rm -rf "$data_dir"
+          ' EXIT
+
+          if [[ ! -x "$binary" ]]; then
+            echo "rustfs binary not found: ${binary}" >&2
+            exit 1
+          fi
+
+          api_port="$(find_free_port)"
+          console_port="$(find_free_port)"
+          console_url="http://127.0.0.1:${console_port}/rustfs/console/"
+
+          "$binary" server \
+            --address "127.0.0.1:${api_port}" \
+            --console-enable \
+            --console-address "127.0.0.1:${console_port}" \
+            --access-key console-smoke \
+            --secret-key console-smoke-secret \
+            "$data_dir" >"$log_file" 2>&1 &
+          server_pid=$!
+
+          response=""
+          for _ in {1..40}; do
+            response="$(curl --silent --output /dev/null --write-out '%{http_code} %{content_type}' "$console_url" || true)"
+            if [[ "$response" == 200\ text/html* ]]; then
+              exit 0
+            fi
+            sleep 0.25
+          done
+
+          echo "Console endpoint did not return 200 text/html: $response" >&2
+          cat "$log_file"
+          exit 1
+
       - name: Build DEB package
         id: deb
         shell: bash
@@ -346,12 +534,12 @@ jobs:
           path: ${{ steps.publish.outputs.candidate_file }}
           if-no-files-found: error
 
-      # Publish the deb/rpm pair to the auto-testing repo's `assets` branch so
-      # engineers can download and install the nightly directly. The branch is
-      # a single-commit orphan rewritten on every build, which keeps the repo
-      # small while the latest files stay reachable at stable raw URLs.
       # Publish the deb/rpm pair as assets of the rolling `nightly` release on
       # rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
+      # The dispatch below runs only after this upload and the immutable R2
+      # candidate both succeed. It does not replace rustfs-functional-chain.yml,
+      # which still starts from this workflow's workflow_run completion.
+      # kms-vault-lane and kms-vault-ha-failover stay parallel with this job.
       - name: Publish packages to auto-testing release assets
         env:
           ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
@@ -360,6 +548,90 @@ jobs:
           DEB_DATE: ${{ steps.deb.outputs.deb_date }}
           BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
         run: bash scripts/release/publish_nightly_assets.sh
+
+      # Cross-repo repository_dispatch cannot use GITHUB_TOKEN. The secret is a
+      # PAT or GitHub App token for rustfs/auto-testing: classic `repo` scope,
+      # or fine-grained Contents write (the permission GitHub checks for
+      # POST /repos/{owner}/{repo}/dispatches). A dispatch failure fails this
+      # step, and therefore this job, so the scheduled failure alert fires.
+      # GitHub allows at most 10 top-level client_payload properties, so the
+      # schema 2 candidate object is nested under `candidate` and the release
+      # fields sit beside it.
+      - name: Dispatch nightly candidate to auto-testing
+        if: ${{ steps.publish.outputs.candidate_file != '' }}
+        env:
+          DISPATCH_TOKEN: ${{ secrets.AUTO_TESTING_DISPATCH_TOKEN }}
+          CANDIDATE_FILE: ${{ steps.publish.outputs.candidate_file }}
+          DEB_DATE: ${{ steps.deb.outputs.deb_date }}
+          SOURCE_REPO: ${{ github.repository }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          : "${DISPATCH_TOKEN:?AUTO_TESTING_DISPATCH_TOKEN is required to dispatch rustfs/auto-testing}"
+          : "${CANDIDATE_FILE:?nightly candidate manifest is required}"
+          : "${DEB_DATE:?DEB_DATE is required}"
+          [[ "${DEB_DATE}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]
+          [[ "${SOURCE_REPO}" == "rustfs/rustfs" ]]
+
+          payload="${RUNNER_TEMP}/nightly-candidate-dispatch-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
+          jq -n --slurpfile candidate "${CANDIDATE_FILE}" \
+            --arg event_type "nightly-candidate-ready" \
+            --arg source_repo "${SOURCE_REPO}" \
+            --arg latest_alias_url "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb" \
+            --arg auto_testing_release "https://github.com/rustfs/auto-testing/releases/tag/nightly" \
+            --arg channel "nightly" \
+            --arg deb_date "${DEB_DATE}" \
+            --arg nightly_workflow ".github/workflows/nightly-gnu.yml" \
+            '{
+              event_type: $event_type,
+              source_repo: $source_repo,
+              latest_alias_url: $latest_alias_url,
+              auto_testing_release: $auto_testing_release,
+              channel: $channel,
+              deb_date: $deb_date,
+              nightly_workflow: $nightly_workflow,
+              candidate: $candidate[0]
+            }' > "${payload}"
+          if ! jq -e '
+            .event_type == "nightly-candidate-ready"
+            and .source_repo == "rustfs/rustfs"
+            and .channel == "nightly"
+            and .deb_date == env.DEB_DATE
+            and .nightly_workflow == ".github/workflows/nightly-gnu.yml"
+            and .latest_alias_url == "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb"
+            and .auto_testing_release == "https://github.com/rustfs/auto-testing/releases/tag/nightly"
+            and .candidate.schema == 2
+            and ((.candidate | keys | sort) == ["build_run_attempt","build_run_id","package_sha256","package_url","schema","source_ref","source_sha","workflow_sha"])
+            and ((keys | sort) == ["auto_testing_release","candidate","channel","deb_date","event_type","latest_alias_url","nightly_workflow","source_repo"])
+          ' "${payload}" >/dev/null; then
+            echo "nightly dispatch payload does not match the nightly-candidate-ready contract" >&2
+            exit 1
+          fi
+
+          body="${RUNNER_TEMP}/nightly-candidate-dispatch-body-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
+          jq -n --slurpfile client_payload "${payload}" \
+            '{event_type: "nightly-candidate-ready", client_payload: $client_payload[0]}' > "${body}"
+          response="${RUNNER_TEMP}/nightly-candidate-dispatch-response-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.txt"
+          status="$(curl -sS --retry 3 --connect-timeout 15 --max-time 60 \
+            -o "${response}" -w '%{http_code}' \
+            -X POST \
+            -H "Authorization: Bearer ${DISPATCH_TOKEN}" \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            -H "Content-Type: application/json" \
+            --data-binary @"${body}" \
+            https://api.github.com/repos/rustfs/auto-testing/dispatches)" || {
+            echo "repository_dispatch request to rustfs/auto-testing failed before an HTTP status" >&2
+            exit 1
+          }
+          if [[ "${status}" != "204" ]]; then
+            echo "repository_dispatch to rustfs/auto-testing failed: HTTP ${status}" >&2
+            if [[ -s "${response}" ]]; then
+              cat "${response}" >&2
+            fi
+            exit 1
+          fi
+          echo "dispatched nightly-candidate-ready to rustfs/auto-testing (deb_date=${DEB_DATE}, run=${GITHUB_RUN_ID}, attempt=${GITHUB_RUN_ATTEMPT})"
   # Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
   #
   # RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and

.github/workflows/package.yml

@@ -225,10 +225,57 @@ jobs:
           echo "  Build run ID: $BUILD_RUN_ID"
           echo "  Build run number: $RUN_NUMBER"
 
+  # Consume the trusted release certification of the resolved build run and
+  # verify the certified input digests before any package is wrapped. A run
+  # without that certification fails closed here; packages built later must
+  # complete their own verification and cannot inherit an unverified
+  # conclusion.
+  release-certification:
+    name: Release Certification
+    needs: resolve
+    if: needs.resolve.result == 'success'
+    runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      actions: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Download and verify release certification
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
+          BUILD_RUN_ID: ${{ needs.resolve.outputs.build_run_id }}
+          BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p certification
+          if [[ "$BUILD_TYPE" == "development" ]]; then
+            # Development artifacts are explicitly uncertified and never touch
+            # official or latest channels; only the staging/dev paths remain.
+            jq -n '{schema_version: 1, certified: false, reason: "development artifacts are explicitly uncertified"}' \
+              > certification/development-uncertified.json
+            exit 0
+          fi
+          if ! gh run download "$BUILD_RUN_ID" -n release-certification -D certification; then
+            echo "Build run ${BUILD_RUN_ID} published no release-certification artifact; failing closed" >&2
+            exit 1
+          fi
+          python3 scripts/release/release_gate.py verify-promotion \
+            --plan .config/release-validation-plan.json \
+            --candidate certification/candidate.json \
+            --summary certification/release-gate-output \
+            --artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
+            --artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
+
   # Build DEB and RPM packages for each architecture and libc variant
   package:
     name: Package (${{ matrix.arch }}-${{ matrix.libc }})
-    needs: resolve
+    needs: [ resolve, release-certification ]
     runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
     timeout-minutes: 30
     strategy:
@@ -494,6 +541,55 @@ jobs:
           echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
           echo "✅ RPM built: $RPM_FILE"
 
+      - name: Download certification evidence
+        if: needs.resolve.outputs.build_type != 'development'
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: release-certification
+          path: ./certification
+
+      # Later-built wrapper verification: the binary inside each package must
+      # match the certified binary digest for this platform before anything is
+      # uploaded, and the package itself stays on staging until promotion.
+      - name: Verify wrapped binary against certification
+        if: needs.resolve.outputs.build_type != 'development'
+        shell: bash
+        env:
+          DEB_FILE: ${{ steps.deb.outputs.deb_file }}
+          RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
+          ARTIFACT_NAME: ${{ matrix.artifact_name }}
+        run: |
+          set -euo pipefail
+          expected="$(jq -r --arg needle "$ARTIFACT_NAME" \
+            '[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
+            certification/candidate.json)"
+          if [[ -z "$expected" ]]; then
+            echo "No certified binary digest recorded for ${ARTIFACT_NAME}" >&2
+            exit 1
+          fi
+          if ! command -v rpm2cpio >/dev/null 2>&1; then
+            sudo apt-get update && sudo apt-get install -y rpm2cpio
+          fi
+          if ! command -v cpio >/dev/null 2>&1; then
+            sudo apt-get update && sudo apt-get install -y cpio
+          fi
+          for package_file in "$DEB_FILE" "$RPM_FILE"; do
+            case "$package_file" in
+              *.deb)
+                dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
+                ;;
+              *.rpm)
+                rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
+                ;;
+            esac
+            python3 scripts/release/release_gate.py verify-promotion \
+              --plan .config/release-validation-plan.json \
+              --candidate certification/candidate.json \
+              --summary certification/release-gate-output \
+              --artifact "$RUNNER_TEMP/wrapped-rustfs" \
+              --artifact-role binary
+          done
+
       - name: Upload packages to artifacts
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
@@ -531,10 +627,14 @@ jobs:
           export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
           export AWS_DEFAULT_REGION="auto"
 
+          # Build-phase writes stay on staging for release types: the official
+          # packages/release and packages/latest paths are written only by the
+          # certification-gated promote-packages job, from these same staged
+          # objects. Development packages keep the uncertified dev channel.
           if [[ "$BUILD_TYPE" == "development" ]]; then
             R2_PREFIX="artifacts/rustfs/packages/dev"
           else
-            R2_PREFIX="artifacts/rustfs/packages/release"
+            R2_PREFIX="artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}"
           fi
           R2_PATH="s3://${R2_BUCKET}/${R2_PREFIX}/"
 
@@ -549,18 +649,6 @@ jobs:
 
           echo "✅ Upload complete"
 
-          # Also upload as latest for release/prerelease
-          if [[ "$BUILD_TYPE" == "release" || "$BUILD_TYPE" == "prerelease" ]]; then
-            LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
-            for f in "$DEB_FILE" "$RPM_FILE"; do
-              if [[ -n "$f" && -f "$f" ]]; then
-                echo "Uploading latest: $(basename "$f")"
-                aws s3 cp "$f" "$LATEST_PATH" --endpoint-url "$R2_ENDPOINT" --only-show-errors
-              fi
-            done
-            echo "✅ Latest packages updated"
-          fi
-
       - name: Upload packages to GitHub Release
         if: needs.resolve.outputs.tag != ''
         env:
@@ -614,6 +702,114 @@ jobs:
 
           echo "✅ GitHub release assets updated"
 
+  # The only writer of the official packages/release and packages/latest R2
+  # paths. Promotion re-checks each package's wrapped binary against the
+  # passed certification list and copies the staged objects byte-for-byte:
+  # no rebuild-after-verify, no mutable latest resolution downstream.
+  promote-packages:
+    name: Promote Packages
+    needs: [ resolve, package, release-certification ]
+    # Target-channel lock for the official package paths.
+    concurrency:
+      group: release-channel-r2-packages
+      cancel-in-progress: false
+    if: needs.resolve.outputs.tag != '' && (needs.resolve.outputs.build_type == 'release' || needs.resolve.outputs.build_type == 'prerelease')
+    runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      actions: read
+    env:
+      R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
+      R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
+      R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
+      R2_BUCKET: ${{ secrets.R2_BUCKET }}
+      AWS_EC2_METADATA_DISABLED: true
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Download packages and certification
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          path: ./packages
+          pattern: packages-*
+          merge-multiple: true
+
+      - name: Download certification evidence
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: release-certification
+          path: ./certification
+
+      - name: Verify and promote staged packages
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
+            echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
+            exit 1
+          fi
+
+          STAGING_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
+          RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/release/"
+          LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
+
+          if ! command -v aws >/dev/null 2>&1; then
+            sudo apt-get update && sudo apt-get install -y awscli
+          fi
+          export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
+          export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
+          export AWS_DEFAULT_REGION="auto"
+
+          if ! command -v rpm2cpio >/dev/null 2>&1; then
+            sudo apt-get install -y rpm2cpio
+          fi
+          if ! command -v cpio >/dev/null 2>&1; then
+            sudo apt-get install -y cpio
+          fi
+
+          promoted=0
+          for package_file in packages/*.deb packages/*.rpm; do
+            [[ -f "$package_file" ]] || continue
+            name="$(basename "$package_file")"
+            case "$package_file" in
+              *.deb)
+                dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
+                ;;
+              *.rpm)
+                rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
+                ;;
+            esac
+            python3 scripts/release/release_gate.py verify-promotion \
+              --plan .config/release-validation-plan.json \
+              --candidate certification/candidate.json \
+              --summary certification/release-gate-output \
+              --artifact "$RUNNER_TEMP/wrapped-rustfs" \
+              --artifact-role binary
+
+            staged="$(mktemp)"
+            aws s3 cp "${STAGING_PREFIX}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+            staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
+            built_sha="$(sha256sum "$package_file" | awk '{print $1}')"
+            if [[ "$staged_sha" != "$built_sha" ]]; then
+              echo "❌ Staged package ${name} does not match the verified bytes" >&2
+              exit 1
+            fi
+            aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+            aws s3 cp "$staged" "${LATEST_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+            rm -f "$staged"
+            promoted=$((promoted + 1))
+            echo "✅ Promoted ${name} (${built_sha})"
+          done
+          if [[ "$promoted" -eq 0 ]]; then
+            echo "No staged packages found to promote" >&2
+            exit 1
+          fi
+
   # Summary
   summary:
     name: Summary

.github/workflows/rustfs-validation-sweep.yml

@@ -0,0 +1,457 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: RustFS Validation Sweep
+
+# Executes the migrated canonical validation cases (rustfs/auto-testing#113)
+# on the shared functional fleet. Everything with a lane executor is swept in
+# rotating nightly batches so the whole set is covered roughly weekly; a
+# single case or one module can be dispatched manually. That is the
+# implemented catalog (66 today) plus the eight planned cases whose dedicated
+# executors already exist and have live PASS evidence - fresh nightly
+# evidence feeds their acceptance review. heal.* cases stay on the
+# interactive validation flow: they require reviewed native/tool collectors
+# built by separate validation-tools runs before fleet execution.
+
+on:
+  schedule:
+    # After the nightly functional chain settles (chain starts ~16:48 UTC and
+    # runs for about two hours).
+    - cron: "30 19 * * *"
+  workflow_dispatch:
+    inputs:
+      case:
+        description: 'Single canonical validation case id (overrides module and rotation), e.g. pool.decommission'
+        type: string
+        required: false
+        default: ''
+      module:
+        description: 'Restrict the sweep to one catalog module, e.g. bucket-replication'
+        type: string
+        required: false
+        default: ''
+      limit:
+        description: 'Maximum cases per sweep (0 = run everything selected)'
+        type: number
+        required: false
+        default: 12
+      build_run_id:
+        description: 'Successful nightly package run; empty resolves the latest'
+        type: string
+        required: false
+        default: ''
+      build_run_attempt:
+        description: 'Attempt of that package run'
+        type: string
+        required: false
+        default: '1'
+
+permissions:
+  contents: read
+
+concurrency:
+  group: rustfs-validation-sweep
+  cancel-in-progress: false
+
+env:
+  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES || 'vm000 vm001 vm002 vm003' }}
+  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER || 'azureuser' }}
+  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
+  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
+  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
+  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+jobs:
+  sweep:
+    if: github.repository == 'rustfs/rustfs'
+    runs-on: smoke-testing
+    timeout-minutes: 150
+    steps:
+      - name: Checkout chain tooling
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Resolve nightly candidate
+        id: candidate
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
+          CHAIN_OUTPUT: ${{ runner.temp }}/validation-sweep-candidate.json
+        run: |
+          set -euo pipefail
+          # Scheduled runs carry no inputs and the resolver only derives a
+          # build from a workflow_run payload (which a schedule cannot
+          # provide), so resolve the newest successful nightly build
+          # explicitly before invoking it.
+          if [ -z "${BUILD_RUN_ID:-}" ]; then
+            LATEST="$(gh api "repos/rustfs/rustfs/actions/workflows/nightly-gnu.yml/runs?branch=main&status=success&per_page=1" --jq '.workflow_runs[0]')"
+            BUILD_RUN_ID="$(printf '%s' "${LATEST}" | jq -r '.id')"
+            BUILD_RUN_ATTEMPT="$(printf '%s' "${LATEST}" | jq -r '.run_attempt')"
+            export BUILD_RUN_ID BUILD_RUN_ATTEMPT
+            echo "scheduled sweep pinned to nightly ${BUILD_RUN_ID} attempt ${BUILD_RUN_ATTEMPT}"
+          fi
+          python3 scripts/resolve_functional_candidate.py
+          python3 - <<'PY'
+          import json
+          import os
+          from pathlib import Path
+          root = Path(os.environ["RUNNER_TEMP"]) / "validation-sweep"
+          root.mkdir(parents=True, exist_ok=True)
+          chain = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
+          manifest = chain["candidate"]["manifest"]
+          (root / "candidate.json").write_text(json.dumps({
+              "package_url": manifest["package_url"],
+              "package_sha256": manifest["package_sha256"],
+              "source_sha": manifest["source_sha"],
+          }, sort_keys=True) + "\n")
+          with open(os.environ["GITHUB_OUTPUT"], "a") as output:
+              output.write("testing_sha=" + chain["testing_sha"] + "\n")
+              output.write("package_url=" + manifest["package_url"] + "\n")
+              output.write("source_sha=" + manifest["source_sha"] + "\n")
+          PY
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.candidate.outputs.testing_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
+
+      - name: Prepare validation Python runtime
+        # The pool / ilm / replication validation runners require a managed
+        # Python, exactly like the chain's validation job provides.
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.12"
+
+      - name: Install validation Python requirements
+        # Case scripts call python3 and import boto3 at startup. Install the
+        # functional chain's client into a per-run env so that interpreter
+        # sees it without changing the runner's shared Python.
+        run: |
+          set -euo pipefail
+          validation_python="${RUNNER_TEMP}/rustfs-validation-python-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          test ! -e "${validation_python}"
+          python3 -m venv "${validation_python}"
+          "${validation_python}/bin/python" -m pip install \
+            -r auto-testing/scripts/validation_ilm/requirements.txt
+          printf '%s\n' "${validation_python}/bin" >> "${GITHUB_PATH}"
+
+      - name: Select sweep cases
+        id: select
+        env:
+          SELECTED_CASE: ${{ inputs.case }}
+          SELECTED_MODULE: ${{ inputs.module }}
+          CASE_LIMIT: ${{ inputs.limit }}
+          SWEEP_EVENT: ${{ github.event_name }}
+          SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
+        run: |
+          set -euo pipefail
+          python3 - <<'PY'
+          import datetime
+          import json
+          import os
+          from pathlib import Path
+          sweep_root = Path(os.environ["SWEEP_ROOT"])
+
+          catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
+          def sweepable(case):
+              status = case.get("source_status")
+              if status == "implemented":
+                  return True
+              # The remaining planned non-heal cases route to the lane
+              # executors and already have live PASS evidence; sweeping them
+              # nightly produces fresh evidence for their acceptance review.
+              return status == "planned" and not case["id"].startswith("heal.")
+
+          selected = [(case["id"], module["id"]) for module in catalog["modules"]
+                      for case in module["cases"]
+                      if sweepable(case)]
+          if not selected:
+              raise SystemExit("catalog has no sweepable cases")
+
+          one_case = os.environ.get("SELECTED_CASE", "").strip()
+          module_filter = os.environ.get("SELECTED_MODULE", "").strip()
+          known = {case_id for case_id, _ in selected}
+          modules = {module_id for _, module_id in selected}
+          if one_case:
+              if one_case not in known:
+                  raise SystemExit(f"unknown case or no lane executor for it: {one_case}")
+              selected = [(one_case, dict(selected)[one_case])]
+          elif module_filter:
+              if module_filter not in modules:
+                  raise SystemExit(f"unknown module or no implemented cases: {module_filter}")
+              selected = [pair for pair in selected if pair[1] == module_filter]
+          elif os.environ.get("SWEEP_EVENT") == "schedule":
+              # Stateless rotation: every night take one window of the catalog
+              # so a full sweep needs len(selected)/limit nights and no state
+              # has to be carried between runs.
+              limit = max(1, int(os.environ.get("CASE_LIMIT") or 12))
+              day = datetime.datetime.now(datetime.timezone.utc).timetuple().tm_yday
+              start = (day * limit) % len(selected)
+              selected = (selected[start:] + selected[:start])[:limit]
+          limit = int(os.environ.get("CASE_LIMIT") or 0)
+          if one_case or module_filter or (not os.environ.get("SWEEP_EVENT") == "schedule"):
+              if limit > 0:
+                  selected = selected[:limit]
+
+          (sweep_root / "sweep-cases.txt").write_text("".join(f"{case_id}\n" for case_id, _ in selected))
+          with open(os.environ["GITHUB_OUTPUT"], "a") as output:
+              output.write(f"count={len(selected)}\n")
+          print(f"selected {len(selected)} case(s):", ", ".join(c for c, _ in selected))
+          PY
+
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
+        run: python3 scripts/functional_environment.py enter --suite heal
+
+      - name: Run validation cases
+        id: run_cases
+        timeout-minutes: 120
+        env:
+          AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
+          SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
+        run: |
+          set -uo pipefail
+          mkdir -p "${SWEEP_ROOT}/results"
+          : > "${SWEEP_ROOT}/results.txt"
+          while read -r CASE_ID; do
+            [ -n "${CASE_ID}" ] || continue
+            echo "::group::case ${CASE_ID}"
+            rm -rf "${SWEEP_ROOT}/run"
+            if ./auto-testing/scripts/run_validation_case.sh \
+                 "${CASE_ID}" "${SWEEP_ROOT}/candidate.json" \
+                 "${AUTO_TESTING_SHA}" "${SWEEP_ROOT}/run" \
+                 > "${SWEEP_ROOT}/results/${CASE_ID//\//_}.log" 2>&1; then
+              printf '%s\tPASS\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
+              echo "case ${CASE_ID}: PASS"
+            else
+              printf '%s\tFAIL\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
+              echo "case ${CASE_ID}: FAIL (log retained in the sweep artifact)"
+            fi
+            if [ -f "${SWEEP_ROOT}/run/report.md" ]; then
+              cp "${SWEEP_ROOT}/run/report.md" "${SWEEP_ROOT}/results/${CASE_ID//\//_}.report.md" 2>/dev/null || true
+            fi
+            echo "::endgroup::"
+            # Fleet hygiene between cases: the case runners set up their own
+            # topology, this purge guarantees the next case starts clean even
+            # when the previous one died mid-run.
+            for node in ${RUSTFS_NODES}; do
+              # -n keeps ssh from consuming the case list this loop reads from stdin
+              ssh -n -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
+                SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+                ${SUDO} systemctl stop rustfs 2>/dev/null || true
+                if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                  ${SUDO} dpkg -P rustfs
+                fi
+                for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+                ${SUDO} rm -f /etc/default/rustfs
+              ' || echo "inter-case cleanup failed on ${node} (continuing)"
+            done
+          done < "${SWEEP_ROOT}/sweep-cases.txt"
+          echo "sweep finished: $(wc -l < "${SWEEP_ROOT}/results.txt") case(s) executed"
+
+      - name: Generate sweep report
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        env:
+          SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
+          PACKAGE_URL: ${{ steps.candidate.outputs.package_url }}
+          SOURCE_SHA: ${{ steps.candidate.outputs.source_sha }}
+          AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
+        run: |
+          set -euo pipefail
+          python3 - <<'PY'
+          import json
+          import os
+          from pathlib import Path
+
+          root = Path(os.environ["SWEEP_ROOT"])
+          catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
+          module_of = {case["id"]: module["id"] for module in catalog["modules"] for case in module["cases"]}
+          rows = []
+          for line in (root / "results.txt").read_text().splitlines():
+            if not line:
+                continue
+            case_id, status = line.split("\t")
+            rows.append((case_id, module_of.get(case_id, "?"), status))
+          passed = sum(1 for _, _, s in rows if s == "PASS")
+          failed = len(rows) - passed
+          run_url = (f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/"
+                     f"{os.environ.get('GITHUB_REPOSITORY', 'rustfs/rustfs')}/actions/runs/"
+                     f"{os.environ.get('GITHUB_RUN_ID', '')}")
+
+          report = [
+              "# RustFS validation sweep report", "",
+              f"- Run: {run_url}",
+              f"- Attempt: {os.environ.get('GITHUB_RUN_ATTEMPT', '')}",
+              f"- Trigger: {os.environ.get('GITHUB_EVENT_NAME', '')}",
+              f"- Package: {os.environ.get('PACKAGE_URL', 'nightly (R2 latest)')}",
+              f"- Source SHA: {os.environ.get('SOURCE_SHA', '')}",
+              f"- auto-testing: {os.environ.get('AUTO_TESTING_SHA', '')}", "",
+              "Canonical validation cases (rustfs/auto-testing#113) executed by the sweep, grouped by catalog module.", "",
+          ]
+          # Group the executed cases by catalog module (the major category:
+          # pool lifecycle, heal, ILM expiration, bucket/site replication, kms,
+          # data services, identity, s3 compatibility). Per-module subtotals
+          # deliberately use "- Subtotal:" so the dashboard overview regexes
+          # (which anchor on "- Total:") keep reading the grand totals below.
+          by_module = {}
+          module_order = []
+          for case_id, module, status in rows:
+              if module not in by_module:
+                  by_module[module] = []
+                  module_order.append(module)
+              by_module[module].append((case_id, status))
+          report += ["## Case Summary", ""]
+          for module in module_order:
+              cases = by_module[module]
+              mod_pass = sum(1 for _, st in cases if st == "PASS")
+              mod_fail = len(cases) - mod_pass
+              report += [f"### {module} ({len(cases)})", "",
+                         "| Case | Status |", "| --- | --- |"]
+              report += [f"| {case_id} | {status} |" for case_id, status in cases]
+              report += ["", f"- Subtotal: {mod_pass} passed, {mod_fail} failed", ""]
+          report += [
+              "## Totals", "",
+              f"- Total: {len(rows)}",
+              f"- PASS: {passed}",
+              f"- FAIL: {failed}",
+              "- UNSUPPORTED: 0", "",
+              "Failing cases keep this workflow red only as a report; the sweep artifact",
+              "retains every case log and per-case report for diagnosis.", "",
+          ]
+          (root / "report.md").write_text("\n".join(report))
+          (root / "cases.md").write_text("\n".join(f"| {c} | {m} | {s} |" for c, m, s in rows) + "\n")
+          PY
+          cat "${SWEEP_ROOT}/report.md" >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Upload functional report to dashboard
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 2
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: ${{ runner.temp }}/validation-sweep/report.md
+          SUITE: validation
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
+            exit 0
+          fi
+          DATE="$(date -u +%Y-%m-%d)"
+          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
+          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${SHA}" ]; then
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          else
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          fi
+          echo "dashboard updated: ${REPORT_PATH}"
+
+      - name: Retain sweep evidence
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: rustfs-validation-sweep-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            ${{ runner.temp }}/validation-sweep/report.md
+            ${{ runner.temp }}/validation-sweep/results.txt
+            ${{ runner.temp }}/validation-sweep/sweep-cases.txt
+            ${{ runner.temp }}/validation-sweep/candidate.json
+            ${{ runner.temp }}/validation-sweep/results/
+            ${{ runner.temp }}/validation-sweep/run/
+          # warn (not error): an early infra failure legitimately leaves no
+          # per-case evidence, and the failing step is already red.
+          if-no-files-found: warn
+          retention-days: 30
+
+      - name: Cleanup environment (after)
+        id: environment_cleanup
+        timeout-minutes: 5
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        run: |
+          set -uo pipefail
+          cleanup_status=0
+          for node in ${RUSTFS_NODES}; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -f /etc/default/rustfs
+            ' || cleanup_status=$?
+          done
+          exit "${cleanup_status}"
+
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 scripts/functional_environment.py finish --suite heal --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Notify on failure
+        if: failure()
+        run: echo "RustFS validation sweep failed (harness/environment breakdown); see the sweep artifact for case logs."
+
+      - name: Enforce sweep result
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        env:
+          SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
+        run: |
+          set -euo pipefail
+          if grep -q $'\tFAIL$' "${SWEEP_ROOT}/results.txt"; then
+            echo "validation sweep has failing cases:"
+            grep $'\tFAIL$' "${SWEEP_ROOT}/results.txt"
+            exit 1
+          fi
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    # A scheduled sweep failure usually means a canonical validation case
+    # regressed or the fleet environment broke; surface it as a tracking
+    # issue instead of leaving it in the Actions tab.
+    needs: [sweep]
+    if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/windows-filesystem.yml

@@ -107,6 +107,32 @@ jobs:
         shell: pwsh
         run: cargo test -p rustfs-ecstore --lib multipart_owned_record_parent_barriers_ -- --nocapture
 
+      # Native multipart completion publishes its payload from a different
+      # parent than its metadata, and abort renames under a precondition; both
+      # need their own Windows handle discipline.
+      - name: Test native multipart complete and abort publication
+        shell: pwsh
+        run: >-
+          cargo test -p rustfs-ecstore --lib --
+          native_abort_missing_state_with_actual_owned_records_never_becomes_legacy_or_missing
+          native_abort_pending_intent_is_not_a_delete_certificate_and_raw_cleanup_preserves_exact_evidence
+          --nocapture
+
+      # Owned-undo reclaim removes a directory it pinned, copy heal persists
+      # fault bytes through a handle, and metadata reads meet a directory.
+      - name: Test native multipart reclaim, copy heal, and directory reads
+        shell: pwsh
+        run: >-
+          cargo test -p rustfs-ecstore --lib --
+          native_multipart_owned_undo
+          native_multipart_copy_heal
+          orphan_multipart_cleanup_rejects_metadata_directory_on_one_disk
+          --nocapture
+
+      - name: Test directory removal fallback
+        shell: pwsh
+        run: "cargo test -p rustfs-ecstore --lib disk::fs::tests::test_remove -- --nocapture"
+
       - name: Test Windows path conversion
         shell: pwsh
         run: "cargo test -p rustfs-utils --no-default-features --features path --lib dunce:: -- --nocapture"

CHANGELOG.md

@@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - IAM snapshot/deletion replay, target-assigned delete-marker purges, timestamp ordering, and best-effort peer broadcast now close the control-plane gaps found by the R6 review ([#7195](https://github.com/rustfs/rustfs/pull/7195)).
 - A `set-remote-target?update=true&creds=true` request on a site-replication peer's bucket target is now rejected with `400 InvalidRequest`, naming the target and why it counts as peer-owned. It used to return success while keeping the stored endpoint, credentials, and TLS settings, so an endpoint edit from the Console's bucket replication form appeared to apply but did not. Change a peer's address with a site-replication edit instead; see "Changing a site's address" in `docs/operations/site-replication-operations.md`. The `sync`, `bandwidth`, `proxy`, and `path` update groups are unchanged.
 - Site-replication edits that omit the site name, such as `mc admin replicate update --deployment-id <ID> --endpoint <URL>`, now keep the stored name on the editing site and on every receiver instead of renaming the site after the new endpoint host. No persisted format changes.
+- The persisted replication MRF journal is now replayed every 5 minutes while the server runs, not only at startup. Delete-marker purges whose targets were unreachable, and replications that overflowed the worker queues, now converge within one replay interval after the target returns instead of waiting for a restart. A target that stays down is re-attempted once per retained entry per interval across the cluster: the start time of the last pass is shared through a small `config/replication/mrf.bin.replay` object, so nodes do not each replay once per interval. Older versions ignore that object; existing persisted formats are unchanged.
 - Upgrade and rollback: upgrade every node in one site consecutively and verify reconciliation before moving to the next site; do not intentionally run a site mixed-version. Target-version ledger keys are harmless on rollback, although old code cannot use their routing. Before rolling back past [#7307](https://github.com/rustfs/rustfs/pull/7307), drain or repair every pending version purge: older code can free a retained version's data directory before its remote purge is acknowledged. See `docs/operations/site-replication-operations.md`.
 
 ### Security

Cargo.toml

@@ -147,7 +147,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
 async-channel = "2.5.0"
 async_zip = { default-features = false, version = "0.0.19" }
 mysql_async = { default-features = false, version = "0.37.1" }
-async-compression = { version = "0.4.51" }
+async-compression = { version = "0.4.52" }
 async-trait = "0.1.92"
 async-nats = { version = "0.51.0", default-features = false }
 axum = "0.8.9"
@@ -168,16 +168,16 @@ http-body-util = "0.1.5"
 memchr = "2.8.3"
 minlz = "1.2.3"
 reqwest = "0.13.5"
-rustfs-kafka-async = { version = "1.4.0" }
+rustfs-kafka-async = { version = "1.4.1" }
 socket2 = { version = "0.6.5" }
 tokio = { version = "1.54.1" }
 tokio-rustls = { default-features = false, version = "0.26.6" }
 tokio-stream = { version = "0.1.19" }
 tokio-test = "0.4.6"
 tokio-util = { version = "0.7.20" }
-tonic = { version = "0.14.8" }
+tonic = { version = "0.14.9" }
-tonic-prost = { version = "0.14.8" }
+tonic-prost = { version = "0.14.9" }
-tonic-prost-build = { version = "0.14.8" }
+tonic-prost-build = { version = "0.14.9" }
 tower = { version = "0.5.3" }
 tower-http = { version = "0.7.1" }
 h3 = "0.0.8"
@@ -191,6 +191,7 @@ quinn = { version = "0.11.12", default-features = false, features = [
 # Serialization and Data Formats
 apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
 bytes = { version = "1.12.1" }
+csv-core = "0.1.13"
 bytesize = "2.7.0"
 byteorder = "1.5.0"
 flatbuffers = "25.12.19"
@@ -212,12 +213,13 @@ serde_urlencoded = "0.7.1"
 # releases.
 aes-gcm = { version = "0.11.1" }
 argon2 = { version = "0.6.0" }
+aws-lc-rs = { version = "1.19.0", default-features = false }
 blake2 = "0.11.0"
 chacha20poly1305 = { version = "0.11.0" }
 crc-fast = "1.10.0"
 hmac = { version = "0.13.0" }
 jsonwebtoken = { version = "11.1.0" }
-openidconnect = { default-features = false, version = "4.0" }
+openidconnect = { default-features = false, version = "4.1.0" }
 pbkdf2 = "0.13.0"
 p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
 rsa = { version = "=0.10.0-rc.18" }
@@ -234,7 +236,7 @@ proptest = "1"
 # Time and Date
 chrono = { version = "0.4.45" }
 humantime = "2.4.0"
-jiff = { version = "0.2.38" }
+jiff = { version = "0.2.39" }
 time = { version = "0.3.55" }
 
 # Database
@@ -253,9 +255,9 @@ atoi = "3.1.1"
 atomic_enum = "0.3.0"
 aws-config = { version = "1.13.0" }
 aws-credential-types = { version = "1.3.1" }
-aws-sdk-kms = { default-features = false, version = "1.128.0" }
+aws-sdk-kms = { default-features = false, version = "1.129.0" }
-aws-sdk-s3 = { default-features = false, version = "1.156.0" }
+aws-sdk-s3 = { default-features = false, version = "1.157.0" }
-aws-sdk-sts = { default-features = false, version = "1.123.0" }
+aws-sdk-sts = { default-features = false, version = "1.124.0" }
 aws-smithy-async = { version = "1.3.1" }
 aws-smithy-http-client = { default-features = false, version = "1.5.2" }
 aws-smithy-runtime-api = { version = "1.19.2" }
@@ -319,7 +321,7 @@ rustfs-erasure-codec = { version = "9.0.0" }
 reed-solomon-simd = "3.1.0"
 regex = { version = "1.13.1" }
 rumqttc = { package = "rumqttc-next", version = "0.34.0" }
-redis = { version = "1.8.0" }
+redis = { version = "1.8.1" }
 rustify = { version = "0.7", default-features = false }
 rustix = { version = "1.1.6" }
 rust-embed = { version = "8.13.0" }
@@ -334,10 +336,10 @@ compact_str = "0.10.0"
 snap = "1.1.2"
 starshard = { version = "2.4.0" }
 strum = { version = "0.28.0" }
-sysinfo = "0.40.0"
+sysinfo = "0.40.1"
 temp-env = "0.3.6"
 tempfile = "3.27.0"
-test-case = "3.4.0"
+test-case = "3.4.1"
 thiserror = "2.0.21"
 tracing = { version = "0.1.44" }
 tracing-appender = "0.2.5"
@@ -356,13 +358,13 @@ winapi-util = "0.1.11"
 windows = { version = "0.62.2" }
 windows-sys = "0.61.2"
 xxhash-rust = { version = "0.8.19" }
-zip = "9.1.3"
+zip = "9.3.0"
 zstd = "0.14.3"
 
 # Observability and Metrics
 metrics = "0.24.6"
 metrics-util = "0.20"
-dial9 = { version = "0.5.6", default-features = false }
+dial9 = { version = "0.5.7", default-features = false }
 opentelemetry = { version = "0.33.1" }
 opentelemetry-appender-tracing = { version = "0.33.1" }
 opentelemetry-otlp = { version = "0.33.1" }
@@ -377,16 +379,16 @@ libunftp = { version = "0.24.0" }
 unftp-core = "0.1.0"
 suppaftp = { version = "12.2.1" }
 rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
-russh = { version = "0.65.0" }
+russh = { version = "0.65.1" }
 russh-sftp = "3.0.1"
 
 # WebDAV
 dav-server = "0.11.0"
 
 # Performance Analysis and Memory Profiling
-rustfs-mimalloc = { version = "0.7.2" }
+rustfs-mimalloc = { version = "0.7.3" }
 # Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
-hotpath = { version = "0.30.1", default-features = false }
+hotpath = { version = "0.30.3", default-features = false }
 # Snapshot testing for output format regression detection
 insta = { version = "1.49" }
 
@@ -427,7 +429,3 @@ strip = "none"
 # Pin upstream fixes until compatible crates.io releases are available.
 [patch.crates-io]
 zstd = { path = "crates/zstd-compat" }
-# Preserve RustFS OIDC flows while migrating the old elliptic-curve dependency chain.
-# Upstream: openidconnect-rs#244 and oauth2-rs#352.
-openidconnect = { git = "https://github.com/ramosbugs/openidconnect-rs", rev = "2f77ee73bc4554e1cb6279adb386005429fb4e41" }
-oauth2 = { git = "https://github.com/ramosbugs/oauth2-rs", rev = "89585f4a280afd7b285eaf1c9be0068874e03d22" }

Dockerfile

@@ -16,6 +16,7 @@ FROM alpine:3.24.2 AS build
 
 ARG TARGETARCH
 ARG RELEASE=latest
+ARG RC_RELEASE=latest
 
 RUN apk add --no-cache ca-certificates curl jq unzip
 WORKDIR /build
@@ -69,6 +70,9 @@ RUN set -eux; \
     chmod +x /build/rustfs; \
     rm -rf rustfs.zip /build/.tmp || true
 
+COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
+RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /build/rc
+
 
 FROM alpine:3.24.2
 
@@ -95,14 +99,18 @@ RUN apk upgrade --no-cache && \
     ca-certificates \
     coreutils \
     curl \
+    jq \
     tzdata \
     && test "$(TZ=Asia/Kolkata date +%z)" = "+0530"
 
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=build /build/rustfs /usr/bin/rustfs
-COPY entrypoint.sh /entrypoint.sh
+COPY --from=build /build/rc /usr/bin/rc
+COPY entrypoint.sh /etc/rustfs/entrypoint.sh
+COPY bootstrap/ /etc/rustfs/bootstrap/
 
-RUN chmod +x /usr/bin/rustfs /entrypoint.sh
+RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
+    ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
 
 RUN addgroup -g 10001 -S rustfs && \
     adduser -u 10001 -G rustfs -S rustfs -D && \
@@ -124,6 +132,6 @@ VOLUME ["/data"]
 
 USER rustfs
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
 
 CMD ["rustfs"]

Dockerfile.decommission-local

@@ -12,7 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM rust:1.97.1-trixie
+FROM rust:1.99.1-trixie
+
+ARG TARGETARCH
+ARG RC_RELEASE=latest
 
 RUN set -eux; \
     export DEBIAN_FRONTEND=noninteractive; \
@@ -21,6 +24,7 @@ RUN set -eux; \
     build-essential \
     ca-certificates \
     curl \
+    jq \
     git \
     pkg-config \
     libssl-dev \
@@ -36,6 +40,7 @@ COPY . .
 RUN ./scripts/static.sh
 RUN cargo run --bin gproto
 RUN cargo build --release --locked --bin rustfs
+RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
 
 RUN set -eux; \
     groupadd -g 10001 rustfs; \
@@ -50,8 +55,12 @@ RUN set -eux; \
     chown -R rustfs:rustfs /data /logs /app; \
     chmod 0750 /data /logs
 
-COPY entrypoint.sh /entrypoint.sh
+COPY entrypoint.sh /etc/rustfs/entrypoint.sh
-RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && chmod +x /entrypoint.sh
+COPY bootstrap/ /etc/rustfs/bootstrap/
+RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && \
+    install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
+    chmod +x /etc/rustfs/entrypoint.sh && \
+    ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
 
 ENV RUSTFS_VOLUMES="/data" \
     RUST_LOG="warn" \
@@ -63,5 +72,5 @@ ENV RUSTFS_VOLUMES="/data" \
 
 EXPOSE 9000 9001
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
 CMD ["/usr/bin/rustfs"]

Dockerfile.glibc

@@ -16,6 +16,7 @@ FROM ubuntu:26.04 AS build
 
 ARG TARGETARCH
 ARG RELEASE=latest
+ARG RC_RELEASE=latest
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     ca-certificates \
@@ -76,6 +77,9 @@ RUN set -eux; \
     chmod +x /build/rustfs; \
     rm -rf rustfs.zip /build/.tmp || true
 
+COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
+RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" gnu "$RC_RELEASE" /build/rc
+
 FROM ubuntu:26.04
 
 ARG RELEASE=latest
@@ -99,14 +103,18 @@ RUN apt-get update && apt-get upgrade -y \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
     ca-certificates \
     curl \
+    jq \
     tzdata \
     && test "$(TZ=Asia/Kolkata date +%z)" = "+0530" \
     && rm -rf /var/lib/apt/lists/*
 
 COPY --from=build /build/rustfs /usr/bin/rustfs
-COPY entrypoint.sh /entrypoint.sh
+COPY --from=build /build/rc /usr/bin/rc
+COPY entrypoint.sh /etc/rustfs/entrypoint.sh
+COPY bootstrap/ /etc/rustfs/bootstrap/
 
-RUN chmod +x /usr/bin/rustfs /entrypoint.sh
+RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
+    ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
 
 RUN groupadd -g 10001 rustfs && \
     useradd -u 10001 -g rustfs -m -s /sbin/nologin rustfs && \
@@ -131,6 +139,6 @@ VOLUME ["/data"]
 
 USER rustfs
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
 
 CMD ["rustfs"]

Dockerfile.source

@@ -28,17 +28,19 @@ ARG TARGETPLATFORM
 ARG BUILDPLATFORM
 ARG TARGETARCH
 ARG RUSTFS_BUILD_FEATURES=""
+ARG RC_RELEASE=latest
 
 # -----------------------------
 # Build stage
 # -----------------------------
-FROM rust:1.97.1-trixie AS builder
+FROM rust:1.99.1-trixie AS builder
 
 # Re-declare args after FROM
 ARG TARGETPLATFORM
 ARG BUILDPLATFORM
 ARG TARGETARCH
 ARG RUSTFS_BUILD_FEATURES
+ARG RC_RELEASE
 
 # Debug: print platforms
 RUN echo "Build info -> BUILDPLATFORM=${BUILDPLATFORM}, TARGETPLATFORM=${TARGETPLATFORM}"
@@ -52,6 +54,7 @@ RUN set -eux; \
     build-essential \
     ca-certificates \
     curl \
+    jq \
     git \
     pkg-config \
     libssl-dev \
@@ -166,6 +169,8 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
     ;; \
     esac
 
+RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
+
 # -----------------------------
 # Development stage (keeps toolchain)
 # -----------------------------
@@ -199,10 +204,13 @@ ENV RUSTFS_VOLUMES="/data" \
 # We rely on cargo run to build and run
 EXPOSE 9000 9001
 
-COPY entrypoint.sh /entrypoint.sh
+COPY entrypoint.sh /etc/rustfs/entrypoint.sh
-RUN chmod +x /entrypoint.sh
+COPY bootstrap/ /etc/rustfs/bootstrap/
+RUN install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
+    chmod +x /etc/rustfs/entrypoint.sh && \
+    ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
 CMD ["cargo", "run", "--bin", "rustfs", "--"]
 
 # -----------------------------
@@ -227,6 +235,7 @@ RUN set -eux; \
     apt-get install -y --no-install-recommends \
     ca-certificates \
     curl \
+    jq \
     tzdata \
     coreutils; \
     rm -rf /var/lib/apt/lists/*
@@ -248,8 +257,11 @@ RUN set -eux; \
 
 # Copy the freshly built binary and the entrypoint
 COPY --from=builder /usr/local/bin/rustfs /usr/bin/rustfs
-COPY entrypoint.sh /entrypoint.sh
+COPY --from=builder /usr/local/bin/rc /usr/bin/rc
-RUN chmod +x /usr/bin/rustfs /entrypoint.sh
+COPY entrypoint.sh /etc/rustfs/entrypoint.sh
+COPY bootstrap/ /etc/rustfs/bootstrap/
+RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
+    ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
 
 # Default environment (override in docker run/compose as needed)
 ENV RUSTFS_VOLUMES="/data" \
@@ -263,5 +275,5 @@ EXPOSE 9000
 VOLUME ["/data"]
 
 # Keep root here; entrypoint will drop privileges using chroot --userspec
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
 CMD ["/usr/bin/rustfs"]

bootstrap/README.md

@@ -0,0 +1,21 @@
+# RustFS image bootstrap
+
+The official image bundles this directory at `/etc/rustfs/bootstrap`. The default `buckets.json` is an empty JSON array, so the image starts RustFS directly and does not require a bootstrap endpoint or bootstrap credentials.
+
+To create buckets, replace `buckets.json` in a derived image with a JSON array of bucket names. The entrypoint creates each bucket if it does not exist and applies `public-access-block.json`, whose default blocks all four public access paths. Edit that file in the derived image only when public access is an explicit requirement.
+
+Optional per-bucket files use the bucket name as the filename without an extension:
+
+| Directory | File format | Applied operation |
+| --- | --- | --- |
+| `policies/` | S3 bucket policy JSON | `rc bucket anonymous set-json` |
+| `versioning/` | JSON boolean (`true` or `false`) | Enable or suspend bucket versioning |
+| `lifecycle/` | `rc bucket lifecycle rule import` JSON | Replace lifecycle rules |
+| `cors/` | JSON or XML accepted by `rc bucket cors set` | Replace CORS rules |
+| `notifications/` | JSON array of `{ "arn": "...", "events": ["..."] }` | Add or replace event rules by ARN (prefix/suffix filters are not currently accepted) |
+
+Every per-bucket file must correspond to a bucket listed in `buckets.json`. Applying these files is repeatable: bucket creation ignores existing buckets, while each configuration operation replaces or reconciles that bucket's requested state.
+
+When `buckets.json` is non-empty, set `RUSTFS_BOOTSTRAP_ENDPOINT` to the RustFS S3 endpoint reachable from the container, for example `http://rustfs:9000` for a Compose service or the internal service DNS name in Kubernetes. The entrypoint waits on that endpoint's `/health/ready`; it never assumes `localhost`. `RUSTFS_BOOTSTRAP_TIMEOUT` controls the readiness wait in seconds (default `120`, maximum `3600`). `RUSTFS_BOOTSTRAP_DIR` can point to a different in-image configuration directory; it defaults to `/etc/rustfs/bootstrap`. Set `RUSTFS_ACCESS_KEY` and `RUSTFS_SECRET_KEY`, or their `_FILE` variants, for bootstrap authentication. The endpoint and credentials are read at runtime and are not baked into the image.
+
+RustFS built-in IAM policies remain unchanged; the default bootstrap installs no custom bucket policies, lifecycle rules, versioning, CORS rules, or notification rules. The bundled RustFS CLI is resolved from a published release and its release checksum is verified. Set the Docker build argument `RC_RELEASE` to a version to pin it. Build the RustFS image after a CLI release containing the `bucket public-access-block` commands is available. For Kubernetes, the configured service must route traffic to the starting pod before its readiness probe succeeds (for example, with `publishNotReadyAddresses: true`); otherwise the service endpoint cannot reach the pod while the entrypoint waits for readiness.

bootstrap/buckets.json

@@ -0,0 +1 @@
+[]

bootstrap/cors/.gitkeep

@@ -0,0 +1 @@
+

bootstrap/lifecycle/.gitkeep

@@ -0,0 +1 @@
+

bootstrap/notifications/.gitkeep

@@ -0,0 +1 @@
+

bootstrap/policies/.gitkeep

@@ -0,0 +1 @@
+

bootstrap/public-access-block.json

@@ -0,0 +1,6 @@
+{
+  "block_public_acls": true,
+  "ignore_public_acls": true,
+  "block_public_policy": true,
+  "restrict_public_buckets": true
+}

bootstrap/versioning/.gitkeep

@@ -0,0 +1 @@
+

crates/config/src/constants/proxy.rs

@@ -30,10 +30,10 @@ pub const ENV_TRUSTED_PROXY_VALIDATION_MODE: &str = "RUSTFS_TRUSTED_PROXY_VALIDA
 /// Default validation mode is "hop_by_hop".
 pub const DEFAULT_TRUSTED_PROXY_VALIDATION_MODE: &str = "hop_by_hop";
 
-/// Environment variable to enable RFC 7239 "Forwarded" header support.
+/// Select RFC 7239 "Forwarded" instead of "X-Forwarded-For" in legacy mode.
 pub const ENV_TRUSTED_PROXY_ENABLE_RFC7239: &str = "RUSTFS_TRUSTED_PROXY_ENABLE_RFC7239";
-/// RFC 7239 support is enabled by default.
+/// Legacy mode uses X-Forwarded-For unless RFC 7239 is explicitly selected.
-pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = true;
+pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = false;
 
 /// Environment variable for the maximum allowed proxy hops.
 pub const ENV_TRUSTED_PROXY_MAX_HOPS: &str = "RUSTFS_TRUSTED_PROXY_MAX_HOPS";

crates/config/src/constants/targets.rs

@@ -146,3 +146,9 @@ pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";
 
 /// Queue-store compression is enabled by default to reduce disk footprint.
 pub const DEFAULT_TARGET_STORE_COMPRESS: bool = true;
+
+/// Maximum on-disk bytes in each target queue, including failed entries and pending writes.
+pub const ENV_TARGET_STORE_MAX_BYTES: &str = "RUSTFS_TARGET_STORE_MAX_BYTES";
+
+/// Queue writes fail at one GiB unless the operator sets another positive byte budget.
+pub const DEFAULT_TARGET_STORE_MAX_BYTES: u64 = 1024 * 1024 * 1024;

crates/config/src/constants/tls.rs

@@ -175,12 +175,17 @@ pub const ENV_HTTP_SOCKET_RECV_BUFFER_BYTES: &str = "RUSTFS_HTTP_SOCKET_RECV_BUF
 pub const DEFAULT_HTTP_SOCKET_RECV_BUFFER_BYTES: usize = 0;
 
 /// Environment variable for the S3 request-body inter-chunk read timeout
-/// (seconds). Default: 300. Set to 0 to disable.
+/// (seconds). Default: 300. Set to 0 to disable the inactivity timeout.
 ///
 /// This bounds how long a `PutObject`/upload may wait for the *next* body chunk
 /// while more bytes are still expected. It resets on every chunk, so it does not
 /// penalize slow-but-progressing uploads — it only fires when a peer sends a
 /// partial body and then goes silent *without* closing the connection (no EOF).
+/// Independently of this knob, `PutObject` and `UploadPart` require at least
+/// 64 KiB of body progress per 60 seconds of server wait (the minimum-progress
+/// floor, logged as `upload_body_progress_stalled`); it bounds trickle uploads
+/// that would otherwise pin upload admission indefinitely, and it stays in
+/// force when this timeout is set to 0.
 /// A reverse proxy or CDN that forwards a truncated body this way would
 /// otherwise make RustFS wait forever for bytes that never arrive; with this
 /// timeout the stalled read is aborted and logged with the received/expected

crates/e2e_test/README.md

@@ -26,7 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
 | **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
 | **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
 | **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
-| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
+| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
 | **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
 | **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
 
@@ -194,9 +194,9 @@ the same profile for membership and execution with one nightly worker.
 | `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
 | ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
 | KMS suite | `e2e-full` job, merge queue + main | **Active** |
-| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
+| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, release tags + weekly schedule + manual dispatch | **Active** |
 | Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
-| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
+| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (nightly / dispatch) |
 | Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
 | Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
 | Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |

crates/e2e_test/src/create_bucket_region_test.rs

@@ -502,10 +502,163 @@ mod tests {
                 assert!(response.contains(if expected == reqwest::StatusCode::FORBIDDEN {
                     "<Code>AccessDenied</Code>"
                 } else {
-                    "<Code>BucketAlreadyExists</Code>"
+                    "<Code>BucketAlreadyOwnedByYou</Code>"
                 }));
             }
         }
         Ok(())
     }
+
+    /// A deployment owns a single bucket namespace, so an IAM user that recreates
+    /// an existing bucket is the owner, not another account. In the default region
+    /// every such request returns the legacy idempotent `200 OK` (AWS CreateBucket
+    /// semantics), including buckets created by root and by another IAM user.
+    #[tokio::test]
+    async fn test_create_bucket_duplicate_in_default_region_is_idempotent_for_iam_user()
+    -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec!["--region", "us-east-1"]).await?;
+        let root = env.create_s3_client();
+
+        admin_create_user(&env, "duplicate-iam", "duplicate-iam-secret").await?;
+        admin_attach_user_policy_via(
+            AdminTransport::Signed,
+            &env.url,
+            &env.access_key,
+            &env.secret_key,
+            "readwrite",
+            "duplicate-iam",
+        )
+        .await?;
+        let iam = Client::from_conf(build_test_s3_config(
+            &env.url,
+            "duplicate-iam",
+            "duplicate-iam-secret",
+            None,
+            "duplicate-create",
+        ));
+
+        root.create_bucket()
+            .bucket("duplicate-root-owned")
+            .send()
+            .await
+            .expect("root creates the first bucket");
+        iam.create_bucket()
+            .bucket("duplicate-root-owned")
+            .send()
+            .await
+            .expect("an IAM user recreating a root-owned bucket in us-east-1 gets 200 OK");
+
+        iam.create_bucket()
+            .bucket("duplicate-iam-owned")
+            .send()
+            .await
+            .expect("the IAM user creates its own bucket");
+        iam.create_bucket()
+            .bucket("duplicate-iam-owned")
+            .send()
+            .await
+            .expect("the same IAM user recreating its bucket gets 200 OK");
+        root.create_bucket()
+            .bucket("duplicate-iam-owned")
+            .send()
+            .await
+            .expect("root recreating an IAM-created bucket gets 200 OK");
+
+        Ok(())
+    }
+
+    /// Outside the default region the same ownership answer becomes
+    /// `409 BucketAlreadyOwnedByYou` for every authorized requester. Requests that
+    /// lack `s3:CreateBucket` still fail authorization before the duplicate check.
+    #[tokio::test]
+    async fn test_create_bucket_duplicate_outside_default_region_reports_owned_by_you() -> Result<(), Box<dyn Error + Send + Sync>>
+    {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec!["--region", "eu-west-1"]).await?;
+        let region = "eu-west-1";
+        let root = Client::from_conf(
+            env.create_s3_client()
+                .config()
+                .to_builder()
+                .region(Region::new(region))
+                .build(),
+        );
+
+        admin_create_user(&env, "duplicate-eu-iam", "duplicate-eu-iam-secret").await?;
+        admin_attach_user_policy_via(
+            AdminTransport::Signed,
+            &env.url,
+            &env.access_key,
+            &env.secret_key,
+            "readwrite",
+            "duplicate-eu-iam",
+        )
+        .await?;
+        let iam = Client::from_conf(
+            build_test_s3_config(&env.url, "duplicate-eu-iam", "duplicate-eu-iam-secret", None, "duplicate-create")
+                .to_builder()
+                .region(Region::new(region))
+                .build(),
+        );
+
+        admin_create_user(&env, "duplicate-eu-readonly", "duplicate-eu-readonly-secret").await?;
+        admin_attach_user_policy_via(
+            AdminTransport::Signed,
+            &env.url,
+            &env.access_key,
+            &env.secret_key,
+            "readonly",
+            "duplicate-eu-readonly",
+        )
+        .await?;
+        let readonly = Client::from_conf(
+            build_test_s3_config(
+                &env.url,
+                "duplicate-eu-readonly",
+                "duplicate-eu-readonly-secret",
+                None,
+                "duplicate-create",
+            )
+            .to_builder()
+            .region(Region::new(region))
+            .build(),
+        );
+
+        for (client, bucket) in [(&root, "duplicate-eu-root"), (&iam, "duplicate-eu-iam")] {
+            client
+                .create_bucket()
+                .bucket(bucket)
+                .send()
+                .await
+                .unwrap_or_else(|error| panic!("{bucket} must be created: {error:?}"));
+            // Both the creator and the other authenticated identity own the namespace.
+            for creator in [&root, &iam] {
+                let error = creator
+                    .create_bucket()
+                    .bucket(bucket)
+                    .send()
+                    .await
+                    .expect_err("a duplicate create outside us-east-1 must report ownership");
+                assert_eq!(
+                    error.as_service_error().and_then(ProvideErrorMetadata::code),
+                    Some("BucketAlreadyOwnedByYou"),
+                    "{bucket}"
+                );
+                assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(409), "{bucket}");
+            }
+        }
+
+        let denied = readonly
+            .create_bucket()
+            .bucket("duplicate-eu-root")
+            .send()
+            .await
+            .expect_err("a requester without s3:CreateBucket must be denied before the duplicate check");
+        assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
+
+        Ok(())
+    }
 }

crates/e2e_test/src/delete_marker_migration_semantics_test.rs

@@ -16,8 +16,193 @@
 mod tests {
     use crate::common::{RustFSTestEnvironment, init_logging};
     use aws_sdk_s3::Client;
+    use aws_sdk_s3::config::http::HttpResponse;
+    use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
     use aws_sdk_s3::primitives::ByteStream;
-    use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
+    use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
+
+    fn assert_not_found<E: ProvideErrorMetadata + std::fmt::Debug>(error: &SdkError<E, HttpResponse>) {
+        assert_eq!(error.raw_response().map(|r| r.status().as_u16()), Some(404), "{error:?}");
+        assert_eq!(
+            error.as_service_error().and_then(ProvideErrorMetadata::code),
+            Some("NoSuchKey"),
+            "{error:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn conditional_writes_cannot_replace_delete_marker_with_if_match() {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
+        env.start_rustfs_server(vec![]).await.expect("start RustFS");
+        let client = env.create_s3_client();
+        let bucket = "conditional-delete-marker-proof";
+        create_versioned_bucket(&client, bucket).await;
+        client
+            .put_object()
+            .bucket(bucket)
+            .key("source")
+            .body(ByteStream::from_static(b"source"))
+            .send()
+            .await
+            .expect("write copy source");
+
+        for operation in ["put", "copy", "multipart"] {
+            let old = client
+                .put_object()
+                .bucket(bucket)
+                .key(operation)
+                .body(ByteStream::from_static(b"old"))
+                .send()
+                .await
+                .expect("write historical version");
+            let old_etag = old.e_tag().expect("historical ETag");
+            let marker = client
+                .delete_object()
+                .bucket(bucket)
+                .key(operation)
+                .send()
+                .await
+                .expect("create delete marker");
+            let marker_id = marker.version_id().expect("delete marker version");
+
+            let (upload_id, completed) = if operation == "multipart" {
+                let upload = client
+                    .create_multipart_upload()
+                    .bucket(bucket)
+                    .key(operation)
+                    .send()
+                    .await
+                    .expect("initiate multipart upload");
+                let upload_id = upload.upload_id().expect("upload ID").to_owned();
+                let part = client
+                    .upload_part()
+                    .bucket(bucket)
+                    .key(operation)
+                    .upload_id(&upload_id)
+                    .part_number(1)
+                    .body(ByteStream::from_static(b"new"))
+                    .send()
+                    .await
+                    .expect("upload part");
+                let completed = CompletedMultipartUpload::builder()
+                    .parts(
+                        CompletedPart::builder()
+                            .part_number(1)
+                            .set_e_tag(part.e_tag().map(str::to_owned))
+                            .build(),
+                    )
+                    .build();
+                (Some(upload_id), Some(completed))
+            } else {
+                (None, None)
+            };
+
+            for etag in [old_etag, "\"00000000000000000000000000000000\""] {
+                match operation {
+                    "put" => {
+                        let err = client
+                            .put_object()
+                            .bucket(bucket)
+                            .key(operation)
+                            .if_match(etag)
+                            .body(ByteStream::from_static(b"new"))
+                            .send()
+                            .await
+                            .expect_err("If-Match must reject marker");
+                        assert_not_found(&err);
+                    }
+                    "copy" => {
+                        let err = client
+                            .copy_object()
+                            .bucket(bucket)
+                            .key(operation)
+                            .copy_source(format!("{bucket}/source"))
+                            .if_match(etag)
+                            .send()
+                            .await
+                            .expect_err("If-Match copy must reject marker");
+                        assert_not_found(&err);
+                    }
+                    _ => {
+                        let err = client
+                            .complete_multipart_upload()
+                            .bucket(bucket)
+                            .key(operation)
+                            .upload_id(upload_id.as_deref().expect("upload ID"))
+                            .multipart_upload(completed.clone().expect("completed parts"))
+                            .if_match(etag)
+                            .send()
+                            .await
+                            .expect_err("If-Match completion must reject marker");
+                        assert_not_found(&err);
+                    }
+                }
+                let versions = client
+                    .list_object_versions()
+                    .bucket(bucket)
+                    .prefix(operation)
+                    .send()
+                    .await
+                    .expect("list versions after rejected write");
+                assert_eq!(versions.versions().len(), 1, "{operation}: failed write created a version");
+                assert_eq!(versions.delete_markers().len(), 1);
+                assert_eq!(versions.delete_markers()[0].version_id(), Some(marker_id));
+                assert_eq!(versions.delete_markers()[0].is_latest(), Some(true));
+            }
+
+            match operation {
+                "put" => {
+                    client
+                        .put_object()
+                        .bucket(bucket)
+                        .key(operation)
+                        .if_none_match("*")
+                        .body(ByteStream::from_static(b"new"))
+                        .send()
+                        .await
+                        .expect("If-None-Match PUT over marker");
+                }
+                "copy" => {
+                    client
+                        .copy_object()
+                        .bucket(bucket)
+                        .key(operation)
+                        .copy_source(format!("{bucket}/source"))
+                        .if_none_match("*")
+                        .send()
+                        .await
+                        .expect("If-None-Match copy over marker");
+                }
+                _ => {
+                    client
+                        .complete_multipart_upload()
+                        .bucket(bucket)
+                        .key(operation)
+                        .upload_id(upload_id.as_deref().expect("upload ID"))
+                        .multipart_upload(completed.expect("completed parts"))
+                        .if_none_match("*")
+                        .send()
+                        .await
+                        .expect("If-None-Match completion over marker");
+                }
+            }
+            let current = client
+                .get_object()
+                .bucket(bucket)
+                .key(operation)
+                .send()
+                .await
+                .expect("read successful conditional write")
+                .body
+                .collect()
+                .await
+                .expect("read object body")
+                .into_bytes();
+            let expected: &[u8] = if operation == "copy" { b"source" } else { b"new" };
+            assert_eq!(current.as_ref(), expected, "{operation}: unexpected current object");
+        }
+    }
 
     async fn create_versioned_bucket(client: &Client, bucket: &str) {
         client

crates/e2e_test/src/distributed/concurrency_stability_test.rs

@@ -12,11 +12,249 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
+use super::harness::{DistCluster, DistLayout, TestResult, payload_for, s3_probe_client, sha256_hex, unique_bucket};
 use crate::common::init_logging;
+use aws_sdk_s3::{
+    Client,
+    error::{ProvideErrorMetadata, SdkError},
+    primitives::ByteStream,
+};
+use std::any::TypeId;
 use std::collections::BTreeSet;
+use std::fmt::Debug;
+use std::future::Future;
 use std::sync::Arc;
+use std::time::Duration;
 use tokio::sync::Barrier;
+use tokio::time::Instant;
+
+const LOCK_ADMISSION_BUDGET: Duration = Duration::from_secs(45);
+
+async fn after_lock_contention<T, E, F, Fut>(operation: &str, mut call: F) -> TestResult<Result<T, SdkError<E>>>
+where
+    E: Debug + ProvideErrorMetadata + 'static,
+    F: FnMut() -> Fut,
+    Fut: Future<Output = Result<T, SdkError<E>>>,
+{
+    let started = Instant::now();
+    let mut attempts = 0;
+    // The workload uses unique unversioned keys. Repeat only explicitly
+    // advertised lock backpressure; every step still needs its real result.
+    let outcome = tokio::time::timeout(LOCK_ADMISSION_BUDGET, async {
+        loop {
+            attempts += 1;
+            let result = call().await;
+            let retry = result.as_ref().err().is_some_and(|error| {
+                let Some(response) = error.raw_response() else {
+                    return false;
+                };
+                let Some(service_error) = error.as_service_error() else {
+                    return false;
+                };
+                // HEAD has no XML error body, so its SDK service error can
+                // lack a code even when the server supplies retry headers.
+                let namespace_code = service_error.code() == Some("ServiceUnavailable")
+                    || (service_error.code().is_none()
+                        && TypeId::of::<E>() == TypeId::of::<aws_sdk_s3::operation::head_object::HeadObjectError>());
+                response.status().as_u16() == 503
+                    && namespace_code
+                    && response.headers().get("x-rustfs-error-cause") == Some("namespace_lock_timeout")
+                    && response.headers().get("retry-after") == Some("1")
+            });
+            if !retry {
+                return result;
+            }
+            tokio::time::sleep(Duration::from_secs(1)).await;
+        }
+    })
+    .await;
+    if attempts > 1 || !matches!(&outcome, Ok(Ok(_))) {
+        eprintln!(
+            "mixed workload request: operation={operation}, attempts={attempts}, elapsed={:?}",
+            started.elapsed()
+        );
+    }
+    outcome.map_err(|_| {
+        format!("{operation} exceeded its {LOCK_ADMISSION_BUDGET:?} request budget after {attempts} attempts").into()
+    })
+}
+
+async fn assert_workload_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
+    let output = after_lock_contention(&format!("GET {key}"), || client.get_object().bucket(bucket).key(key).send()).await??;
+    let actual = output.body.collect().await?.into_bytes();
+    if actual.as_ref() != expected {
+        return Err(format!(
+            "object {key} bytes mismatch: expected {} bytes sha256={} got {} bytes sha256={}",
+            expected.len(),
+            sha256_hex(expected),
+            actual.len(),
+            sha256_hex(&actual)
+        )
+        .into());
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use aws_smithy_http_client::test_util::infallible_client_fn;
+    use aws_smithy_types::body::SdkBody;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+
+    type Reply = (u16, &'static str, Option<&'static str>, Option<&'static str>, &'static str);
+
+    fn client(respond: impl Fn(usize) -> Reply + Send + Sync + 'static) -> (Client, Arc<AtomicUsize>) {
+        let requests = Arc::new(AtomicUsize::new(0));
+        let seen = Arc::clone(&requests);
+        let transport = infallible_client_fn(move |_| {
+            let (status, code, cause, retry_after, body) = respond(seen.fetch_add(1, Ordering::SeqCst));
+            let body = if code.is_empty() {
+                body.to_string()
+            } else {
+                format!("<Error><Code>{code}</Code></Error>")
+            };
+            let mut response = http::Response::builder()
+                .status(status)
+                .header("content-type", "application/xml")
+                .header("content-length", body.len());
+            if let Some(cause) = cause {
+                response = response.header("x-rustfs-error-cause", cause);
+            }
+            if let Some(retry_after) = retry_after {
+                response = response.header("retry-after", retry_after);
+            }
+            response.body(SdkBody::from(body)).expect("scripted S3 response")
+        });
+        let config = crate::common::build_test_s3_config(
+            "http://contention.example.invalid",
+            "synthetic-access",
+            "synthetic-secret",
+            None,
+            "contention-test",
+        );
+        let client = Client::from_conf(config.to_builder().http_client(transport).build());
+        (s3_probe_client(&client), requests)
+    }
+
+    fn contention() -> Reply {
+        (503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("1"), "")
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn advertised_contention_recovers_after_standard_sdk_attempts_are_exhausted() -> TestResult {
+        let (client, requests) = client(|attempt| {
+            if attempt < 3 {
+                contention()
+            } else {
+                (200, "", None, None, "")
+            }
+        });
+        let standard = Client::from_conf(
+            client
+                .config()
+                .to_builder()
+                .retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(3))
+                .build(),
+        );
+        standard
+            .put_object()
+            .bucket("bucket")
+            .key("key")
+            .body(ByteStream::from_static(b"body"))
+            .send()
+            .await
+            .expect_err("three advertised lock responses exhaust the standard SDK policy");
+        assert_eq!(requests.load(Ordering::SeqCst), 3);
+        requests.store(0, Ordering::SeqCst);
+        let started = Instant::now();
+        after_lock_contention("PUT recovery", || {
+            client
+                .put_object()
+                .bucket("bucket")
+                .key("key")
+                .body(ByteStream::from_static(b"body"))
+                .send()
+        })
+        .await??;
+        assert_eq!(requests.load(Ordering::SeqCst), 4);
+        assert!(started.elapsed() >= Duration::from_secs(3), "honor each Retry-After response");
+        Ok(())
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn bodyless_head_contention_uses_explicit_retry_headers() -> TestResult {
+        let (client, requests) = client(|attempt| {
+            if attempt == 0 {
+                (503, "", Some("namespace_lock_timeout"), Some("1"), "")
+            } else {
+                (200, "", None, None, "")
+            }
+        });
+        after_lock_contention("HEAD recovery", || client.head_object().bucket("bucket").key("key").send()).await??;
+        assert_eq!(requests.load(Ordering::SeqCst), 2);
+        Ok(())
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn unrelated_or_malformed_responses_are_not_retried() -> TestResult {
+        for reply in [
+            (500, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
+            (403, "AccessDenied", None, None, ""),
+            (404, "NoSuchKey", None, None, ""),
+            (503, "ServiceUnavailable", Some("storage_write_quorum"), None, ""),
+            (503, "ServiceUnavailable", Some("namespace_lock_unavailable"), Some("1"), ""),
+            (503, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
+            (503, "", Some("namespace_lock_timeout"), Some("1"), ""),
+            (503, "ServiceUnavailable", None, Some("1"), ""),
+            (503, "ServiceUnavailable", Some("namespace_lock_timeout"), None, ""),
+            (503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("2"), ""),
+        ] {
+            let (client, requests) = client(move |_| reply);
+            let result = after_lock_contention("PUT fatal response", || {
+                client
+                    .put_object()
+                    .bucket("bucket")
+                    .key("key")
+                    .body(ByteStream::from_static(b"body"))
+                    .send()
+            })
+            .await?;
+            assert!(result.is_err(), "fatal response cannot become success: {reply:?}");
+            assert_eq!(requests.load(Ordering::SeqCst), 1, "fatal response must remain visible: {reply:?}");
+        }
+        Ok(())
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn persistent_lock_contention_exhausts_the_request_budget() {
+        let (client, requests) = client(|_| contention());
+        let started = Instant::now();
+        let result = after_lock_contention("PUT unavailable", || {
+            client
+                .put_object()
+                .bucket("bucket")
+                .key("key")
+                .body(ByteStream::from_static(b"body"))
+                .send()
+        })
+        .await;
+        assert!(result.is_err(), "persistent contention cannot acknowledge an operation");
+        assert!(started.elapsed() >= LOCK_ADMISSION_BUDGET);
+        assert!(requests.load(Ordering::SeqCst) > 1);
+        assert!(requests.load(Ordering::SeqCst) <= LOCK_ADMISSION_BUDGET.as_secs() as usize + 1);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn acknowledged_wrong_bytes_fail_without_retry() {
+        let (client, requests) = client(|_| (200, "", None, None, "wrong bytes"));
+        let error = assert_workload_bytes(&client, "bucket", "key", b"expected bytes")
+            .await
+            .expect_err("a successful but incorrect read is a consistency failure");
+        assert!(error.to_string().contains("bytes mismatch"));
+        assert_eq!(requests.load(Ordering::SeqCst), 1);
+    }
+}
 
 #[tokio::test]
 async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
@@ -24,7 +262,9 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
     let dist = DistCluster::start(DistLayout::FourByFour).await?;
     let bucket = unique_bucket("conc");
     dist.create_bucket(&bucket).await?;
-    let clients = Arc::new(dist.clients()?);
+    // This consistency probe owns its admission policy; unrelated responses
+    // must not be hidden by the SDK's general-purpose retries.
+    let clients = Arc::new(dist.clients()?.iter().map(s3_probe_client).collect::<Vec<_>>());
     let worker_count = 24;
     let rounds = 4;
     let barrier = Arc::new(Barrier::new(worker_count));
@@ -44,31 +284,46 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
                 let key = format!("source/worker-{idx:02}-round-{round}.bin");
                 let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
                 let body = payload_for(&key, 64 * 1024);
-                put_object(writer, &bucket, &key, body.clone()).await?;
+                let context = format!("worker={idx} round={round}");
+                after_lock_contention(&format!("{context} PUT source"), || {
+                    writer
+                        .put_object()
+                        .bucket(&bucket)
+                        .key(&key)
+                        .body(ByteStream::from(body.clone()))
+                        .send()
+                })
+                .await??;
 
-                let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
+                let head = after_lock_contention(&format!("{context} HEAD source"), || {
+                    reader.head_object().bucket(&bucket).key(&key).send()
+                })
+                .await??;
                 if head.content_length() != Some(body.len() as i64) {
                     return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
                 }
-                assert_object_bytes(reader, &bucket, &key, &body).await?;
+                assert_workload_bytes(reader, &bucket, &key, &body).await?;
 
-                copier
+                after_lock_contention(&format!("{context} COPY retained"), || {
-                    .copy_object()
+                    copier
-                    .bucket(&bucket)
+                        .copy_object()
-                    .key(&copy_key)
+                        .bucket(&bucket)
-                    .copy_source(format!("{bucket}/{key}"))
+                        .key(&copy_key)
-                    .send()
+                        .copy_source(format!("{bucket}/{key}"))
-                    .await?;
+                        .send()
-                assert_object_bytes(writer, &bucket, &copy_key, &body).await?;
+                })
+                .await??;
+                assert_workload_bytes(writer, &bucket, &copy_key, &body).await?;
 
-                writer.delete_object().bucket(&bucket).key(&key).send().await?;
+                after_lock_contention(&format!("{context} DELETE source"), || {
-                let missing = reader
+                    writer.delete_object().bucket(&bucket).key(&key).send()
-                    .head_object()
+                })
-                    .bucket(&bucket)
+                .await??;
-                    .key(&key)
+                let missing = after_lock_contention(&format!("{context} HEAD deleted source"), || {
-                    .send()
+                    reader.head_object().bucket(&bucket).key(&key).send()
-                    .await
+                })
-                    .expect_err("deleted source key must not remain visible");
+                .await?
+                .expect_err("deleted source key must not remain visible");
                 if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
                     return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
                 }
@@ -82,14 +337,22 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
     for handle in handles {
         inventory.extend(handle.await??);
     }
+    assert_eq!(
+        inventory.len(),
+        worker_count * rounds,
+        "every worker round must retain its acknowledged copy"
+    );
 
     let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
     for (node_idx, client) in clients.iter().enumerate() {
-        let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
+        let listed = after_lock_contention(&format!("node={node_idx} LIST retained"), || {
+            client.list_objects_v2().bucket(&bucket).prefix("retained/").send()
+        })
+        .await??;
         let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
         assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
         for (key, body) in &inventory {
-            assert_object_bytes(client, &bucket, key, body)
+            assert_workload_bytes(client, &bucket, key, body)
                 .await
                 .map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
         }

crates/e2e_test/src/distributed/scanner_heal_topology_fixture.rs

@@ -25,7 +25,8 @@
 
 use super::harness::{
     DistCluster, DistLayout, TestResult, cluster_admin, cluster_admin_ok, decommission_status_json, get_object_bytes,
-    payload_for, put_object, s3_probe_client, s3_probe_error_code, sha256_hex, unique_bucket, wait_until,
+    payload_for, put_object, rebalance_complete, rebalance_status_json, s3_probe_client, s3_probe_error_code, sha256_hex,
+    unique_bucket, wait_until,
 };
 use crate::chaos::{VersionShardCensus, census_object_version_on_disk};
 use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment};
@@ -47,6 +48,7 @@ use tokio::time::{Instant, sleep};
 use uuid::Uuid;
 
 pub(crate) const MOVEMENT_BARRIER_DIR_ENV: &str = "RUSTFS_E2E_DATA_MOVEMENT_BARRIER_DIR";
+pub(crate) const STAGE_AFTER_ENTRY_ADMISSION: &str = "after_entry_admission";
 pub(crate) const STAGE_BEFORE_TARGET_COMMIT: &str = "before_target_commit";
 pub(crate) const STAGE_BEFORE_SOURCE_CLEANUP: &str = "before_source_cleanup";
 pub(crate) const STAGE_AFTER_RESPONSIBILITY_PERSISTED: &str = "after_responsibility_persisted";
@@ -1166,8 +1168,21 @@ pub(crate) async fn clear_decommission(cluster: &RustFSTestClusterEnvironment, p
     .await
 }
 
-pub(crate) async fn stop_rebalance(cluster: &RustFSTestClusterEnvironment) -> TestResult<String> {
+pub(crate) async fn stop_rebalance(cluster: &RustFSTestClusterEnvironment, expected_id: &str) -> TestResult<String> {
-    cluster_admin_ok(cluster, Method::POST, "/rustfs/admin/v3/rebalance/stop", None).await
+    let path = "/rustfs/admin/v3/rebalance/stop";
+    let (status, response) = cluster_admin(cluster, Method::POST, path, None).await?;
+    if status.is_success() {
+        return Ok(response);
+    }
+    // The run can finish between the active observation and the stop request.
+    // Only completion of that same run makes a not-started response acceptable.
+    if status == StatusCode::NOT_FOUND
+        && response.contains("<Code>NoSuchResource</Code>")
+        && rebalance_complete(&rebalance_status_json(cluster).await?, expected_id)?
+    {
+        return Ok(response);
+    }
+    Err(format!("POST {path} failed: {status} {response}").into())
 }
 
 pub(crate) fn decommission_pool_entry(status: &Value, pool_id: usize) -> TestResult<Value> {
@@ -1585,6 +1600,129 @@ pub(crate) async fn assert_every_node_serves(dist: &DistCluster, bucket: &str, l
 mod tests {
     use super::*;
 
+    const REBALANCE_NOT_STARTED: &str =
+        "<Error><Code>NoSuchResource</Code><Message>pool rebalance is not started</Message></Error>";
+
+    async fn mock_rebalance_stop(
+        stop_status: StatusCode,
+        stop_body: &str,
+        run_status: Value,
+    ) -> TestResult<(TestResult<String>, Vec<String>)> {
+        use bytes::Bytes;
+        use http_body_util::Full;
+        use hyper::service::service_fn;
+        use hyper_util::rt::TokioIo;
+        use tokio::net::TcpListener;
+        use tokio::sync::mpsc;
+        use tokio::task::JoinSet;
+
+        let listener = TcpListener::bind("127.0.0.1:0").await?;
+        let mut cluster = RustFSTestClusterEnvironment::new(1).await?;
+        cluster.nodes[0].url = format!("http://{}", listener.local_addr()?);
+        let stop_body = stop_body.to_string();
+        let (requests_tx, mut requests_rx) = mpsc::unbounded_channel();
+        let mut server = JoinSet::new();
+        server.spawn(async move {
+            loop {
+                let (stream, _) = match listener.accept().await {
+                    Ok(connection) => connection,
+                    Err(error) => return Err::<(), Box<dyn std::error::Error + Send + Sync>>(error.into()),
+                };
+                let stop_body = stop_body.clone();
+                let run_status = run_status.clone();
+                let requests_tx = requests_tx.clone();
+                let handler = service_fn(move |request: hyper::Request<hyper::body::Incoming>| {
+                    let route = format!("{} {}", request.method(), request.uri().path());
+                    requests_tx.send(route.clone()).expect("record mock admin request");
+                    let (status, body) = match route.as_str() {
+                        "POST /rustfs/admin/v3/rebalance/stop" => (stop_status, stop_body.clone()),
+                        "GET /rustfs/admin/v3/rebalance/status" => (StatusCode::OK, run_status.to_string()),
+                        _ => (StatusCode::BAD_REQUEST, "unexpected admin request".to_string()),
+                    };
+                    async move { http::Response::builder().status(status).body(Full::new(Bytes::from(body))) }
+                });
+                hyper::server::conn::http1::Builder::new()
+                    .keep_alive(false)
+                    .serve_connection(TokioIo::new(stream), handler)
+                    .await?;
+            }
+        });
+        let outcome = tokio::time::timeout(Duration::from_secs(5), stop_rebalance(&cluster, "run-1")).await?;
+        server.abort_all();
+        let requests = std::iter::from_fn(|| requests_rx.try_recv().ok()).collect();
+        Ok((outcome, requests))
+    }
+
+    fn completed_rebalance() -> Value {
+        serde_json::json!({
+            "id": "run-1",
+            "pools": [
+                { "id": 0, "status": "Completed", "progress": { "versions": 113 } },
+                { "id": 1, "status": "None" }
+            ]
+        })
+    }
+
+    #[tokio::test]
+    async fn rebalance_stop_success_does_not_query_completion() -> TestResult {
+        let (outcome, requests) = mock_rebalance_stop(StatusCode::OK, "", Value::Null).await?;
+        assert_eq!(outcome?, "");
+        assert_eq!(requests, ["POST /rustfs/admin/v3/rebalance/stop"]);
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn rebalance_stop_accepts_completed_run_after_not_started_response() -> TestResult {
+        let (outcome, requests) =
+            mock_rebalance_stop(StatusCode::NOT_FOUND, REBALANCE_NOT_STARTED, completed_rebalance()).await?;
+        outcome?;
+        assert_eq!(
+            requests,
+            [
+                "POST /rustfs/admin/v3/rebalance/stop",
+                "GET /rustfs/admin/v3/rebalance/status"
+            ]
+        );
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn rebalance_stop_rejects_unproven_completion_and_other_errors() -> TestResult {
+        let complete = completed_rebalance();
+        let mut invalid = vec![Value::Null];
+        for (pointer, value) in [
+            ("/id", serde_json::json!("run-2")),
+            ("/pools/0/status", serde_json::json!("Started")),
+            ("/pools/0/status", serde_json::json!("Failed")),
+            ("/pools/0/status", serde_json::json!("Stopped")),
+            ("/pools/0/progress/versions", serde_json::json!(0)),
+        ] {
+            let mut status = complete.clone();
+            *status.pointer_mut(pointer).expect("completion fixture field") = value;
+            invalid.push(status);
+        }
+        let mut pending = complete.clone();
+        pending["stopPropagation"] = serde_json::json!({ "pendingTerminalReload": true });
+        invalid.push(pending);
+        let mut failed = complete.clone();
+        failed["pools"][0]["lastError"] = serde_json::json!("migration failed");
+        invalid.push(failed);
+        for status in invalid {
+            let (outcome, _) = mock_rebalance_stop(StatusCode::NOT_FOUND, REBALANCE_NOT_STARTED, status.clone()).await?;
+            assert!(outcome.is_err(), "unproven completion accepted: {status}");
+        }
+        for (code, body) in [
+            (StatusCode::FORBIDDEN, REBALANCE_NOT_STARTED),
+            (StatusCode::INTERNAL_SERVER_ERROR, REBALANCE_NOT_STARTED),
+            (StatusCode::NOT_FOUND, "<Error><Code>NoSuchBucket</Code></Error>"),
+        ] {
+            let (outcome, requests) = mock_rebalance_stop(code, body, complete.clone()).await?;
+            assert!(outcome.is_err(), "unrelated error accepted: {code} {body}");
+            assert_eq!(requests, ["POST /rustfs/admin/v3/rebalance/stop"]);
+        }
+        Ok(())
+    }
+
     #[test]
     fn version_ids_normalize_null_spellings() {
         for spelling in [None, Some(""), Some("null"), Some("00000000-0000-0000-0000-000000000000")] {

crates/e2e_test/src/distributed/scanner_heal_topology_test.rs

@@ -28,7 +28,7 @@ use super::harness::{
     wait_for_rebalance_active, wait_for_rebalance_complete,
 };
 use super::scanner_heal_topology_fixture::{
-    LiveWorkload, MOVEMENT_ACTIVE_TIMEOUT, MOVEMENT_COMPLETE_TIMEOUT, MovementOperation, SEED_POOL,
+    LiveWorkload, MOVEMENT_ACTIVE_TIMEOUT, MOVEMENT_COMPLETE_TIMEOUT, MovementOperation, SEED_POOL, STAGE_AFTER_ENTRY_ADMISSION,
     STAGE_AFTER_RESPONSIBILITY_PERSISTED, STAGE_BEFORE_SOURCE_CLEANUP, STAGE_BEFORE_TARGET_COMMIT, TopologyManifest, TopologyRun,
     assert_bucket_durability_strict, assert_every_node_serves, assert_probe_objects, assert_process_durability_strict,
     assert_seed_pool_census_unchanged, assert_single_pool_placement, cancel_decommission, capture_seed_pool_census,
@@ -274,6 +274,7 @@ async fn wait_for_rebalance_stopped_or_completed(run: &TopologyRun, rebalance_id
 #[tokio::test]
 async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart() -> TestResult {
     init_logging();
+    require_test_hooks();
     let mut run = TopologyRun::start("rebalance").await?;
     let manifest = seed_manifest(&run).await?;
     assert_bucket_durability_strict(&run.dist.cluster, &manifest.buckets()).await?;
@@ -288,23 +289,52 @@ async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart()
         .map(|entry| (entry.key.clone(), entry.sha256.clone()))
         .collect();
 
-    let first_id = start_rebalance(&run.dist.cluster).await?;
+    // Keep an admitted entry on the source until stop cancels admission. Start
-    wait_for_rebalance_active(&run.dist.cluster, &first_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
+    // propagation may await that entry, so observe and stop before awaiting ACK.
-    let during_rebalance = put_probe_objects(&run.dist, &scratch, "during-rebalance", 4).await?;
+    let first_key = manifest.plain_key("external", 0)?;
-    let workload = tokio::spawn(run_live_workload(LiveWorkload {
+    run.movement_barrier
-        clients: run.dist.clients()?,
+        .arm_movement("rebalance", &manifest.plain_bucket, &first_key, STAGE_AFTER_ENTRY_ADMISSION)?;
-        bucket: scratch.clone(),
+    let observe_first = async {
-        prefix: "live".to_string(),
+        let (observed_id, reached) = tokio::time::timeout(MOVEMENT_ACTIVE_TIMEOUT, async {
-        writers: LIVE_WRITERS,
+            let reached = run.movement_barrier.wait_reached(MOVEMENT_ACTIVE_TIMEOUT).await?;
-        rounds: LIVE_ROUNDS,
+            let status = rebalance_status_json(&run.dist.cluster).await?;
-        baseline_bucket: manifest.plain_bucket.clone(),
+            let observed_id = status
-        baseline: baseline.clone(),
+                .get("id")
-        pause: Duration::from_millis(100),
+                .and_then(Value::as_str)
-    }));
+                .ok_or("rebalance status omitted id")?;
-    sleep(Duration::from_secs(2)).await;
+            wait_for_rebalance_active(&run.dist.cluster, observed_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
-    stop_rebalance(&run.dist.cluster).await?;
+            Ok::<_, Box<dyn std::error::Error + Send + Sync>>((observed_id.to_string(), reached))
-    let (first_outcome, first_status) = wait_for_rebalance_stopped_or_completed(&run, &first_id).await?;
+        })
-    let ops = workload.await??;
+        .await??;
+        let probes = put_probe_objects(&run.dist, &scratch, "during-rebalance", 4).await?;
+        let workload = run_live_workload(LiveWorkload {
+            clients: run.dist.clients()?,
+            bucket: scratch.clone(),
+            prefix: "live".to_string(),
+            writers: LIVE_WRITERS,
+            rounds: LIVE_ROUNDS,
+            baseline_bucket: manifest.plain_bucket.clone(),
+            baseline: baseline.clone(),
+            pause: Duration::from_millis(100),
+        });
+        let stop = async {
+            sleep(Duration::from_secs(2)).await;
+            stop_rebalance(&run.dist.cluster, &observed_id).await?;
+            let (outcome, status) = wait_for_rebalance_stopped_or_completed(&run, &observed_id).await?;
+            if outcome != "stopped" {
+                return Err("rebalance completed before stop; mid-run stop was not exercised".into());
+            }
+            Ok::<_, Box<dyn std::error::Error + Send + Sync>>((outcome, status))
+        };
+        let (ops, (outcome, status)) = tokio::try_join!(workload, stop)?;
+        Ok::<_, Box<dyn std::error::Error + Send + Sync>>((observed_id, reached, probes, ops, outcome, status))
+    };
+    let (first_id, (observed_id, first_reached, during_rebalance, ops, first_outcome, first_status)) =
+        tokio::try_join!(start_rebalance(&run.dist.cluster), observe_first)?;
+    if first_id != observed_id {
+        return Err(format!("rebalance start acknowledged {first_id}, but the stopped run was {observed_id}").into());
+    }
+    run.movement_barrier.disarm()?;
     let live_state = expected_live_state(&ops);
 
     for node_idx in 0..run.dist.cluster.nodes.len() {
@@ -333,8 +363,40 @@ async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart()
     verify_from_every_node(&run.dist, &manifest, "after restart following the stop").await?;
     verify_live_state(&run.dist.client(0)?, &scratch, &live_state, "after restart following the stop").await?;
 
-    let second_id = start_rebalance(&run.dist.cluster).await?;
+    // A short second run can finish before the start response reaches us.
-    wait_for_rebalance_active(&run.dist.cluster, &second_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
+    // Hold one remaining source entry until its active state is observed.
+    let mut second_key = None;
+    for entry in manifest.plain_entries() {
+        if pools_holding(&run.dist, &manifest.plain_bucket, &entry.key, None)?.contains(&SEED_POOL) {
+            second_key = Some(entry.key.clone());
+            break;
+        }
+    }
+    let second_key = second_key.ok_or("no seed-pool object remains for the second rebalance")?;
+    let second_barrier = run.movement_barrier.next_phase();
+    second_barrier.arm_movement("rebalance", &manifest.plain_bucket, &second_key, STAGE_BEFORE_SOURCE_CLEANUP)?;
+    let observe_second = async {
+        let reached = second_barrier.wait_reached(MOVEMENT_ACTIVE_TIMEOUT).await?;
+        let status = rebalance_status_json(&run.dist.cluster).await?;
+        let observed_id = status
+            .get("id")
+            .and_then(Value::as_str)
+            .ok_or("rebalance status omitted id")?;
+        if observed_id == first_id {
+            return Err("the second rebalance reused the stopped run id".into());
+        }
+        wait_for_rebalance_active(&run.dist.cluster, observed_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
+        second_barrier.disarm()?;
+        second_barrier.release_movement()?;
+        Ok::<_, Box<dyn std::error::Error + Send + Sync>>((observed_id.to_string(), reached))
+    };
+    // Worker propagation can hold the start response until movement drains.
+    let (second_id, (observed_id, second_reached)) = tokio::try_join!(start_rebalance(&run.dist.cluster), async {
+        tokio::time::timeout(MOVEMENT_ACTIVE_TIMEOUT, observe_second).await?
+    })?;
+    if second_id != observed_id {
+        return Err(format!("rebalance start acknowledged {second_id}, but the active run was {observed_id}").into());
+    }
     wait_for_rebalance_complete(&run.dist.cluster, &second_id, MOVEMENT_COMPLETE_TIMEOUT).await?;
     verify_from_every_node(&run.dist, &manifest, "after the second rebalance completed").await?;
     verify_live_state(&run.dist.client(3)?, &scratch, &live_state, "after the second rebalance completed").await?;
@@ -351,8 +413,8 @@ async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart()
         &json!({
             "gate": "G-REB",
             "manifest": manifest_summary(&manifest),
-            "first_run": { "id": first_id, "outcome": first_outcome, "status": first_status, "versions_held_by_two_pools_after_stop": retained_after_stop },
+            "first_run": { "id": first_id, "outcome": first_outcome, "status": first_status, "versions_held_by_two_pools_after_stop": retained_after_stop, "barrier_object": first_key, "barrier": first_reached },
-            "second_run": { "id": second_id, "versions_off_seed_pool": moved },
+            "second_run": { "id": second_id, "versions_off_seed_pool": moved, "barrier_object": second_key, "barrier": second_reached },
             "live_ops": ops,
             "probe_placement_during_rebalance": during_placement,
             "process_durability_strict_nodes": strict_nodes,

crates/e2e_test/src/distributed/scanner_peer_bucket_test.rs

@@ -1636,20 +1636,53 @@ async fn hot_window_with_old_followers(
     let mixed_initial_physical = if mixed { json!(converged_shards(&run)?) } else { Value::Null };
     let scheduler_config = run.assert_default_cadence().await?;
     let baseline = run.quiet_baseline().await?;
-    let leader = run.leader().await?;
+    // A leader-lock refresh gap during old-binary replacement can make
-    let leader_pid = run.pid(leader)?;
+    // `leader()` briefly observe a different node. Poll until the coordinator
+    // is stable before comparing against the measured fleet. A persistent
+    // change to an old-binary node is a genuine election outcome and fails.
+    let (leader, leader_pid) = if mixed {
+        let expected_leader = usize::try_from(
+            mixed_fleet["leader"]
+                .as_u64()
+                .ok_or("mixed fleet omitted its measured leader")?,
+        )?;
+        let expected_pid = mixed_fleet["leader_pid"]
+            .as_u64()
+            .ok_or("mixed fleet omitted its measured leader pid")?;
+        let mut resolved = None;
+        for _ in 0..20 {
+            let candidate = run.leader().await?;
+            let candidate_pid = u64::from(run.pid(candidate)?);
+            if Some(u64::try_from(candidate)?) == mixed_fleet["leader"].as_u64() && Some(candidate_pid) == Some(expected_pid) {
+                resolved = Some((candidate, run.pid(candidate)?));
+                break;
+            }
+            tokio::time::sleep(std::time::Duration::from_millis(500)).await;
+        }
+        match resolved {
+            Some(pair) => pair,
+            None => {
+                let current = match run.leader().await {
+                    Ok(node) => Some((node, run.pid(node).ok())),
+                    Err(_) => None,
+                };
+                return Err(format!(
+                    "the hot window did not retain its measured new coordinator after stabilization \
+                     (expected node {expected_leader} pid {expected_pid}, observed {current:?})"
+                )
+                .into());
+            }
+        }
+    } else {
+        let leader = run.leader().await?;
+        (leader, run.pid(leader)?)
+    };
     let producer = (leader + 1) % 4;
     let producers = if fault.is_some() {
         vec![producer, (leader + 2) % 4]
     } else {
         vec![producer]
     };
-    if mixed
-        && (mixed_fleet["leader"].as_u64() != Some(u64::try_from(leader)?)
-            || mixed_fleet["leader_pid"].as_u64() != Some(u64::from(leader_pid)))
-    {
-        return Err("the hot window did not retain its measured new coordinator".into());
-    }
     let producer_pid = run.pid(producer)?;
     let producer_address = run.peer_address(producer);
     let [cold_a, cold_b, hot] = run.buckets.clone();

crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs

@@ -33,7 +33,7 @@ mod tests {
     use sha2::{Digest, Sha256};
     use std::collections::HashSet;
     use std::error::Error;
-    use std::io::{Read, Write};
+    use std::io::{Read, Seek, SeekFrom, Write};
     use std::net::SocketAddr;
     use std::path::{Path, PathBuf};
     use std::process::Command;
@@ -1654,6 +1654,89 @@ mod tests {
         TargetEndpointBlackhole,
     }
 
+    async fn release_heal_commit_barrier_after_shutdown(
+        log: &Path,
+        log_offset: u64,
+        barrier: &Path,
+    ) -> Result<serde_json::Value, Box<dyn Error + Send + Sync>> {
+        // Observe within the existing graceful process-stop budget.
+        timeout(Duration::from_secs(45), async {
+            loop {
+                let mut file = std::fs::File::open(log)?;
+                file.seek(SeekFrom::Start(log_offset))?;
+                let mut tail = String::new();
+                file.read_to_string(&mut tail)?;
+                if let Some(event) = tail
+                    .lines()
+                    .filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
+                    .find(|event| {
+                        event["event"] == "server_shutdown_state"
+                            && event["component"] == "main"
+                            && event["subsystem"] == "startup"
+                            && event["state"] == "stopping"
+                    })
+                {
+                    // Local heal has stopped and HTTP shutdown has been signalled.
+                    // Let admitted peer RPCs finish; keeping this synthetic barrier
+                    // until process exit would make the fixture block its own drain.
+                    std::fs::remove_file(barrier)?;
+                    return Ok(event);
+                }
+                sleep(Duration::from_millis(10)).await;
+            }
+        })
+        .await?
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn heal_commit_barrier_waits_for_new_shutdown_event() -> Result<(), Box<dyn Error + Send + Sync>> {
+        let directory = tempfile::tempdir()?;
+        let log = directory.path().join("server.log");
+        let barrier = directory.path().join("barrier");
+        let event = serde_json::json!({
+            "event": "server_shutdown_state", "component": "main", "subsystem": "startup", "state": "stopping"
+        });
+        std::fs::write(&log, format!("{event}\n"))?;
+        std::fs::write(&barrier, "bucket/prefix/")?;
+        let log_offset = log.metadata()?.len();
+        let mut writer = std::fs::OpenOptions::new().append(true).open(&log)?;
+        writeln!(writer, "{}", serde_json::json!({"event": "shutdown_signal_received"}))?;
+        writeln!(writer, "{}", serde_json::json!({"event": "server_shutdown_state", "state": "stopping"}))?;
+
+        let release = release_heal_commit_barrier_after_shutdown(&log, log_offset, &barrier);
+        tokio::pin!(release);
+        assert!(
+            futures::poll!(release.as_mut()).is_pending(),
+            "old or unrelated events must not release the barrier"
+        );
+        assert!(barrier.is_file(), "the partial-rebuild barrier must remain armed before shutdown");
+        tokio::time::advance(Duration::from_secs(6)).await;
+        assert!(
+            futures::poll!(release.as_mut()).is_pending(),
+            "background shutdown may take longer than five seconds within the process-stop budget"
+        );
+        writeln!(writer, "{event}")?;
+        tokio::time::advance(Duration::from_millis(10)).await;
+        assert_eq!(release.await?, event, "retain the actual new shutdown witness");
+        assert!(!barrier.exists(), "shutdown must release the synthetic hold");
+        Ok(())
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn heal_commit_barrier_missing_shutdown_fails_without_releasing() -> Result<(), Box<dyn Error + Send + Sync>> {
+        let directory = tempfile::tempdir()?;
+        let log = directory.path().join("server.log");
+        let barrier = directory.path().join("barrier");
+        std::fs::write(&log, "")?;
+        std::fs::write(&barrier, "bucket/prefix/")?;
+        let error = release_heal_commit_barrier_after_shutdown(&log, 0, &barrier)
+            .await
+            .expect_err("missing shutdown evidence must fail the fixture");
+        assert!(error.is::<tokio::time::error::Elapsed>());
+        assert!(barrier.is_file(), "a missing shutdown witness must not release repairs");
+        Ok(())
+    }
+
     async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
         let server_binary = rustfs_binary_path();
         let evidence_case = match scenario {
@@ -1791,6 +1874,7 @@ mod tests {
         let mut created_online_objects = Vec::with_capacity(online_object_count);
         let mut expected_manifests = Vec::with_capacity(online_object_count);
         let mut unclean_shutdown_marker_observed = None;
+        let mut shutdown_barrier_release = None;
         let mut attempt_count = 0usize;
         let max_online_attempts = online_object_count.saturating_mul(topology.total_drives().max(1));
         while expected_manifests.len() < online_object_count {
@@ -2373,7 +2457,21 @@ mod tests {
                     | InterruptionScenario::BackgroundTargetRestartEc84MultiSet
                     | InterruptionScenario::BackgroundCoordinatorRestart
             ) {
-                cluster.stop_node_gracefully(interruption_node).await?;
+                if interruption_node == 1
+                    && let Some(barrier) = &commit_barrier
+                {
+                    let target_log = PathBuf::from(format!("{log_dir}/node1.log"));
+                    let log_offset = target_log.metadata()?.len();
+                    let (stopped, released) = tokio::join!(
+                        cluster.stop_node_gracefully(interruption_node),
+                        release_heal_commit_barrier_after_shutdown(&target_log, log_offset, barrier),
+                    );
+                    // Await both branches even on error so the child is reaped.
+                    stopped?;
+                    shutdown_barrier_release = Some(released?);
+                } else {
+                    cluster.stop_node_gracefully(interruption_node).await?;
+                }
             } else {
                 cluster.stop_node(interruption_node)?;
             }
@@ -2415,7 +2513,9 @@ mod tests {
                     }
                 }
             }
-            if let Some(barrier) = &commit_barrier {
+            if shutdown_barrier_release.is_none()
+                && let Some(barrier) = &commit_barrier
+            {
                 std::fs::remove_file(barrier)?;
             }
             cluster.start_node_from_binary(interruption_node, &server_binary).await?;
@@ -2746,6 +2846,7 @@ mod tests {
                 "all_peers_bound_to_generation_window": true,
                 "pid_before": target_pid, "pid_after": restarted_pid,
                 "unclean_shutdown_marker": unclean_shutdown_marker_observed.unwrap_or(false),
+                "graceful_barrier_release": shutdown_barrier_release,
                 "objects": evidence_objects, "node_listings": node_listings,
             });
             if let Some(outage_write) = outage_write_diagnostic {

crates/e2e_test/src/lib.rs

@@ -283,6 +283,9 @@ mod ssec_copy_test;
 #[cfg(test)]
 mod multipart_storage_class_test;
 
+#[cfg(test)]
+mod multipart_upload_id_test;
+
 #[cfg(test)]
 mod storage_class_capability_test;
 

crates/e2e_test/src/multipart_upload_id_test.rs

@@ -0,0 +1,175 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Multipart upload ID regression tests.
+//!
+//! Every operation that addresses an existing multipart upload must report an
+//! upload ID that does not name a live upload as `404 NoSuchUpload`, whether
+//! the ID is malformed or is a well-formed ID of an aborted upload.
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging};
+    use aws_sdk_s3::Client;
+    use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
+    use aws_sdk_s3::primitives::ByteStream;
+    use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
+
+    type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
+
+    const BUCKET: &str = "multipart-upload-id";
+    const KEY: &str = "target.bin";
+    const COPY_SOURCE_KEY: &str = "copy-source.bin";
+    // RustFS upload IDs are URL-safe base64; '.' is outside that alphabet, so
+    // the object layer rejects this ID as malformed instead of looking it up.
+    const MALFORMED_UPLOAD_ID: &str = "malformed.upload.id";
+    const PART_NUMBER: i32 = 1;
+    const PART_BODY: &[u8] = b"multipart upload id regression part";
+    const PART_ETAG: &str = "\"00000000000000000000000000000000\"";
+    const NO_SUCH_UPLOAD_STATUS: u16 = 404;
+    const NO_SUCH_UPLOAD_CODE: &str = "NoSuchUpload";
+    const OBJECT_NOT_FOUND_STATUS: u16 = 404;
+
+    /// Describes how `result` differs from `404 NoSuchUpload`, or returns `None` when it matches.
+    fn no_such_upload_mismatch<T, E>(operation: &str, result: Result<T, SdkError<E>>) -> Option<String>
+    where
+        E: ProvideErrorMetadata + std::fmt::Debug,
+    {
+        let Err(error) = result else {
+            return Some(format!("{operation}: succeeded"));
+        };
+        let status = error.raw_response().map(|response| response.status().as_u16());
+        let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
+        (status != Some(NO_SUCH_UPLOAD_STATUS) || code != Some(NO_SUCH_UPLOAD_CODE))
+            .then(|| format!("{operation}: status {status:?}, code {code:?}: {error:?}"))
+    }
+
+    async fn start_server_with_copy_source() -> Result<(RustFSTestEnvironment, Client), Box<dyn std::error::Error + Send + Sync>>
+    {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(Vec::new()).await?;
+        env.create_test_bucket(BUCKET).await?;
+        let client = env.create_s3_client();
+        client
+            .put_object()
+            .bucket(BUCKET)
+            .key(COPY_SOURCE_KEY)
+            .body(ByteStream::from_static(PART_BODY))
+            .send()
+            .await?;
+        Ok((env, client))
+    }
+
+    /// Sends every operation that addresses an existing upload and asserts that all of them report `404 NoSuchUpload`.
+    async fn assert_every_upload_operation_reports_no_such_upload(client: &Client, upload_id: &str) {
+        let upload_part = client
+            .upload_part()
+            .bucket(BUCKET)
+            .key(KEY)
+            .upload_id(upload_id)
+            .part_number(PART_NUMBER)
+            .body(ByteStream::from_static(PART_BODY))
+            .send()
+            .await;
+        let upload_part_copy = client
+            .upload_part_copy()
+            .bucket(BUCKET)
+            .key(KEY)
+            .upload_id(upload_id)
+            .part_number(PART_NUMBER)
+            .copy_source(format!("/{BUCKET}/{COPY_SOURCE_KEY}"))
+            .send()
+            .await;
+        let list_parts = client.list_parts().bucket(BUCKET).key(KEY).upload_id(upload_id).send().await;
+        let complete = client
+            .complete_multipart_upload()
+            .bucket(BUCKET)
+            .key(KEY)
+            .upload_id(upload_id)
+            .multipart_upload(
+                CompletedMultipartUpload::builder()
+                    .parts(CompletedPart::builder().part_number(PART_NUMBER).e_tag(PART_ETAG).build())
+                    .build(),
+            )
+            .send()
+            .await;
+        let abort = client
+            .abort_multipart_upload()
+            .bucket(BUCKET)
+            .key(KEY)
+            .upload_id(upload_id)
+            .send()
+            .await;
+
+        let mismatches: Vec<String> = [
+            no_such_upload_mismatch("UploadPart", upload_part),
+            no_such_upload_mismatch("UploadPartCopy", upload_part_copy),
+            no_such_upload_mismatch("ListParts", list_parts),
+            no_such_upload_mismatch("CompleteMultipartUpload", complete),
+            no_such_upload_mismatch("AbortMultipartUpload", abort),
+        ]
+        .into_iter()
+        .flatten()
+        .collect();
+        assert!(
+            mismatches.is_empty(),
+            "upload ID {upload_id:?}: operations not reporting {NO_SUCH_UPLOAD_STATUS} {NO_SUCH_UPLOAD_CODE}:\n{}",
+            mismatches.join("\n")
+        );
+    }
+
+    /// Asserts that the rejected operations left neither a multipart upload nor an object under `KEY`.
+    async fn assert_no_upload_or_object_published(client: &Client) -> TestResult {
+        let uploads = client.list_multipart_uploads().bucket(BUCKET).prefix(KEY).send().await?;
+        assert!(uploads.uploads().is_empty(), "rejected operations must not leave an upload: {uploads:?}");
+        let head = client.head_object().bucket(BUCKET).key(KEY).send().await;
+        let status = head
+            .as_ref()
+            .err()
+            .and_then(|error| error.raw_response())
+            .map(|response| response.status().as_u16());
+        assert_eq!(
+            status,
+            Some(OBJECT_NOT_FOUND_STATUS),
+            "rejected operations must not publish {KEY}: {head:?}"
+        );
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn malformed_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
+        let (_env, client) = start_server_with_copy_source().await?;
+
+        assert_every_upload_operation_reports_no_such_upload(&client, MALFORMED_UPLOAD_ID).await;
+        assert_no_upload_or_object_published(&client).await
+    }
+
+    #[tokio::test]
+    async fn aborted_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
+        let (_env, client) = start_server_with_copy_source().await?;
+        let created = client.create_multipart_upload().bucket(BUCKET).key(KEY).send().await?;
+        let upload_id = created.upload_id().ok_or("CreateMultipartUpload returned no upload ID")?;
+        client
+            .abort_multipart_upload()
+            .bucket(BUCKET)
+            .key(KEY)
+            .upload_id(upload_id)
+            .send()
+            .await?;
+
+        assert_every_upload_operation_reports_no_such_upload(&client, upload_id).await;
+        assert_no_upload_or_object_published(&client).await
+    }
+}

crates/e2e_test/src/replication_extension_test.rs

@@ -31,8 +31,8 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{
-    BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DeleteMarkerEntry, ObjectVersion, ServerSideEncryption,
+    BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Delete, DeleteMarkerEntry, ObjectIdentifier, ObjectVersion,
-    VersioningConfiguration,
+    ServerSideEncryption, VersioningConfiguration,
 };
 use base64_simd::STANDARD as BASE64_STANDARD;
 use bytes::Bytes;
@@ -3729,6 +3729,8 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
     init_logging();
 
     let mut source_env = RustFSTestEnvironment::new().await?;
+    let source_log = format!("{}/source.log", source_env.temp_dir);
+    source_env.capture_log_path = Some(source_log.clone());
     let mut source_env_vars = replication_fast_env();
     source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
     source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
@@ -3791,19 +3793,66 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
         "target did not preserve the source delete-marker version ID"
     );
 
-    source_client
+    let wait_for_source_writeback = || async {
-        .delete_object()
+        let metrics_url = format!("{}/{source_bucket}?replication-metrics", source_env.url);
+        timeout(Duration::from_secs(30), async {
+            loop {
+                let response = signed_request(
+                    http::Method::GET,
+                    &metrics_url,
+                    &source_env.access_key,
+                    &source_env.secret_key,
+                    None,
+                    None,
+                )
+                .await?;
+                let metrics: serde_json::Value = response.error_for_status()?.json().await?;
+                let queued = metrics["queued"]["curr"]["count"]
+                    .as_f64()
+                    .ok_or("missing replication queue count")?;
+                if metrics["provider_available"] == true && queued == 0.0 {
+                    return Ok::<(), Box<dyn Error + Send + Sync>>(());
+                }
+                sleep(Duration::from_millis(50)).await;
+            }
+        })
+        .await?
+    };
+    wait_for_source_writeback().await?;
+
+    let purged = source_client
+        .delete_objects()
         .bucket(source_bucket)
-        .key(object_key)
+        .delete(
-        .version_id(&purged_version_id)
+            Delete::builder()
+                .objects(
+                    ObjectIdentifier::builder()
+                        .key(object_key)
+                        .version_id(&purged_version_id)
+                        .build()?,
+                )
+                .objects(
+                    ObjectIdentifier::builder()
+                        .key(object_key)
+                        .version_id(delete_marker_version_id)
+                        .build()?,
+                )
+                .build()?,
+        )
         .send()
         .await?;
+    assert!(purged.errors().is_empty(), "batch version purge failed: {:?}", purged.errors());
+    assert_eq!(purged.deleted().len(), 2, "both the data version and marker must be purged");
     assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
     let target_state = list_replication_state(&target_client, target_bucket).await?;
     assert!(
         target_state.iter().all(|entry| entry.version_id != purged_version_id),
         "target retained the explicitly purged object version"
     );
+    assert!(
+        target_state.iter().all(|entry| entry.version_id != delete_marker_version_id),
+        "target retained the explicitly purged delete marker"
+    );
     assert!(
         target_state
             .iter()
@@ -3819,6 +3868,19 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
         .await?;
     assert_eq!(retained.body.collect().await?.into_bytes().as_ref(), b"versioned replication payload v2");
 
+    // Wait for the source workers to finish their final writeback, which runs
+    // after the remote purge has already become visible (rustfs#9121).
+    wait_for_source_writeback().await?;
+    let logs = fs::read_to_string(source_log).await?;
+    let writeback_failures: Vec<_> = logs
+        .lines()
+        .filter(|line| line.contains("apply_replication_delete_state"))
+        .collect();
+    assert!(
+        writeback_failures.is_empty(),
+        "a completed batch purge must not report a source writeback failure: {writeback_failures:?}"
+    );
+
     Ok(())
 }
 

crates/e2e_test/src/rpc_fault_proxy.rs

@@ -2146,3 +2146,4 @@ mod tests {
         );
     }
 }
+// event-drill: isolated harness-only change

crates/e2e_test/src/storage_api.rs

@@ -94,3 +94,12 @@ pub(crate) mod scanner_peer_bucket {
     pub(crate) use ecstore_erasure::{ReedSolomonEncoder, calc_shard_size};
     pub(crate) use ecstore_rpc::PeerRestClient;
 }
+
+/// Storage handles used to seed a pre-validation bucket record while the fixture server is stopped.
+#[cfg(test)]
+pub(crate) mod bucket_tag_fixture {
+    pub(crate) use rustfs_ecstore::api::bucket::metadata::load_bucket_metadata;
+    pub(crate) use rustfs_ecstore::api::layout::{DisksLayout, EndpointServerPools};
+    pub(crate) use rustfs_ecstore::api::runtime::InstanceContext;
+    pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks_with_instance_ctx};
+}

crates/e2e_test/src/sts_query_compat_test.rs

@@ -360,12 +360,26 @@ impl OpaMock {
 
     pub(crate) async fn expect_bucket_tags(&mut self, action: &str, bucket: &str, department: Option<&str>) -> TestResult {
         let payload = self.next_request().await?;
+        Self::assert_bucket_tags(&payload, action, bucket, department);
+        Ok(())
+    }
+
+    async fn expect_versioned_tag_permission(&mut self, bucket: &str, object: &str, version: &str) -> TestResult {
+        let payload = self.next_request().await?;
+        Self::assert_bucket_tags(&payload, "s3:GetObjectVersionTagging", bucket, Some("finance"));
+        assert_eq!(payload["input"]["resource"]["object"], object);
+        assert_eq!(payload["input"]["context"]["conditions"]["versionid"], serde_json::json!([version]));
+        Ok(())
+    }
+
+    fn assert_bucket_tags(payload: &Value, action: &str, bucket: &str, department: Option<&str>) {
         let input = &payload["input"];
+        assert_eq!(input["identity"]["account"], "opabuckettags");
+        assert_eq!(input["identity"]["is_owner"], false);
         assert_eq!(input["action"], action);
         assert_eq!(input["resource"]["bucket"], bucket);
         let expected = department.map(|value| serde_json::json!([value]));
         assert_eq!(input["context"]["conditions"].get("ExistingBucketTag/department"), expected.as_ref());
-        Ok(())
     }
 
     async fn wait_for_validation(&mut self) -> TestResult {
@@ -678,22 +692,61 @@ async fn test_list_buckets_opa_contract() -> TestResult {
     Ok(())
 }
 
+/// Stop and reap the directory's owning server before opening the fixture store.
+async fn seed_legacy_duplicate_bucket_tags(env: &mut RustFSTestEnvironment, bucket: &str) -> TestResult {
+    use crate::storage_api::bucket_tag_fixture::{
+        DisksLayout, ECStore, EndpointServerPools, InstanceContext, init_local_disks_with_instance_ctx, load_bucket_metadata,
+    };
+
+    let server = env
+        .process
+        .as_mut()
+        .ok_or("missing server process before offline fixture write")?;
+    server.kill()?;
+    server.wait()?;
+    env.process = None;
+    let layout = DisksLayout::from_volumes(std::slice::from_ref(&env.temp_dir))?;
+    let (pools, _) = EndpointServerPools::create_server_endpoints(&env.address, &layout).await?;
+    let context = std::sync::Arc::new(InstanceContext::new());
+    init_local_disks_with_instance_ctx(&context, pools.clone()).await?;
+    let store = ECStore::new_with_instance_ctx(env.address.parse()?, pools, Default::default(), context.clone()).await?;
+    let _cancel = context
+        .background_cancel_token()
+        .expect("store binds its shutdown token")
+        .drop_guard();
+    let mut metadata = load_bucket_metadata(store.clone(), bucket).await?;
+    let incarnation = metadata.bucket_incarnation_id;
+    let policy = metadata.policy_config_json.clone();
+    assert!(!incarnation.is_nil());
+    assert!(!policy.is_empty(), "the bucket-policy Allow must survive legacy seeding");
+    // Encode the duplicate tag set with the same bucket record and erasure writer used by the store.
+    let xml = b"<Tagging><TagSet><Tag><Key>department</Key><Value>finance</Value></Tag><Tag><Key>department</Key><Value>engineering</Value></Tag></TagSet></Tagging>";
+    metadata.tagging_config_xml = xml.to_vec();
+    metadata.save_with_store(store.clone()).await?;
+    let persisted = load_bucket_metadata(store, bucket).await?;
+    assert_eq!(persisted.bucket_incarnation_id, incarnation);
+    assert_eq!(persisted.policy_config_json, policy);
+    assert_eq!(persisted.tagging_config_xml.as_slice(), xml);
+    Ok(())
+}
+
 #[tokio::test]
 async fn test_bucket_tags_opa_contract() -> TestResult {
     use aws_sdk_s3::primitives::ByteStream;
     use aws_sdk_s3::types::{
-        BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, VersioningConfiguration,
+        BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, TaggingDirective, VersioningConfiguration,
     };
 
     init_logging();
     let mut opa = OpaMock::start().await?;
     let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server_without_cleanup_with_env(&[
+    let opa_url = opa.url.clone();
-        ("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str()),
+    let server_env = [
+        ("RUSTFS_POLICY_PLUGIN_URL", opa_url.as_str()),
         ("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", OPA_AUTH_TOKEN),
         ("NO_PROXY", "127.0.0.1,localhost"),
-    ])
+    ];
-    .await?;
+    env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
     let secret = uuid::Uuid::new_v4().to_string();
     create_user(&env, "opabuckettags", &secret).await?;
     let client = aws_sdk_s3::Client::from_conf(build_test_s3_config(&env.url, "opabuckettags", &secret, None, "e2e-bucket-tags"));
@@ -733,7 +786,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
         .await?;
     assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
     opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
-    client
+    let head = client
         .head_object()
         .bucket(source)
         .key("report")
@@ -741,6 +794,8 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
         .send()
         .await?;
     opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
+    assert_eq!(head.tag_count(), Some(1));
+    opa.expect_versioned_tag_permission(source, "report", version).await?;
     client.list_objects_v2().bucket(source).send().await?;
     opa.expect_bucket_tags("s3:ListBucket", source, Some("finance")).await?;
     client.list_object_versions().bucket(source).send().await?;
@@ -750,28 +805,65 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
     admin.create_bucket().bucket(destination).send().await?;
     set_department(&admin, destination, "engineering").await?;
     let copy_source = format!("{source}/report?versionId={version}");
-    let error = client
+    for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
-        .copy_object()
+        let error = client
-        .bucket(destination)
+            .copy_object()
-        .key("copied")
+            .bucket(destination)
-        .copy_source(&copy_source)
+            .key("copied")
-        .send()
+            .copy_source(&copy_source)
-        .await
+            .set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
-        .expect_err("source tags must not authorize a different destination");
+            .set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
-    assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
+            .send()
-    opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
+            .await
-    opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
+            .expect_err("source tags must not authorize a different destination");
-        .await?;
+        assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
+        opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
+        if directive == TaggingDirective::Copy {
+            opa.expect_versioned_tag_permission(source, "report", version).await?;
+        }
+        opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
+            .await?;
+        let missing = admin
+            .head_object()
+            .bucket(destination)
+            .key("copied")
+            .send()
+            .await
+            .expect_err("denied COPY and REPLACE must leave no destination object");
+        assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
+    }
     set_department(&admin, destination, "finance").await?;
-    client
+    for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
-        .copy_object()
+        client
-        .bucket(destination)
+            .copy_object()
-        .key("copied")
+            .bucket(destination)
-        .copy_source(&copy_source)
+            .key("copied")
-        .send()
+            .copy_source(&copy_source)
-        .await?;
+            .set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
-    opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
+            .set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
-    opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
+            .send()
+            .await?;
+        opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
+        // The access hook checks preliminary tags, then the copy use case checks
+        // the exact source reader's tags before publishing the destination.
+        for _ in 0..2 {
+            if directive == TaggingDirective::Copy {
+                opa.expect_versioned_tag_permission(source, "report", version).await?;
+            }
+            opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
+        }
+        let tags = admin.get_object_tagging().bucket(destination).key("copied").send().await?;
+        assert_eq!(tags.tag_set().len(), 1);
+        assert_eq!(tags.tag_set()[0].key(), "department");
+        assert_eq!(
+            tags.tag_set()[0].value(),
+            if directive == TaggingDirective::Copy {
+                "engineering"
+            } else {
+                "legal"
+            }
+        );
+    }
 
     let upload = client
         .create_multipart_upload()
@@ -847,6 +939,14 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
     assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
     opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("engineering"))
         .await?;
+    let missing = admin
+        .head_object()
+        .bucket(destination)
+        .key("denied-source")
+        .send()
+        .await
+        .expect_err("a denied source must leave no destination object");
+    assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
     set_department(&client, source, "finance")
         .await
         .expect_err("requested tags must not authorize their own mutation");
@@ -885,7 +985,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
     assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
     opa.expect_bucket_tags("s3:GetObject", destination, Some("engineering"))
         .await?;
-    admin
+    let invalid_write = admin
         .put_bucket_tagging()
         .bucket(destination)
         .tagging(
@@ -895,7 +995,25 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
                 .build()?,
         )
         .send()
+        .await
+        .expect_err("new duplicate tags must be rejected before publication");
+    assert_eq!(invalid_write.as_service_error().and_then(ProvideErrorMetadata::code), Some("InvalidTag"));
+    assert_eq!(invalid_write.raw_response().map(|response| response.status().as_u16()), Some(400));
+
+    set_department(&admin, source, "finance").await?;
+    seed_legacy_duplicate_bucket_tags(&mut env, destination).await?;
+    env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
+    // Prove this process can authorize the same caller and read a healthy bucket's persisted tags.
+    let read = client
+        .get_object()
+        .bucket(source)
+        .key("report")
+        .version_id(version)
+        .send()
         .await?;
+    assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
+    opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
+
     let error = client
         .get_object()
         .bucket(destination)

crates/e2e_test/src/upgrade_compatibility_test.rs

@@ -48,6 +48,7 @@ const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
 const G09_EVIDENCE_DIR_ENV: &str = "RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR";
 const RC5_COMMIT: &str = "40a2470feb567201165a5b809b7598bb4b1f68f5";
 const STABLE_1_0_1_COMMIT: &str = "6de965ae3c965a78ff819fbcd7acd4aa44177d92";
+const STABLE_1_0_4_COMMIT: &str = "46337942d55f3a8e6a97b147ca98d9ea8115555d";
 const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
 const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI=";
 const PLAIN_BUCKET: &str = "upgrade-plain-data";
@@ -954,7 +955,8 @@ async fn exercise_mixed_cluster(
     phase: &str,
     current_node: usize,
     previous_node: usize,
-) -> TestResult {
+    previous_has_delete_recovery: bool,
+) -> Result<Option<String>, BoxError> {
     let clients = cluster.create_all_clients()?;
     wait_for_upgrade_write_readiness(&clients, MIXED_BUCKET, phase, LISTING_CONVERGENCE_TIMEOUT).await?;
     wait_for_drive_readmission(&clients, MIXED_BUCKET, phase, DRIVE_READMISSION_TIMEOUT).await?;
@@ -1009,6 +1011,30 @@ async fn exercise_mixed_cluster(
         vec![u8::try_from(MULTIPART_WORKERS - 1)?; 64 * 1024]
     );
 
+    // 1.0.4 already implements the delete decision protocol. Use a separate
+    // old-writer key so the original seed still spans the entire rollout.
+    if previous_has_delete_recovery {
+        let key = format!(".upgrade-delete/{phase}/previous-writer");
+        previous_client
+            .put_object()
+            .bucket(MIXED_BUCKET)
+            .key(&key)
+            .body(ByteStream::from_static(UPGRADE_READINESS_BODY))
+            .send()
+            .await?;
+        for client in &clients {
+            assert_eq!(read_object(client, MIXED_BUCKET, &key, None).await?.1, UPGRADE_READINESS_BODY);
+        }
+        upgrade_probe_client(current_client)
+            .delete_object()
+            .bucket(MIXED_BUCKET)
+            .key(&key)
+            .send()
+            .await?;
+        assert_mixed_deletes_visible(&clients, std::slice::from_ref(&key)).await?;
+        return Ok(Some(key));
+    }
+
     // Every online participant must understand the delete decision protocol.
     // An upgrade phase can serve reads/writes while intentionally refusing a
     // modern DELETE; rejecting it must leave the old writer's seed intact.
@@ -1029,6 +1055,31 @@ async fn exercise_mixed_cluster(
         );
     }
 
+    Ok(None)
+}
+
+async fn assert_mixed_deletes_visible(clients: &[Client], keys: &[String]) -> TestResult {
+    for (node, client) in clients.iter().enumerate() {
+        for key in keys {
+            let error = upgrade_probe_client(client)
+                .get_object()
+                .bucket(MIXED_BUCKET)
+                .key(key)
+                .send()
+                .await
+                .expect_err("a successful mixed-version DELETE must remain visible through every reader");
+            assert_eq!(
+                error.raw_response().map(|response| response.status().as_u16()),
+                Some(404),
+                "node {node}: {key}"
+            );
+            assert_eq!(
+                error.as_service_error().and_then(ProvideErrorMetadata::code),
+                Some("NoSuchKey"),
+                "node {node}: {key}"
+            );
+        }
+    }
     Ok(())
 }
 
@@ -1112,8 +1163,9 @@ async fn wait_for_upgrade_writes(
             match response {
                 Ok(Ok(_)) => break,
                 Ok(Err(error)) => {
+                    let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
                     if error.raw_response().map(|response| response.status().as_u16()) != Some(503)
-                        || error.as_service_error().and_then(ProvideErrorMetadata::code) != Some("ServiceUnavailable")
+                        || !matches!(code, Some("ServiceUnavailable" | "SlowDown" | "SlowDownRead"))
                     {
                         return Err(format!("{phase}: node {node} {probe} failed: {error:?}").into());
                     }
@@ -2239,6 +2291,9 @@ async fn direct_upgrade_from_previous_release_preserves_object_contracts() -> Te
 async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts() -> TestResult {
     init_logging();
     let previous_binary = source_binary()?;
+    // Bind the expected protocol to the verified release source, never to the
+    // DELETE result itself. The older CI source pins retain their 503 contract.
+    let previous_has_delete_recovery = binary_source_revision(&previous_binary).await? == STABLE_1_0_4_COMMIT;
     let evidence_context = g09_evidence_context(&previous_binary).await?;
     let current_binary = rustfs_binary_path();
     let mut cluster = RustFSTestClusterEnvironment::new(MIXED_NODE_COUNT).await?;
@@ -2259,7 +2314,10 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
 
     cluster.stop_node(0)?;
     cluster.start_node_from_binary(0, &current_binary).await?;
-    exercise_mixed_cluster(&cluster, "one-current-node", 0, 1).await?;
+    let mut mixed_deleted_keys = Vec::new();
+    if let Some(key) = exercise_mixed_cluster(&cluster, "one-current-node", 0, 1, previous_has_delete_recovery).await? {
+        mixed_deleted_keys.push(key);
+    }
     // One controlled out-of-policy attempt proves fail-closed behavior below
     // native write quorum; positive MPU writers remain paused while mixed.
     assert_mixed_pending_refusal(&cluster, &pending, &sources).await?;
@@ -2267,7 +2325,9 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
     for (node_idx, phase) in [(1, "two-current-nodes"), (2, "one-previous-node")] {
         cluster.stop_node(node_idx)?;
         cluster.start_node_from_binary(node_idx, &current_binary).await?;
-        exercise_mixed_cluster(&cluster, phase, 0, 3).await?;
+        if let Some(key) = exercise_mixed_cluster(&cluster, phase, 0, 3, previous_has_delete_recovery).await? {
+            mixed_deleted_keys.push(key);
+        }
     }
 
     cluster.stop_node(3)?;
@@ -2277,6 +2337,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
     let completed_pending = finish_pending_multipart(&clients, &pending, pending_parts.concat()).await?;
     exercise_homogeneous_multipart(&cluster, "homogeneous-current").await?;
     verify_load_bodies(&clients, "homogeneous-previous", "multipart").await?;
+    assert_mixed_deletes_visible(&clients, &mixed_deleted_keys).await?;
 
     for (node_idx, client) in clients.iter().enumerate() {
         assert_eq!(
@@ -2325,6 +2386,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
     cluster.start_with_binary(&previous_binary).await?;
     prepare_release_baseline(&mut cluster, &previous_binary, "rollback").await?;
     let old_clients = cluster.create_all_clients()?;
+    assert_mixed_deletes_visible(&old_clients, &mixed_deleted_keys).await?;
     assert_resumed_multipart(&old_clients, &completed_pending).await?;
     for phase in ["homogeneous-previous", "homogeneous-current"] {
         verify_load_bodies(&old_clients, phase, "multipart").await?;
@@ -2389,6 +2451,8 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
                 "phases": phases,
                 "objects_per_phase": objects_per_phase,
                 "mixed_load": "plain-put",
+                "mixed_delete_protocol_supported": previous_has_delete_recovery,
+                "mixed_deleted_keys": mixed_deleted_keys,
                 "homogeneous_multipart_phases": ["homogeneous-previous", "homogeneous-current", "homogeneous-rollback"],
                 "multipart_objects_per_homogeneous_phase": MULTIPART_WORKERS * MULTIPART_UPLOADS_PER_WORKER,
                 "assertions": [
@@ -2421,7 +2485,11 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
                     "16x16 multipart load runs in homogeneous previous, current and rollback phases",
                     "one-current-node Complete returns typed 503 and preserves actual source authority and parts",
                     "the same legacy upload completes and replays once fully upgraded without adding a version",
-                    "modern DELETE fails closed and preserves every byte while old peers remain"
+                    if previous_has_delete_recovery {
+                        "mixed-version DELETE succeeds and stays absent through every reader after upgrade and rollback"
+                    } else {
+                        "modern DELETE fails closed and preserves every byte while old peers remain"
+                    }
                 ],
             }),
         )?;

crates/ecstore/Cargo.toml

@@ -31,6 +31,7 @@ workspace = true
 
 [features]
 default = []
+fips = ["rustfs-s3-client/fips", "rustfs-utils/fips"]
 diagnostic-read-version-caller = ["rustfs-io-metrics/diagnostic-read-version-caller"]
 # Sampled wall/CPU diagnostics only; does not change the read backend.
 diagnostic-local-batch = ["rustix/time"]

crates/ecstore/src/api/mod.rs

@@ -192,6 +192,7 @@ pub mod bucket {
             table_catalog_path_hash, unreadable_config_refusal,
         };
         pub use crate::bucket::metadata::{BUCKET_DURABILITY_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
+        pub use crate::bucket::metadata::{MAX_BUCKET_TAGS, validate_tag_set_entries};
     }
 
     pub mod config_parse_mode {
@@ -331,8 +332,8 @@ pub mod bucket {
     pub mod utils {
         pub use crate::bucket::utils::{
             check_bucket_and_object_names, check_list_objs_args, check_object_name_for_length_and_slash,
-            check_valid_bucket_name_strict, deserialize, has_bad_path_component, is_meta_bucketname, is_valid_object_prefix,
+            check_valid_bucket_name_strict, deserialize, has_bad_path_component, has_xml_invalid_control_characters,
-            serialize,
+            is_meta_bucketname, is_valid_object_prefix, serialize,
         };
     }
 
@@ -451,9 +452,15 @@ pub mod disk {
     pub use crate::cluster::rpc::background_reader_stream::BackgroundReaderStream;
     pub use crate::disk::background_memory::BackgroundEncodeBuffer;
     pub use crate::disk::disk_store::get_object_disk_read_timeout;
+    #[cfg(feature = "test-util")]
+    pub use crate::disk::disk_store::wait_for_multipart_index_bootstrap;
     pub use crate::disk::local::{ReplacementExecutionLease, ScanGuard};
     #[cfg(feature = "test-util")]
     pub use crate::disk::local::{local_durability_modes, set_local_rename_data_fail_after_metadata_commit};
+    pub use crate::disk::multipart_index::{
+        MULTIPART_INDEX_RPC_MAX_BYTES, MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES, MultipartIndexPage, MultipartIndexRpcReply,
+        MultipartIndexRpcRequest,
+    };
     pub use crate::disk::multipart_source::{
         MULTIPART_SOURCE_PROTOCOL_VERSION, MultipartAbortCertificate, MultipartAbortProof, MultipartAbortProofKind,
         MultipartAbortRound, MultipartSourceOperation, MultipartSourcePhase, MultipartSourceRequest, MultipartSourceResponse,
@@ -466,13 +473,15 @@ pub mod disk {
     pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage};
     pub use crate::disk::{
         BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
-        CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration, CurrentDeleteTransition,
+        BoundedDeleteRecoveryDiscovery, BoundedDeleteRecoveryInspection, BoundedDeleteRecoverySettlement,
+        BoundedDeleteRecoverySettlementReply, CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration,
+        CurrentDeleteTransition, DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES, DELETE_RECOVERY_MAX_INSPECTION_ENTRIES,
         DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest,
         DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DeleteRecoveryState,
         Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, FileInfoVersions, FileReader, FileWriter,
-        GuardedDeleteVersionsPayload, GuardedFileInfoVersions, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize,
+        GuardedDeleteVersionsPayload, GuardedFileInfoVersions, HEAL_REPLACEMENT_RECOVERY_DIR, HEALING_MARKER_PATH,
-        POOL_META_NATIVE_CAS_METADATA_SUFFIX, PartTransactionAction, PoolMetaNativeBackup, PoolMetaNativeInspection,
+        NsScannerOpenRequest, OldCurrentSize, POOL_META_NATIVE_CAS_METADATA_SUFFIX, PartTransactionAction, PoolMetaNativeBackup,
-        PoolMetaNativeObject, PoolMetaNativeRecoveryRequest, PoolMetaNativeRecoveryResponse,
+        PoolMetaNativeInspection, PoolMetaNativeObject, PoolMetaNativeRecoveryRequest, PoolMetaNativeRecoveryResponse,
         QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET,
         ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, SnapshotLeaseToken,
         UpdateMetadataOpts, VolumeInfo, WalkDirOptions, is_quota_mutation_fence_path, new_disk,
@@ -617,7 +626,7 @@ pub mod rebalance {
 pub mod rio {
     pub use crate::io_support::rio::{
         DecryptReader, DynReader, EncryptReader, HardLimitReader, HashReader, ReadStream, Reader, WriteEncryption, WritePlan,
-        boxed_reader, compression_metadata_value, wrap_reader,
+        boxed_reader, compression_metadata_value, validate_compression_scheme, wrap_reader,
     };
 }
 
@@ -654,7 +663,9 @@ pub mod set_disk {
 
     #[cfg(feature = "test-util")]
     pub mod test_util {
-        pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test;
+        pub use crate::bucket::quota::reservation::{
+            fail_next_quota_ledger_save_for_test, initialize_quota_ledger_for_test, retry_quota_reconciliation_for_test,
+        };
         pub use crate::set_disk::ConditionalReadBenchmarkMetadataGuard;
         pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause};
 

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -16,7 +16,8 @@ use crate::bucket::metadata::BucketMetadata;
 use crate::bucket::metadata_sys::get_bucket_targets_config;
 use crate::bucket::metadata_sys::get_replication_config;
 use crate::bucket::remote_s3_client::{
-    PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, build_remote_s3_client,
+    PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, RemoteS3RetryPolicy,
+    UploadResponseDeadline, build_remote_s3_client,
 };
 use crate::bucket::replication::{ObjectLockIntegrity, object_lock_put_integrity, replication_etags_match};
 use crate::bucket::replication::{ReplicationStatusType, ReplicationTargetConfigBridge};
@@ -119,6 +120,7 @@ impl From<&BucketTarget> for RemoteS3EndpointSpec {
             ca_cert_pem: (!target.ca_cert_pem.trim().is_empty()).then(|| target.ca_cert_pem.clone()),
             connect_timeout: None,
             read_timeout: None,
+            operation_timeout: Some(TargetRequestTimeouts::from_env().request),
             // Replication has no retry budget of its own on the request path,
             // so it keeps the SDK's standard three attempts; stating it here
             // pins the behaviour to this line instead of an SDK default.
@@ -1191,6 +1193,7 @@ impl BucketTargetSys {
             health_check_duration: target.health_check_duration,
             replicate_sync: target.replication_sync,
             client: Arc::new(client),
+            timeouts: TargetRequestTimeouts::from_env(),
         })
     }
 
@@ -1922,6 +1925,83 @@ impl std::fmt::Display for S3ClientError {
     }
 }
 
+/// Replication request budgets, snapshotted when a target client is built.
+/// Streamed GET bodies are not subject to the upload-response budget.
+#[derive(Debug, Clone, Copy)]
+pub struct TargetRequestTimeouts {
+    request: Duration,
+    complete: Duration,
+    minimum_transfer_rate: u64,
+}
+
+impl Default for TargetRequestTimeouts {
+    fn default() -> Self {
+        Self {
+            request: Duration::from_secs(60),
+            complete: Duration::from_secs(15 * 60),
+            minimum_transfer_rate: 1024 * 1024,
+        }
+    }
+}
+
+impl TargetRequestTimeouts {
+    fn from_env() -> Self {
+        let defaults = Self::default();
+        let seconds = |name: &str, fallback: Duration| {
+            let value = rustfs_utils::get_env_u64(name, fallback.as_secs());
+            Duration::from_secs(if (1..=86400).contains(&value) {
+                value
+            } else {
+                fallback.as_secs()
+            })
+        };
+        let minimum_transfer_rate =
+            rustfs_utils::get_env_u64("RUSTFS_REPLICATION_MIN_TRANSFER_RATE_BYTES_PER_SEC", defaults.minimum_transfer_rate);
+        Self {
+            request: seconds("RUSTFS_REPLICATION_REQUEST_TIMEOUT_SECS", defaults.request),
+            complete: seconds("RUSTFS_REPLICATION_COMPLETE_TIMEOUT_SECS", defaults.complete),
+            minimum_transfer_rate: if minimum_transfer_rate == 0 {
+                defaults.minimum_transfer_rate
+            } else {
+                minimum_transfer_rate
+            },
+        }
+    }
+
+    fn upload_timeout(self, size: i64) -> Result<Duration, S3ClientError> {
+        let size = u64::try_from(size).map_err(|_| S3ClientError::new("replication upload size must be non-negative"))?;
+        // The SDK can make three attempts. Leave transmission headroom for
+        // all three; the separate response deadline still bounds a trickling
+        // target after each upload finishes.
+        let attempts = match REPLICATION_TARGET_RETRY_POLICY {
+            RemoteS3RetryPolicy::Disabled => 1,
+            RemoteS3RetryPolicy::Standard { max_attempts } => u64::from(max_attempts.max(1)),
+        };
+        let seconds = size
+            .div_ceil(self.minimum_transfer_rate)
+            .checked_mul(attempts)
+            .ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
+        let timeout = self
+            .request
+            .checked_add(Duration::from_secs(seconds))
+            .ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
+        if Instant::now().checked_add(timeout).is_none() {
+            return Err(S3ClientError::new("replication upload timeout exceeds supported duration"));
+        }
+        Ok(timeout)
+    }
+
+    fn upload_override(self, size: i64) -> Result<aws_sdk_s3::config::Builder, S3ClientError> {
+        Ok(aws_sdk_s3::config::Builder::new()
+            .timeout_config(
+                aws_smithy_types::timeout::TimeoutConfig::builder()
+                    .operation_timeout(self.upload_timeout(size)?)
+                    .build(),
+            )
+            .interceptor(UploadResponseDeadline(self.request)))
+    }
+}
+
 #[derive(Debug)]
 pub struct TargetClient {
     pub endpoint: String,
@@ -1935,6 +2015,7 @@ pub struct TargetClient {
     pub health_check_duration: Duration,
     pub replicate_sync: bool,
     pub client: Arc<S3Client>,
+    pub timeouts: TargetRequestTimeouts,
 }
 
 impl TargetClient {
@@ -2360,12 +2441,11 @@ impl TargetClient {
 
                 Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
             });
+        let mut config = self.timeouts.upload_override(size)?;
         if forwards_source_checksum {
-            operation = operation.config_override(
+            config = config.request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired);
-                aws_sdk_s3::config::Builder::new()
-                    .request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired),
-            );
         }
+        operation = operation.config_override(config);
         match operation.send().await {
             Ok(output) => {
                 // Under SSE-KMS/DSSE or SSE-C the target's ETag is not the MD5
@@ -2501,6 +2581,7 @@ impl TargetClient {
                 }
                 Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
             })
+            .config_override(self.timeouts.upload_override(size)?)
             .send()
             .await
         {
@@ -2538,6 +2619,13 @@ impl TargetClient {
                 }
                 Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
             })
+            .config_override(
+                aws_sdk_s3::config::Builder::new().timeout_config(
+                    aws_smithy_types::timeout::TimeoutConfig::builder()
+                        .operation_timeout(self.timeouts.complete)
+                        .build(),
+                ),
+            )
             .send()
             .await
         {
@@ -2966,11 +3054,241 @@ mod tests {
                 health_check_duration: Duration::from_secs(5),
                 replicate_sync: false,
                 client: Arc::new(client),
+                timeouts: TargetRequestTimeouts::default(),
             },
             request_headers,
         )
     }
 
+    #[derive(Debug, Clone)]
+    struct TimedResponseConnector {
+        upload_delay: Duration,
+        response_delay: Duration,
+        prefix_bytes: usize,
+        response: bytes::Bytes,
+        frames: Arc<std::sync::atomic::AtomicUsize>,
+        dropped: Arc<std::sync::atomic::AtomicUsize>,
+    }
+
+    struct ResponseDrop(Arc<std::sync::atomic::AtomicUsize>);
+    impl Drop for ResponseDrop {
+        fn drop(&mut self) {
+            self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+        }
+    }
+
+    impl SmithyHttpConnector for TimedResponseConnector {
+        fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
+            let settings = self.clone();
+            HttpConnectorFuture::new(async move {
+                http_body_util::BodyExt::collect(request.into_body())
+                    .await
+                    .map_err(aws_smithy_runtime_api::client::result::ConnectorError::io)?;
+                tokio::time::sleep(settings.upload_delay).await;
+                let guard = ResponseDrop(Arc::clone(&settings.dropped));
+                let stream = futures::stream::unfold(
+                    (settings.prefix_bytes, Some(settings.response), true, guard),
+                    move |(remaining, mut body, first, guard)| {
+                        let frames = Arc::clone(&settings.frames);
+                        async move {
+                            if remaining == 0 && body.is_none() {
+                                return None;
+                            }
+                            if !first {
+                                tokio::time::sleep(settings.response_delay).await;
+                            }
+                            let bytes = if remaining > 0 {
+                                bytes::Bytes::from_static(b" ")
+                            } else {
+                                body.take()?
+                            };
+                            frames.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+                            Some((
+                                Ok::<_, std::io::Error>(http_body::Frame::data(bytes)),
+                                (remaining.saturating_sub(1), body, false, guard),
+                            ))
+                        }
+                    },
+                )
+                .fuse();
+                let body = SdkBody::from_body_1_x(http_body_util::StreamBody::new(stream));
+                let mut response =
+                    HttpResponse::new(aws_smithy_runtime_api::http::StatusCode::try_from(200u16).expect("status"), body);
+                response.headers_mut().insert("etag", "\"test-etag\"");
+                Ok(response)
+            })
+        }
+    }
+
+    fn timed_response_target(
+        upload_delay: Duration,
+        response_delay: Duration,
+        prefix_bytes: usize,
+        response: &'static [u8],
+    ) -> (TargetClient, Arc<std::sync::atomic::AtomicUsize>, Arc<std::sync::atomic::AtomicUsize>) {
+        let frames = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let dropped = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let connector = SharedHttpConnector::new(TimedResponseConnector {
+            upload_delay,
+            response_delay,
+            prefix_bytes,
+            response: bytes::Bytes::from_static(response),
+            frames: Arc::clone(&frames),
+            dropped: Arc::clone(&dropped),
+        });
+        let http_client = http_client_fn(move |_settings, _components| connector.clone());
+        let (mut target, _) = header_recording_target_client(Vec::new());
+        target.timeouts = TargetRequestTimeouts {
+            request: Duration::from_secs(2),
+            complete: Duration::from_secs(10),
+            minimum_transfer_rate: 1024 * 1024,
+        };
+        target.client = Arc::new(S3Client::from_conf(
+            target
+                .client
+                .config()
+                .to_builder()
+                .http_client(http_client)
+                .retry_config(aws_sdk_s3::config::retry::RetryConfig::disabled())
+                .timeout_config(
+                    aws_smithy_types::timeout::TimeoutConfig::builder()
+                        .operation_timeout(target.timeouts.request)
+                        .build(),
+                )
+                .build(),
+        ));
+        (target, frames, dropped)
+    }
+
+    #[test]
+    fn replication_upload_budgets_preserve_large_transfers_and_reject_invalid_sizes() {
+        let policy = TargetRequestTimeouts::default();
+        assert_eq!(policy.upload_timeout(0).expect("empty upload"), Duration::from_secs(60));
+        assert_eq!(
+            policy.upload_timeout(5 * 1024 * 1024 * 1024).expect("maximum single PUT"),
+            Duration::from_secs(15420)
+        );
+        assert!(policy.upload_timeout(-1).is_err());
+        let slow = TargetRequestTimeouts {
+            minimum_transfer_rate: 1,
+            ..policy
+        };
+        assert!(slow.upload_timeout(i64::MAX).is_err());
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn replication_upload_response_deadline_is_independent_of_size_and_checksum_override() {
+        for operation in ["put", "forwarded-checksum", "part"] {
+            let (target, frames, dropped) = timed_response_target(Duration::from_secs(4), Duration::from_millis(500), 100, b"");
+            let body = vec![b'x'; 2 * 1024 * 1024];
+            let size = i64::try_from(body.len()).expect("size");
+            let started = tokio::time::Instant::now();
+            let error = if operation == "part" {
+                target
+                    .put_object_part(
+                        "bucket",
+                        "key",
+                        "upload",
+                        1,
+                        size,
+                        ByteStream::from(body),
+                        &PutObjectPartOptions::default(),
+                    )
+                    .await
+                    .expect_err("response deadline")
+            } else {
+                let mut opts = PutObjectOptions::default();
+                if operation == "forwarded-checksum" {
+                    opts.user_metadata.insert(
+                        "x-amz-checksum-sha256".to_owned(),
+                        "OoJ3yNhRwv3wwtZoGqEIPrPX9xwTnfLl+ka0wStN1g0=".to_owned(),
+                    );
+                }
+                target
+                    .put_object("bucket", "key", size, ByteStream::from(body), &opts)
+                    .await
+                    .expect_err("response deadline")
+            };
+            assert!(
+                started.elapsed() >= Duration::from_secs(6),
+                "upload gets its full independent window: {error}"
+            );
+            assert!(
+                started.elapsed() < target.timeouts.upload_timeout(size).expect("budget"),
+                "response must end before the larger total deadline"
+            );
+            assert!(
+                frames.load(std::sync::atomic::Ordering::SeqCst) >= 2,
+                "trickle delivered progress before failure"
+            );
+            assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1, "transport body released");
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn replication_slow_upload_preserves_transfer_window_and_streaming_get_body() {
+        let (target, _, _) = timed_response_target(Duration::from_secs(4), Duration::from_millis(100), 5, b"");
+        let body = vec![b'x'; 2 * 1024 * 1024];
+        let size = i64::try_from(body.len()).expect("size");
+        target
+            .put_object("bucket", "key", size, ByteStream::from(body), &PutObjectOptions::default())
+            .await
+            .expect("slow upload, prompt response");
+        let (target, _, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), 5, b"payload");
+        let response = target
+            .get_object("bucket", "key", None, None, None, HeaderMap::new())
+            .await
+            .expect("GET headers");
+        let started = tokio::time::Instant::now();
+        let body = response
+            .body
+            .collect()
+            .await
+            .expect("streamed GET remains outside upload deadline")
+            .into_bytes();
+        assert!(started.elapsed() > target.timeouts.request);
+        assert_eq!(&body[..], b"     payload");
+        assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn replication_complete_processing_whitespace_has_a_separate_bounded_window() {
+        const RESULT: &[u8] = b"<CompleteMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><ETag>\"etag\"</ETag></CompleteMultipartUploadResult>";
+        for prefix in [5, 100] {
+            let (target, frames, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), prefix, RESULT);
+            let started = tokio::time::Instant::now();
+            let result = target
+                .complete_multipart_upload(
+                    "bucket",
+                    "key",
+                    "upload",
+                    vec![CompletedPart::builder().part_number(1).e_tag("etag").build()],
+                    &PutObjectOptions::default(),
+                )
+                .await;
+            if prefix == 5 {
+                assert!(result.is_ok(), "legal processing whitespace: {result:?}");
+            } else {
+                assert!(result.is_err());
+                assert!(started.elapsed() <= Duration::from_secs(11));
+            }
+            assert!(started.elapsed() > target.timeouts.request);
+            assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
+            assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn replication_control_operation_deadline_covers_response_trickle() {
+        let (target, frames, dropped) =
+            timed_response_target(Duration::ZERO, Duration::from_millis(500), 100, b"<VersioningConfiguration/>");
+        let started = tokio::time::Instant::now();
+        assert!(target.get_bucket_versioning("bucket").await.is_err());
+        assert!(started.elapsed() <= Duration::from_secs(3));
+        assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
+        assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
+    }
+
     fn streaming_test_body(payload: &'static [u8]) -> ByteStream {
         let stream = tokio_util::io::ReaderStream::new(std::io::Cursor::new(payload));
         let body = http_body_util::StreamBody::new(futures::StreamExt::map(stream, |r| r.map(http_body::Frame::data)));
@@ -3279,6 +3597,7 @@ mod tests {
                 health_check_duration: Duration::from_secs(5),
                 replicate_sync: false,
                 client: Arc::new(client),
+                timeouts: TargetRequestTimeouts::default(),
             },
             request_uris,
         )
@@ -3591,6 +3910,7 @@ mod tests {
             health_check_duration: Duration::from_secs(5),
             replicate_sync: false,
             client: Arc::new(client),
+            timeouts: TargetRequestTimeouts::default(),
         })
     }
 

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -5487,6 +5487,14 @@ pub async fn put_restore_opts(
     if !oi.user_tags.is_empty() {
         meta.insert(AMZ_OBJECT_TAGGING.to_string(), (*oi.user_tags).clone());
     }
+    if let Some(expires) = oi.expires {
+        meta.insert(
+            "expires".to_string(),
+            expires
+                .format(&time::format_description::well_known::Rfc3339)
+                .map_err(std::io::Error::other)?,
+        );
+    }
     let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), rreq.days.unwrap_or(1));
     meta.insert(
         metadata_keys::RESTORE.to_string(),
@@ -5506,7 +5514,6 @@ pub async fn put_restore_opts(
         // computed MD5 is not the object's public plaintext ETag.
         preserve_etag: oi.etag.clone(),
         shard_integrity_write_mode: Some(oi.shard_integrity_write_mode()),
-        //expires:           oi.expires,
         ..Default::default()
     })
 }
@@ -7144,6 +7151,25 @@ mod tests {
             0,
             "tier generation lease should release after EOF"
         );
+        let unread = get_transitioned_object_reader_with_tier_manager(
+            &object_info.bucket,
+            &object_info.name,
+            &None,
+            &HeaderMap::new(),
+            &object_info,
+            &ObjectOptions::default(),
+            &manager,
+            None,
+        )
+        .await
+        .expect("open an unread transitioned response");
+        assert_eq!(TierConfigMgr::active_operation_lease_count(&manager, &tier).await, 1);
+        drop(unread);
+        assert_eq!(
+            TierConfigMgr::active_operation_lease_count(&manager, &tier).await,
+            0,
+            "canceling an unread response must release the tier lease"
+        );
     }
 
     #[cfg(feature = "test-util")]

crates/ecstore/src/bucket/metadata.rs

@@ -256,6 +256,38 @@ pub const BUCKET_POLICY_CONFIG: &str = "policy.json";
 pub const BUCKET_NOTIFICATION_CONFIG: &str = "notification.xml";
 pub const BUCKET_LIFECYCLE_CONFIG: &str = "lifecycle.xml";
 pub const BUCKET_SSECONFIG: &str = "bucket-encryption.xml";
+/// Shared tag-entry validation for object requests and persisted bucket tags.
+pub fn validate_tag_set_entries(tag_set: &[s3s::dto::Tag]) -> std::result::Result<(), &'static str> {
+    fn is_valid_tag_text(value: &str) -> bool {
+        static VALID_TAG_TEXT: std::sync::OnceLock<std::result::Result<regex::Regex, regex::Error>> = std::sync::OnceLock::new();
+        VALID_TAG_TEXT
+            .get_or_init(|| regex::Regex::new(r"^[\p{L}\p{N}\p{Z}+\-=._:/@]*$"))
+            .as_ref()
+            .is_ok_and(|pattern| pattern.is_match(value))
+    }
+    let mut keys = std::collections::HashSet::with_capacity(tag_set.len());
+    for tag in tag_set {
+        let key = tag
+            .key
+            .as_deref()
+            .filter(|key| !key.is_empty())
+            .ok_or("Tag key cannot be empty")?;
+        if key.encode_utf16().count() > 128 || !is_valid_tag_text(key) {
+            return Err("The TagKey you have provided is invalid");
+        }
+        let value = tag.value.as_deref().ok_or("Tag value cannot be null")?;
+        if value.encode_utf16().count() > 256 || !is_valid_tag_text(value) {
+            return Err("The TagValue you have provided is invalid");
+        }
+        if !keys.insert(key) {
+            return Err("Cannot provide multiple Tags with the same key");
+        }
+    }
+    Ok(())
+}
+
+pub const MAX_BUCKET_TAGS: usize = 50;
+
 pub const BUCKET_TAGGING_CONFIG: &str = "tagging.xml";
 pub const BUCKET_QUOTA_CONFIG_FILE: &str = "quota.json";
 pub const OBJECT_LOCK_CONFIG: &str = "object-lock.xml";
@@ -990,6 +1022,20 @@ impl BucketMetadata {
                 self.encryption_config_updated_at = updated;
             }
             BUCKET_TAGGING_CONFIG => {
+                if !data.is_empty() {
+                    let tagging = deserialize::<Tagging>(&data).map_err(|_| {
+                        Error::InvalidArgument(self.name.clone(), String::new(), "Invalid bucket tagging XML".to_owned())
+                    })?;
+                    if tagging.tag_set.len() > MAX_BUCKET_TAGS {
+                        return Err(Error::InvalidArgument(
+                            self.name.clone(),
+                            String::new(),
+                            "Cannot have more than 50 tags per bucket".to_owned(),
+                        ));
+                    }
+                    validate_tag_set_entries(&tagging.tag_set)
+                        .map_err(|message| Error::InvalidArgument(self.name.clone(), String::new(), message.to_owned()))?;
+                }
                 self.tagging_config_xml = data;
                 // Drop the parsed form (like lifecycle above) so clearing the
                 // payload can't leave stale parsed tags to be cached.
@@ -1784,6 +1830,31 @@ mod test {
     /// backlog#2292: a replicated config is stamped with the source
     /// `updated_at` it was given, not the local clock, while the plain
     /// `update_config` entry keeps stamping the local clock.
+    #[test]
+    fn bucket_tag_updates_reject_unreadable_shapes_without_replacing_metadata() {
+        let mut metadata = BucketMetadata::new("validated-tags");
+        let valid = br#"<Tagging><TagSet><Tag><Key>team</Key><Value></Value></Tag></TagSet></Tagging>"#.to_vec();
+        metadata
+            .update_config(BUCKET_TAGGING_CONFIG, valid.clone())
+            .expect("empty value is valid");
+        let updated = metadata.tagging_config_updated_at;
+        for invalid in [
+            "<Tagging><TagSet><Tag><Key>team</Key></Tag></TagSet></Tagging>".to_owned(),
+            "<Tagging><TagSet><Tag><Key></Key><Value>x</Value></Tag></TagSet></Tagging>".to_owned(),
+            "<Tagging><TagSet><Tag><Key>team</Key><Value>x</Value></Tag><Tag><Key>team</Key><Value>y</Value></Tag></TagSet></Tagging>".to_owned(),
+            format!("<Tagging><TagSet>{}</TagSet></Tagging>", (0..51).map(|i| format!("<Tag><Key>k{i}</Key><Value>v</Value></Tag>")).collect::<String>()),
+        ] {
+            assert!(metadata.update_config_at(BUCKET_TAGGING_CONFIG, invalid.into_bytes(), updated + time::Duration::hours(1)).is_err());
+            assert_eq!(metadata.tagging_config_xml, valid);
+            assert_eq!(metadata.tagging_config_updated_at, updated);
+        }
+        metadata.tagging_config_xml = b"legacy corrupt tags".to_vec();
+        metadata
+            .update_config(BUCKET_TAGGING_CONFIG, Vec::new())
+            .expect("deletion repairs legacy corruption");
+        assert!(metadata.tagging_config_xml.is_empty());
+    }
+
     #[test]
     fn update_config_at_stamps_the_given_time_and_update_config_stamps_now() {
         let source_time = OffsetDateTime::now_utc() - time::Duration::hours(3);

crates/ecstore/src/bucket/metadata_sys.rs

@@ -683,6 +683,7 @@ pub(crate) async fn update_validated_in<F>(
     bucket: &str,
     config_file: &str,
     data: Vec<u8>,
+    expected_incarnation_id: Option<Uuid>,
     validate: F,
 ) -> Result<OffsetDateTime>
 where
@@ -690,7 +691,7 @@ where
 {
     Box::pin(async {
         let sys = require_bucket_metadata_sys_in(ctx)?;
-        let guard = acquire_config_write_guard(sys.clone(), bucket).await?;
+        let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?;
         validate()?;
         update_under_config_write_guard(sys, &guard, config_file, data, None).await
     })
@@ -1901,6 +1902,15 @@ impl BucketMetadataSys {
         self.api.upgrade()
     }
 
+    /// Whether the backing `ECStore` has been dropped.
+    ///
+    /// Used by `InstanceContext::clear_bucket_metadata_sys_if_stale` during
+    /// `ECStore::drop` so a shared bootstrap context does not retain a
+    /// `BucketMetadataSys` whose weak handle is dangling.
+    pub(crate) fn is_store_stale(&self) -> bool {
+        self.api.upgrade().is_none()
+    }
+
     fn metadata_publish_lock(&self, bucket: &str) -> Arc<Mutex<MetadataPublishLockState>> {
         Self::bucket_lock_in(&self.metadata_publish_locks, bucket)
     }
@@ -3418,7 +3428,7 @@ mod tests {
         ),
         (
             BUCKET_TAGGING_CONFIG,
-            r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-🦀</Value></Tag></TagSet></Tagging>"#.as_bytes(),
+            r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-rust</Value></Tag></TagSet></Tagging>"#.as_bytes(),
         ),
         (BUCKET_REPLICATION_CONFIG, NEW_WRITER_REPLICATION_XML),
         (
@@ -4546,8 +4556,13 @@ mod tests {
             std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created");
         }
         let mut bm = BucketMetadata::new(bucket);
-        bm.update_config(config_file, raw.to_vec())
+        if config_file == crate::bucket::metadata::BUCKET_TAGGING_CONFIG {
-            .expect("raw config should be stored");
+            // Model legacy persisted corruption without passing the write-time validator.
+            bm.tagging_config_xml = raw.to_vec();
+        } else {
+            bm.update_config(config_file, raw.to_vec())
+                .expect("raw config should be stored");
+        }
         sys.persist_new_and_set(bm).await.expect("initial metadata should persist");
     }
 
@@ -5200,6 +5215,61 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn validated_config_update_rejects_a_different_bucket_incarnation() {
+        let (_dirs, store) = isolated_store_over_temp_disks().await;
+        init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let bucket = "catalog-cleanup-incarnation";
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create bucket");
+        let incarnation = store
+            .bucket_incarnation_id_from_disk(bucket)
+            .await
+            .expect("read bucket incarnation");
+        let called = std::sync::atomic::AtomicBool::new(false);
+        let result = store
+            .update_bucket_metadata_config_for_incarnation_validated(
+                bucket,
+                "quota.json",
+                br#"{"quota":1}"#.to_vec(),
+                Uuid::new_v4(),
+                || {
+                    called.store(true, std::sync::atomic::Ordering::SeqCst);
+                    Ok(())
+                },
+            )
+            .await;
+        assert!(matches!(result, Err(Error::BucketNotFound(name)) if name == bucket));
+        assert!(
+            !called.load(std::sync::atomic::Ordering::SeqCst),
+            "incarnation must be checked before the external fence"
+        );
+        let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
+            .await
+            .expect("read unchanged bucket metadata");
+        assert!(
+            metadata.quota_config_json.is_empty(),
+            "a stale cleanup must not modify the replacement bucket"
+        );
+        store
+            .update_bucket_metadata_config_for_incarnation_validated(
+                bucket,
+                "quota.json",
+                br#"{"quota":1}"#.to_vec(),
+                incarnation,
+                || Ok(()),
+            )
+            .await
+            .expect("the current incarnation permits the validated update");
+        let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
+            .await
+            .expect("read updated bucket metadata");
+        assert_eq!(metadata.quota_config_json, br#"{"quota":1}"#);
+    }
+
     #[tokio::test]
     #[serial]
     async fn quota_admission_rejects_stale_cached_no_quota() {
@@ -5673,7 +5743,8 @@ mod tests {
         // update only needs one interleaving to show up.
         const ROUNDS: usize = 8;
         for round in 0..ROUNDS {
-            let tagging = format!("<Tagging><Round>{round}</Round></Tagging>").into_bytes();
+            let tagging =
+                format!("<Tagging><TagSet><Tag><Key>round</Key><Value>{round}</Value></Tag></TagSet></Tagging>").into_bytes();
             let policy = format!(r#"{{"Version":"2012-10-17","Round":{round}}}"#).into_bytes();
 
             let start = Arc::new(tokio::sync::Barrier::new(2));
@@ -5746,7 +5817,7 @@ mod tests {
 
         let blocked = tokio::spawn({
             let node_b = node_b.clone();
-            async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging/>".to_vec()).await }
+            async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging><TagSet/></Tagging>".to_vec()).await }
         });
 
         // Long enough for the write to have finished had it not waited: the

crates/ecstore/src/bucket/quota/checker/tests/reporting_retirement.rs

@@ -119,6 +119,21 @@ async fn durable_quota_reporting_retires_degraded_floors_without_changing_ledger
             // Exercise the real storage entry point before testing the new
             // reporting semantics: the same ledger must admit and refuse
             // these writes even when a scanner floor claims a million bytes.
+            if protocol == 1 {
+                let mut bootstrap = PutObjReader::from_vec(vec![0x5a; 512]);
+                assert!(
+                    matches!(
+                        store
+                            .put_object(&bucket, "bootstrap", &mut bootstrap, &ObjectOptions::default())
+                            .await,
+                        Err(StorageError::SlowDown)
+                    ),
+                    "a missing authoritative ledger must not admit a write from scanner reporting"
+                );
+                crate::bucket::quota::reservation::retry_quota_reconciliation_for_test(store.clone(), &bucket)
+                    .await
+                    .expect("bootstrap authoritative usage through the production worker");
+            }
             let mut too_large = PutObjReader::from_vec(vec![0x5a; 8192]);
             assert!(
                 matches!(

crates/ecstore/src/bucket/remote_s3_client.rs

@@ -40,7 +40,9 @@ use aws_smithy_runtime_api::client::http::{
     HttpConnector as SmithyHttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn,
 };
 use aws_smithy_runtime_api::client::interceptors::Intercept;
-use aws_smithy_runtime_api::client::interceptors::context::BeforeTransmitInterceptorContextMut;
+use aws_smithy_runtime_api::client::interceptors::context::{
+    BeforeDeserializationInterceptorContextMut, BeforeTransmitInterceptorContextMut,
+};
 use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse};
 use aws_smithy_runtime_api::client::result::ConnectorError;
 use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents;
@@ -155,6 +157,8 @@ pub struct RemoteS3EndpointSpec {
     pub ca_cert_pem: Option<String>,
     pub connect_timeout: Option<Duration>,
     pub read_timeout: Option<Duration>,
+    /// Entire SDK call, including non-streaming response bodies and retries.
+    pub operation_timeout: Option<Duration>,
     /// How many wire requests one logical call may cost. Every consumer
     /// declares it; see [`RemoteS3RetryPolicy`].
     pub retry: RemoteS3RetryPolicy,
@@ -252,6 +256,78 @@ pub(crate) fn remote_sdk_credentials(credentials: &RemoteCredentials, now: Syste
     Ok(builder.build())
 }
 
+/// Only installed on non-streaming upload operations. The budget begins at
+/// response headers, independently of the potentially long request upload.
+#[derive(Debug)]
+pub(crate) struct UploadResponseDeadline(pub Duration);
+
+impl Intercept for UploadResponseDeadline {
+    fn name(&self) -> &'static str {
+        "RustfsUploadResponseDeadline"
+    }
+
+    fn modify_before_deserialization(
+        &self,
+        context: &mut BeforeDeserializationInterceptorContextMut<'_>,
+        _runtime_components: &RuntimeComponents,
+        _cfg: &mut ConfigBag,
+    ) -> Result<(), BoxError> {
+        let response = context.response_mut();
+        let inner = std::mem::replace(response.body_mut(), SdkBody::taken());
+        *response.body_mut() = SdkBody::from_body_1_x(UploadResponseBody {
+            inner: Some(inner),
+            deadline: Box::pin(tokio::time::sleep(self.0)),
+        });
+        Ok(())
+    }
+}
+
+struct UploadResponseBody {
+    inner: Option<SdkBody>,
+    deadline: std::pin::Pin<Box<tokio::time::Sleep>>,
+}
+
+impl http_body::Body for UploadResponseBody {
+    type Data = bytes::Bytes;
+    type Error = BoxError;
+
+    fn poll_frame(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Option<Result<http_body::Frame<Self::Data>, Self::Error>>> {
+        use std::future::Future as _;
+        use std::task::Poll;
+        if self.inner.is_none() {
+            return Poll::Ready(None);
+        }
+        // The absolute response budget never renews on data, empty frames,
+        // or trailers. A retry gets its own bounded response window.
+        if self.deadline.as_mut().poll(cx).is_ready() {
+            self.inner.take();
+            return Poll::Ready(Some(Err(std::io::Error::new(
+                std::io::ErrorKind::TimedOut,
+                "replication response body deadline exceeded",
+            )
+            .into())));
+        }
+        let Some(inner) = self.inner.as_mut() else {
+            return Poll::Ready(None);
+        };
+        let result = std::pin::Pin::new(inner).poll_frame(cx);
+        if matches!(result, Poll::Ready(None) | Poll::Ready(Some(Err(_)))) {
+            self.inner.take();
+        }
+        result
+    }
+
+    fn is_end_stream(&self) -> bool {
+        self.inner.as_ref().is_none_or(http_body::Body::is_end_stream)
+    }
+    fn size_hint(&self) -> http_body::SizeHint {
+        self.inner.as_ref().map(http_body::Body::size_hint).unwrap_or_default()
+    }
+}
+
 /// Appends a caller-identifying token to the SDK `User-Agent`. Runs after
 /// signing: SigV4 excludes `user-agent` from the canonical request, so the
 /// signature stays valid.
@@ -309,7 +385,7 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
         config_builder = config_builder.http_client(http_client);
     }
 
-    if spec.connect_timeout.is_some() || spec.read_timeout.is_some() {
+    if spec.connect_timeout.is_some() || spec.read_timeout.is_some() || spec.operation_timeout.is_some() {
         let mut timeouts = TimeoutConfig::builder();
         if let Some(connect_timeout) = spec.connect_timeout {
             timeouts = timeouts.connect_timeout(connect_timeout);
@@ -317,6 +393,9 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
         if let Some(read_timeout) = spec.read_timeout {
             timeouts = timeouts.read_timeout(read_timeout);
         }
+        if let Some(operation_timeout) = spec.operation_timeout {
+            timeouts = timeouts.operation_timeout(operation_timeout);
+        }
         config_builder = config_builder.timeout_config(timeouts.build());
     }
 
@@ -674,6 +753,7 @@ mod tests {
             ca_cert_pem: None,
             connect_timeout: None,
             read_timeout: None,
+            operation_timeout: None,
             retry: RemoteS3RetryPolicy::Disabled,
             user_agent_suffix: "",
         }
@@ -1027,11 +1107,16 @@ mod tests {
         spec.user_agent_suffix = "RustFS-Test/0.0";
         spec.connect_timeout = Some(Duration::from_secs(5));
         spec.read_timeout = Some(Duration::from_secs(5));
+        spec.operation_timeout = Some(Duration::from_secs(12));
         let config = build_remote_s3_config(&spec)
             .await
             .expect("spec should build")
             .http_client(http_client)
             .build();
+        assert_eq!(
+            config.timeout_config().and_then(TimeoutConfig::operation_timeout),
+            Some(Duration::from_secs(12))
+        );
         S3Client::from_conf(config)
             .head_bucket()
             .bucket("bucket")

crates/ecstore/src/bucket/replication/replication_metadata_boundary.rs

@@ -33,6 +33,7 @@ pub(crate) struct ReplicationMetadataStore;
 impl ReplicationMetadataStore {
     pub(crate) const MRF_REPLICATION_FILE: &'static str = "config/replication/mrf.bin";
     pub(crate) const MRF_REPLICATION_RECOVERY_LOCK: &'static str = "config/replication/mrf.bin.recovery";
+    pub(crate) const MRF_REPLICATION_REPLAY_STAMP: &'static str = "config/replication/mrf.bin.replay";
     pub(crate) const FORCE_DELETE_REPLICATION_FILE: &'static str = "config/replication/force-delete.bin";
     pub(crate) const FORCE_DELETE_REPLICATION_TRANSACTION_LOCK: &'static str = "config/replication/force-delete.bin.transaction";
 
@@ -116,6 +117,10 @@ mod tests {
             ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
             "config/replication/mrf.bin.recovery"
         );
+        assert_eq!(
+            ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
+            "config/replication/mrf.bin.replay"
+        );
         assert_eq!(
             ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE,
             "config/replication/force-delete.bin"

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -16,8 +16,8 @@ use super::replication_config_store::ReplicationConfigStore;
 use super::replication_error_boundary::{Error as EcstoreError, is_err_object_not_found, is_err_version_not_found};
 use super::replication_filemeta_boundary::{
     MrfOpKind, MrfReplicateEntry, REPLICATE_HEAL_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicatedTargetInfo,
-    ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision, replicate_decision_for_admitted_targets,
+    ReplicationGenerationSnapshot, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision,
-    replication_statuses_map, version_purge_statuses_map,
+    replicate_decision_for_admitted_targets, replication_statuses_map, version_purge_statuses_map,
 };
 use super::replication_lock_boundary::ReplicationLockTiming;
 use super::replication_logging::{EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REPLICATION};
@@ -86,6 +86,10 @@ const MRF_CORRUPT_FILE_PREFIX: &str = "config/replication/mrf.corrupt";
 const MRF_PENDING_CAP: usize = 200_000;
 const MRF_RETRY_INITIAL_DELAY: Duration = Duration::from_millis(100);
 const MRF_RETRY_MAX_DELAY: Duration = Duration::from_secs(5);
+/// Interval between runtime replays of the MRF journal, cluster-wide. A pass
+/// re-attempts every retained entry, so a target that stays down costs one
+/// attempt per entry per interval.
+const MRF_REPLAY_INTERVAL: Duration = Duration::from_secs(300);
 
 #[derive(Debug, Default)]
 pub struct DurableMrfBacklog {
@@ -615,7 +619,7 @@ async fn read_mrf_entries<S: ReplicationObjectIO>(storage: Arc<S>) -> Result<Vec
 }
 
 /// Acknowledge only the generation read by the recovery leader. Entries appended
-/// after that generation are retained as a suffix and replayed on the next startup.
+/// after that generation are retained as a suffix and replayed on the next pass.
 /// Lock order: recovery leader lock -> MRF journal object lock.
 async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
     storage: Arc<S>,
@@ -667,7 +671,7 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
     Err(EcstoreError::PreconditionFailed)
 }
 
-/// Acquires the MRF recovery leader lock for the startup replay.
+/// Acquires the MRF recovery leader lock for one journal replay pass.
 /// Returns `None` (after logging) when the lock cannot be created or another
 /// node is already processing the backlog.
 async fn acquire_mrf_recovery_guard<S: ReplicationStorage>(storage: &Arc<S>) -> Option<rustfs_lock::NamespaceLockGuard> {
@@ -1018,7 +1022,7 @@ fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_t
 }
 
 /// Acknowledges the replayed MRF prefix and returns the retained backlog.
-/// On acknowledgement failure the backlog is preserved for the next startup and
+/// On acknowledgement failure the backlog is preserved for the next replay pass and
 /// re-read (falling back to the replayed snapshot) so the published summary stays accurate.
 async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
     storage: &Arc<S>,
@@ -1033,7 +1037,7 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
                 component = LOG_COMPONENT_ECSTORE,
                 subsystem = LOG_SUBSYSTEM_REPLICATION,
                 error = %error,
-                "Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
+                "Failed to acknowledge the MRF recovery prefix; preserving it for the next replay pass"
             );
             match read_mrf_entries(storage.clone()).await {
                 Ok(current) => current,
@@ -1051,6 +1055,120 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
     }
 }
 
+fn unix_millis_now() -> i64 {
+    i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000).unwrap_or(i64::MAX)
+}
+
+/// Whether a periodic pass may run, given the start time of the last pass on
+/// any node. A stamp more than one interval ahead of this node's clock is skew
+/// or a bad value, so it does not suppress replay.
+fn mrf_replay_due(last_started_ms: Option<i64>, now_ms: i64, interval: Duration) -> bool {
+    let Some(last_started_ms) = last_started_ms else {
+        return true;
+    };
+    let interval_ms = i64::try_from(interval.as_millis()).unwrap_or(i64::MAX);
+    now_ms.saturating_sub(last_started_ms) >= interval_ms || last_started_ms.saturating_sub(now_ms) >= interval_ms
+}
+
+async fn read_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>) -> Option<i64> {
+    let data = ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP)
+        .await
+        .ok()?;
+    std::str::from_utf8(&data).ok()?.trim().parse().ok()
+}
+
+async fn write_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>, started_ms: i64) {
+    if let Err(error) = ReplicationConfigStore::save(
+        storage.clone(),
+        ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
+        started_ms.to_string().into_bytes(),
+    )
+    .await
+    {
+        warn!(
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REPLICATION,
+            error = %error,
+            "Failed to record the MRF replay start time"
+        );
+    }
+}
+
+/// One MRF journal replay pass. The recovery leader reads the journal,
+/// re-drives every entry, and acknowledges the replayed prefix; entries that
+/// could not be admitted, and entries appended meanwhile, stay for the next pass.
+///
+/// Every node runs the periodic loop, and the leader lock only serializes
+/// passes. With `min_interval`, a pass is skipped when a pass on any node
+/// started within that interval, read from the shared replay stamp, so a
+/// cluster replays once per interval rather than once per node. The startup
+/// pass passes `None` and always runs. A pass that finds entries records its
+/// start time. Returns whether the journal was read.
+/// Lock order: recovery leader lock -> replay stamp object lock.
+async fn replay_mrf_journal<S: ReplicationStorage>(storage: &Arc<S>, min_interval: Option<Duration>) -> bool {
+    let Some(recovery_guard) = acquire_mrf_recovery_guard(storage).await else {
+        return false;
+    };
+    let started_ms = unix_millis_now();
+    if let Some(interval) = min_interval
+        && !mrf_replay_due(read_mrf_replay_stamp(storage).await, started_ms, interval)
+    {
+        return false;
+    }
+
+    let Some(entries) = load_mrf_recovery_entries(storage).await else {
+        return true;
+    };
+    if !entries.is_empty() {
+        write_mrf_replay_stamp(storage, started_ms).await;
+    }
+    set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
+
+    let total = entries.len();
+    let mut queued_count = 0usize;
+    let mut retry_entries = Vec::new();
+
+    for entry in entries.iter() {
+        let Some(admission) = replay_mrf_entry(entry, storage, &mut retry_entries).await else {
+            continue;
+        };
+
+        if admission == ReplicationQueueAdmission::Missed {
+            retry_entries.push(entry.clone());
+        } else if admission == ReplicationQueueAdmission::Queued {
+            queued_count += 1;
+        }
+    }
+
+    let retained = resolve_retained_mrf_entries(storage, &recovery_guard, &entries, &retry_entries).await;
+    let retained_count = retained.len();
+    set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
+
+    if queued_count > 0 {
+        info!(
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REPLICATION,
+            recovered = queued_count,
+            total,
+            retained = retained_count,
+            "Replayed MRF entries admitted for retry"
+        );
+    }
+    true
+}
+
+/// Re-runs [`replay_mrf_journal`] every `period`. The recovery leader lock keeps
+/// one pass at a time across the cluster, including the startup pass, and the
+/// shared replay stamp spaces passes from different nodes by `period`.
+fn spawn_mrf_replay_loop<S: ReplicationStorage>(storage: Arc<S>, period: Duration) -> JoinHandle<()> {
+    tokio::spawn(async move {
+        loop {
+            tokio::time::sleep(period).await;
+            replay_mrf_journal(&storage, Some(period)).await;
+        }
+    })
+}
+
 #[derive(Debug, thiserror::Error)]
 #[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
 struct ResyncActiveConflictError {
@@ -1171,6 +1289,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
         // Start background tasks
         pool.start_mrf_persister().await;
         pool.start_mrf_processor().await;
+        pool.start_mrf_replayer().await;
         pool.start_force_delete_processor().await;
 
         pool
@@ -1462,7 +1581,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 
     /// Queues a replica task
     pub async fn queue_replica_task(&self, ri: ReplicateObjectInfo) -> ReplicationQueueAdmission {
-        // A version that is already queued or being uploaded is not driven a
+        // A source generation already queued or being uploaded is not driven a
         // second time: the scanner heal pass sees it as PENDING until the
         // first upload lands and would otherwise re-queue it every cycle
         // (backlog#2362). The key is released when the worker finishes, or
@@ -1639,55 +1758,26 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 
     /// Starts the MRF processor — one-shot at startup.
     ///
-    /// Reads the on-disk MRF file, re-injects admitted entries as Heal operations, and
+    /// Replays the on-disk MRF journal once; [`Self::start_mrf_replayer`] repeats
-    /// rewrites any entries that could not be admitted for a later startup retry.
+    /// the pass while the server runs.
     async fn start_mrf_processor(&self) {
         let storage = self.storage.clone();
-
         let handle = tokio::spawn(async move {
-            let Some(recovery_guard) = acquire_mrf_recovery_guard(&storage).await else {
+            replay_mrf_journal(&storage, None).await;
-                return;
-            };
-
-            let Some(entries) = load_mrf_recovery_entries(&storage).await else {
-                return;
-            };
-            set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
-
-            let total = entries.len();
-            let mut queued_count = 0usize;
-            let mut retry_entries = Vec::new();
-
-            for entry in entries.iter() {
-                let Some(admission) = replay_mrf_entry(entry, &storage, &mut retry_entries).await else {
-                    continue;
-                };
-
-                if admission == ReplicationQueueAdmission::Missed {
-                    retry_entries.push(entry.clone());
-                } else if admission == ReplicationQueueAdmission::Queued {
-                    queued_count += 1;
-                }
-            }
-
-            let retained = resolve_retained_mrf_entries(&storage, &recovery_guard, &entries, &retry_entries).await;
-            let retained_count = retained.len();
-            set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
-
-            if queued_count > 0 {
-                info!(
-                    component = LOG_COMPONENT_ECSTORE,
-                    subsystem = LOG_SUBSYSTEM_REPLICATION,
-                    recovered = queued_count,
-                    total,
-                    retained = retained_count,
-                    "Replayed MRF entries admitted for retry"
-                );
-            }
         });
         self.task_handles.lock().await.push(handle);
     }
 
+    /// Starts the runtime MRF replayer — ongoing background task.
+    ///
+    /// Entries reach the journal while the server runs (worker-queue overflow,
+    /// delete-marker purge intents whose watch window drained with targets still
+    /// dirty). Replaying only at startup left them waiting for a restart.
+    async fn start_mrf_replayer(&self) {
+        let handle = spawn_mrf_replay_loop(self.storage.clone(), MRF_REPLAY_INTERVAL);
+        self.task_handles.lock().await.push(handle);
+    }
+
     async fn start_force_delete_processor(&self) {
         let storage = self.storage.clone();
         let handle =
@@ -1755,7 +1845,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
     /// writes them to the on-disk MRF file every flush interval (default 10s,
     /// overridable via `RUSTFS_REPL_MRF_FLUSH_INTERVAL_MS`) or when 1 000 new
     /// entries accumulate. Runtime writers append under the journal lock; only the
-    /// startup recovery leader may remove an acknowledged prefix.
+    /// recovery leader of a replay pass may remove an acknowledged prefix.
     async fn start_mrf_persister(&self) {
         let Some(mut rx) = self.mrf_save_rx.lock().await.take() else {
             return;
@@ -2314,28 +2404,47 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 }
 
 /// Object versions currently queued or being uploaded, keyed by bucket,
-/// object name and version. `queue_replica_task` admits a version only once
+/// object name, version and source generation. `queue_replica_task` admits a generation only once
 /// while it is in flight; the scanner heal pass and MRF replays that arrive
 /// in the meantime are `Skipped` instead of driving a second complete upload
 /// (backlog#2362). Entries are removed when the worker finishes the task or
 /// when no worker accepted it.
 #[derive(Debug, Default)]
 pub(crate) struct ReplicationInFlight {
-    keys: std::sync::Mutex<std::collections::HashSet<(String, String, Option<uuid::Uuid>)>>,
+    keys: std::sync::Mutex<std::collections::HashSet<ReplicationInFlightKey>>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+struct ReplicationInFlightKey {
+    bucket: String,
+    object: String,
+    version_id: Option<uuid::Uuid>,
+    generation: ReplicationGenerationSnapshot,
+}
+
+impl From<&ReplicateObjectInfo> for ReplicationInFlightKey {
+    fn from(info: &ReplicateObjectInfo) -> Self {
+        Self {
+            bucket: info.bucket.clone(),
+            object: info.name.clone(),
+            version_id: info.version_id,
+            generation: info.replication_generation.clone(),
+        }
+    }
 }
 
 impl ReplicationInFlight {
-    fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<(String, String, Option<uuid::Uuid>)>> {
+    fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<ReplicationInFlightKey>> {
         self.keys.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
     }
 
-    /// Claim `ri`; `false` when the same version is already in flight.
+    /// Claim `ri`; a metadata mutation is independent of its earlier generation.
     fn try_begin(&self, ri: &ReplicateObjectInfo) -> bool {
-        self.lock().insert((ri.bucket.clone(), ri.name.clone(), ri.version_id))
+        self.lock().insert(ReplicationInFlightKey::from(ri))
     }
 
     fn finish(&self, ri: &ReplicateObjectInfo) {
-        self.lock().remove(&(ri.bucket.clone(), ri.name.clone(), ri.version_id));
+        self.lock().remove(&ReplicationInFlightKey::from(ri));
     }
 
     #[cfg(test)]
@@ -2348,26 +2457,21 @@ impl ReplicationInFlight {
 /// including when replication panics.
 struct ReplicationInFlightGuard {
     in_flight: Arc<ReplicationInFlight>,
-    key: ReplicateObjectInfo,
+    key: ReplicationInFlightKey,
 }
 
 impl ReplicationInFlightGuard {
     fn new(in_flight: Arc<ReplicationInFlight>, ri: &ReplicateObjectInfo) -> Self {
         Self {
             in_flight,
-            key: ReplicateObjectInfo {
+            key: ReplicationInFlightKey::from(ri),
-                bucket: ri.bucket.clone(),
-                name: ri.name.clone(),
-                version_id: ri.version_id,
-                ..Default::default()
-            },
         }
     }
 }
 
 impl Drop for ReplicationInFlightGuard {
     fn drop(&mut self) {
-        self.in_flight.finish(&self.key);
+        self.in_flight.lock().remove(&self.key);
     }
 }
 
@@ -3421,6 +3525,8 @@ mod tests {
         allow_first_read: Notify,
         read_count: AtomicUsize,
         delete_storage_calls: AtomicUsize,
+        delete_error: StdMutex<EcstoreError>,
+        replay_stamp: StdMutex<Option<Vec<u8>>>,
         reported_size: StdMutex<Option<i64>>,
         stream_read_bytes: Arc<AtomicUsize>,
         write_count: AtomicUsize,
@@ -3488,6 +3594,24 @@ mod tests {
             _h: Self::HeaderMap,
             _opts: &Self::ObjectOptions,
         ) -> Result<Self::GetObjectReader, Self::Error> {
+            if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
+                let stamp = self
+                    .shared
+                    .replay_stamp
+                    .lock()
+                    .expect("test replay stamp lock should not be poisoned")
+                    .clone()
+                    .ok_or(EcstoreError::FileNotFound)?;
+                return Ok(Self::GetObjectReader {
+                    object_info: ObjectInfo {
+                        size: i64::try_from(stamp.len()).expect("test stamp length should fit i64"),
+                        ..Default::default()
+                    },
+                    stream: Box::new(Cursor::new(stamp)),
+                    buffered_body: None,
+                    body_source: Default::default(),
+                });
+            }
             if !object.ends_with("/.replication/resync.bin")
                 && !object.ends_with("config/replication/mrf.bin")
                 && !object.ends_with("config/replication/force-delete.bin")
@@ -3551,6 +3675,16 @@ mod tests {
                 let lock = self.new_ns_lock(bucket, object).await?;
                 Some(lock.get_write_lock(Duration::from_secs(10)).await?)
             };
+            if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
+                let mut stamp = Vec::new();
+                data.stream.read_to_end(&mut stamp).await.map_err(EcstoreError::from)?;
+                *self
+                    .shared
+                    .replay_stamp
+                    .lock()
+                    .expect("test replay stamp lock should not be poisoned") = Some(stamp);
+                return Ok(ObjectInfo::default());
+            }
             if opts.http_preconditions.is_some()
                 && let Some(replacement) = self
                     .shared
@@ -3672,7 +3806,12 @@ mod tests {
             _opts: Self::ObjectOptions,
         ) -> Result<Self::ObjectInfo, Self::Error> {
             self.shared.delete_storage_calls.fetch_add(1, Ordering::SeqCst);
-            Err(EcstoreError::NotImplemented)
+            Err(self
+                .shared
+                .delete_error
+                .lock()
+                .expect("delete error lock should not be poisoned")
+                .clone())
         }
 
         async fn delete_objects(
@@ -3929,6 +4068,48 @@ mod tests {
         assert_eq!(current_queue(&pool, "in-flight-bucket").await, (3, 3 * 4096));
     }
 
+    #[tokio::test]
+    async fn queue_replica_task_admits_metadata_changes_during_an_initial_upload() {
+        let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
+        let (tx, mut rx) = mpsc::channel(4);
+        pool.workers.write().await.push(tx);
+        let initial = ReplicateObjectInfo {
+            bucket: "pending-tags-bucket".to_string(),
+            name: "object".to_string(),
+            version_id: Some(uuid::Uuid::new_v4()),
+            size: 4096,
+            replication_generation: ReplicationGenerationSnapshot {
+                mutation_id: Some(uuid::Uuid::new_v4().to_string()),
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        let metadata = ReplicateObjectInfo {
+            op_type: ReplicationType::Metadata,
+            user_tags: String::new(),
+            replication_generation: ReplicationGenerationSnapshot {
+                mutation_id: Some(uuid::Uuid::new_v4().to_string()),
+                ..Default::default()
+            },
+            ..initial.clone()
+        };
+        assert_eq!(pool.queue_replica_task(initial.clone()).await, ReplicationQueueAdmission::Queued);
+        assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Queued);
+        assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
+        assert_eq!(pool.in_flight.len(), 2);
+        drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &initial));
+        assert_eq!(pool.in_flight.len(), 1);
+        assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
+        for expected in [&initial, &metadata] {
+            let ReplicationOperation::Object(queued) = rx.recv().await.expect("queued operation") else {
+                panic!("expected object replication");
+            };
+            assert_eq!(queued.replication_generation, expected.replication_generation);
+        }
+        drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &metadata));
+        assert_eq!(pool.in_flight.len(), 0);
+    }
+
     #[tokio::test]
     async fn queue_replica_task_releases_the_version_when_no_worker_accepts_it() {
         let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
@@ -4171,6 +4352,8 @@ mod tests {
             allow_first_read: Notify::new(),
             read_count: AtomicUsize::new(0),
             delete_storage_calls: AtomicUsize::new(0),
+            delete_error: StdMutex::new(EcstoreError::NotImplemented),
+            replay_stamp: StdMutex::new(None),
             reported_size: StdMutex::new(None),
             stream_read_bytes: Arc::new(AtomicUsize::new(0)),
             write_count: AtomicUsize::new(0),
@@ -4221,6 +4404,103 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn delete_worker_already_purged_source_completes_without_retry() {
+        let arn = "arn:test";
+        for delete_marker_version in [false, true] {
+            for version_id in [Uuid::new_v4(), Uuid::nil()] {
+                for error in [
+                    EcstoreError::VersionNotFound("source".to_string(), "purged".to_string(), version_id.to_string()),
+                    EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
+                    EcstoreError::FileVersionNotFound,
+                    EcstoreError::FileNotFound,
+                ] {
+                    let shared = empty_resync_shared_state();
+                    *shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
+                    let storage = Arc::new(LoadResyncNodeStore::new("purged-source", shared.clone()));
+                    let delete = DeletedObjectReplicationInfo {
+                        bucket: "source".to_string(),
+                        delete_object: ReplicationDeletedObject {
+                            object_name: "purged".to_string(),
+                            version_id: (!delete_marker_version).then_some(version_id),
+                            delete_marker_version_id: delete_marker_version.then_some(version_id),
+                            replication_state: Some(ReplicationState {
+                                replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
+                                purge_targets: HashMap::from([(arn.to_string(), VersionPurgeStatusType::Complete)]),
+                                ..Default::default()
+                            }),
+                            ..Default::default()
+                        },
+                        op_type: ReplicationType::Delete,
+                        ..Default::default()
+                    };
+
+                    assert!(
+                        replicate_delete_with_outcome(delete, storage).await,
+                        "a confirmed target purge must complete when the source is already absent (marker={delete_marker_version}, version={version_id})"
+                    );
+                    assert_eq!(
+                        shared.delete_storage_calls.load(Ordering::SeqCst),
+                        2 + usize::from(!delete_marker_version && !version_id.is_nil()),
+                        "the worker must lock and attempt writeback, reading a non-null data version first"
+                    );
+                }
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn delete_worker_missing_source_does_not_hide_unfinished_replication() {
+        let arn = "arn:test";
+        for (version_purge, target_status, error) in [
+            (
+                true,
+                VersionPurgeStatusType::Pending,
+                EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
+            ),
+            (true, VersionPurgeStatusType::Complete, EcstoreError::ErasureWriteQuorum),
+            (
+                false,
+                VersionPurgeStatusType::Empty,
+                EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
+            ),
+        ] {
+            let shared = empty_resync_shared_state();
+            *shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
+            let storage = Arc::new(LoadResyncNodeStore::new("unfinished-delete", shared));
+            let delete = DeletedObjectReplicationInfo {
+                bucket: "source".to_string(),
+                delete_object: ReplicationDeletedObject {
+                    object_name: "purged".to_string(),
+                    version_id: version_purge.then(Uuid::new_v4),
+                    delete_marker: !version_purge,
+                    replication_state: Some(ReplicationState {
+                        replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
+                        targets: if version_purge {
+                            HashMap::new()
+                        } else {
+                            HashMap::from([(arn.to_string(), ReplicationStatusType::Completed)])
+                        },
+                        purge_targets: if version_purge {
+                            HashMap::from([(arn.to_string(), target_status)])
+                        } else {
+                            HashMap::new()
+                        },
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                },
+                op_type: ReplicationType::Delete,
+                ..Default::default()
+            };
+
+            assert!(
+                !replicate_delete_with_outcome(delete, storage).await,
+                "unfinished target work, source I/O failures, and marker creation must remain retryable"
+            );
+        }
+    }
+
     #[tokio::test]
     async fn delete_worker_preserves_persisted_admission_without_live_config() {
         let arn = "arn:test";
@@ -5304,6 +5584,8 @@ mod tests {
                 allow_first_read: Notify::new(),
                 read_count: AtomicUsize::new(0),
                 delete_storage_calls: AtomicUsize::new(0),
+                delete_error: StdMutex::new(EcstoreError::NotImplemented),
+                replay_stamp: StdMutex::new(None),
                 reported_size: StdMutex::new(None),
                 stream_read_bytes: Arc::new(AtomicUsize::new(0)),
                 write_count: AtomicUsize::new(0),
@@ -6268,6 +6550,108 @@ mod tests {
         assert_eq!(retained.target_arns, entry.target_arns);
     }
 
+    #[tokio::test]
+    async fn mrf_replay_loop_drains_entries_persisted_after_startup() {
+        let shared = empty_resync_shared_state();
+        let storage = Arc::new(LoadResyncNodeStore::new("mrf-runtime-replay", shared.clone()));
+        // A force-delete intent from a superseded generation replays as
+        // Skipped, so a pass that reaches it acknowledges it off the journal.
+        let stale_intent = |object: &str| MrfReplicateEntry {
+            bucket: "mrf-runtime-replay".to_string(),
+            object: object.to_string(),
+            op: MrfOpKind::Delete,
+            force_delete_id: Some(uuid::Uuid::new_v4()),
+            force_delete_local_commit: false,
+            target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
+            ..Default::default()
+        };
+        let journal_drained = || async {
+            tokio::time::timeout(Duration::from_secs(5), async {
+                while !shared.data.lock().expect("test data lock should not be poisoned").is_empty() {
+                    tokio::time::sleep(Duration::from_millis(10)).await;
+                }
+            })
+            .await
+            .is_ok()
+        };
+
+        let handle = spawn_mrf_replay_loop(storage, Duration::from_millis(20));
+        for object in ["first", "second"] {
+            *shared.data.lock().expect("test data lock should not be poisoned") =
+                encode_mrf_file(&[stale_intent(object)]).expect("MRF entry should encode");
+            assert!(
+                journal_drained().await,
+                "an MRF entry persisted while the server runs must be replayed without a restart ({object})"
+            );
+        }
+        handle.abort();
+    }
+
+    #[tokio::test]
+    async fn mrf_periodic_replay_runs_once_per_interval_across_nodes() {
+        let shared = empty_resync_shared_state();
+        let node_a = Arc::new(LoadResyncNodeStore::new("mrf-node-a", shared.clone()));
+        let node_b = Arc::new(LoadResyncNodeStore::new("mrf-node-b", shared.clone()));
+        let interval = Duration::from_secs(1);
+        let persist_backlog = |object: &str| {
+            let entry = MrfReplicateEntry {
+                bucket: "mrf-node-replay".to_string(),
+                object: object.to_string(),
+                op: MrfOpKind::Delete,
+                force_delete_id: Some(uuid::Uuid::new_v4()),
+                force_delete_local_commit: false,
+                target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
+                ..Default::default()
+            };
+            *shared.data.lock().expect("test data lock should not be poisoned") =
+                encode_mrf_file(&[entry]).expect("MRF entry should encode");
+        };
+        let journal_is_empty = || shared.data.lock().expect("test data lock should not be poisoned").is_empty();
+
+        persist_backlog("first");
+        assert!(replay_mrf_journal(&node_a, Some(interval)).await, "node A's periodic pass should replay");
+        assert!(journal_is_empty(), "node A's pass should acknowledge the replayed entry");
+
+        // Node B's timer fires within the same interval with a backlog present.
+        persist_backlog("second");
+        assert!(
+            !replay_mrf_journal(&node_b, Some(interval)).await,
+            "a node must not replay within one interval of another node's pass"
+        );
+        assert!(!journal_is_empty(), "the backlog should wait for the next interval");
+
+        assert!(
+            replay_mrf_journal(&node_b, None).await,
+            "the startup pass should replay regardless of the shared stamp"
+        );
+        assert!(journal_is_empty(), "the startup pass should acknowledge the replayed entry");
+
+        persist_backlog("third");
+        tokio::time::sleep(interval).await;
+        assert!(
+            replay_mrf_journal(&node_a, Some(interval)).await,
+            "a periodic pass should replay once the interval has elapsed"
+        );
+        assert!(journal_is_empty(), "the due pass should acknowledge the replayed entry");
+    }
+
+    #[test]
+    fn mrf_replay_due_tolerates_missing_and_skewed_stamps() {
+        let interval = Duration::from_secs(300);
+        let now = 1_000_000_000;
+        assert!(mrf_replay_due(None, now, interval));
+        assert!(!mrf_replay_due(Some(now - 299_999), now, interval));
+        assert!(mrf_replay_due(Some(now - 300_000), now, interval));
+        assert!(
+            !mrf_replay_due(Some(now + 1_000), now, interval),
+            "small clock skew should still space passes"
+        );
+        assert!(
+            mrf_replay_due(Some(now + 300_000), now, interval),
+            "a stamp an interval ahead must not suppress replay"
+        );
+    }
+
     #[test]
     fn mrf_entry_delete_marker_roundtrip() {
         let dm_vid = Uuid::new_v4();

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -2809,17 +2809,22 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
             true
         }
         Err(e) => {
-            error!(
+            // A batch purge or another worker may have removed this version
-                event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
+            // already. Remote failures still control the final outcome below.
-                component = LOG_COMPONENT_ECSTORE,
+            let source_already_purged = is_version_purge && (is_err_object_not_found(&e) || is_err_version_not_found(&e));
-                subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
+            if !source_already_purged {
-                bucket = %bucket,
+                error!(
-                arn = %dobj.target_arn,
+                    event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
-                object = %dobj.delete_object.object_name,
+                    component = LOG_COMPONENT_ECSTORE,
-                operation = "apply_replication_delete_state",
+                    subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
-                error = %e,
+                    bucket = %bucket,
-                "Replication target operation failed"
+                    arn = %dobj.target_arn,
-            );
+                    object = %dobj.delete_object.object_name,
+                    operation = "apply_replication_delete_state",
+                    error = %e,
+                    "Replication target operation failed"
+                );
+            }
             send_event(EventArgs {
                 event_name,
                 bucket_name: bucket.clone(),
@@ -2832,7 +2837,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
                 },
                 ..Default::default()
             });
-            false
+            source_already_purged
         }
     };
 
@@ -3010,7 +3015,7 @@ const DELETE_MARKER_PURGE_WATCH_INTERVAL: TokioDuration = TokioDuration::from_se
 /// If the marker disappears (deleted before or while the replica landed),
 /// purge the replicated marker from the targets, retrying failed targets on
 /// later rounds. When the window drains with targets still dirty, persist the
-/// purge intent as a durable MRF entry so the next startup replays it through
+/// purge intent as a durable MRF entry so the next MRF replay pass re-drives it through
 /// `purge_stale_delete_marker_targets`.
 async fn watch_and_purge_source_delete_marker<S: ReplicationStorage>(
     bucket: String,
@@ -3101,7 +3106,7 @@ async fn enqueue_delete_marker_purge_mrf(dobj: &DeletedObjectReplicationInfo, fa
 /// Heal callers retry for the full watch window because the startup MRF
 /// processor runs before bucket metadata (and thus target clients) finishes
 /// initializing — the first pass can see `target_client_missing` and a later
-/// round resolves the client; the replay loop is serial and startup-only, so
+/// round resolves the client; replay passes are serial background work, so
 /// blocking it for up to the window per dirty entry is acceptable. Live
 /// callers run on replication workers where a down target would pin a worker
 /// for the whole window, so they attempt once and lean on the durable intent
@@ -6604,6 +6609,7 @@ mod tests {
             health_check_duration: std::time::Duration::from_secs(5),
             replicate_sync: false,
             client: Arc::new(aws_sdk_s3::Client::from_conf(config)),
+            timeouts: Default::default(),
         })
     }
 

crates/ecstore/src/bucket/utils.rs

@@ -259,6 +259,10 @@ pub fn check_object_name_for_length_and_slash(bucket: &str, object: &str) -> Res
 pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) -> Result<()> {
     check_put_object_args(bucket, object)?;
 
+    if !data_movement && !is_meta_bucketname(bucket) && has_xml_invalid_control_characters(object) {
+        return Err(StorageError::ObjectNameInvalid(bucket.to_owned(), object.to_owned()));
+    }
+
     if cfg!(target_os = "windows") && !data_movement && !is_meta_bucketname(bucket) && object.contains('\\') {
         return Err(StorageError::InvalidArgument(
             bucket.to_owned(),
@@ -270,6 +274,14 @@ pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) ->
     Ok(())
 }
 
+/// New public keys must remain representable in an unencoded XML listing.
+/// Existing keys still use the read/delete validators so they can be removed.
+pub fn has_xml_invalid_control_characters(object: &str) -> bool {
+    object
+        .chars()
+        .any(|character| character < ' ' && !matches!(character, '\t' | '\n' | '\r'))
+}
+
 pub fn check_copy_obj_args(bucket: &str, object: &str) -> Result<()> {
     check_bucket_and_object_names(bucket, object)
 }

crates/ecstore/src/cache_value/metacache_set.rs

@@ -34,7 +34,7 @@ use tokio::sync::Mutex as TokioMutex;
 use tokio::task::JoinSet;
 use tokio::time::timeout;
 use tokio_util::sync::CancellationToken;
-use tracing::{debug, error, warn};
+use tracing::{Instrument, debug, error, warn};
 
 const LOG_COMPONENT_ECSTORE: &str = "ecstore";
 const LOG_SUBSYSTEM_METACACHE: &str = "metacache";
@@ -224,6 +224,8 @@ pub struct ListPathRawOptions {
     pub recursive: bool,
     pub incl_deleted: bool,
     pub skip_hidden_prefix_check: bool,
+    pub hidden_entry_budget: usize,
+    pub pending_prefix: Option<String>,
     pub filter_prefix: Option<String>,
     pub forward_to: Option<String>,
     pub min_disks: usize,
@@ -260,6 +262,8 @@ impl Clone for ListPathRawOptions {
             recursive: self.recursive,
             incl_deleted: self.incl_deleted,
             skip_hidden_prefix_check: self.skip_hidden_prefix_check,
+            hidden_entry_budget: self.hidden_entry_budget,
+            pending_prefix: self.pending_prefix.clone(),
             filter_prefix: self.filter_prefix.clone(),
             forward_to: self.forward_to.clone(),
             min_disks: self.min_disks,
@@ -289,6 +293,8 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
         recursive: opts.recursive,
         incl_deleted: opts.incl_deleted,
         skip_hidden_prefix_check: opts.skip_hidden_prefix_check,
+        hidden_entry_budget: opts.hidden_entry_budget,
+        pending_prefix: opts.pending_prefix.clone(),
         report_notfound: opts.report_not_found,
         filter_prefix: opts.filter_prefix.clone(),
         forward_to: opts.forward_to.clone(),
@@ -301,13 +307,15 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
 }
 
 /// How a merged raw listing stream ended.
-#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
 pub struct ListPathRawOutcome {
     /// At least one producer stopped at its per-disk limit instead of at the
     /// end of its namespace, so the closed stream is not proof of exhaustion.
     /// Carried in-band by the metacache stream trailer, so remote producers
     /// report it as well.
     pub limit_reached: bool,
+    /// Inclusive sorted boundary proved by every reader before a bounded stop.
+    pub scan_cut: Option<String>,
 }
 
 pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> disk::error::Result<()> {
@@ -363,10 +371,15 @@ async fn list_path_raw_inner(
     let producer_errs: Arc<[OnceLock<DiskError>]> = (0..opts.disks.len()).map(|_| OnceLock::new()).collect::<Vec<_>>().into();
 
     let cancel_rx = CancellationToken::new();
+    let walk_options = Arc::new(walk_dir_options(&opts));
 
     for (disk_idx, disk) in opts.disks.iter().enumerate() {
         let opdisk = disk.clone();
-        let opts_clone = opts.clone();
+        let walk_options = walk_options.clone();
+        #[cfg(test)]
+        let test_reader_behavior = opts.test_reader_behaviors.get(disk_idx).cloned();
+        #[cfg(test)]
+        let test_disk_io_context_observed = opts.test_disk_io_context_observed.clone();
         let fallback_claim_tracker = fallback_claim_tracker.clone();
         let fds_clone = fds.clone();
         #[cfg(test)]
@@ -376,17 +389,17 @@ async fn list_path_raw_inner(
         let disk_io_context_clone = disk_io_context.clone();
         let (rd, wr) = tokio::io::duplex(64 * 1024);
         readers.push(MetacacheReader::new(rd));
-        jobs.spawn(with_optional_disk_io_context(disk_io_context_clone, async move {
+        let producer = with_optional_disk_io_context(disk_io_context_clone, async move {
             let mut wr = PublishedBytesWriter::new(wr);
             #[cfg(test)]
-            if let Some(observer) = &opts_clone.test_disk_io_context_observed {
+            if let Some(observer) = &test_disk_io_context_observed {
                 observer.store(
                     crate::disk::current_disk_io_context().is_some_and(|context| context.class == DiskIoClass::Background),
                     std::sync::atomic::Ordering::Release,
                 );
             }
             #[cfg(test)]
-            let test_primary_error = if let Some(behavior) = opts_clone.test_reader_behaviors.get(disk_idx).cloned() {
+            let test_primary_error = if let Some(behavior) = test_reader_behavior {
                 match behavior {
                     TestReaderBehavior::Eof => return Ok(()),
                     TestReaderBehavior::Entries(entries) => {
@@ -443,8 +456,6 @@ async fn list_path_raw_inner(
                 None
             };
 
-            let wakl_opts = walk_dir_options(&opts_clone);
-
             let mut need_fallback = false;
             let mut last_err = None;
             #[cfg(test)]
@@ -459,7 +470,7 @@ async fn list_path_raw_inner(
 
             if !need_fallback && let Some(disk) = opdisk {
                 let primary_walk_started = std::time::Instant::now();
-                match disk.walk_dir(wakl_opts, &mut wr).await {
+                match disk.walk_dir((*walk_options).clone(), &mut wr).await {
                     Ok(_res) => {
                         rustfs_io_metrics::record_stage_duration(
                             "metacache_walk_dir_primary",
@@ -483,8 +494,8 @@ async fn list_path_raw_inner(
                                 event = EVENT_METACACHE_LISTING,
                                 component = LOG_COMPONENT_ECSTORE,
                                 subsystem = LOG_SUBSYSTEM_METACACHE,
-                                bucket = %opts_clone.bucket,
+                                bucket = %walk_options.bucket,
-                                path = %opts_clone.path,
+                                path = %walk_options.base_dir,
                                 disk_index = disk_idx,
                                 state = "walk_dir_missing_path",
                                 error = ?err,
@@ -495,8 +506,8 @@ async fn list_path_raw_inner(
                                 event = EVENT_METACACHE_LISTING,
                                 component = LOG_COMPONENT_ECSTORE,
                                 subsystem = LOG_SUBSYSTEM_METACACHE,
-                                bucket = %opts_clone.bucket,
+                                bucket = %walk_options.bucket,
-                                path = %opts_clone.path,
+                                path = %walk_options.base_dir,
                                 disk_index = disk_idx,
                                 state = "walk_dir_failed",
                                 error = ?err,
@@ -598,8 +609,8 @@ async fn list_path_raw_inner(
                             event = EVENT_METACACHE_LISTING,
                             component = LOG_COMPONENT_ECSTORE,
                             subsystem = LOG_SUBSYSTEM_METACACHE,
-                            bucket = %opts_clone.bucket,
+                            bucket = %walk_options.bucket,
-                            path = %opts_clone.path,
+                            path = %walk_options.base_dir,
                             disk_index = disk_idx,
                             state = "fallback_disk_missing_for_path",
                             error = ?err,
@@ -610,8 +621,8 @@ async fn list_path_raw_inner(
                             event = EVENT_METACACHE_LISTING,
                             component = LOG_COMPONENT_ECSTORE,
                             subsystem = LOG_SUBSYSTEM_METACACHE,
-                            bucket = %opts_clone.bucket,
+                            bucket = %walk_options.bucket,
-                            path = %opts_clone.path,
+                            path = %walk_options.base_dir,
                             disk_index = disk_idx,
                             state = "fallback_disk_missing",
                             error = ?err,
@@ -626,7 +637,7 @@ async fn list_path_raw_inner(
                 }
 
                 let fallback_walk_started = std::time::Instant::now();
-                match disk.as_ref().walk_dir(walk_dir_options(&opts_clone), &mut wr).await {
+                match disk.as_ref().walk_dir((*walk_options).clone(), &mut wr).await {
                     Ok(_r) => {
                         rustfs_io_metrics::record_stage_duration(
                             "metacache_walk_dir_fallback",
@@ -652,8 +663,8 @@ async fn list_path_raw_inner(
                                 event = EVENT_METACACHE_LISTING,
                                 component = LOG_COMPONENT_ECSTORE,
                                 subsystem = LOG_SUBSYSTEM_METACACHE,
-                                bucket = %opts_clone.bucket,
+                                bucket = %walk_options.bucket,
-                                path = %opts_clone.path,
+                                path = %walk_options.base_dir,
                                 disk_index = disk_idx,
                                 state = "fallback_walk_dir_missing_path",
                                 error = ?err,
@@ -664,8 +675,8 @@ async fn list_path_raw_inner(
                                 event = EVENT_METACACHE_LISTING,
                                 component = LOG_COMPONENT_ECSTORE,
                                 subsystem = LOG_SUBSYSTEM_METACACHE,
-                                bucket = %opts_clone.bucket,
+                                bucket = %walk_options.bucket,
-                                path = %opts_clone.path,
+                                path = %walk_options.base_dir,
                                 disk_index = disk_idx,
                                 state = "fallback_walk_dir_failed",
                                 error = ?err,
@@ -687,12 +698,13 @@ async fn list_path_raw_inner(
 
             // warn!("list_path_raw: while need_fallback done");
             Ok(())
-        }));
+        });
+        jobs.spawn(producer.in_current_span());
     }
 
     let revjob_rx = rx.clone();
     let mut revjobs = JoinSet::new();
-    revjobs.spawn(async move {
+    let merge = async move {
         // Consumer-side peek timeout, and a caveat worth understanding
         // (rustfs/backlog#1217).
         //
@@ -722,6 +734,8 @@ async fn list_path_raw_inner(
         // Sticky across rounds: once any reader's trailer says its producer
         // stopped at the per-disk limit, the merged stream may hide more keys.
         let mut limit_reached = false;
+        let mut scan_cut = None;
+        let mut last_scanned_key: Option<String> = None;
 
         loop {
             let mut current = MetaCacheEntry::default();
@@ -733,7 +747,7 @@ async fn list_path_raw_inner(
             // );
 
             if revjob_rx.is_cancelled() {
-                return Ok(ListPathRawOutcome { limit_reached });
+                return Ok(ListPathRawOutcome { limit_reached, scan_cut });
             }
 
             let mut top_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
@@ -747,6 +761,16 @@ async fn list_path_raw_inner(
             // Start every missing head read in the same round so one stalled
             // disk cannot multiply the wait budget by the erasure-set width.
             // Outcomes are still consumed below in stable disk-index order.
+            if !revjob_rx.is_cancelled() {
+                for (i, reader) in readers.iter_mut().enumerate() {
+                    if errs[i].is_none()
+                        && pending_entries[i].is_none()
+                        && let Some(Ok(Some(entry))) = reader.try_peek_buffered()
+                    {
+                        pending_entries[i] = Some(entry);
+                    }
+                }
+            }
             let concurrent_peeks = readers.iter_mut().enumerate().filter_map(|(i, reader)| {
                 if errs[i].is_some() || pending_entries[i].is_some() {
                     return None;
@@ -783,7 +807,12 @@ async fn list_path_raw_inner(
                                     continue;
                                 }
                                 // eof
-                                limit_reached |= r.limit_reached() == Some(true);
+                                if r.limit_reached() == Some(true) {
+                                    limit_reached = true;
+                                    scan_cut = Some(last_scanned_key.clone().ok_or_else(|| {
+                                        DiskError::other("bounded listing reader ended without a scan boundary")
+                                    })?);
+                                }
                                 at_eof += 1;
                                 // warn!("list_path_raw: peek eof, disk: {}", i);
                                 continue;
@@ -857,7 +886,7 @@ async fn list_path_raw_inner(
                             *r = MetacacheReader::new(detached_rd);
                             continue;
                         }
-                        PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached }),
+                        PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
                     }
                 };
 
@@ -878,7 +907,7 @@ async fn list_path_raw_inner(
                 }
                 if entry.name == current.name {
                     // If exact match, we agree.
-                    if current.matches_strict(&entry) {
+                    if current.matches_strict_borrowed(&entry) {
                         top_entries[i] = Some(entry);
                         agree += 1;
                     } else {
@@ -916,7 +945,7 @@ async fn list_path_raw_inner(
                 if let Some(finished_fn) = opts.finished.as_ref() {
                     tokio::select! {
                         biased;
-                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
+                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
                         _ = finished_fn(&errs) => {}
                     }
                 }
@@ -966,6 +995,14 @@ async fn list_path_raw_inner(
                 return Err(classify_listing_quorum_failure(&failures));
             }
 
+            // The next head from a bounded reader is unknown. The previous
+            // globally consumed group is a safe cut; consuming another reader's
+            // later head would turn this absence of evidence into a skipped key.
+            // Check quorum failures above before accepting the bounded result.
+            if limit_reached {
+                break;
+            }
+
             // Break if all at EOF or error.
             if at_eof + has_err == readers.len() {
                 if has_err > 0
@@ -973,7 +1010,7 @@ async fn list_path_raw_inner(
                 {
                     tokio::select! {
                         biased;
-                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
+                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
                         _ = finished_fn(&errs) => {}
                     }
                 }
@@ -984,16 +1021,19 @@ async fn list_path_raw_inner(
                 break;
             }
 
+            last_scanned_key = Some(current.name.clone());
             if agree == readers.len() && !opts.preserve_replica_metadata {
                 for r in readers.iter_mut() {
-                    let _ = r.skip(1).await;
+                    if !r.skip_current() {
+                        let _ = r.skip(1).await;
+                    }
                 }
 
                 if let Some(agreed_fn) = opts.agreed.as_ref() {
                     // warn!("list_path_raw: agreed_fn start, current: {:?}", &current.name);
                     tokio::select! {
                         biased;
-                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
+                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
                         result = agreed_fn(current) => result?
                     }
                     // warn!("list_path_raw: agreed_fn done");
@@ -1009,7 +1049,7 @@ async fn list_path_raw_inner(
             }
 
             for (i, r) in readers.iter_mut().enumerate() {
-                if top_entries[i].is_some() {
+                if top_entries[i].is_some() && !r.skip_current() {
                     let _ = r.skip(1).await;
                 }
             }
@@ -1017,13 +1057,14 @@ async fn list_path_raw_inner(
             if let Some(partial_fn) = opts.partial.as_ref() {
                 tokio::select! {
                     biased;
-                    _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
+                    _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
                     _ = partial_fn(MetaCacheEntries(top_entries), &errs) => {}
                 }
             }
         }
-        Ok(ListPathRawOutcome { limit_reached })
+        Ok(ListPathRawOutcome { limit_reached, scan_cut })
-    });
+    };
+    revjobs.spawn(merge.in_current_span());
 
     let merge_started = std::time::Instant::now();
     let Some(revjob) = revjobs.join_next().await else {
@@ -1187,6 +1228,24 @@ mod tests {
         assert!(!options.skip_total_timeout);
     }
 
+    #[test]
+    fn cloned_reader_options_preserve_hidden_scan_budget_and_pending_prefix() {
+        let request = ListPathRawOptions {
+            bucket: "versioned".to_owned(),
+            hidden_entry_budget: 100,
+            pending_prefix: Some("pending/".to_owned()),
+            forward_to: Some("pending/0099".to_owned()),
+            ..Default::default()
+        };
+        // Both walks are built once from the shared options; the per-job clone
+        // must carry every field that the walk derives from the request.
+        let job = request.clone();
+        let walk = walk_dir_options(&job);
+        assert_eq!(walk.hidden_entry_budget, 100);
+        assert_eq!(walk.pending_prefix.as_deref(), request.pending_prefix.as_deref());
+        assert_eq!(walk.forward_to.as_deref(), Some("pending/0099"));
+    }
+
     #[tokio::test]
     async fn list_path_raw_empty_disks_returns_read_quorum() {
         let err = list_path_raw(CancellationToken::new(), ListPathRawOptions::default())
@@ -1237,6 +1296,163 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn list_path_raw_fanout_retains_only_linear_disk_references() {
+        let root = tempfile::tempdir().expect("create disk root");
+        let endpoint = crate::disk::endpoint::Endpoint::try_from(root.path().to_str().expect("UTF-8 disk path"))
+            .expect("parse disk endpoint");
+        let local = Arc::new(
+            crate::disk::local::LocalDisk::new(&endpoint, false)
+                .await
+                .expect("open local disk"),
+        );
+        let wrapper = crate::disk::disk_store::LocalDiskWrapper::new(local, false);
+        let disk = Arc::new(crate::disk::Disk::Local(Box::new(wrapper)));
+        let baseline = Arc::strong_count(&disk);
+
+        for primary_count in [1, 4, 16] {
+            for fallback_count in [0, 2] {
+                let entered: Vec<_> = (0..primary_count).map(|_| Arc::new(tokio::sync::Notify::new())).collect();
+                let stopped: Vec<_> = (0..primary_count).map(|_| Arc::new(tokio::sync::Notify::new())).collect();
+                let behaviors = entered
+                    .iter()
+                    .zip(&stopped)
+                    .map(|(entered, stopped)| TestReaderBehavior::StallWithSignals {
+                        entered: entered.clone(),
+                        stopped: stopped.clone(),
+                    })
+                    .collect();
+                let cancel = CancellationToken::new();
+                let listing = tokio::spawn(list_path_raw(
+                    cancel.clone(),
+                    ListPathRawOptions {
+                        disks: vec![Some(disk.clone()); primary_count],
+                        fallback_disks: vec![Some(disk.clone()); fallback_count],
+                        min_disks: primary_count,
+                        test_reader_behaviors: behaviors,
+                        peek_timeout: Some(Duration::from_secs(60)),
+                        ..Default::default()
+                    },
+                ));
+                for signal in &entered {
+                    timeout(Duration::from_secs(2), signal.notified())
+                        .await
+                        .expect("every producer must enter the real fanout before counting references");
+                }
+
+                // The merge owns the configured primary/fallback slots, each producer
+                // owns its primary, and the shared fallback queue owns each fallback.
+                let expected = baseline + 2 * (primary_count + fallback_count);
+                let retained = Arc::strong_count(&disk);
+                cancel.cancel();
+                timeout(Duration::from_secs(2), listing)
+                    .await
+                    .expect("cancellation must stop the listing")
+                    .expect("listing task must join")
+                    .expect("external cancellation must finish cleanly");
+                for signal in &stopped {
+                    timeout(Duration::from_secs(2), signal.notified())
+                        .await
+                        .expect("cancellation must drop every producer");
+                }
+                assert_eq!(Arc::strong_count(&disk), baseline, "cancellation must release every fanout reference");
+                assert_eq!(
+                    retained, expected,
+                    "fanout must retain linear references for {primary_count} primary and {fallback_count} fallback slots"
+                );
+            }
+        }
+    }
+
+    #[tokio::test(flavor = "current_thread")]
+    async fn list_path_raw_producer_and_merge_preserve_request_span() {
+        use tracing_subscriber::{Layer, layer::SubscriberExt};
+
+        struct ProducerSpan(Arc<AtomicBool>);
+        #[derive(Default)]
+        struct StateField(bool);
+        impl tracing::field::Visit for StateField {
+            fn record_debug(&mut self, _: &tracing::field::Field, _: &dyn std::fmt::Debug) {}
+
+            fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
+                self.0 |= field.name() == "state" && value == "walk_dir_missing_path";
+            }
+        }
+        impl<S> Layer<S> for ProducerSpan
+        where
+            S: tracing::Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a>,
+        {
+            fn on_event(&self, event: &tracing::Event<'_>, context: tracing_subscriber::layer::Context<'_, S>) {
+                let mut state = StateField::default();
+                event.record(&mut state);
+                if state.0
+                    && context
+                        .lookup_current()
+                        .is_some_and(|span| span.name() == "flink_list_request")
+                {
+                    self.0.store(true, std::sync::atomic::Ordering::Release);
+                }
+            }
+        }
+
+        // The subscriber is thread-local; keep the spawned tasks on this runtime thread.
+        let producer_observed = Arc::new(AtomicBool::new(false));
+        let subscriber = tracing_subscriber::registry().with(ProducerSpan(producer_observed.clone()));
+        let _subscriber = tracing::subscriber::set_default(subscriber);
+        let request = tracing::info_span!("flink_list_request", request_id = "list-span-regression");
+        let root = tempfile::tempdir().expect("create disk root");
+        let endpoint = crate::disk::endpoint::Endpoint::try_from(root.path().to_str().expect("UTF-8 disk path"))
+            .expect("parse disk endpoint");
+        let local = Arc::new(
+            crate::disk::local::LocalDisk::new(&endpoint, false)
+                .await
+                .expect("open local disk"),
+        );
+        let wrapper = crate::disk::disk_store::LocalDiskWrapper::new(local, false);
+        let disk = Arc::new(crate::disk::Disk::Local(Box::new(wrapper)));
+        let error = list_path_raw(
+            CancellationToken::new(),
+            ListPathRawOptions {
+                disks: vec![Some(disk)],
+                bucket: "missing-bucket".to_owned(),
+                min_disks: 1,
+                ..Default::default()
+            },
+        )
+        .instrument(request.clone())
+        .await
+        .expect_err("missing bucket must fail the real disk producer");
+        assert_eq!(error, DiskError::VolumeNotFound);
+        assert!(
+            producer_observed.load(std::sync::atomic::Ordering::Acquire),
+            "disk error must retain the request span"
+        );
+
+        let merge_observed = Arc::new(AtomicBool::new(false));
+        let merge_sink = merge_observed.clone();
+        let request_id = request.id();
+        list_path_raw(
+            CancellationToken::new(),
+            ListPathRawOptions {
+                disks: vec![None],
+                min_disks: 1,
+                test_reader_behaviors: vec![TestReaderBehavior::Entries(vec![fallback_test_entry()])],
+                agreed: Some(Box::new(move |_| {
+                    merge_sink.store(tracing::Span::current().id() == request_id, std::sync::atomic::Ordering::Release);
+                    Box::pin(async { Ok(()) })
+                })),
+                ..Default::default()
+            },
+        )
+        .instrument(request)
+        .await
+        .expect("merge should deliver the test entry");
+        assert!(
+            merge_observed.load(std::sync::atomic::Ordering::Acquire),
+            "merge callback must retain the request span"
+        );
+    }
+
     #[tokio::test]
     async fn scanner_joinset_walk_uses_the_local_disk_background_gate() {
         let root = tempfile::tempdir().expect("local disk root should be created");
@@ -1304,6 +1520,7 @@ mod tests {
             outcome.limit_reached,
             "a reader whose trailer reports a limit stop must mark the merged stream as possibly incomplete"
         );
+        assert_eq!(outcome.scan_cut.as_deref(), Some("bucket/object"));
     }
 
     #[tokio::test]

crates/ecstore/src/cluster/rpc/http_auth.rs

@@ -32,7 +32,6 @@ use crate::storage_api_contracts::internode::{
     PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_CAPABILITY_VERSION,
 };
 
-use hmac::{Hmac, KeyInit, Mac};
 use http::uri::Authority;
 use http::{HeaderMap, HeaderValue, Method, Uri};
 #[cfg(test)]
@@ -47,6 +46,7 @@ use rustfs_io_metrics::internode_metrics::{
     global_internode_metrics,
 };
 use rustfs_object_data_cache::{MemoryBasis, resolve_effective_memory};
+use rustfs_utils::crypto::{HmacSha256, KeyInit, Mac};
 use rustfs_utils::get_env_bool;
 use sha2::Digest as _;
 use sha2::Sha256;
@@ -58,8 +58,6 @@ use time::OffsetDateTime;
 use tracing::{error, info, warn};
 use uuid::Uuid;
 
-type HmacSha256 = Hmac<Sha256>;
-
 const SIGNATURE_HEADER: &str = "x-rustfs-signature";
 pub const RPC_BACKGROUND_ADMISSION_HEADER: &str = "x-rustfs-rpc-background-admission";
 const RPC_BACKGROUND_ADMISSION_V1: &str = "v1";
@@ -1590,7 +1588,13 @@ fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &Hea
     if boot_epoch != tonic_rpc_boot_epoch() {
         return Err(std::io::Error::other("RPC boot epoch is stale"));
     }
-    check_and_record_tonic_nonce(nonce, signed_at, path)
+    check_and_record_tonic_nonce(nonce, signed_at, path)?;
+    // A body-bound request also carries a usable v2 signature. Consume both
+    // nonces so stripping the replay-scope headers cannot authorize it again.
+    if content_sha256 != UNSIGNED_PAYLOAD {
+        verify_tonic_rpc_signature_with_strictness(audience, path, headers, true)?;
+    }
+    Ok(())
 }
 
 /// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
@@ -2581,6 +2585,43 @@ mod tests {
         assert_eq!(path_error.to_string(), "Invalid RPC replay scope signature");
     }
 
+    #[test]
+    fn replay_scope_consumes_the_body_bound_v2_nonce() {
+        ensure_test_rpc_secret();
+        let audience = "node-a:9000";
+        let path = "/node_service.NodeService/RecoverDeleteTransaction";
+        let digest = "a".repeat(64);
+        for v3_first in [true, false] {
+            let mut headers =
+                gen_tonic_signature_headers(audience, "node_service.NodeService", "RecoverDeleteTransaction", Some(&digest))
+                    .expect("body-bound v2 headers should build");
+            let timestamp = headers[TIMESTAMP_HEADER].to_str().expect("timestamp").to_string();
+            headers.extend(
+                gen_tonic_replay_scope_headers(audience, path, &timestamp, &digest, tonic_rpc_boot_epoch())
+                    .expect("replay-scope headers should build"),
+            );
+            let mut downgraded = headers.clone();
+            for header in [
+                RPC_REPLAY_SCOPE_VERSION_HEADER,
+                RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
+                RPC_REPLAY_SCOPE_NONCE_HEADER,
+                RPC_BOOT_EPOCH_HEADER,
+            ] {
+                downgraded.remove(header);
+            }
+            let (first, replay) = if v3_first {
+                (&headers, &downgraded)
+            } else {
+                (&downgraded, &headers)
+            };
+            verify_tonic_rpc_signature_with_policy(audience, path, first, true, false, false)
+                .expect("the first delivery must be accepted");
+            let error = verify_tonic_rpc_signature_with_policy(audience, path, replay, true, false, false)
+                .expect_err("a mutation cannot replay through the other signature generation");
+            assert_eq!(error.to_string(), "RPC request replay detected");
+        }
+    }
+
     #[test]
     fn replay_scope_rejects_partial_metadata_and_stale_epoch_without_fallback() {
         ensure_test_rpc_secret();

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -2014,9 +2014,21 @@ impl PeerRestClient {
             }
             result
         })
-        .await
+        .await;
-        .unwrap_or_else(|_| Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))));
+        self.settle_bucket_metadata_reload(bucket, result).await
-        self.finalize_result(result).await
+    }
+
+    async fn settle_bucket_metadata_reload(
+        &self,
+        bucket: &str,
+        result: std::result::Result<Result<()>, tokio::time::error::Elapsed>,
+    ) -> Result<()> {
+        match result {
+            Ok(result) => self.finalize_result(result).await,
+            // A local operation deadline does not establish that the peer is
+            // unreachable. In particular, it must not gate IAM revocations.
+            Err(_) => Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))),
+        }
     }
 
     async fn load_bucket_metadata_once(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
@@ -4753,6 +4765,31 @@ mod tests {
         assert!(client.offline.load(Ordering::Acquire));
     }
 
+    #[tokio::test]
+    async fn bucket_metadata_deadline_does_not_gate_healthy_peer_notifications() {
+        let client = test_peer_client();
+        let elapsed = tokio::time::timeout(std::time::Duration::ZERO, std::future::pending::<Result<()>>()).await;
+        let error = client
+            .settle_bucket_metadata_reload("slow-policy", elapsed)
+            .await
+            .expect_err("local budget expires");
+        assert!(error.to_string().contains("timed out"));
+        assert!(
+            !client.offline.load(Ordering::Acquire),
+            "local work must not suppress subsequent IAM RPCs"
+        );
+        client
+            .settle_bucket_metadata_reload("healthy", Ok(Ok(())))
+            .await
+            .expect("peer reply");
+        let error = client
+            .settle_bucket_metadata_reload("unreachable", Ok(Err(Error::from(tonic::Status::unavailable("connection refused")))))
+            .await
+            .expect_err("real transport failure");
+        assert!(error.to_string().contains("connection refused"));
+        assert!(client.offline.load(Ordering::Acquire), "transport failures must still enter recovery");
+    }
+
     #[tokio::test]
     async fn peer_rest_client_finalize_result_keeps_online_for_business_errors() {
         let client = test_peer_client();

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -1082,7 +1082,10 @@ impl RemoteDisk {
         if payload.len() > operation.max_wire_payload_bytes() {
             return Err(Error::other("delete recovery operation exceeds payload limit"));
         }
-        let settle_timeout = if matches!(&operation, DeleteRecoveryRpcOperation::Settle { .. }) {
+        let settle_timeout = if matches!(
+            &operation,
+            DeleteRecoveryRpcOperation::Settle { .. } | DeleteRecoveryRpcOperation::SettleBounded(_)
+        ) {
             Some(crate::disk::get_delete_recovery_timeout()?)
         } else {
             None
@@ -1637,6 +1640,7 @@ impl RemoteDisk {
         let initial_probe_ok = Self::perform_connectivity_check(&addr).await.is_ok();
         if initial_probe_ok {
             health.record_operation_success(&endpoint, "connectivity_probe_success");
+            rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
         } else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
             warn!(
                 event = EVENT_REMOTE_DISK_HEALTH,
@@ -1708,6 +1712,7 @@ impl RemoteDisk {
                     // Perform basic connectivity check
                     if Self::perform_connectivity_check(&addr).await.is_ok() {
                         health.record_operation_success(&endpoint, "connectivity_probe_success");
+                        rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
                     } else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
                         warn!(
                             event = EVENT_REMOTE_DISK_HEALTH,
@@ -1778,6 +1783,7 @@ impl RemoteDisk {
                 }
                 _ = interval.tick() => {
                     if Self::perform_recovery_probe(&addr, &endpoint).await.is_ok() {
+                        rustfs_io_metrics::internode_metrics::record_peer_reachable(&addr);
                         let became_online = health.mark_recovery_success(&endpoint, "disk_info_probe_success");
                         debug!(
                             event = EVENT_REMOTE_DISK_HEALTH,
@@ -2009,6 +2015,7 @@ impl RemoteDisk {
             let operation_result = operation().await;
             if operation_result.is_ok() {
                 self.health.record_operation_success(&self.endpoint, "operation_success");
+                rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
             }
             self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
                 .await;
@@ -2023,6 +2030,7 @@ impl RemoteDisk {
                 // Log success; the waiting guard balances every exit path.
                 if operation_result.is_ok() {
                     self.health.record_operation_success(&self.endpoint, "operation_success");
+                    rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
                 }
                 self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
                     .await;
@@ -2711,6 +2719,54 @@ impl RemoteDisk {
 
 #[async_trait::async_trait]
 impl DiskAPI for RemoteDisk {
+    async fn list_multipart_index_roots(
+        &self,
+        bucket: &str,
+        marker: Option<&str>,
+    ) -> Result<crate::disk::multipart_index::MultipartIndexPage> {
+        use crate::disk::multipart_index::{MULTIPART_INDEX_RPC_MAX_BYTES, MultipartIndexRpcReply, MultipartIndexRpcRequest};
+        let expected_disk = self
+            .id
+            .lock()
+            .await
+            .filter(|id| !id.is_nil())
+            .ok_or(DiskError::DiskNotFound)?;
+        let query = MultipartIndexRpcRequest::new(bucket, marker);
+        let endpoint = self.endpoint.to_string();
+        let canonical = query.canonical_request(&endpoint)?;
+        let opts = serde_json::to_string(&query)?;
+        self.execute_with_timeout(
+            || async {
+                let mut client = self.get_client().await?;
+                let mut request = Request::new(DiskInfoRequest {
+                    disk: endpoint.clone(),
+                    opts: opts.clone(),
+                });
+                crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
+                let response = client.disk_info(request).await?.into_inner();
+                if !response.success {
+                    return Err(response.error.unwrap_or_default().into());
+                }
+                if response.disk_info.len() > MULTIPART_INDEX_RPC_MAX_BYTES {
+                    return Err(DiskError::FileCorrupt);
+                }
+                // An old server ignores the additive query and returns ordinary DiskInfo.
+                // It cannot satisfy the dedicated reply schema or the fresh challenge.
+                let reply: MultipartIndexRpcReply =
+                    serde_json::from_str(&response.disk_info).map_err(|_| DiskError::MethodNotAllowed)?;
+                reply.validate(&query, expected_disk)?;
+                crate::cluster::rpc::verify_tonic_rpc_response_proof(
+                    &query.canonical_response(&endpoint, &reply.page)?,
+                    &reply.proof,
+                )
+                .map_err(Error::other)?;
+                Ok(reply.page)
+            },
+            get_drive_disk_info_timeout(),
+        )
+        .await
+    }
+
     async fn supports_guarded_delete_versions(&self) -> Result<()> {
         let response = self.guarded_delete_versions_rpc("", Vec::new(), true).await?;
         if !response.errors.is_empty() || !response.item_errors.is_empty() {
@@ -4294,6 +4350,38 @@ impl DiskAPI for RemoteDisk {
         }
     }
 
+    async fn discover_delete_transactions_bounded(
+        &self,
+        volume: &str,
+        object: &str,
+        max_entries: u32,
+    ) -> Result<crate::disk::BoundedDeleteRecoveryDiscovery> {
+        // Older peers reject this variant. Do not retry through the unbounded protocol;
+        // they are reported as unsupported participants (rolling-upgrade contract).
+        match self
+            .recover_delete_transaction(
+                volume,
+                DeleteRecoveryRpcOperation::DiscoverBounded {
+                    object: object.to_owned(),
+                    max_entries,
+                },
+            )
+            .await
+        {
+            Ok(DeleteRecoveryRpcReply::DiscoveredBounded(page)) => Ok(page),
+            Ok(_) => Err(Error::FileCorrupt),
+            Err(error)
+                if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
+                    || matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
+                    .is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
+            {
+                // No paginated fallback through the unbounded protocol; the peer
+                // is an unsupported participant until it is upgraded.
+                Err(Error::UnsupportedDisk)
+            }
+            Err(error) => Err(error),
+        }
+    }
     async fn discover_delete_transactions(
         &self,
         volume: &str,
@@ -4329,6 +4417,62 @@ impl DiskAPI for RemoteDisk {
             _ => Err(Error::other("unexpected delete discovery reply")),
         }
     }
+    async fn settle_delete_transaction_bounded(
+        &self,
+        volume: &str,
+        request: crate::disk::BoundedDeleteRecoverySettlement,
+        _namespace_owner: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<crate::disk::BoundedDeleteRecoverySettlementReply> {
+        match self
+            .recover_delete_transaction(volume, DeleteRecoveryRpcOperation::SettleBounded(request))
+            .await
+        {
+            Ok(DeleteRecoveryRpcReply::SettledBounded(reply)) => Ok(reply),
+            Ok(_) => Err(Error::FileCorrupt),
+            Err(error)
+                if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
+                    || matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
+                    .is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
+            {
+                // No paginated fallback through the unbounded protocol; the peer
+                // is an unsupported participant until it is upgraded.
+                Err(Error::UnsupportedDisk)
+            }
+            Err(error) => Err(error),
+        }
+    }
+    async fn inspect_delete_transaction_bounded(
+        &self,
+        volume: &str,
+        object: &str,
+        transaction: Uuid,
+        max_entries: u32,
+    ) -> Result<crate::disk::BoundedDeleteRecoveryInspection> {
+        match self
+            .recover_delete_transaction(
+                volume,
+                DeleteRecoveryRpcOperation::InspectBounded {
+                    object: object.to_owned(),
+                    transaction,
+                    max_entries,
+                },
+            )
+            .await
+        {
+            Ok(DeleteRecoveryRpcReply::InspectedBounded(page)) => Ok(page),
+            Ok(_) => Err(Error::FileCorrupt),
+            Err(error)
+                if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
+                    || matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
+                    .is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
+            {
+                // No paginated fallback through the unbounded protocol; the peer
+                // is an unsupported participant until it is upgraded.
+                Err(Error::UnsupportedDisk)
+            }
+            Err(error) => Err(error),
+        }
+    }
     async fn inspect_delete_transaction(
         &self,
         volume: &str,
@@ -4849,6 +4993,113 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn bounded_delete_discovery_lost_remote_reply_leaves_no_cursor() {
+        runtime_sources::ensure_test_rpc_secret();
+        let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
+            .await
+            .expect("test peer must bind");
+        let directory = tempfile::tempdir().expect("temporary disk");
+        let endpoint = Endpoint::try_from(directory.path().to_str().expect("disk path")).expect("endpoint");
+        let disk = Arc::new(
+            crate::disk::local::LocalDisk::new(&endpoint, false)
+                .await
+                .expect("local disk"),
+        );
+        disk.make_volume("bucket").await.expect("bucket");
+        let prefix = directory.path().join("bucket/prefix");
+        std::fs::create_dir(&prefix).expect("prefix");
+        for index in 0..64 {
+            std::fs::create_dir(prefix.join(format!("child-{index}"))).expect("legitimate child");
+        }
+        let fixture = Arc::new(TestBoundedDiscovery {
+            disk: Arc::clone(&disk),
+            response_gate: TestFormatReadGate::default(),
+        });
+        *peer.peer.bounded_discovery.lock().expect("bounded fixture") = Some(Arc::clone(&fixture));
+        let remote = remote_disk_for_addr(&peer.addr).await;
+        let task = tokio::spawn(async move { remote.discover_delete_transactions_bounded("bucket", "prefix", 8).await });
+        tokio::time::timeout(Duration::from_secs(5), fixture.response_gate.entered.notified())
+            .await
+            .expect("disk completed bounded scan");
+        task.abort();
+        assert!(task.await.expect_err("caller cancelled").is_cancelled());
+        assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
+        fixture.response_gate.release.cancel();
+        peer.stop().await;
+        assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
+        assert!(prefix.join("child-63").is_dir());
+    }
+
+    #[tokio::test]
+    async fn multipart_index_old_disk_info_handler_is_not_a_ready_empty_index() {
+        runtime_sources::ensure_test_rpc_secret();
+        let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
+            .await
+            .expect("old peer must bind");
+        let remote = remote_disk_for_addr(&peer.addr).await;
+        remote
+            .set_disk_id(Some(Uuid::new_v4()))
+            .await
+            .expect("expected disk identity");
+        assert!(matches!(
+            remote.list_multipart_index_roots("bucket", None).await,
+            Err(DiskError::MethodNotAllowed)
+        ));
+        assert_eq!(
+            peer.peer.disk_info_calls.load(Ordering::Acquire),
+            1,
+            "exercise the actual legacy JSON handler"
+        );
+        peer.stop().await;
+    }
+
+    #[tokio::test]
+    async fn bounded_delete_discovery_old_peer_rejects_without_paginated_fallback() {
+        runtime_sources::ensure_test_rpc_secret();
+        let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
+            .await
+            .expect("old peer must bind");
+        let remote = remote_disk_for_addr(&peer.addr).await;
+        assert_eq!(
+            remote
+                .discover_delete_transactions_bounded("bucket", "prefix", 4096)
+                .await
+                .expect_err("old peer cannot prove a bounded inventory"),
+            DiskError::UnsupportedDisk
+        );
+        let transaction = Uuid::new_v4();
+        assert_eq!(
+            remote
+                .inspect_delete_transaction_bounded("bucket", "prefix", transaction, 65536)
+                .await
+                .expect_err("old peer cannot inspect with a budget"),
+            DiskError::UnsupportedDisk
+        );
+        assert_eq!(
+            remote
+                .settle_delete_transaction_bounded(
+                    "bucket",
+                    crate::disk::BoundedDeleteRecoverySettlement {
+                        object: "prefix".to_owned(),
+                        transaction,
+                        expected_fingerprint: [0; 32],
+                        action: DeleteRecoveryAction::PublishCommit,
+                        max_entries: 65536,
+                    },
+                    None
+                )
+                .await
+                .expect_err("old peer cannot settle with a budget"),
+            DiskError::UnsupportedDisk
+        );
+        assert_eq!(
+            *peer.peer.unsupported_methods.lock().expect("observed methods"),
+            vec!["/node_service.NodeService/RecoverDeleteTransaction"; 3]
+        );
+        peer.stop().await;
+    }
+
     #[tokio::test]
     async fn multipart_source_recovery_old_peer_rejects_without_generic_fallback() {
         runtime_sources::ensure_test_rpc_secret();
@@ -5740,10 +5991,18 @@ mod tests {
         object_read_all_disks: Arc<StdMutex<Vec<String>>>,
         format_data: Bytes,
         read_all_data: Bytes,
+        fail_format_reads: Arc<AtomicBool>,
         next_format_read_gate: Arc<StdMutex<Option<Arc<TestFormatReadGate>>>>,
+        bounded_discovery: Arc<StdMutex<Option<Arc<TestBoundedDiscovery>>>>,
         unsupported_methods: Arc<StdMutex<Vec<String>>>,
     }
 
+    #[derive(Debug)]
+    struct TestBoundedDiscovery {
+        disk: Arc<crate::disk::local::LocalDisk>,
+        response_gate: TestFormatReadGate,
+    }
+
     #[derive(Debug, Default)]
     struct TestFormatReadGate {
         entered: tokio::sync::Notify,
@@ -5759,7 +6018,9 @@ mod tests {
                 object_read_all_disks: Arc::default(),
                 format_data,
                 read_all_data,
+                fail_format_reads: Arc::default(),
                 next_format_read_gate: Arc::default(),
+                bounded_discovery: Arc::default(),
                 unsupported_methods: Arc::default(),
             }
         }
@@ -5859,6 +6120,56 @@ mod tests {
                         Ok(grpc.unary(method, request).await)
                     })
                 }
+                "/node_service.NodeService/RecoverDeleteTransaction"
+                    if self.peer.bounded_discovery.lock().expect("bounded fixture").is_some() =>
+                {
+                    #[derive(Clone)]
+                    struct BoundedSvc(Arc<TestBoundedDiscovery>);
+                    impl tonic::server::UnaryService<RecoverDeleteTransactionRequest> for BoundedSvc {
+                        type Response = rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse;
+                        type Future = Pin<Box<dyn Future<Output = std::result::Result<Response<Self::Response>, Status>> + Send>>;
+                        fn call(&mut self, request: Request<RecoverDeleteTransactionRequest>) -> Self::Future {
+                            let fixture = Arc::clone(&self.0);
+                            Box::pin(async move {
+                                let request = request.into_inner();
+                                let operation: DeleteRecoveryRpcOperation =
+                                    rustfs_protos::decode_delete_recovery_payload(&request.operation)
+                                        .map_err(|_| Status::invalid_argument("invalid bounded request"))?;
+                                let DeleteRecoveryRpcOperation::DiscoverBounded { object, max_entries } = operation else {
+                                    return Err(Status::invalid_argument("unexpected fallback"));
+                                };
+                                let page = fixture
+                                    .disk
+                                    .discover_delete_transactions_bounded(&request.volume, &object, max_entries)
+                                    .await
+                                    .map_err(|_| Status::internal("bounded discovery failed"))?;
+                                // Lose the response only after the real disk operation has completed.
+                                fixture.response_gate.entered.notify_one();
+                                fixture.response_gate.release.cancelled().await;
+                                let result = rmp_serde::to_vec_named(&DeleteRecoveryRpcReply::DiscoveredBounded(page))
+                                    .map_err(|_| Status::internal("encode bounded reply"))?;
+                                Ok(Response::new(rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse {
+                                    success: true,
+                                    protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
+                                    result: result.into(),
+                                    error: None,
+                                }))
+                            })
+                        }
+                    }
+                    let fixture = self
+                        .peer
+                        .bounded_discovery
+                        .lock()
+                        .expect("bounded fixture")
+                        .clone()
+                        .expect("configured fixture");
+                    Box::pin(async move {
+                        let codec = tonic_prost::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec);
+                        Ok(grpc.unary(BoundedSvc(fixture), request).await)
+                    })
+                }
                 "/node_service.NodeService/ReadAll" => {
                     #[derive(Clone)]
                     struct ReadAllSvc(AuthenticatedReadPeer);
@@ -5882,6 +6193,9 @@ mod tests {
                                         gate.entered.notify_one();
                                         gate.release.cancelled().await;
                                     }
+                                    if peer.fail_format_reads.load(Ordering::Acquire) {
+                                        return Err(Status::unavailable("format read temporarily unavailable"));
+                                    }
                                     peer.format_data.clone()
                                 } else {
                                     peer.object_read_all_disks
@@ -8847,6 +9161,124 @@ mod tests {
         peer.stop().await;
     }
 
+    #[tokio::test]
+    #[serial(remote_disk_recovery_probe)]
+    async fn endpoint_reconnect_failure_preserves_registered_remote_recovery() {
+        temp_env::async_with_vars(
+            [
+                (crate::disk::disk_store::ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, Some("true")),
+                (rustfs_config::ENV_DRIVE_RETURNING_SUCCESS_THRESHOLD, Some("1")),
+            ],
+            async {
+                runtime_sources::ensure_test_rpc_secret();
+                let mut reference = crate::layout::format::FormatV3::new(1, 1);
+                reference.erasure.this = reference.erasure.sets[0][0];
+                let peer = TestGrpcPeer::spawn(
+                    Bytes::from(reference.to_json().expect("format should serialize")),
+                    Bytes::from_static(b"recovered-peer-data"),
+                )
+                .await
+                .expect("authenticated regression peer is required");
+                let set = missing_read_set(&peer, reference).await;
+                let disk = crate::disk::new_disk(
+                    &set.set_endpoints[0],
+                    &DiskOption {
+                        cleanup: false,
+                        health_check: true,
+                    },
+                )
+                .await
+                .expect("registered remote disk should construct");
+                let crate::disk::Disk::Remote(remote) = disk.as_ref() else {
+                    panic!("fixture must create a remote disk");
+                };
+                assert!(remote.health_check);
+                remote.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
+                set.disks.write().await[0] = Some(Arc::clone(&disk));
+
+                peer.peer.fail_format_reads.store(true, Ordering::Release);
+                set.connect_disks().await;
+                peer.peer.fail_format_reads.store(false, Ordering::Release);
+                let cancelled_after_failure = remote.cancel_token.is_cancelled();
+                assert!(peer.peer.read_all_calls() > 0, "reconnect must attempt the failing format RPC");
+                assert!(
+                    Arc::ptr_eq(&disk, set.disks.read().await[0].as_ref().expect("existing slot must remain")),
+                    "a failed reconnect must not publish its unvalidated candidate"
+                );
+                remote.spawn_recovery_monitor_if_needed();
+                let recovery = time::timeout(Duration::from_secs(5), async {
+                    while !disk.is_online().await {
+                        tokio::task::yield_now().await;
+                    }
+                    disk.read_all("bucket", "object").await
+                })
+                .await;
+                disk.close().await.expect("stop the registered disk's monitoring");
+                peer.stop().await;
+                assert!(!cancelled_after_failure, "failed replacement must retain the registered monitor");
+                assert_eq!(
+                    recovery
+                        .expect("failed reconnect must leave the registered disk able to recover without another sweep")
+                        .expect("the recovered registered disk must serve real I/O"),
+                    Bytes::from_static(b"recovered-peer-data")
+                );
+            },
+        )
+        .await;
+    }
+
+    #[tokio::test]
+    #[serial(remote_disk_recovery_probe)]
+    async fn endpoint_reconnect_stops_old_remote_monitor_only_after_publication() {
+        runtime_sources::ensure_test_rpc_secret();
+        let mut reference = crate::layout::format::FormatV3::new(1, 1);
+        reference.erasure.this = reference.erasure.sets[0][0];
+        let peer = TestGrpcPeer::spawn(
+            Bytes::from(reference.to_json().expect("format should serialize")),
+            Bytes::from_static(b"replacement-peer-data"),
+        )
+        .await
+        .expect("authenticated regression peer is required");
+        let set = missing_read_set(&peer, reference).await;
+        let disk = crate::disk::new_disk(
+            &set.set_endpoints[0],
+            &DiskOption {
+                cleanup: false,
+                health_check: true,
+            },
+        )
+        .await
+        .expect("registered remote disk should construct");
+        let crate::disk::Disk::Remote(remote) = disk.as_ref() else {
+            panic!("fixture must create a remote disk");
+        };
+        remote.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
+        set.disks.write().await[0] = Some(Arc::clone(&disk));
+        let gate = peer.peer.pause_next_format_read();
+        let pending_set = Arc::clone(&set);
+        let pending = tokio::spawn(async move { pending_set.connect_disks().await });
+        time::timeout(Duration::from_secs(5), gate.entered.notified())
+            .await
+            .expect("replacement must reach the format validation barrier");
+        let cancelled_before_publication = remote.cancel_token.is_cancelled();
+        gate.release.cancel();
+        pending.await.expect("validated replacement must finish publication");
+        let replacement = set.disks.read().await[0].clone().expect("replacement must occupy the slot");
+        let readback = replacement.read_all("bucket", "object").await;
+        replacement.close().await.expect("stop replacement monitoring");
+        peer.stop().await;
+        assert!(
+            !cancelled_before_publication,
+            "the registered monitor must survive pending format validation"
+        );
+        assert!(!Arc::ptr_eq(&disk, &replacement), "validated replacement must be published");
+        assert!(remote.cancel_token.is_cancelled(), "the displaced monitor must stop after publication");
+        assert_eq!(
+            readback.expect("replacement must serve real I/O"),
+            Bytes::from_static(b"replacement-peer-data")
+        );
+    }
+
     #[tokio::test]
     #[serial(remote_disk_recovery_probe)]
     async fn endpoint_reconnect_admits_ready_drive_while_another_format_probe_is_blocked() {

crates/ecstore/src/cluster/rpc/remote_locker.rs

@@ -18,6 +18,7 @@ use crate::cluster::rpc::client::{
 use crate::cluster::rpc::set_tonic_rolling_mutation_body_digest;
 use async_trait::async_trait;
 use bytes::Bytes;
+use futures::{FutureExt, future::BoxFuture};
 use rustfs_lock::{
     LockClient, LockError, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
     types::{LockId, LockMetadata, LockPriority},
@@ -32,16 +33,17 @@ use rustfs_protos::{
 };
 use std::collections::HashMap;
 use std::future::Future;
+use std::panic::AssertUnwindSafe;
 use std::pin::Pin;
 use std::sync::{Mutex, OnceLock};
 use std::time::Duration;
-use tokio::task::JoinHandle;
 use tokio::time::{Instant, timeout};
 use tonic::service::interceptor::InterceptedService;
 use tonic::{Request, Response};
 use tracing::{debug, info, warn};
 
 const EVENT_NAMESPACE_LOCK_RELEASE: &str = "namespace_lock_release";
+const EVENT_REMOTE_LOCK_RPC_CLEANUP: &str = "remote_lock_rpc_cleanup";
 const LOG_COMPONENT_ECSTORE: &str = "ecstore";
 const LOG_SUBSYSTEM_REMOTE_LOCK: &str = "remote_lock";
 
@@ -49,18 +51,38 @@ fn attach_lock_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(req
     set_tonic_rolling_mutation_body_digest(request)
 }
 
-/// Work to run if an RPC that already timed out for its caller completes later.
+/// Work to run if an RPC completes after its caller stops waiting.
 type LateCompletion<T> = Option<Box<dyn FnOnce(T) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send>>;
 type LateFailure = LateCompletion<()>;
+type RpcFuture<T> = BoxFuture<'static, std::thread::Result<std::result::Result<T, tonic::Status>>>;
 
-struct RpcCleanup<T> {
+/// Own admission before connecting as well as while an RPC is in flight.
-    late: LateCompletion<T>,
+struct LockRequestGuard<'a> {
-    late_failure: LateFailure,
+    client: &'a RemoteClient,
+    op: &'static str,
     admission: Option<LockRequestAdmission>,
 }
 
-impl<T> RpcCleanup<T> {
+impl LockRequestGuard<'_> {
-    fn new(late: LateCompletion<T>, late_failure: LateFailure, admission: Option<LockRequestAdmission>) -> Self {
+    fn into_admission(mut self) -> Option<LockRequestAdmission> {
+        self.admission.take()
+    }
+}
+
+impl Drop for LockRequestGuard<'_> {
+    fn drop(&mut self) {
+        self.client.release_lock_request(self.op, self.admission.take());
+    }
+}
+
+struct RpcCleanup<'a, T> {
+    late: LateCompletion<T>,
+    late_failure: LateFailure,
+    admission: Option<LockRequestGuard<'a>>,
+}
+
+impl<'a, T> RpcCleanup<'a, T> {
+    fn new(late: LateCompletion<T>, late_failure: LateFailure, admission: Option<LockRequestGuard<'a>>) -> Self {
         Self {
             late,
             late_failure,
@@ -69,6 +91,25 @@ impl<T> RpcCleanup<T> {
     }
 }
 
+/// A dropped caller hands its RPC and cleanup to the same bounded owner as a timeout.
+struct InFlightRpc<'a, T: Send + 'static> {
+    client: &'a RemoteClient,
+    op: &'static str,
+    resource_summary: &'a str,
+    future: Option<RpcFuture<T>>,
+    cleanup: Option<RpcCleanup<'a, T>>,
+}
+
+impl<T: Send + 'static> Drop for InFlightRpc<'_, T> {
+    fn drop(&mut self) {
+        if let Some(future) = self.future.take()
+            && let Some(cleanup) = self.cleanup.take()
+        {
+            self.client.detach_rpc(self.op, self.resource_summary, future, cleanup);
+        }
+    }
+}
+
 /// The liveness window is this many RPC deadlines: a peer that completed a
 /// lock RPC within it is slow, not gone, and keeps its channel on a timeout.
 const LOCK_RPC_LIVENESS_WINDOW_DEADLINES: u32 = 2;
@@ -137,6 +178,9 @@ fn with_lock_peer_health<R>(addr: &str, update: impl FnOnce(&mut LockPeerChannel
     let mut peers = lock_peer_channel_health()
         .lock()
         .unwrap_or_else(std::sync::PoisonError::into_inner);
+    if let Some(health) = peers.get_mut(addr) {
+        return update(health);
+    }
     update(peers.entry(addr.to_string()).or_default())
 }
 
@@ -158,6 +202,15 @@ fn reset_lock_peer_health_for_test(addr: &str) {
         .remove(addr);
 }
 
+#[cfg(test)]
+type EvictionPause = (tokio::sync::oneshot::Sender<()>, tokio::sync::oneshot::Receiver<()>);
+
+#[cfg(test)]
+fn eviction_pauses_for_test() -> &'static Mutex<HashMap<String, EvictionPause>> {
+    static PAUSES: OnceLock<Mutex<HashMap<String, EvictionPause>>> = OnceLock::new();
+    PAUSES.get_or_init(Mutex::default)
+}
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 enum EvictionTrigger {
     /// The caller's deadline expired while the stream was still open.
@@ -445,6 +498,10 @@ impl RemoteClient {
                 }
             }
         });
+        // A successful lock RPC over a cached or fresh channel proves the peer is
+        // reachable; reconcile the internode peer-health gauge so it does not stay
+        // stale when no fresh dial fires (issue #9119).
+        rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
     }
 
     fn admit_late_release(&self) -> bool {
@@ -502,6 +559,14 @@ impl RemoteClient {
         admission
     }
 
+    fn admit_lock_request_guard(&self, op: &'static str) -> Option<LockRequestGuard<'_>> {
+        self.admit_lock_request(op).map(|admission| LockRequestGuard {
+            client: self,
+            op,
+            admission: Some(admission),
+        })
+    }
+
     fn release_lock_request(&self, op: &'static str, admission: Option<LockRequestAdmission>) {
         if is_request_breaker_operation(op) && admission.is_some() {
             with_lock_peer_health(&self.addr, |health| {
@@ -583,6 +648,17 @@ impl RemoteClient {
         trigger: EvictionTrigger,
         deadline: Duration,
     ) {
+        #[cfg(test)]
+        let pause = eviction_pauses_for_test()
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner)
+            .remove(&self.addr);
+        #[cfg(test)]
+        if let Some((entered, resume)) = pause {
+            let _ = entered.send(());
+            let _ = resume.await;
+        }
+
         let now = Instant::now();
         let cooldown = Self::eviction_cooldown();
         let liveness_window = Self::liveness_window(deadline);
@@ -613,22 +689,26 @@ impl RemoteClient {
         );
     }
 
-    /// Keep a timed-out RPC running instead of cancelling its stream.
+    /// Keep an abandoned RPC running instead of cancelling its stream.
     ///
     /// Dropping the future sends `RST_STREAM`; under load those resets pile up
     /// in the server's pending-accept queue until it answers `GOAWAY
     /// too_many_resets` and kills every stream on the connection. A detached
     /// stream ends on its own within the internode RPC timeout, the number per
     /// peer is bounded, and a lock granted after its caller gave up is released.
-    fn detach_timed_out_rpc<T: Send + 'static>(
+    fn detach_rpc<T: Send + 'static>(
         &self,
         op: &'static str,
         resource_summary: &str,
-        handle: JoinHandle<std::result::Result<T, tonic::Status>>,
+        future: RpcFuture<T>,
-        late: LateCompletion<T>,
+        cleanup: RpcCleanup<'_, T>,
-        late_failure: LateFailure,
-        admission: Option<LockRequestAdmission>,
     ) {
+        let RpcCleanup {
+            late,
+            late_failure,
+            admission,
+        } = cleanup;
+        let admission = admission.and_then(LockRequestGuard::into_admission);
         let limit = Self::detached_rpc_limit();
         let admitted = with_lock_peer_health(&self.addr, |health| {
             if health.detached_rpcs >= limit {
@@ -639,25 +719,29 @@ impl RemoteClient {
             }
         });
         if !admitted {
-            handle.abort();
+            drop(future);
             self.release_lock_request(op, admission);
             if let Some(late_failure) = late_failure {
                 drop(tokio::spawn(late_failure(())));
             }
             rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_detached(op, "aborted");
             debug!(
+                event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
+                state = "detached_budget_exhausted",
                 addr = %self.addr,
                 op,
                 resource_summary,
                 limit,
-                "Cancelled timed-out remote lock RPC because the detached stream budget is exhausted"
+                "Abandoned remote lock RPC cancelled"
             );
             return;
         }
         rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_detached(op, "detached");
         let addr = self.addr.clone();
         tokio::spawn(async move {
-            let outcome = handle.await;
+            let outcome = future.await;
             with_lock_peer_health(&addr, |health| {
                 health.detached_rpcs = health.detached_rpcs.saturating_sub(1);
                 if is_request_breaker_operation(op) && admission.is_some() {
@@ -672,6 +756,7 @@ impl RemoteClient {
                     with_lock_peer_health(&addr, |health| {
                         health.last_success = Some(Instant::now());
                     });
+                    rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
                     rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "success");
                     if let Some(late) = late {
                         late(response).await;
@@ -683,19 +768,31 @@ impl RemoteClient {
                     }
                     rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "error");
                     debug!(
+                        event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
+                        state = "late_rpc_failed",
                         addr = %addr,
                         op,
                         tonic_code = ?status.code(),
                         tonic_message = status.message(),
-                        "Detached remote lock RPC failed after its caller timed out"
+                        "Abandoned remote lock RPC failed"
                     );
                 }
-                Err(join_error) => {
+                Err(_) => {
                     if let Some(late_failure) = late_failure {
                         late_failure(()).await;
                     }
                     rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "join_error");
-                    debug!(addr = %addr, op, error = %join_error, "Detached remote lock RPC task ended abnormally");
+                    debug!(
+                        event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
+                        state = "late_rpc_panicked",
+                        addr = %addr,
+                        op,
+                        "Abandoned remote lock RPC panicked"
+                    );
                 }
             }
         });
@@ -783,16 +880,28 @@ impl RemoteClient {
         self.release_late_release();
         rustfs_io_metrics::lock_metrics::record_remote_lock_late_release(outcome);
         if outcome == "released" {
-            debug!(addr = %self.addr, count = original_count, "Released remote locks granted after their caller timed out");
+            debug!(
+                event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
+                state = "late_grants_released",
+                addr = %self.addr,
+                count = original_count,
+                "Released remote locks granted after their caller stopped waiting"
+            );
         } else {
             let (log_warning, suppressed_timeouts) = self.timeout_log_decision("late_release", Self::rpc_timeout());
             if log_warning {
                 warn!(
+                    event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
+                    state = "late_grants_unreleased",
                     addr = %self.addr,
                     count = original_count,
                     outcome,
                     suppressed_timeouts,
-                    "Could not release every remote lock granted after its caller timed out; the server lease will expire it"
+                    "Could not release every remote lock granted after its caller stopped waiting; the server lease will expire it"
                 );
             } else {
                 debug!(addr = %self.addr, count = original_count, outcome, suppressed_timeouts, "Suppressed repeated late remote lock release warning");
@@ -806,25 +915,47 @@ impl RemoteClient {
         resource_summary: &str,
         deadline: Duration,
         future: Fut,
-        cleanup: RpcCleanup<T>,
+        cleanup: RpcCleanup<'_, T>,
     ) -> std::result::Result<T, LockError>
     where
         Fut: Future<Output = std::result::Result<T, tonic::Status>> + Send + 'static,
         T: Send + 'static,
     {
-        let RpcCleanup {
+        let mut rpc = InFlightRpc {
-            late,
+            client: self,
+            op,
+            resource_summary,
+            future: Some(AssertUnwindSafe(future).catch_unwind().boxed()),
+            cleanup: Some(cleanup),
+        };
+        let outcome = match rpc.future.as_mut() {
+            Some(future) => tokio::select! {
+                // A response becoming ready on the deadline tick must not turn an
+                // expired caller budget into a transport error and channel eviction.
+                biased;
+                _ = tokio::time::sleep(deadline) => None,
+                outcome = future => Some(outcome),
+            },
+            None => return Err(LockError::internal("remote lock RPC has no in-flight future")),
+        };
+        if outcome.is_some() {
+            rpc.future.take();
+        }
+        let Some(RpcCleanup {
+            late: _,
             late_failure,
             admission,
-        } = cleanup;
+        }) = rpc.cleanup.as_mut()
-        let mut handle = tokio::spawn(future);
+        else {
-        match timeout(deadline, &mut handle).await {
+            return Err(LockError::internal("remote lock RPC has no cleanup owner"));
-            Ok(Ok(Ok(response))) => {
+        };
-                self.record_rpc_success(op, admission);
+        let token = admission.as_ref().and_then(|guard| guard.admission);
-                self.release_lock_request(op, admission);
+        let result = match outcome {
+            Some(Ok(Ok(response))) => {
+                self.record_rpc_success(op, token);
                 Ok(response)
             }
-            Ok(Ok(Err(err))) => {
+            Some(Ok(Err(err))) => {
                 let reason = err.to_string();
                 if err.code() == tonic::Code::DeadlineExceeded {
                     rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_timeout(&self.addr, op);
@@ -857,12 +988,11 @@ impl RemoteClient {
                             "Suppressed repeated remote lock RPC deadline"
                         );
                     }
-                    self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
+                    if let Some(late_failure) = late_failure.take() {
-                        .await;
-                    self.release_lock_request(op, admission);
-                    if let Some(late_failure) = late_failure {
                         drop(tokio::spawn(late_failure(())));
                     }
+                    self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
+                        .await;
                     return Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline));
                 }
                 // Only evict (and re-dial) the cached channel when the failure is a genuine
@@ -923,30 +1053,32 @@ impl RemoteClient {
                     );
                 }
                 if transport_failure {
+                    // Ambiguous-grant cleanup owns its work before channel eviction can suspend.
+                    if let Some(late_failure) = late_failure.take() {
+                        drop(tokio::spawn(late_failure(())));
+                    }
                     self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Transport, deadline)
                         .await;
                 }
-                self.release_lock_request(op, admission);
-                if transport_failure && let Some(late_failure) = late_failure {
-                    drop(tokio::spawn(late_failure(())));
-                }
                 Err(LockError::internal(format!("{op} RPC failed: {reason}")))
             }
-            Ok(Err(join_error)) => {
+            Some(Err(_)) => {
                 warn!(
+                    event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
+                    state = "rpc_panicked",
                     addr = %self.addr,
                     op,
                     resource_summary,
-                    error = %join_error,
+                    "Remote lock RPC panicked"
-                    "Remote lock RPC task ended abnormally"
                 );
-                self.release_lock_request(op, admission);
+                if let Some(late_failure) = late_failure.take() {
-                if let Some(late_failure) = late_failure {
                     drop(tokio::spawn(late_failure(())));
                 }
-                Err(LockError::internal(format!("{op} RPC task failed: {join_error}")))
+                Err(LockError::internal(format!("{op} RPC future panicked")))
             }
-            Err(_) => {
+            None => {
                 let reason = format!("RPC timed out after {deadline:?}");
                 rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_timeout(&self.addr, op);
                 self.record_request_timeout(op, deadline);
@@ -978,12 +1110,17 @@ impl RemoteClient {
                         "Suppressed repeated remote lock RPC timeout"
                     );
                 }
+                // Continue draining the abandoned stream while channel eviction waits.
+                drop(rpc);
                 self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
                     .await;
-                self.detach_timed_out_rpc(op, resource_summary, handle, late, late_failure, admission);
+                return Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline));
-                Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline))
             }
+        };
+        if rpc.future.is_none() {
+            rpc.cleanup.take();
         }
+        result
     }
 
     fn rpc_timeout_failure_response(request: &LockRequest, err: &LockError) -> LockResponse {
@@ -1074,7 +1211,7 @@ impl RemoteClient {
 #[async_trait]
 impl LockClient for RemoteClient {
     async fn acquire_lock(&self, request: &LockRequest) -> Result<LockResponse> {
-        let admission = match self.admit_lock_request("lock") {
+        let admission = match self.admit_lock_request_guard("lock") {
             Some(admission) => admission,
             None => return Ok(Self::rpc_backoff_failure_response(request)),
         };
@@ -1085,7 +1222,6 @@ impl LockClient for RemoteClient {
             Ok(client) => client,
             Err(err) => {
                 self.record_request_timeout("lock", rpc_timeout);
-                self.release_lock_request("lock", Some(admission));
                 return Err(err);
             }
         };
@@ -1093,14 +1229,12 @@ impl LockClient for RemoteClient {
         let args = match serde_json::to_string(&bounded_request) {
             Ok(args) => args,
             Err(err) => {
-                self.release_lock_request("lock", Some(admission));
                 return Err(LockError::internal(format!("Failed to serialize request: {err}")));
             }
         };
         let mut req = Request::new(GenerallyLockRequest { args });
         req.set_timeout(rpc_timeout);
         if let Err(err) = attach_lock_mutation_body_digest(&mut req) {
-            self.release_lock_request("lock", Some(admission));
             return Err(err.into());
         }
         let late = self.late_release_hook(request.lock_id.clone());
@@ -1142,7 +1276,7 @@ impl LockClient for RemoteClient {
             return Ok(Vec::new());
         }
 
-        let admission = match self.admit_lock_request("lock_batch") {
+        let admission = match self.admit_lock_request_guard("lock_batch") {
             Some(admission) => admission,
             None => return Ok(Self::rpc_backoff_failure_batch(requests)),
         };
@@ -1156,7 +1290,6 @@ impl LockClient for RemoteClient {
             Ok(client) => client,
             Err(err) => {
                 self.record_request_timeout("lock_batch", rpc_timeout);
-                self.release_lock_request("lock_batch", Some(admission));
                 return Err(err);
             }
         };
@@ -1170,14 +1303,12 @@ impl LockClient for RemoteClient {
         {
             Ok(args) => args,
             Err(err) => {
-                self.release_lock_request("lock_batch", Some(admission));
                 return Err(err);
             }
         };
         let mut req = Request::new(BatchGenerallyLockRequest { args });
         req.set_timeout(rpc_timeout);
         if let Err(err) = attach_lock_mutation_body_digest(&mut req) {
-            self.release_lock_request("lock_batch", Some(admission));
             return Err(err.into());
         }
         let late = self.late_release_batch_hook(requests.iter().map(|request| request.lock_id.clone()).collect());
@@ -1705,6 +1836,351 @@ mod tests {
         assert!(acquired_lock_ids(&lock_ids, &[]).is_empty());
     }
 
+    #[tokio::test]
+    async fn test_remote_client_cancelled_connection_releases_request_admission() {
+        ensure_test_rpc_secret();
+        let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind pending-handshake peer");
+        let addr = format!("http://{}", listener.local_addr().expect("pending-handshake peer address"));
+        let client = RemoteClient::new(addr.clone());
+        let request = test_lock_request(Duration::from_secs(30));
+
+        for batch in [false, true] {
+            reset_lock_peer_health_for_test(&addr);
+            with_lock_peer_health(&addr, |health| {
+                health.request_backoff_until = Some(Instant::now() - Duration::from_secs(1));
+            });
+            let mut acquisition = Box::pin(async {
+                if batch {
+                    client.acquire_locks_batch(std::slice::from_ref(&request)).await.map(|_| ())
+                } else {
+                    client.acquire_lock(&request).await.map(|_| ())
+                }
+            });
+            std::future::poll_fn(|cx| {
+                assert!(acquisition.as_mut().poll(cx).is_pending(), "peer must wait for its HTTP/2 handshake");
+                std::task::Poll::Ready(())
+            })
+            .await;
+            let pending = lock_peer_health_for_test(&addr);
+            assert_eq!(pending.request_in_flight, 1, "pending connection owns one request admission");
+            assert!(pending.request_probe_generation.is_some(), "the request is the half-open probe");
+
+            drop(acquisition);
+            let cancelled = lock_peer_health_for_test(&addr);
+            assert_eq!(cancelled.request_in_flight, 0, "cancellation must return request admission");
+            assert!(cancelled.request_probe_generation.is_none(), "cancellation must unblock the next probe");
+            let probe = client.admit_lock_request("lock").expect("cancelled peer can be probed again");
+            client.release_lock_request("lock", Some(probe));
+        }
+        reset_lock_peer_health_for_test(&addr);
+    }
+
+    #[tokio::test]
+    async fn test_remote_client_cancelled_rpc_runs_late_grant_cleanup() {
+        let addr = "http://cancelled-rpc-late-grant";
+        reset_lock_peer_health_for_test(addr);
+        let client = RemoteClient::new(addr.to_string());
+        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
+        let (grant_tx, grant_rx) = tokio::sync::oneshot::channel();
+        let (released_tx, released_rx) = tokio::sync::oneshot::channel();
+        let caller = tokio::spawn(async move {
+            let late: LateCompletion<bool> = Some(Box::new(move |granted| {
+                Box::pin(async move {
+                    released_tx.send(granted).expect("late grant observer remains alive");
+                })
+            }));
+            client
+                .execute_rpc(
+                    "lock",
+                    "bucket/object",
+                    Duration::from_secs(30),
+                    async move {
+                        started_tx.send(()).expect("caller observes RPC dispatch");
+                        Ok(grant_rx.await.expect("test decides when peer grants lock"))
+                    },
+                    RpcCleanup::new(late, None, None),
+                )
+                .await
+        });
+        timeout(Duration::from_secs(1), started_rx)
+            .await
+            .expect("RPC must dispatch")
+            .expect("dispatch signal");
+        caller.abort();
+        assert!(caller.await.expect_err("caller was cancelled").is_cancelled());
+        assert_eq!(
+            lock_peer_health_for_test(addr).detached_rpcs,
+            1,
+            "cancelled RPC uses bounded detached ownership"
+        );
+        grant_tx.send(true).expect("cancellation must preserve the running RPC");
+        assert!(
+            timeout(Duration::from_secs(1), released_rx)
+                .await
+                .expect("late grant must be cleaned up")
+                .expect("cleanup result")
+        );
+        assert_eq!(lock_peer_health_for_test(addr).detached_rpcs, 0, "completed cleanup returns its slot");
+        reset_lock_peer_health_for_test(addr);
+    }
+
+    #[tokio::test]
+    async fn test_remote_client_cancelled_rpc_runs_ambiguous_failure_cleanup() {
+        let addr = "http://cancelled-rpc-late-failure";
+        reset_lock_peer_health_for_test(addr);
+        let client = RemoteClient::new(addr.to_string());
+        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
+        let (failure_tx, failure_rx) = tokio::sync::oneshot::channel();
+        let (released_tx, released_rx) = tokio::sync::oneshot::channel();
+        let caller = tokio::spawn(async move {
+            let late_failure: LateFailure = Some(Box::new(move |()| {
+                Box::pin(async move {
+                    released_tx.send(()).expect("ambiguous cleanup observer remains alive");
+                })
+            }));
+            client
+                .execute_rpc(
+                    "lock_batch",
+                    "bucket/object",
+                    Duration::from_secs(30),
+                    async move {
+                        started_tx.send(()).expect("caller observes RPC dispatch");
+                        failure_rx.await.expect("test decides when transport fails");
+                        Err::<(), _>(tonic::Status::cancelled("transport failure after caller cancellation"))
+                    },
+                    RpcCleanup::new(None, late_failure, None),
+                )
+                .await
+        });
+        timeout(Duration::from_secs(1), started_rx)
+            .await
+            .expect("RPC must dispatch")
+            .expect("dispatch signal");
+        caller.abort();
+        assert!(caller.await.expect_err("caller was cancelled").is_cancelled());
+        failure_tx.send(()).expect("cancellation must preserve the running RPC");
+        timeout(Duration::from_secs(1), released_rx)
+            .await
+            .expect("ambiguous grant must be cleaned up")
+            .expect("cleanup signal");
+        assert_eq!(lock_peer_health_for_test(addr).detached_rpcs, 0, "failed detached RPC returns its slot");
+        reset_lock_peer_health_for_test(addr);
+    }
+
+    #[tokio::test]
+    async fn test_remote_client_cancelled_completed_failure_runs_cleanup() {
+        for (label, status) in [
+            ("transport", tonic::Status::cancelled("ambiguous transport failure")),
+            ("deadline", tonic::Status::deadline_exceeded("ambiguous deadline failure")),
+        ] {
+            let addr = format!("http://cancelled-completed-{label}");
+            reset_lock_peer_health_for_test(&addr);
+            let client = RemoteClient::new(addr.clone());
+            let (evicting_tx, evicting_rx) = tokio::sync::oneshot::channel();
+            let (_resume_tx, resume_rx) = tokio::sync::oneshot::channel();
+            eviction_pauses_for_test()
+                .lock()
+                .expect("eviction pause registry")
+                .insert(addr.clone(), (evicting_tx, resume_rx));
+            let (released_tx, released_rx) = tokio::sync::oneshot::channel();
+            let caller = tokio::spawn(async move {
+                let late_failure: LateFailure = Some(Box::new(move |()| {
+                    Box::pin(async move {
+                        released_tx
+                            .send(())
+                            .expect("completed-failure cleanup observer remains alive");
+                    })
+                }));
+                client
+                    .execute_rpc(
+                        "lock",
+                        "bucket/object",
+                        Duration::from_secs(30),
+                        async move { Err::<(), _>(status) },
+                        RpcCleanup::new(None, late_failure, None),
+                    )
+                    .await
+            });
+            timeout(Duration::from_secs(1), evicting_rx)
+                .await
+                .expect("completed error must reach channel eviction")
+                .expect("eviction signal");
+            timeout(Duration::from_secs(1), released_rx)
+                .await
+                .expect("cleanup must transfer before channel eviction suspends")
+                .expect("cleanup signal");
+            caller.abort();
+            assert!(caller.await.expect_err("caller is cancelled during eviction").is_cancelled());
+            assert_eq!(
+                lock_peer_health_for_test(&addr).detached_rpcs,
+                0,
+                "completed RPC cannot consume a detached slot"
+            );
+            reset_lock_peer_health_for_test(&addr);
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn test_remote_client_deadline_wins_over_same_tick_transport_timeout() {
+        let addr = "http://same-tick-rpc-deadline";
+        reset_lock_peer_health_for_test(addr);
+        with_lock_peer_health(addr, |health| health.last_success = Some(Instant::now()));
+        let client = RemoteClient::new(addr.to_string());
+        let budget = Duration::from_millis(50);
+        let (released_tx, released_rx) = tokio::sync::oneshot::channel();
+        let late_failure: LateFailure = Some(Box::new(move |()| {
+            Box::pin(async move {
+                released_tx.send(()).expect("same-tick cleanup observer remains alive");
+            })
+        }));
+        let result = client
+            .execute_rpc(
+                "lock",
+                "bucket/object",
+                budget,
+                async move {
+                    tokio::time::sleep(budget).await;
+                    Err::<(), _>(tonic::Status::cancelled("inner transport deadline expired"))
+                },
+                RpcCleanup::new(None, late_failure, None),
+            )
+            .await;
+        assert!(
+            matches!(result, Err(LockError::Timeout { .. })),
+            "expired caller budget must retain timeout classification"
+        );
+        let health = lock_peer_health_for_test(addr);
+        assert_eq!(health.consecutive_timeouts, 1, "same-tick completion must follow timeout eviction policy");
+        assert!(
+            health.last_eviction.is_none(),
+            "a recently serving peer keeps its channel after a caller timeout"
+        );
+        timeout(Duration::from_secs(1), released_rx)
+            .await
+            .expect("same-tick ambiguous completion must trigger cleanup")
+            .expect("cleanup signal");
+        assert_eq!(
+            lock_peer_health_for_test(addr).detached_rpcs,
+            0,
+            "same-tick completion returns detached admission"
+        );
+        reset_lock_peer_health_for_test(addr);
+    }
+
+    #[tokio::test]
+    async fn test_remote_client_timeout_drains_late_grant_during_eviction() {
+        let addr = "http://timeout-late-grant-during-eviction";
+        reset_lock_peer_health_for_test(addr);
+        let client = RemoteClient::new(addr.to_string());
+        let (evicting_tx, evicting_rx) = tokio::sync::oneshot::channel();
+        let (resume_tx, resume_rx) = tokio::sync::oneshot::channel();
+        eviction_pauses_for_test()
+            .lock()
+            .expect("eviction pause registry")
+            .insert(addr.to_string(), (evicting_tx, resume_rx));
+        let (grant_tx, grant_rx) = tokio::sync::oneshot::channel();
+        let (released_tx, released_rx) = tokio::sync::oneshot::channel();
+        let caller = tokio::spawn(async move {
+            let late: LateCompletion<bool> = Some(Box::new(move |granted| {
+                Box::pin(async move {
+                    released_tx.send(granted).expect("late grant observer remains alive");
+                })
+            }));
+            client
+                .execute_rpc(
+                    "lock",
+                    "bucket/object",
+                    Duration::from_millis(25),
+                    async move { Ok(grant_rx.await.expect("test controls the late grant")) },
+                    RpcCleanup::new(late, None, None),
+                )
+                .await
+        });
+        timeout(Duration::from_secs(1), evicting_rx)
+            .await
+            .expect("timed-out RPC must reach channel eviction")
+            .expect("eviction signal");
+        assert_eq!(
+            lock_peer_health_for_test(addr).detached_rpcs,
+            1,
+            "eviction waits with a detached RPC owner"
+        );
+        grant_tx
+            .send(true)
+            .expect("timed-out stream remains available for a late grant");
+        assert!(
+            timeout(Duration::from_secs(1), released_rx)
+                .await
+                .expect("late grant must be cleaned up before eviction resumes")
+                .expect("late cleanup result")
+        );
+        assert!(!caller.is_finished(), "channel eviction is still paused");
+        assert_eq!(
+            lock_peer_health_for_test(addr).detached_rpcs,
+            0,
+            "late completion returns detached admission"
+        );
+        resume_tx.send(()).expect("resume channel eviction");
+        let result = caller.await.expect("timed-out caller completes without panicking");
+        assert!(
+            matches!(result, Err(LockError::Timeout { .. })),
+            "caller preserves its RPC timeout result"
+        );
+        reset_lock_peer_health_for_test(addr);
+    }
+
+    #[tokio::test]
+    async fn test_remote_client_success_polls_rpc_on_callers_task() {
+        tokio::spawn(async {
+            let client = RemoteClient::new("http://direct-poll-success".to_string());
+            let caller_id = tokio::task::id();
+            let rpc_id = client
+                .execute_rpc(
+                    "ping",
+                    "health-lock-online",
+                    Duration::from_secs(1),
+                    async { Ok(tokio::task::id()) },
+                    RpcCleanup::new(None, None, None),
+                )
+                .await
+                .expect("successful RPC must return its result");
+            assert_eq!(rpc_id, caller_id, "successful RPC must not allocate and schedule a separate task");
+        })
+        .await
+        .expect("caller task must complete");
+    }
+
+    #[tokio::test]
+    async fn test_remote_client_rpc_panic_isolated_and_cleans_up() {
+        let (released_tx, released_rx) = tokio::sync::oneshot::channel();
+        let caller = tokio::spawn(async move {
+            let client = RemoteClient::new("http://direct-poll-panic".to_string());
+            let late_failure: LateFailure = Some(Box::new(move |()| {
+                Box::pin(async move {
+                    released_tx.send(()).expect("panic cleanup observer remains alive");
+                })
+            }));
+            client
+                .execute_rpc::<(), _>(
+                    "lock",
+                    "bucket/object",
+                    Duration::from_secs(1),
+                    async { panic!("test peer RPC panic") },
+                    RpcCleanup::new(None, late_failure, None),
+                )
+                .await
+        });
+        let result = caller.await.expect("RPC panic must not escape to its caller");
+        assert!(
+            matches!(result, Err(LockError::Internal { .. })),
+            "RPC panic remains an internal lock failure"
+        );
+        timeout(Duration::from_secs(1), released_rx)
+            .await
+            .expect("panic must trigger ambiguous-grant cleanup")
+            .expect("cleanup signal");
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn test_remote_client_timeout_keeps_channel_of_recently_serving_peer() {

crates/ecstore/src/config/com.rs

@@ -3506,6 +3506,7 @@ mod tests {
                 replication_decision: String::new(),
                 checksum: None,
                 multipart_completion_replayed: false,
+                backfilled_old_current_size: None,
             }
         }
     }

crates/ecstore/src/core/pool_meta_native_integration_tests.rs

@@ -18,6 +18,7 @@ use crate::storage_api_contracts::namespace::NamespaceLocking as _;
 use codec::{fixture_inline_pool_meta_replicas as replicas, fixture_split_inline_pool_meta_replicas as split};
 use std::path::PathBuf;
 use tokio::fs;
+use uuid::Uuid;
 
 struct PhysicalFixture {
     _directories: Vec<tempfile::TempDir>,
@@ -216,6 +217,264 @@ async fn pool_meta_native_physical_split_without_ordinary_commit_performs_no_mut
         "all current heads, identities and complete backups must remain unchanged"
     );
     assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
+    state
+        .ensure_write_safe("ordinary authority temporarily unavailable")
+        .expect("missing ordinary authority alone must not install a permanent corruption latch");
+}
+
+async fn interrupted_first_commit_fixture() -> (PhysicalFixture, PoolMetaCommittedCandidate) {
+    let fixture = fixture(3).await;
+    let PoolMetaReplica::Valid {
+        previous: Some(previous),
+        committed: false,
+        ..
+    } = decode_pool_meta_replica(fixture.pending.clone())
+    else {
+        panic!("fixture must retain its acknowledged predecessor");
+    };
+    let physical = disks(&fixture.store).await;
+    let prepared = replicas(&fixture.pending, OffsetDateTime::now_utc()).await;
+    for (index, disk) in physical[0].iter().enumerate() {
+        fs::write(object_path(disk).join("xl.meta"), &prepared[index])
+            .await
+            .expect("one pool must retain an ordinary-readable PREPARE");
+    }
+    (fixture, *previous)
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_physical_interrupted_first_commit_restores_proven_predecessor() {
+    let (mut fixture, previous) = interrupted_first_commit_fixture().await;
+    let before = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+    assert!(
+        before[1..]
+            .iter()
+            .all(|read| matches!(read.replica, PoolMetaReplica::Unreadable(_)))
+    );
+    assert!(matches!(before[0].replica, PoolMetaReplica::Valid { committed: false, .. }));
+    let mut state = clean_state(&fixture);
+    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
+    assert!(
+        repair(&fixture.store, &mut state, &fence)
+            .await
+            .expect("ordinary PREPARE must authorize restoring only its acknowledged predecessor")
+    );
+    state
+        .ensure_write_safe("first commit rollback")
+        .expect("rollback must finish its transaction");
+    assert_ne!(
+        previous.canonical, fixture.canonical,
+        "the interrupted COMMIT must not become authoritative"
+    );
+    fixture.canonical = previous.canonical;
+    fixture.revision = previous.revision;
+    assert_committed(&fixture).await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_physical_interrupted_rollback_retains_authority_and_resumes() {
+    let (mut fixture, previous) = interrupted_first_commit_fixture().await;
+    let pause = Arc::new(NativeApplyPause::default());
+    let reached = pause.reached.notified();
+    tokio::pin!(reached);
+    reached.as_mut().enable();
+    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
+    let mut state = clean_state(&fixture);
+    let recovery = tokio::spawn({
+        let store = Arc::clone(&fixture.store);
+        let pause = Arc::clone(&pause);
+        async move {
+            AFTER_NATIVE_POOL_APPLY
+                .scope(pause, async move { repair(&store, &mut state, &fence).await })
+                .await
+        }
+    });
+    tokio::time::timeout(std::time::Duration::from_secs(10), reached)
+        .await
+        .expect("real rollback should pause after one pool is durably restored");
+    let partial = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+    assert!(matches!(partial[0].replica, PoolMetaReplica::Valid { committed: false, .. }));
+    assert!(matches!(&partial[1].replica, PoolMetaReplica::Valid { canonical, committed: true, .. }
+        if canonical == &previous.canonical));
+    assert!(matches!(partial[2].replica, PoolMetaReplica::Unreadable(_)));
+    recovery.abort();
+    assert!(
+        recovery
+            .await
+            .expect_err("the paused repair must be cancelled")
+            .is_cancelled()
+    );
+    let mut state = clean_state(&fixture);
+    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
+    assert!(
+        repair(&fixture.store, &mut state, &fence)
+            .await
+            .expect("restart must resume from the retained ordinary PREPARE")
+    );
+    fixture.canonical = previous.canonical;
+    fixture.revision = previous.revision;
+    assert_committed(&fixture).await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_physical_rollback_resumes_after_last_prepare_loses_read_quorum() {
+    let (mut fixture, previous) = interrupted_first_commit_fixture().await;
+    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
+    let inspected = inspect(&fixture.store.pools, &fence)
+        .await
+        .expect("physical rollback inventory");
+    let replacements = plan(
+        &inspected,
+        &fixture.canonical,
+        fixture.revision,
+        &fixture.identity,
+        Some(&previous.canonical),
+    )
+    .await
+    .expect("the ordinary PREPARE must authorize its predecessor");
+    for (pool, count) in [(1, 4), (2, 4), (0, 2)] {
+        for (index, replacement) in replacements[pool].iter().enumerate().take(count) {
+            let snapshot = &inspected[pool].snapshots[index];
+            assert_eq!(
+                inspected[pool].disks[index]
+                    .pool_meta_native_recovery(Request::Apply {
+                        object: PoolMetaNativeObject::PoolBin,
+                        expected_disk_id: snapshot.disk_id,
+                        expected_current_sha256: snapshot.current.as_ref().map(|raw| Sha256::digest(raw).into()),
+                        expected_backup_inventory_sha256: snapshot.inventory_sha256().expect("original inventory hash"),
+                        replacement: replacement.clone(),
+                    })
+                    .await
+                    .expect("partial rollback must use the real conditional publication and sync path"),
+                Response::Applied
+            );
+        }
+    }
+    let partial = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+    assert!(matches!(partial[0].replica, PoolMetaReplica::Unreadable(_)));
+    assert!(partial[1..].iter().all(|read| matches!(&read.replica,
+        PoolMetaReplica::Valid { canonical, committed: true, .. } if canonical == &previous.canonical)));
+    let mut state = clean_state(&fixture);
+    assert!(
+        repair(&fixture.store, &mut state, &fence)
+            .await
+            .expect("restored ordinary COMMIT must finish rollback after the last PREPARE becomes split")
+    );
+    fixture.canonical = previous.canonical;
+    fixture.revision = previous.revision;
+    assert_committed(&fixture).await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_physical_rollback_backup_does_not_poison_reused_generation() {
+    let (mut fixture, previous) = interrupted_first_commit_fixture().await;
+    let mut state = clean_state(&fixture);
+    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
+    assert!(repair(&fixture.store, &mut state, &fence).await.expect("initial rollback"));
+    let revision = PoolMetaRevision {
+        transaction_id: Some(Uuid::new_v4()),
+        ..fixture.revision
+    };
+    let mut meta = previous.meta.clone();
+    meta.pools[0].last_update += Duration::seconds(1);
+    let canonical = encode_pool_meta_v3_envelope(&meta, revision, true, None).expect("later COMMIT");
+    let pending = encode_pool_meta_v3_envelope(&meta, revision, false, Some(&previous)).expect("later PREPARE");
+    let modified = OffsetDateTime::now_utc() + Duration::seconds(1);
+    let split = split(&pending, &canonical, modified).await;
+    let commit = replicas(&canonical, modified).await;
+    let prepared = replicas(&pending, modified).await;
+    let directory = Uuid::new_v4();
+    let retained = snapshots(&fixture.store).await;
+    for (pool_index, pool) in disks(&fixture.store).await.into_iter().enumerate() {
+        for (disk_index, disk) in pool.iter().enumerate() {
+            assert!(
+                !retained[pool_index][disk_index].backups.is_empty(),
+                "the abandoned PREPARE must remain as real on-disk evidence"
+            );
+            let object = object_path(disk);
+            fs::write(object.join("xl.meta"), &if pool_index < 2 { &split } else { &commit }[disk_index])
+                .await
+                .expect("install later partial COMMIT without clearing old backups");
+            fs::create_dir(object.join(directory.to_string()))
+                .await
+                .expect("later backup directory");
+            fs::write(object.join(directory.to_string()).join("xl.meta.bkp"), &prepared[disk_index])
+                .await
+                .expect("later PREPARE backup");
+        }
+    }
+    assert!(
+        repair(&fixture.store, &mut state, &fence)
+            .await
+            .expect("an ordinary later COMMIT must survive the abandoned sibling PREPARE")
+    );
+    fixture.canonical = canonical;
+    fixture.revision = revision;
+    assert_committed(&fixture).await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_physical_rollback_rejects_retained_commit_without_mutation() {
+    let (fixture, _) = interrupted_first_commit_fixture().await;
+    let physical = disks(&fixture.store).await;
+    let commit = replicas(&fixture.canonical, OffsetDateTime::now_utc()).await;
+    let directory = Uuid::new_v4();
+    let object = object_path(&physical[0][0]);
+    fs::create_dir(object.join(directory.to_string()))
+        .await
+        .expect("retained COMMIT directory");
+    fs::write(object.join(directory.to_string()).join("xl.meta.bkp"), &commit[0])
+        .await
+        .expect("retain evidence that the interrupted transaction may already have committed");
+    let before = snapshots(&fixture.store).await;
+    let mut state = clean_state(&fixture);
+    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
+    assert!(repair(&fixture.store, &mut state, &fence).await.is_err());
+    assert_eq!(snapshots(&fixture.store).await, before, "ambiguous COMMIT evidence must remain untouched");
+    assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_physical_rollback_rejects_competing_prepares_without_mutation() {
+    let (fixture, previous) = interrupted_first_commit_fixture().await;
+    let pending = encode_pool_meta_v3_envelope(
+        &previous.meta,
+        PoolMetaRevision {
+            transaction_id: Some(Uuid::new_v4()),
+            ..fixture.revision
+        },
+        false,
+        Some(&previous),
+    )
+    .expect("competing PREPARE must remain structurally valid");
+    let physical = disks(&fixture.store).await;
+    let prepared = replicas(&pending, OffsetDateTime::now_utc()).await;
+    for (index, disk) in physical[1].iter().enumerate() {
+        fs::write(object_path(disk).join("xl.meta"), &prepared[index])
+            .await
+            .expect("second pool must expose the competing transaction through ordinary reads");
+    }
+    let before = snapshots(&fixture.store).await;
+    let mut state = clean_state(&fixture);
+    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
+    let error = repair(&fixture.store, &mut state, &fence)
+        .await
+        .expect_err("two distinct prepared transactions must not authorize a guessed rollback");
+    assert_eq!(
+        error
+            .pool_metadata_failure()
+            .expect("transaction conflict must be typed")
+            .kind,
+        crate::error::PoolMetadataFailure::RecoveryRequired
+    );
+    assert_eq!(snapshots(&fixture.store).await, before, "competing transactions must remain untouched");
+    assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
 }
 
 struct IdentityEpochFork {
@@ -467,7 +726,7 @@ async fn pool_meta_native_physical_partial_apply_resumes_after_conditional_rejec
     let inspected = inspect(&fixture.store.pools, &fence)
         .await
         .expect("real native snapshots should be obtained");
-    let replacements = plan(&inspected, &fixture.canonical, fixture.revision, &fixture.identity)
+    let replacements = plan(&inspected, &fixture.canonical, fixture.revision, &fixture.identity, None)
         .await
         .expect("physical recovery plan should validate");
     let snapshot = &inspected[0].snapshots[0];
@@ -628,3 +887,559 @@ async fn pool_meta_native_physical_three_plus_one_checkpoint_converges_all_shard
         );
     }
 }
+
+async fn owner_repair_no_backups_fixture() -> PhysicalFixture {
+    let fixture = fixture(0).await;
+    for pool in disks(&fixture.store).await {
+        for disk in pool {
+            let object = object_path(&disk);
+            let mut entries = fs::read_dir(&object).await.expect("fixture metadata directory");
+            while let Some(entry) = entries.next_entry().await.expect("fixture directory entry") {
+                if entry.file_type().await.expect("fixture entry type").is_dir() {
+                    fs::remove_dir_all(entry.path())
+                        .await
+                        .expect("remove task-owned fixture rollback directory");
+                }
+            }
+        }
+    }
+    fixture
+}
+
+async fn owner_repair_identity_heads(store: &ECStore) -> Vec<Vec<(Uuid, Bytes)>> {
+    let mut all = Vec::new();
+    for pool in &store.pools {
+        let set = pool.get_disks_by_key(POOL_META_IDENTITY_NAME);
+        let members = set
+            .disks
+            .read()
+            .await
+            .iter()
+            .cloned()
+            .map(|disk| disk.expect("identity fixture disk must be online"))
+            .collect::<Vec<_>>();
+        let mut heads = Vec::new();
+        for disk in members {
+            match disk
+                .pool_meta_native_recovery(Request::InspectIdentity {})
+                .await
+                .expect("physical identity inspection")
+            {
+                Response::IdentityInspected { disk_id, raw } => heads.push((disk_id, raw)),
+                _ => panic!("identity inspection must return its physical head"),
+            }
+        }
+        all.push(heads);
+    }
+    all
+}
+
+async fn legacy_native_owner_fixture(version: u16, stale: bool) -> (PhysicalFixture, Vec<u8>) {
+    let fixture = owner_repair_no_backups_fixture().await;
+    let mut meta = match decode_pool_meta_replica(fixture.canonical.clone()) {
+        PoolMetaReplica::Valid {
+            meta, committed: true, ..
+        } => meta,
+        _ => panic!("fixture must provide valid committed pool metadata"),
+    };
+    meta.version = version;
+    let raw = meta
+        .encode_config_data_for_v2_gate(version == POOL_META_VERSION)
+        .expect("legacy fixture should encode in its original version");
+    assert_eq!(u16::from_le_bytes([raw[2], raw[3]]), version);
+    let all = disks(&fixture.store).await;
+    let now = OffsetDateTime::now_utc();
+    for pool in &all {
+        let heads = replicas(&raw, now).await;
+        for (index, disk) in pool.iter().enumerate() {
+            fs::write(object_path(disk).join("xl.meta"), &heads[index])
+                .await
+                .expect("install exact legacy head");
+        }
+    }
+    if stale {
+        let old = replicas(&raw, now - Duration::seconds(10)).await;
+        fs::write(object_path(&all[0][0]).join("xl.meta"), &old[0])
+            .await
+            .expect("install stale same-raw legacy head");
+    }
+    (fixture, raw)
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_v1_with_all_upgrade_gates_disabled_keeps_exact_raw() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            let (fixture, raw) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+            let guard = owner(&fixture.store).await;
+            let mut state = fixture.store.pool_meta_save_gate.lock().await;
+            let result = tokio::time::timeout(
+                std::time::Duration::from_secs(30),
+                fixture.store.heal_pool_meta_replica_under_fence(0, guard, &mut state),
+            )
+            .await
+            .expect("legacy native heal must terminate");
+            assert!(result.is_ok(), "fully online legacy native heal should succeed: {result:?}");
+            drop(state);
+            let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            for read in &reads {
+                assert!(
+                    matches!(&read.replica, PoolMetaReplica::Valid { raw: bytes, revision, .. }
+                    if bytes == &raw && revision.version == POOL_META_V1_VERSION),
+                    "native heal must not upgrade a V1 raw payload when both upgrade gates are disabled"
+                );
+            }
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_legacy_source_with_trailing_bytes_without_writes() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            for version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
+                let (fixture, raw) = legacy_native_owner_fixture(version, false).await;
+                let mut source_raw = raw.clone();
+                // A second MessagePack value is readable by legacy decoding but
+                // must not be copied as part of an unverified repair payload.
+                source_raw.push(0xc0);
+                assert!(
+                    matches!(decode_pool_meta_replica(source_raw.clone()), PoolMetaReplica::Valid { raw, .. } if raw == source_raw),
+                    "ordinary legacy reads must remain compatible with trailing bytes"
+                );
+                let all_disks = disks(&fixture.store).await;
+                let source_heads = replicas(&source_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+                for (index, disk) in all_disks[0].iter().enumerate() {
+                    fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
+                        .await
+                        .expect("install readable legacy source with trailing bytes");
+                }
+                let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
+                    .await
+                    .expect("legacy semantic selection must remain readable");
+                assert_eq!(selection.legacy_raw.as_deref(), Some(source_raw.as_slice()));
+                let before = snapshots(&fixture.store).await;
+                let identities = owner_repair_identity_heads(&fixture.store).await;
+                let guard = owner(&fixture.store).await;
+                let mut state = fixture.store.pool_meta_save_gate.lock().await;
+                let result = tokio::time::timeout(
+                    std::time::Duration::from_secs(30),
+                    fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
+                )
+                .await
+                .expect("legacy native heal must terminate");
+                drop(state);
+                assert!(result.is_err(), "version {version}: unparsed trailing source bytes must not be republished");
+                assert_eq!(snapshots(&fixture.store).await, before, "version {version}: no pool.bin head or backup may change");
+                assert_eq!(
+                    owner_repair_identity_heads(&fixture.store).await,
+                    identities,
+                    "version {version}: no identity shard may change"
+                );
+                let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+                assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &source_raw));
+                assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw: target, .. } if target == &raw));
+            }
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_mixed_v1_v2_preserves_observed_format_floor() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            for first_version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
+                for target_pool in [0, 1] {
+                    let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+                    let mut meta = match decode_pool_meta_replica(raw_v1.clone()) {
+                        PoolMetaReplica::Valid { meta, committed: true, .. } => meta,
+                        _ => panic!("initial V1 metadata must decode"),
+                    };
+                    meta.version = POOL_META_VERSION;
+                    let raw_v2 = meta.encode_config_data_for_v2_gate(true).expect("equivalent V2 payload must encode");
+                    assert_eq!(u16::from_le_bytes([raw_v1[2], raw_v1[3]]), POOL_META_V1_VERSION);
+                    assert_eq!(u16::from_le_bytes([raw_v2[2], raw_v2[3]]), POOL_META_VERSION);
+                    let semantic_v1 = match decode_pool_meta_replica(raw_v1.clone()) {
+                        PoolMetaReplica::Valid { canonical, .. } => canonical,
+                        _ => panic!("V1 payload must remain valid"),
+                    };
+                    let semantic_v2 = match decode_pool_meta_replica(raw_v2.clone()) {
+                        PoolMetaReplica::Valid { canonical, .. } => canonical,
+                        _ => panic!("V2 payload must remain valid"),
+                    };
+                    assert_eq!(semantic_v1, semantic_v2, "the two formats must carry equivalent metadata");
+                    let other_version = if first_version == POOL_META_V1_VERSION { POOL_META_VERSION } else { POOL_META_V1_VERSION };
+                    // Pool two follows pool zero; pools zero and one cover both
+                    // orderings and both target formats in separate runs.
+                    let versions = [first_version, other_version, first_version];
+                    let physical = disks(&fixture.store).await;
+                    let timestamp = OffsetDateTime::now_utc();
+                    for (pool_index, pool) in physical.iter().enumerate() {
+                        let raw = if versions[pool_index] == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
+                        let heads = replicas(raw, timestamp).await;
+                        for (index, disk) in pool.iter().enumerate() {
+                            fs::write(object_path(disk).join("xl.meta"), &heads[index]).await.expect("install mixed legacy pool head");
+                        }
+                    }
+                    let before = snapshots(&fixture.store).await;
+                    let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
+                        .await.expect("equivalent legacy pools should be selectable");
+                    assert_eq!(selection.revision.version, POOL_META_VERSION, "the observed V2 format floor must be retained");
+                    assert_eq!(selection.canonical.as_deref(), Some(semantic_v2.as_slice()));
+                    let guard = owner(&fixture.store).await;
+                    let mut state = fixture.store.pool_meta_save_gate.lock().await;
+                    tokio::time::timeout(std::time::Duration::from_secs(30),
+                        fixture.store.heal_pool_meta_replica_under_fence(target_pool, guard, &mut state))
+                        .await.expect("mixed legacy heal must terminate")
+                        .expect("equivalent legacy metadata should heal without downgrading the format floor");
+                    drop(state);
+                    let after = snapshots(&fixture.store).await;
+                    let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+                    for (pool_index, read) in reads.iter().enumerate() {
+                        let expected_version = if pool_index == target_pool { POOL_META_VERSION } else { versions[pool_index] };
+                        let expected_raw = if expected_version == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
+                        assert!(matches!(&read.replica,
+                            PoolMetaReplica::Valid { raw, canonical, revision, committed: true, .. }
+                                if raw == expected_raw && canonical == &semantic_v2 && revision.version == expected_version
+                        ), "pool {pool_index}, first={first_version}, target={target_pool}: preserve equivalent payload and expected legacy version");
+                        for (index, snapshot) in after[pool_index].iter().enumerate() {
+                            codec::validate_inline_pool_meta_shard(snapshot.current.as_ref().expect("physical head"), index, expected_raw)
+                                .await.expect("every physical shard must match its selected legacy payload");
+                        }
+                        if pool_index != target_pool {
+                            assert_eq!(after[pool_index], before[pool_index], "healing one pool must not modify other pools");
+                        }
+                    }
+                }
+            }
+        },
+    ).await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_v1_and_v3_prepare_in_either_pool_order() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            for prepare_pool in [0, 1] {
+                let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+                let (legacy_meta, legacy_canonical) = match decode_pool_meta_replica(raw_v1.clone()) {
+                    PoolMetaReplica::Valid {
+                        meta,
+                        canonical,
+                        revision,
+                        committed: true,
+                        ..
+                    } if revision.version == POOL_META_V1_VERSION => (meta, canonical),
+                    _ => panic!("baseline V1 metadata must be committed and readable"),
+                };
+                let previous = PoolMetaCommittedCandidate {
+                    canonical: legacy_canonical,
+                    meta: legacy_meta.clone(),
+                    revision: PoolMetaRevision::legacy(POOL_META_V1_VERSION),
+                };
+                let mut next = legacy_meta;
+                next.version = POOL_META_GENERATION_VERSION;
+                next.pools[0].last_update += Duration::seconds(1);
+                let revision = PoolMetaRevision {
+                    version: POOL_META_GENERATION_VERSION,
+                    cluster_id: fixture.revision.cluster_id,
+                    epoch: fixture.revision.epoch,
+                    generation: 1,
+                    transaction_id: Some(Uuid::new_v4()),
+                };
+                let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&previous))
+                    .expect("V3 PREPARE with V1 predecessor must encode");
+                assert!(matches!(decode_pool_meta_replica(pending.clone()),
+                    PoolMetaReplica::Valid { committed: false, revision: actual, .. } if actual == revision));
+                let heads = replicas(&pending, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+                let physical = disks(&fixture.store).await;
+                for (index, disk) in physical[prepare_pool].iter().enumerate() {
+                    fs::write(object_path(disk).join("xl.meta"), &heads[index])
+                        .await
+                        .expect("install task-owned V3 PREPARE evidence");
+                }
+                let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+                assert!(matches!(&reads[prepare_pool].replica,
+                    PoolMetaReplica::Valid { raw, committed: false, revision: actual, .. }
+                        if raw == &pending && *actual == revision));
+                let other_pool = 1 - prepare_pool;
+                assert!(matches!(&reads[other_pool].replica,
+                    PoolMetaReplica::Valid { raw, committed: true, revision: actual, .. }
+                        if raw == &raw_v1 && actual.version == POOL_META_V1_VERSION));
+                let before = snapshots(&fixture.store).await;
+                let identities = owner_repair_identity_heads(&fixture.store).await;
+                let guard = owner(&fixture.store).await;
+                let mut state = fixture.store.pool_meta_save_gate.lock().await;
+                let result = tokio::time::timeout(
+                    std::time::Duration::from_secs(30),
+                    fixture
+                        .store
+                        .heal_pool_meta_replica_under_fence(prepare_pool, guard, &mut state),
+                )
+                .await
+                .expect("mixed V1/V3 PREPARE heal must terminate");
+                drop(state);
+                assert!(
+                    result.is_err(),
+                    "pool {prepare_pool}: native heal must not overwrite a V3 PREPARE with legacy data"
+                );
+                assert_eq!(
+                    snapshots(&fixture.store).await,
+                    before,
+                    "pool {prepare_pool}: rejection must retain all physical head and backup evidence"
+                );
+                assert_eq!(
+                    owner_repair_identity_heads(&fixture.store).await,
+                    identities,
+                    "pool {prepare_pool}: rejection must not mutate identity evidence"
+                );
+            }
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_v1_legacy_dont_save_true_without_writes() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+            let meta = match decode_pool_meta_replica(safe_v1.clone()) {
+                PoolMetaReplica::Valid {
+                    meta, committed: true, ..
+                } => meta,
+                _ => panic!("baseline V1 metadata must decode"),
+            };
+            let statuses = meta
+                .pools
+                .iter()
+                .map(|pool| {
+                    assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
+                    LegacyPoolStatus {
+                        id: pool.id,
+                        cmd_line: pool.cmd_line.clone(),
+                        last_update: pool.last_update,
+                        decommission: None,
+                    }
+                })
+                .collect();
+            let legacy = LegacyPoolMeta {
+                version: POOL_META_V1_VERSION,
+                pools: statuses,
+                dont_save: true,
+            };
+            let mut dangerous_raw = Vec::new();
+            dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
+            dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
+            dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
+            let old_reader: LegacyPoolMeta =
+                rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode source");
+            assert!(old_reader.dont_save, "the old reader would treat this source as permanently do-not-save");
+            assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
+            let source = match decode_pool_meta_replica(dangerous_raw.clone()) {
+                PoolMetaReplica::Valid {
+                    canonical,
+                    revision,
+                    committed: true,
+                    ..
+                } => {
+                    assert_eq!(revision.version, POOL_META_V1_VERSION);
+                    canonical
+                }
+                _ => panic!("current reader must recognize the legacy source"),
+            };
+            assert!(matches!(decode_pool_meta_replica(safe_v1.clone()),
+                PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
+                    if canonical == source && revision.version == POOL_META_V1_VERSION));
+            let physical = disks(&fixture.store).await;
+            let source_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+            for (index, disk) in physical[0].iter().enumerate() {
+                fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
+                    .await
+                    .expect("install legacy source on pool zero");
+            }
+            let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
+            assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
+            let before = snapshots(&fixture.store).await;
+            let identities = owner_repair_identity_heads(&fixture.store).await;
+            let guard = owner(&fixture.store).await;
+            let mut state = fixture.store.pool_meta_save_gate.lock().await;
+            let result = tokio::time::timeout(
+                std::time::Duration::from_secs(30),
+                fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
+            )
+            .await
+            .expect("unsafe legacy heal must terminate");
+            drop(state);
+            let error = result.expect_err("native heal must reject a source whose old-reader dont_save is true");
+            assert!(
+                error.to_string().contains("dont_save"),
+                "rejection must identify the unsafe legacy field: {error}"
+            );
+            assert_eq!(
+                snapshots(&fixture.store).await,
+                before,
+                "rejection must preserve every original head and backup"
+            );
+            assert_eq!(
+                owner_repair_identity_heads(&fixture.store).await,
+                identities,
+                "rejection must not write identity shards"
+            );
+            let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw));
+            assert!(
+                matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1),
+                "the dangerous source bytes must not be copied into the target pool"
+            );
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_target_v1_dont_save_true_without_writes() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+            let meta = match decode_pool_meta_replica(safe_v1.clone()) {
+                PoolMetaReplica::Valid {
+                    meta, committed: true, ..
+                } => meta,
+                _ => panic!("baseline V1 metadata must decode"),
+            };
+            let statuses = meta
+                .pools
+                .iter()
+                .map(|pool| {
+                    assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
+                    LegacyPoolStatus {
+                        id: pool.id,
+                        cmd_line: pool.cmd_line.clone(),
+                        last_update: pool.last_update,
+                        decommission: None,
+                    }
+                })
+                .collect();
+            let legacy = LegacyPoolMeta {
+                version: POOL_META_V1_VERSION,
+                pools: statuses,
+                dont_save: true,
+            };
+            let mut dangerous_raw = Vec::new();
+            dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
+            dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
+            dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
+            let old_reader: LegacyPoolMeta =
+                rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode target");
+            assert!(old_reader.dont_save, "the old reader interprets the target as do-not-save");
+            assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
+            let source_canonical = match decode_pool_meta_replica(safe_v1.clone()) {
+                PoolMetaReplica::Valid {
+                    canonical,
+                    revision,
+                    committed: true,
+                    ..
+                } => {
+                    assert_eq!(revision.version, POOL_META_V1_VERSION);
+                    canonical
+                }
+                _ => panic!("safe pool-zero source must decode"),
+            };
+            assert!(
+                matches!(decode_pool_meta_replica(dangerous_raw.clone()),
+                PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
+                    if canonical == source_canonical && revision.version == POOL_META_V1_VERSION),
+                "the dangerous target must remain semantically equivalent to the safe source"
+            );
+            let physical = disks(&fixture.store).await;
+            let target_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+            for (index, disk) in physical[1].iter().enumerate() {
+                fs::write(object_path(disk).join("xl.meta"), &target_heads[index])
+                    .await
+                    .expect("install legacy dangerous target head");
+            }
+            let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
+            assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
+            let before = snapshots(&fixture.store).await;
+            let identities = owner_repair_identity_heads(&fixture.store).await;
+            let guard = owner(&fixture.store).await;
+            let mut state = fixture.store.pool_meta_save_gate.lock().await;
+            let result = tokio::time::timeout(
+                std::time::Duration::from_secs(30),
+                fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
+            )
+            .await
+            .expect("unsafe legacy target heal must terminate");
+            drop(state);
+            let error = result.expect_err("native heal must reject a target whose old-reader dont_save is true");
+            assert!(
+                error.to_string().contains("dont_save"),
+                "rejection must identify the unsafe legacy field: {error}"
+            );
+            assert_eq!(
+                snapshots(&fixture.store).await,
+                before,
+                "rejection must preserve all physical heads and backups"
+            );
+            assert_eq!(
+                owner_repair_identity_heads(&fixture.store).await,
+                identities,
+                "rejection must not write identity shards"
+            );
+            let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1));
+            assert!(
+                matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw),
+                "unsafe target evidence must not be overwritten by the safe source"
+            );
+        },
+    )
+    .await;
+}

crates/ecstore/src/core/pool_meta_native_recovery.rs

@@ -30,6 +30,7 @@ pub(super) struct NativeApplyPause {
 #[cfg(test)]
 tokio::task_local! {
     pub(super) static AFTER_NATIVE_APPLY: Arc<NativeApplyPause>;
+    pub(super) static AFTER_NATIVE_POOL_APPLY: Arc<NativeApplyPause>;
 }
 
 struct InspectedPool {
@@ -168,6 +169,7 @@ async fn plan(
     canonical: &[u8],
     revision: PoolMetaRevision,
     identity: &[u8],
+    rollback: Option<&[u8]>,
 ) -> Result<Vec<Vec<Bytes>>> {
     let mut pending = Vec::<Vec<u8>>::new();
     let mut predecessors = Vec::<Vec<u8>>::new();
@@ -223,23 +225,83 @@ async fn plan(
             }
         }
     }
-    let mut obsolete = Vec::<Vec<u8>>::new();
+    // The last ordinary PREPARE can itself become split during rollback. In
-    for payload in &decoded {
+    // that case another pool already exposes the restored ordinary COMMIT.
-        match decode_pool_meta_replica(payload.clone()) {
+    // Its authority permits continuing the rollback, never promoting a child
-            PoolMetaReplica::Valid {
+    // transaction reconstructed from disk evidence.
+    let mut continuation: Option<(Vec<u8>, PoolMetaRevision)> = None;
+    if rollback.is_none() {
+        let mut outstanding = Vec::new();
+        for pool in inspected {
+            for (index, snapshot) in pool.snapshots.iter().enumerate() {
+                let raw = snapshot.current.as_ref().ok_or_else(invalid)?;
+                if codec::validate_inline_pool_meta_shard(raw, index, canonical).await.is_err() {
+                    outstanding.push((index, raw));
+                }
+            }
+        }
+        for payload in &decoded {
+            if let PoolMetaReplica::Valid {
                 canonical: candidate,
                 revision: candidate_revision,
                 committed: false,
                 previous: Some(previous),
                 ..
-            } if candidate_revision == revision && candidate == canonical => {
+            } = decode_pool_meta_replica(payload.clone())
-                if !pending.contains(payload) {
+                && previous.revision == revision
-                    pending.push(payload.clone());
+                && previous.canonical == canonical
+            {
+                let mut matches = true;
+                for (index, raw) in &outstanding {
+                    matches &= codec::validate_inline_pool_meta_shard(raw, *index, payload).await.is_ok()
+                        || codec::validate_inline_pool_meta_shard(raw, *index, &candidate).await.is_ok();
                 }
-                if !predecessors.contains(&previous.canonical) {
+                if matches {
-                    predecessors.push(previous.canonical);
+                    if !outstanding.is_empty()
+                        && continuation.as_ref().is_some_and(|(selected, selected_revision)| {
+                            selected != &candidate || *selected_revision != candidate_revision
+                        })
+                    {
+                        return Err(known_validation(invalid(), "native_transaction_validation"));
+                    }
+                    continuation.get_or_insert((candidate, candidate_revision));
                 }
             }
+        }
+    }
+    let (canonical, revision, rollback) = match &continuation {
+        Some((transaction, next_revision)) => (transaction.as_slice(), *next_revision, Some(canonical)),
+        None => (canonical, revision, rollback),
+    };
+    for payload in &decoded {
+        if let PoolMetaReplica::Valid {
+            canonical: candidate,
+            revision: candidate_revision,
+            committed: false,
+            previous: Some(previous),
+            ..
+        } = decode_pool_meta_replica(payload.clone())
+            && candidate_revision == revision
+            && candidate == canonical
+        {
+            pending.push(payload.clone());
+            if !predecessors.contains(&previous.canonical) {
+                predecessors.push(previous.canonical);
+            }
+        }
+    }
+    if pending.len() > 1 {
+        return Err(known_validation(
+            Error::other("native V3 prepare records disagree on the predecessor"),
+            "native_transaction_validation",
+        ));
+    }
+    let mut obsolete = Vec::<Vec<u8>>::new();
+    for payload in &decoded {
+        if pending.contains(payload) {
+            continue;
+        }
+        match decode_pool_meta_replica(payload.clone()) {
             PoolMetaReplica::Valid { revision: older, .. }
                 if older.is_generation_protocol()
                     && older.cluster_id == revision.cluster_id
@@ -251,6 +313,23 @@ async fn plan(
                 // fully reconstructed and bound into the native CAS.
                 obsolete.push(payload.clone());
             }
+            PoolMetaReplica::Valid {
+                revision: abandoned,
+                committed: false,
+                previous: Some(previous),
+                ..
+            } if abandoned.is_generation_protocol()
+                && abandoned.cluster_id == revision.cluster_id
+                && abandoned.epoch == revision.epoch
+                && abandoned.generation == revision.generation
+                && abandoned.transaction_id != revision.transaction_id
+                && predecessors.contains(&previous.canonical) =>
+            {
+                // A rollback retains the failed PREPARE as evidence. A later
+                // transaction may reuse that generation, but only a backup of
+                // an uncommitted sibling with the same predecessor is obsolete.
+                obsolete.push(payload.clone());
+            }
             PoolMetaReplica::Valid {
                 canonical: candidate,
                 revision: candidate_revision,
@@ -272,13 +351,7 @@ async fn plan(
             _ => {}
         }
     }
-    if pending.len() > 1 {
+    if pending.is_empty() || rollback.is_some_and(|previous| !predecessors.iter().any(|candidate| candidate == previous)) {
-        return Err(known_validation(
-            Error::other("native V3 prepare records disagree on the predecessor"),
-            "native_transaction_validation",
-        ));
-    }
-    if pending.is_empty() {
         return Err(invalid());
     }
     for payload in &decoded {
@@ -294,13 +367,25 @@ async fn plan(
         for (index, snapshot) in pool.snapshots.iter().enumerate() {
             let raw = snapshot.current.as_ref().ok_or_else(invalid)?;
             let mut matched = false;
-            for payload in std::iter::once(canonical).chain(pending.iter().map(Vec::as_slice)) {
+            for payload in std::iter::once(canonical)
+                .chain(pending.iter().map(Vec::as_slice))
+                .chain(rollback)
+            {
                 matched |= codec::validate_inline_pool_meta_shard(raw, index, payload).await.is_ok();
             }
             if !matched {
                 return Err(invalid());
             }
             for backup in &snapshot.backups {
+                if rollback.is_some()
+                    && codec::validate_inline_pool_meta_shard(&backup.data, index, canonical)
+                        .await
+                        .is_ok()
+                {
+                    // A retained COMMIT might have been authoritative before
+                    // this interruption. Never roll it back from PREPARE alone.
+                    return Err(invalid());
+                }
                 let mut matched = false;
                 for payload in std::iter::once(canonical).chain(
                     pending
@@ -323,7 +408,7 @@ async fn plan(
     let mut replacements = Vec::with_capacity(inspected.len());
     for pool in inspected {
         replacements.push(
-            codec::encode_inline_pool_meta_replicas(&current(pool)?, canonical, modified)
+            codec::encode_inline_pool_meta_replicas(&current(pool)?, rollback.unwrap_or(canonical), modified)
                 .await
                 .map_err(|error| codec_validation(error, "native_metadata_validation"))?,
         );
@@ -338,10 +423,12 @@ pub(super) async fn repair(
 ) -> Result<bool> {
     fence.ensure_held()?;
     let reads = read_pool_meta_replicas(store.pools.clone(), true).await;
-    if !reads
+    let unreadable = reads
         .iter()
-        .any(|read| matches!(read.replica, PoolMetaReplica::Unreadable(_)))
+        .enumerate()
-    {
+        .filter_map(|(index, read)| matches!(read.replica, PoolMetaReplica::Unreadable(_)).then_some(index))
+        .collect::<Vec<_>>();
+    if unreadable.is_empty() {
         return Ok(false);
     }
     state.ensure_write_safe("native pool metadata recovery")?;
@@ -360,8 +447,9 @@ pub(super) async fn repair(
     {
         return Err(invalid());
     }
-    // Only an ordinary, successfully read COMMIT grants authority. Rebuilding
+    // Ordinary reads grant authority: a COMMIT selects its own state, while a
-    // two erasure shards below proves integrity, never a commit decision.
+    // PREPARE can select only its acknowledged predecessor. Reconstruction
+    // below proves integrity and never promotes an interrupted transaction.
     let ordinary_commits = reads
         .iter()
         .filter_map(|read| match &read.replica {
@@ -375,7 +463,22 @@ pub(super) async fn repair(
             _ => None,
         })
         .collect::<Vec<_>>();
-    if ordinary_commits.is_empty() {
+    let ordinary_prepares = reads
+        .iter()
+        .filter_map(|read| match &read.replica {
+            PoolMetaReplica::Valid {
+                canonical,
+                revision,
+                committed: false,
+                previous: Some(previous),
+                ..
+            } if revision.is_generation_protocol() && previous.revision.is_generation_protocol() => {
+                Some((canonical.clone(), *revision, previous.canonical.clone()))
+            }
+            _ => None,
+        })
+        .collect::<Vec<_>>();
+    if ordinary_commits.is_empty() && ordinary_prepares.is_empty() {
         return Err(invalid());
     }
     let selection = select_pool_meta_replica_reads(reads)
@@ -395,10 +498,20 @@ pub(super) async fn repair(
         return Err(invalid());
     }
     let canonical = selection.canonical.as_ref().ok_or_else(invalid)?;
+    let mut rollback = None;
+    for (pending, revision, previous) in &ordinary_prepares {
+        if previous == canonical {
+            if rollback.is_some_and(|candidate| candidate != (pending, *revision)) {
+                return Err(latch_validation(state, known_validation(invalid(), "native_transaction_validation")));
+            }
+            rollback = Some((pending, *revision));
+        }
+    }
     if !selection.revision.is_generation_protocol()
-        || !ordinary_commits
+        || (rollback.is_none()
-            .iter()
+            && !ordinary_commits
-            .any(|(raw, revision)| raw == canonical && *revision == selection.revision)
+                .iter()
+                .any(|(raw, revision)| raw == canonical && *revision == selection.revision))
     {
         return Err(invalid());
     }
@@ -445,13 +558,18 @@ pub(super) async fn repair(
         .await
         .map_err(|error| latch_validation(state, error))?;
     let inspected = inspect(&store.pools, fence).await?;
-    let replacements = plan(&inspected, canonical, selection.revision, &identity_bytes)
+    let (transaction, revision) = rollback.unwrap_or((canonical, selection.revision));
+    let replacements = plan(&inspected, transaction, revision, &identity_bytes, rollback.map(|_| canonical.as_slice()))
         .await
         .map_err(|error| latch_validation(state, error))?;
     fence.ensure_held()?;
     let mut arm = state.arm_transaction();
     arm.phase = Some("native_repair_apply");
-    for (pool, replacements) in inspected.iter().zip(replacements) {
+    let mut apply_order = inspected.iter().zip(replacements).enumerate().collect::<Vec<_>>();
+    // Preserve ordinary authority until all unreadable pools are repaired.
+    // Cancellation can then resume from a PREPARE or a restored COMMIT.
+    apply_order.sort_by_key(|(index, _)| !unreadable.contains(index));
+    for (_, (pool, replacements)) in apply_order {
         for ((disk, snapshot), replacement) in pool.disks.iter().zip(&pool.snapshots).zip(replacements) {
             fence.ensure_held()?;
             let request = Request::Apply {
@@ -466,6 +584,11 @@ pub(super) async fn repair(
                 _ => return Err(invalid()),
             }
         }
+        #[cfg(test)]
+        if let Ok(pause) = AFTER_NATIVE_POOL_APPLY.try_with(Arc::clone) {
+            pause.reached.notify_one();
+            pause.resume.notified().await;
+        }
     }
     #[cfg(test)]
     if let Ok(pause) = AFTER_NATIVE_APPLY.try_with(Arc::clone) {
@@ -627,7 +750,7 @@ mod tests {
     #[tokio::test]
     async fn pool_meta_native_plan_restores_physical_split_with_fresh_common_identity() {
         let f = fixture().await;
-        let repaired = plan(&f.pools, &f.commit, f.revision, &f.identity).await.unwrap();
+        let repaired = plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.unwrap();
         assert_eq!(repaired.len(), 4);
         for pool in repaired {
             assert_eq!(
@@ -648,7 +771,7 @@ mod tests {
         let mut f = fixture().await;
         let wrong = replicas(b"different identity", f.modified).await;
         f.pools[0].snapshots[0].identity = Some(wrong[0].clone());
-        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
+        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
     }
 
     #[tokio::test]
@@ -671,10 +794,10 @@ mod tests {
         .unwrap();
         let wrong = replicas(&higher, f.modified + Duration::seconds(2)).await;
         f.pools[0].snapshots[0].current = Some(wrong[0].clone());
-        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
+        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
         f.pools[0].snapshots[0].current = Some(replicas(&f.pending, f.modified).await[0].clone());
         f.pools[0].snapshots[0].backups[0].data = wrong[0].clone();
-        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
+        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
     }
 
     #[tokio::test]
@@ -717,17 +840,17 @@ mod tests {
             }
         }
         assert!(older_pending != f.pending);
-        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_ok());
+        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_ok());
     }
 
     #[tokio::test]
     async fn pool_meta_native_plan_rejects_missing_or_corrupt_evidence() {
         let mut f = fixture().await;
         let before = f.pools[1].snapshots[2].current.take();
-        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
+        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
         f.pools[1].snapshots[2].current = before;
         f.pools[1].snapshots[2].backups[0].data = Bytes::from_static(b"invalid metadata");
-        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
+        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
     }
 }
 

crates/ecstore/src/core/pools.rs

@@ -94,7 +94,6 @@ use serde::{Deserialize, Serialize};
 use sha2::{Digest, Sha256};
 use std::collections::{HashMap, HashSet};
 use std::fmt::Display;
-#[cfg(test)]
 use std::io::Cursor;
 use std::io::Write;
 use std::path::PathBuf;
@@ -3720,33 +3719,45 @@ fn is_decommission_copy_quorum_deferral(err: &Error) -> bool {
     data_movement::data_movement_stage_source_as::<DecommissionCopyQuorumFailure>(err).is_some()
 }
 
-/// The storage error of a version copy that kept timing out on a namespace
+/// A copy whose object namespace lock remained unavailable through its
-/// lock through its in-place retries, kept typed so the entry failure policy
+/// in-place retries, retained as a typed deferral.
-/// defers it.
 #[derive(Debug, thiserror::Error)]
 #[error("{0}")]
-struct DecommissionCopyLockTimeout(#[source] Error);
+struct DecommissionCopyLockUnavailable(#[source] Error);
 
-fn is_decommission_copy_lock_timeout_error(err: &Error) -> bool {
+fn is_decommission_copy_lock_unavailable_error(err: &Error, bucket: &str, object: &str) -> bool {
-    let err = data_movement::data_movement_stage_source(err).unwrap_or(err);
+    let source = data_movement::data_movement_stage_source(err).unwrap_or(err);
-    matches!(err, Error::Lock(rustfs_lock::LockError::Timeout { .. }))
+    match source {
+        Error::Lock(rustfs_lock::LockError::Timeout { .. }) => true,
+        Error::NamespaceLockQuorumUnavailable {
+            mode: "read",
+            bucket: lock_bucket,
+            object: lock_object,
+            ..
+        } => {
+            data_movement::data_movement_error_is_stage(err, "prepare_put_object")
+                && lock_bucket == bucket
+                && lock_object == object
+        }
+        _ => false,
+    }
 }
 
-/// A version copy that timed out acquiring a namespace lock leaves the source
+/// A version copy that could not acquire its namespace lock leaves the source
 /// in place; the next attempt copies the version again or finds an equivalent
 /// committed target. The lock is held by another mutation of the key or, when
 /// this node restarted mid-copy, by its previous incarnation until that lock's
 /// lease expires on the peers. Either clears by waiting, so the entry is
 /// deferred instead of being counted as a permanent per-object failure.
-fn decommission_copy_lock_timeout_deferral(stage: &str, bucket: &str, object: &str, err: Error) -> Error {
+fn decommission_copy_lock_unavailable_deferral(stage: &str, bucket: &str, object: &str, err: Error) -> Error {
     data_movement::data_movement_context_error(
-        format!("decommission {stage} timed out on a namespace lock for {bucket}/{object}: {err}"),
+        format!("decommission {stage} could not acquire a namespace lock for {bucket}/{object}: {err}"),
-        DecommissionCopyLockTimeout(err),
+        DecommissionCopyLockUnavailable(err),
     )
 }
 
-fn is_decommission_copy_lock_timeout_deferral(err: &Error) -> bool {
+fn is_decommission_copy_lock_unavailable_deferral(err: &Error) -> bool {
-    data_movement::data_movement_stage_source_as::<DecommissionCopyLockTimeout>(err).is_some()
+    data_movement::data_movement_stage_source_as::<DecommissionCopyLockUnavailable>(err).is_some()
 }
 
 /// The deferral of a version copy whose in-place retries ended on `error`, if
@@ -3756,8 +3767,8 @@ fn take_decommission_copy_deferral(stage: &str, bucket: &str, object: &str, erro
         return Some(decommission_copy_quorum_deferral(stage, bucket, object, err));
     }
     error
-        .take_if(|err| is_decommission_copy_lock_timeout_error(err))
+        .take_if(|err| is_decommission_copy_lock_unavailable_error(err, bucket, object))
-        .map(|err| decommission_copy_lock_timeout_deferral(stage, bucket, object, err))
+        .map(|err| decommission_copy_lock_unavailable_deferral(stage, bucket, object, err))
 }
 
 fn resolve_decommission_entry_reload_result(result: Result<()>, bucket: &str, object_name: &str) -> Result<()> {
@@ -5822,13 +5833,13 @@ enum DecommissionEntryOutcome {
 /// entry is done and must not be counted as a failure or escalated. A quorum
 /// failure of the source cleanup is deferred too: the copied source is still
 /// in place and the next attempt cleans it up again. So is a version copy that
-/// lost drive quorum or timed out on a namespace lock through its in-place
+/// lost drive quorum or could not acquire an object namespace lock through its in-place
 /// retries: the source still holds the version and the next attempt copies it
 /// again. Every other failure stays terminal.
 fn resolve_decommission_entry_failure(err: Error) -> Result<DecommissionEntryOutcome> {
     if is_decommission_source_cleanup_retryable_error(&err)
         || is_decommission_copy_quorum_deferral(&err)
-        || is_decommission_copy_lock_timeout_deferral(&err)
+        || is_decommission_copy_lock_unavailable_deferral(&err)
     {
         return Ok(DecommissionEntryOutcome::Deferred(err));
     }
@@ -6827,11 +6838,47 @@ impl Drop for PoolMetaTransactionArm {
     }
 }
 
+fn ensure_legacy_pool_meta_republish_safe(data: &[u8]) -> Result<()> {
+    if data.len() >= 4
+        && LittleEndian::read_u16(&data[2..4]) == POOL_META_V1_VERSION
+        // The selection already validates the payload. The runtime-only
+        // flag can only appear in the legacy three-field sequence or map.
+        && rmp::decode::read_array_len(&mut &data[4..])
+            .or_else(|_| rmp::decode::read_map_len(&mut &data[4..]))
+            .ok()
+            == Some(3)
+        && let Ok(legacy) = rmp_serde::from_slice::<LegacyPoolMeta>(&data[4..])
+        && legacy.dont_save
+    {
+        return Err(Error::other("pool metadata legacy source contains dont_save=true"));
+    }
+    Ok(())
+}
+
+// Ordinary legacy reads retain their compatibility behavior, but bytes that
+// will be copied to another pool must not contain an unparsed extension.
+fn ensure_legacy_pool_meta_payload_complete(data: &[u8]) -> Result<()> {
+    let payload = data
+        .get(4..)
+        .ok_or_else(|| Error::other("pool metadata legacy source is too short"))?;
+    let mut decoder = rmp_serde::Deserializer::new(Cursor::new(payload));
+    serde::de::IgnoredAny::deserialize(&mut decoder).map_err(|_| Error::other("pool metadata legacy source is invalid"))?;
+    let payload_len = u64::try_from(payload.len()).map_err(|_| Error::other("pool metadata legacy source is too large"))?;
+    if decoder.get_ref().position() != payload_len {
+        return Err(Error::other("pool metadata legacy source has trailing bytes"));
+    }
+    Ok(())
+}
+
 #[derive(Debug)]
 struct PoolMetaSelection {
     meta: PoolMeta,
     revision: PoolMetaRevision,
     canonical: Option<Vec<u8>>,
+    // Legacy semantic comparison canonicalizes to V2; a heal must instead
+    // preserve verified raw bytes at the highest observed format floor.
+    legacy_raw: Option<Vec<u8>>,
+    legacy_has_unsafe_flag: bool,
     replica_state: PoolMetaReplicaState,
     cas_tokens: Vec<PoolMetaCasToken>,
     absent: bool,
@@ -7222,6 +7269,8 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
             meta: PoolMeta::default(),
             revision: PoolMetaRevision::legacy(0),
             canonical: None,
+            legacy_raw: None,
+            legacy_has_unsafe_flag: false,
             replica_state: PoolMetaReplicaState {
                 needs_repair: false,
                 repair_write_safe: true,
@@ -7276,6 +7325,8 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
             meta: selected.meta.clone(),
             revision: selected.revision,
             canonical: Some(selected.canonical.clone()),
+            legacy_raw: None,
+            legacy_has_unsafe_flag: false,
             replica_state: PoolMetaReplicaState {
                 needs_repair,
                 repair_write_safe,
@@ -7289,6 +7340,10 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
 
     // Legacy V1/V2 has no durable generation. Pool zero remains the commit
     // record; divergent backups without it are ambiguous and fail closed.
+    let legacy_has_unsafe_flag = committed
+        .iter()
+        .any(|(_, raw, _)| ensure_legacy_pool_meta_republish_safe(raw).is_err());
+    let mut floor_raw = None;
     let mut selected: Option<(usize, Vec<u8>, PoolMetaCommittedCandidate)> = None;
     for (idx, raw, candidate) in committed {
         if let Some((selected_idx, selected_raw, selected_candidate)) = selected.as_ref() {
@@ -7302,19 +7357,36 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
                 }
             } else {
                 needs_repair |= selected_raw != &raw;
+                if floor_raw.is_none()
+                    && candidate.revision.version == observed_version
+                    && raw
+                        .get(2..4)
+                        .is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
+                {
+                    floor_raw = Some(raw);
+                }
             }
         } else {
             selected = Some((idx, raw, candidate));
         }
     }
-    let (_, _, mut selected) =
+    let (_, selected_raw, mut selected) =
         selected.ok_or_else(|| Error::other("pool metadata recovery required: no valid legacy replica is available"))?;
+    if selected.revision.version == observed_version
+        && selected_raw
+            .get(2..4)
+            .is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
+    {
+        floor_raw = Some(selected_raw);
+    }
     selected.meta.version = observed_version;
     selected.revision.version = observed_version;
     Ok(PoolMetaSelection {
         meta: selected.meta,
         revision: selected.revision,
         canonical: Some(selected.canonical),
+        legacy_raw: floor_raw,
+        legacy_has_unsafe_flag,
         replica_state: PoolMetaReplicaState {
             needs_repair,
             repair_write_safe,
@@ -8308,8 +8380,9 @@ where
         })
     });
     let result = save_config_with_opts_and_metadata(pool, object, data, &opts).await;
-    let may_have_mutated = !matches!(&result, Err(Error::PreconditionFailed));
+    let write_not_started = matches!(&result, Err(Error::Io(inner)) if inner.get_ref().is_some_and(|source| source.is::<crate::set_disk::SystemMetadataWriteNotStarted>()));
-    if !may_have_mutated {
+    let may_have_mutated = !write_not_started && !matches!(&result, Err(Error::PreconditionFailed));
+    if matches!(&result, Err(Error::PreconditionFailed)) {
         record_pool_meta_stale_write_rejection(phase);
     }
     let mut source = None;
@@ -8320,6 +8393,8 @@ where
             source = Some(Arc::clone(&original));
             if matches!(original.as_ref(), Error::PreconditionFailed) {
                 Err(Error::PreconditionFailed)
+            } else if write_not_started {
+                Err(Error::other(original))
             } else {
                 Err(Error::other(pool_metadata_error(
                     crate::error::PoolMetadataFailure::TransactionUnknown,
@@ -14534,6 +14609,23 @@ impl ECStore {
             .canonical
             .as_ref()
             .ok_or_else(|| Error::other("pool metadata heal has no committed source"))?;
+        if selection.generation_protocol_observed && !selection.revision.is_generation_protocol() {
+            return Err(Error::other("pool metadata heal requires recovery of the observed generation protocol"));
+        }
+        if selection.legacy_has_unsafe_flag {
+            return Err(Error::other("pool metadata legacy replica contains dont_save=true"));
+        }
+        let repair_data = if selection.revision.is_generation_protocol() {
+            canonical.clone()
+        } else {
+            let raw = selection
+                .legacy_raw
+                .clone()
+                .ok_or_else(|| Error::other("pool metadata heal cannot prove the legacy format floor"))?;
+            ensure_legacy_pool_meta_payload_complete(&raw)?;
+            raw
+        };
+        ensure_legacy_pool_meta_republish_safe(&repair_data)?;
         let pool = self
             .pools
             .get(pool_index)
@@ -14544,10 +14636,10 @@ impl ECStore {
             .get(pool_index)
             .ok_or_else(|| Error::other("pool metadata heal has no target revision"))?;
         let mut arm = state.arm_transaction();
-        save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, canonical.clone(), token, &fence, "native_heal", &mut arm)
+        save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, repair_data.clone(), token, &fence, "native_heal", &mut arm)
             .await?;
         let (confirmed, _) = read_config_no_lock_preserve_empty_with_metadata(pool, POOL_META_NAME).await?;
-        if confirmed != *canonical {
+        if confirmed != repair_data {
             return Err(Error::other("pool metadata native heal readback diverged"));
         }
         fence.ensure_held()?;
@@ -28353,7 +28445,7 @@ mod tests {
             "one version must commit before the next version times out"
         );
         assert!(
-            matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_timeout_deferral(err)),
+            matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
             "{outcome:?}"
         );
         assert_eq!(
@@ -28387,6 +28479,88 @@ mod tests {
         assert_eq!(actual, b"written after versioning was enabled");
     }
 
+    /// A target lookup that loses its read-lock quorum must retain the source
+    /// and resume an entry after an earlier version has already committed.
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn decommission_defers_target_lookup_read_lock_quorum_and_resumes() {
+        use tokio::io::AsyncReadExt as _;
+
+        let fixture = NullVersionDecommission::new(NullVersionTop::NewerVersion).await;
+        let object = fixture.object;
+        let attempted = Arc::new(AtomicUsize::new(0));
+        let fault = DecommissionTestFaultGuard::install_with_error(
+            Arc::new({
+                let attempted = Arc::clone(&attempted);
+                move |stage, _, faulted, _, success| {
+                    stage == "before_target_commit"
+                        && faulted == object
+                        && success
+                        && attempted.fetch_add(1, Ordering::SeqCst) > 0
+                }
+            }),
+            Some(Arc::new(|_, bucket, object| {
+                data_movement::data_movement_stage_error_for_test(
+                    "decommission_object",
+                    "prepare_put_object",
+                    bucket,
+                    object,
+                    Error::NamespaceLockQuorumUnavailable {
+                        mode: "read",
+                        bucket: bucket.to_owned(),
+                        object: object.to_owned(),
+                        required: 1,
+                        achieved: 0,
+                    },
+                )
+            })),
+        );
+        let (outcome, _) = fixture.run_entry().await;
+        assert!(
+            matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
+            "{outcome:?}"
+        );
+        fixture.assert_source_retained().await;
+        assert_eq!(
+            fixture
+                .target_versions()
+                .await
+                .expect("first version committed")
+                .versions
+                .len(),
+            1
+        );
+        assert_eq!(
+            fixture.store.pool_meta.read().await.pools[0]
+                .decommission
+                .as_ref()
+                .expect("active decommission")
+                .items_decommission_failed,
+            0
+        );
+
+        drop(fault);
+        let (outcome, _) = fixture.run_entry().await;
+        assert!(matches!(outcome, DecommissionEntryOutcome::Complete), "{outcome:?}");
+        fixture.assert_moved().await;
+        let mut reader = fixture.store.pools[1]
+            .get_object_reader(
+                fixture.bucket,
+                fixture.object,
+                None,
+                HeaderMap::new(),
+                &ObjectOptions {
+                    version_id: Some(fixture.seeded.versions[0].version_id.expect("newer version").to_string()),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("the newer version must remain readable");
+        let mut actual = Vec::new();
+        reader.stream.read_to_end(&mut actual).await.expect("read the newer version");
+        assert_eq!(actual, b"written after versioning was enabled");
+    }
+
     /// A namespace lock still held on the target key, as by a copy whose node
     /// was killed mid-write, makes target selection fail with a typed
     /// object-lock timeout. A real held lock, not an injected error, must defer
@@ -28429,7 +28603,7 @@ mod tests {
 
             let (outcome, _) = fixture.run_entry().await;
             assert!(
-                matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_timeout_deferral(err)),
+                matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
                 "{outcome:?}"
             );
             assert_eq!(
@@ -28838,17 +29012,17 @@ mod tests {
             timeout(),
         );
         for err in [timeout(), prepare] {
-            assert!(is_decommission_copy_lock_timeout_error(&err), "{err}");
+            assert!(is_decommission_copy_lock_unavailable_error(&err, "bucket-a", "object-a"), "{err}");
             assert!(!is_decommission_copy_quorum_error(&err), "{err}");
             let mut error = Some(err);
             let deferred = take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error)
                 .expect("a lock timeout copy must be deferred");
             assert!(error.is_none());
-            assert!(is_decommission_copy_lock_timeout_deferral(&deferred));
+            assert!(is_decommission_copy_lock_unavailable_deferral(&deferred));
             assert!(!is_decommission_copy_quorum_deferral(&deferred));
-            let failure = data_movement::data_movement_stage_source_as::<DecommissionCopyLockTimeout>(&deferred)
+            let failure = data_movement::data_movement_stage_source_as::<DecommissionCopyLockUnavailable>(&deferred)
                 .expect("the deferred error must keep its lock-timeout origin");
-            assert!(is_decommission_copy_lock_timeout_error(&failure.0));
+            assert!(is_decommission_copy_lock_unavailable_error(&failure.0, "bucket-a", "object-a"));
             assert!(matches!(
                 resolve_decommission_entry_failure(deferred),
                 Ok(DecommissionEntryOutcome::Deferred(_))
@@ -28873,6 +29047,43 @@ mod tests {
         }
     }
 
+    #[test]
+    fn decommission_copy_defers_only_the_matching_object_read_lock_quorum() {
+        let quorum = |mode, bucket: &str, object: &str| Error::NamespaceLockQuorumUnavailable {
+            mode,
+            bucket: bucket.to_owned(),
+            object: object.to_owned(),
+            required: 1,
+            achieved: 0,
+        };
+        let wrap = |stage, err| {
+            data_movement::data_movement_stage_error_for_test("decommission_object", stage, "bucket-a", "object-a", err)
+        };
+        let mut error = Some(wrap("prepare_put_object", quorum("read", "bucket-a", "object-a")));
+        let deferred = take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error)
+            .expect("matching read-lock quorum must defer");
+        assert!(error.is_none());
+        assert!(matches!(
+            resolve_decommission_entry_failure(deferred),
+            Ok(DecommissionEntryOutcome::Deferred(_))
+        ));
+        for err in [
+            quorum("read", "bucket-a", "object-a"),
+            wrap("source_recheck", quorum("read", "bucket-a", "object-a")),
+            wrap("prepare_put_object", quorum("write", "bucket-a", "object-a")),
+            wrap(
+                "prepare_put_object",
+                quorum("exact_delete_capacity_reconciliation", "bucket-a", "object-a"),
+            ),
+            wrap("prepare_put_object", quorum("read", "other-bucket", "object-a")),
+            wrap("prepare_put_object", quorum("read", "bucket-a", "other-object")),
+        ] {
+            let mut error = Some(err);
+            assert!(take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error).is_none());
+            assert!(error.is_some(), "unrelated lock failures must stay terminal");
+        }
+    }
+
     #[test]
     fn decommission_target_capacity_error_accepts_wrapped_capacity_errors() {
         let disk_full = Error::other(format!("decommission_object: put_object failed for bucket/object: {}", Error::DiskFull));
@@ -29351,6 +29562,53 @@ mod tests {
         );
     }
 
+    #[test]
+    fn legacy_republish_guard_checks_only_three_field_sequence_or_map() {
+        let frame = |payload: Vec<u8>| {
+            let mut data = Vec::new();
+            data.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
+            data.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
+            data.extend_from_slice(&payload);
+            data
+        };
+        let two_fields = PersistedPoolMetaV1 {
+            version: POOL_META_V1_VERSION,
+            pools: Vec::new(),
+        };
+        for payload in [rmp_serde::to_vec(&two_fields), rmp_serde::to_vec_named(&two_fields)] {
+            let data = frame(payload.expect("two-field V1 payload must serialize"));
+            assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
+            assert!(ensure_legacy_pool_meta_republish_safe(&data).is_ok());
+            assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
+            let mut trailing = data;
+            trailing.push(0xc0);
+            assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
+        }
+        for dont_save in [false, true] {
+            let legacy = LegacyPoolMeta {
+                version: POOL_META_V1_VERSION,
+                pools: Vec::new(),
+                dont_save,
+            };
+            for payload in [rmp_serde::to_vec(&legacy), rmp_serde::to_vec_named(&legacy)] {
+                let data = frame(payload.expect("three-field V1 payload must serialize"));
+                assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
+                assert_eq!(ensure_legacy_pool_meta_republish_safe(&data).is_err(), dont_save);
+                assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
+                let mut trailing = data;
+                trailing.push(0xc0);
+                assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
+            }
+        }
+        // This guard is only called for already-decoded replicas; malformed
+        // containers must fail selection rather than becoming repair sources.
+        let truncated = frame(vec![0x93, 0x01]);
+        assert!(matches!(
+            decode_pool_meta_replica(truncated),
+            PoolMetaReplica::Corrupt(_) | PoolMetaReplica::Incompatible(_)
+        ));
+    }
+
     #[test]
     fn pool_meta_decode_supports_legacy_payload() {
         let start_time = OffsetDateTime::now_utc();
@@ -30571,6 +30829,154 @@ mod pools_tests {
         );
     }
 
+    #[tokio::test]
+    async fn pool_meta_prewrite_quorum_rejection_allows_retry_after_disks_return() {
+        let (dirs, disks, pool) = crate::set_disk::hermetic_set_disks_isolated(4).await;
+        pool.disks.write().await[2..].fill(None);
+        let state = super::PoolMetaWriteState::default();
+        let data =
+            super::initialized_pool_meta_identity_for_test(uuid::Uuid::new_v4(), 1).expect("encode the deployment identity");
+        let mut transaction = state.arm_transaction();
+        save_pool_meta_object_cas(
+            pool.clone(),
+            POOL_META_IDENTITY_NAME,
+            data.clone(),
+            &PoolMetaCasToken::Missing,
+            &PoolMetaPersistenceFence::OfflineRecovery,
+            "identity_cas",
+            &mut transaction,
+        )
+        .await
+        .expect_err("two of four disks cannot satisfy the identity write quorum");
+        drop(transaction);
+        for dir in &dirs {
+            assert!(
+                !dir.path()
+                    .join(crate::disk::RUSTFS_META_BUCKET)
+                    .join(POOL_META_IDENTITY_NAME)
+                    .exists(),
+                "the rejected write must not create an identity replica"
+            );
+        }
+        state
+            .ensure_write_safe("retry after disks return")
+            .expect("a rejected inline write must not poison the startup transaction");
+
+        let ordinary_error = crate::config::com::save_config_with_opts(
+            pool.clone(),
+            "ordinary-config",
+            data.clone(),
+            &ObjectOptions {
+                max_parity: true,
+                no_lock: true,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect_err("ordinary writes must still fail without quorum");
+        assert!(
+            matches!(ordinary_error, Error::InsufficientWriteQuorum(_, _)),
+            "ordinary callers must retain their existing quorum error"
+        );
+
+        *pool.disks.write().await = disks.into_iter().map(Some).collect();
+        let mut retry = state.arm_transaction();
+        save_pool_meta_object_cas(
+            pool.clone(),
+            POOL_META_IDENTITY_NAME,
+            data.clone(),
+            &PoolMetaCasToken::Missing,
+            &PoolMetaPersistenceFence::OfflineRecovery,
+            "identity_cas",
+            &mut retry,
+        )
+        .await
+        .expect("the same transaction must succeed when the disks return");
+        retry.disarm();
+        drop(retry);
+        assert_eq!(
+            crate::config::com::read_config(pool, POOL_META_IDENTITY_NAME)
+                .await
+                .expect("read back the committed identity"),
+            data
+        );
+        state
+            .ensure_write_safe("after successful retry")
+            .expect("retry must leave writes enabled");
+    }
+
+    #[tokio::test]
+    async fn pool_meta_prewrite_quorum_rejection_keeps_an_earlier_write_armed() {
+        let (_dirs, disks, pool) = crate::set_disk::hermetic_set_disks_isolated(4).await;
+        let state = super::PoolMetaWriteState::default();
+        let data =
+            super::initialized_pool_meta_identity_for_test(uuid::Uuid::new_v4(), 1).expect("encode the deployment identity");
+        let mut transaction = state.arm_transaction();
+        let written = save_pool_meta_object_cas(
+            pool.clone(),
+            POOL_META_IDENTITY_NAME,
+            data.clone(),
+            &PoolMetaCasToken::Missing,
+            &PoolMetaPersistenceFence::OfflineRecovery,
+            "prepare_cas",
+            &mut transaction,
+        )
+        .await
+        .expect("the earlier phase must really commit");
+        pool.disks.write().await[2..].fill(None);
+        save_pool_meta_object_cas(
+            pool.clone(),
+            POOL_META_IDENTITY_NAME,
+            data.clone(),
+            &PoolMetaCasToken::Existing(written.etag.expect("committed identity has an ETag")),
+            &PoolMetaPersistenceFence::OfflineRecovery,
+            "identity_cas",
+            &mut transaction,
+        )
+        .await
+        .expect_err("the next phase must fail without a write quorum");
+        drop(transaction);
+        *pool.disks.write().await = disks.into_iter().map(Some).collect();
+        assert_eq!(
+            crate::config::com::read_config(pool, POOL_META_IDENTITY_NAME)
+                .await
+                .expect("the earlier commit must survive the rejected write"),
+            data
+        );
+        let error = state
+            .ensure_write_safe("after a committed phase")
+            .expect_err("a later rejection cannot discharge the earlier write");
+        assert_eq!(
+            error.pool_metadata_failure().expect("retain typed uncertainty").kind,
+            crate::error::PoolMetadataFailure::TransactionUnknown
+        );
+        assert_eq!(error.pool_metadata_failure().expect("retain the earlier mutation").phase, "prepare_cas");
+    }
+
+    #[tokio::test]
+    async fn pool_meta_prewrite_noninline_failure_keeps_transaction_armed() {
+        let (_dirs, _disks, pool) = crate::set_disk::hermetic_set_disks_isolated(4).await;
+        pool.disks.write().await[2..].fill(None);
+        let state = super::PoolMetaWriteState::default();
+        let mut transaction = state.arm_transaction();
+        save_pool_meta_object_cas(
+            pool,
+            POOL_META_IDENTITY_NAME,
+            vec![0x51; 2 * 1024 * 1024],
+            &PoolMetaCasToken::Missing,
+            &PoolMetaPersistenceFence::OfflineRecovery,
+            "identity_cas",
+            &mut transaction,
+        )
+        .await
+        .expect_err("the noninline write must fail without quorum");
+        drop(transaction);
+        assert!(
+            state.ensure_write_safe("after writer setup").is_err(),
+            "a failure after disk writers were created must remain fail-closed"
+        );
+    }
+
     #[derive(Clone, Copy, Debug, PartialEq, Eq)]
     enum IdentityRepairFault {
         Healthy,

crates/ecstore/src/core/sets.rs

@@ -317,8 +317,9 @@ impl Sets {
         &self.ctx
     }
 
-    #[cfg(test)]
+    /// Stop endpoint reconnection and wait for any in-flight pass before a test owns disk probes.
-    pub(crate) async fn stop_endpoint_monitor_for_test(&self) {
+    #[cfg(any(test, feature = "test-util"))]
+    pub async fn stop_endpoint_monitor_for_test(&self) {
         if let Some(exit_signal) = &self.exit_signal {
             let _ = exit_signal.send(());
             // Wait for an in-flight reconnect pass as well as its receiver to exit.

crates/ecstore/src/data_movement/e2e_barrier.rs

@@ -31,6 +31,8 @@ use tokio::fs;
 use tokio::time::Instant;
 
 pub(crate) const BARRIER_DIR_ENV: &str = "RUSTFS_E2E_DATA_MOVEMENT_BARRIER_DIR";
+/// Rebalance holds its admission fence; no version of this entry has moved.
+pub(crate) const STAGE_AFTER_ENTRY_ADMISSION: &str = "after_entry_admission";
 /// Every target part is staged and the target version is not yet published.
 pub(crate) const STAGE_BEFORE_TARGET_COMMIT: &str = "before_target_commit";
 /// The target commit is durable and the source preflight matched; the source

crates/ecstore/src/data_movement/mod.rs

@@ -605,6 +605,7 @@ fn resolve_data_movement_abort_result(
 #[derive(Debug)]
 struct DataMovementStageError {
     rendered: String,
+    stage: Option<String>,
     source: Box<dyn std::error::Error + Send + Sync>,
 }
 
@@ -626,6 +627,7 @@ where
 {
     Error::other(DataMovementStageError {
         rendered,
+        stage: None,
         source: Box::new(err),
     })
 }
@@ -634,7 +636,21 @@ fn data_movement_stage_error<E>(op_label: &str, stage: &str, bucket: &str, objec
 where
     E: std::error::Error + Send + Sync + 'static,
 {
-    data_movement_context_error(format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"), err)
+    Error::other(DataMovementStageError {
+        rendered: format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"),
+        stage: Some(stage.to_owned()),
+        source: Box::new(err),
+    })
+}
+
+pub(crate) fn data_movement_error_is_stage(err: &Error, stage: &str) -> bool {
+    let Error::Io(io_err) = err else {
+        return false;
+    };
+    io_err
+        .get_ref()
+        .and_then(|source| source.downcast_ref::<DataMovementStageError>())
+        .is_some_and(|source| source.stage.as_deref() == Some(stage))
 }
 
 #[cfg(test)]

crates/ecstore/src/disk/disk_store.rs

@@ -55,6 +55,20 @@ use tokio_util::sync::CancellationToken;
 use tracing::{info, warn};
 use uuid::Uuid;
 
+/// Join the local disk's actual bootstrap worker for deterministic coverage tests.
+/// Completion may mean legacy fallback after a failed scan; callers must assert
+/// the expected index result separately. Remote disks have no local worker.
+#[cfg(any(test, feature = "test-util"))]
+pub async fn wait_for_multipart_index_bootstrap(disk: &crate::disk::DiskStore) -> Result<()> {
+    match disk.as_ref() {
+        crate::disk::Disk::Local(local) => {
+            local.disk.wait_for_multipart_index_bootstrap().await;
+            Ok(())
+        }
+        crate::disk::Disk::Remote(_) => Err(DiskError::MethodNotAllowed),
+    }
+}
+
 /// Disk health status constants
 const DISK_HEALTH_OK: u32 = 0;
 const DISK_HEALTH_FAULTY: u32 = 1;
@@ -853,6 +867,8 @@ pub(crate) struct DiskHealthMetricEpoch {
     fallback_operation_metrics: StdRwLock<HashMap<&'static str, Arc<DiskOperationMetrics>>>,
     #[cfg(test)]
     multipart_cleanup_observer: std::sync::Mutex<Option<std::sync::Weak<ScopedMultipartCleanupCompletions>>>,
+    #[cfg(test)]
+    operation_timeout_override: std::sync::Mutex<Option<Duration>>,
     /// Caller API operations currently executing through the disk health wrapper.
     api_waiting: AtomicU32,
     /// Operations rejected because the disk was unavailable for the caller.
@@ -880,6 +896,8 @@ impl Default for DiskHealthMetricEpoch {
             fallback_operation_metrics: StdRwLock::new(HashMap::new()),
             #[cfg(test)]
             multipart_cleanup_observer: std::sync::Mutex::new(None),
+            #[cfg(test)]
+            operation_timeout_override: std::sync::Mutex::new(None),
             api_waiting: AtomicU32::new(0),
             total_errors_availability: AtomicU64::new(0),
             total_errors_timeout: AtomicU64::new(0),
@@ -1573,6 +1591,20 @@ impl Default for DiskHealthTracker {
     }
 }
 
+#[cfg(test)]
+#[derive(Debug)]
+pub(crate) struct ScopedDiskOperationTimeout {
+    metrics: Arc<DiskHealthMetricEpoch>,
+    previous: Option<Duration>,
+}
+
+#[cfg(test)]
+impl Drop for ScopedDiskOperationTimeout {
+    fn drop(&mut self) {
+        *self.metrics.operation_timeout_override.lock().expect("test timeout lock") = self.previous;
+    }
+}
+
 /// LocalDiskWrapper wraps a DiskStore with health tracking capabilities.
 /// This is similar to Go's xlStorageDiskIDCheck.
 #[derive(Debug, Clone)]
@@ -1914,6 +1946,23 @@ impl LocalDiskWrapper {
         self.health_check
     }
 
+    #[cfg(test)]
+    pub(crate) fn override_operation_timeout_for_test(&self, timeout: Duration) -> ScopedDiskOperationTimeout {
+        assert!(!timeout.is_zero(), "fault injection needs a positive timeout");
+        // The metrics epoch follows this drive across clones and reconnects.
+        // Its scoped override cannot shorten unrelated healthy drives' I/O.
+        let previous = self
+            .metrics
+            .operation_timeout_override
+            .lock()
+            .expect("test timeout lock")
+            .replace(timeout);
+        ScopedDiskOperationTimeout {
+            metrics: Arc::clone(&self.metrics),
+            previous,
+        }
+    }
+
     /// Enable health monitoring after disk creation.
     /// Used to defer health checks until after startup format loading completes.
     pub fn enable_health_check(&self) {
@@ -2398,12 +2447,17 @@ impl LocalDiskWrapper {
         .await
     }
 
+    pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
+        self.disk.has_clean_active_multipart_source(upload_or_part_path, part_path)
+    }
+
     pub(crate) async fn try_publish_prepared_delete_local(
         &self,
         volume: &str,
         object: &str,
         transaction: uuid::Uuid,
         namespace_owner: Option<Arc<dyn Send + Sync>>,
+        work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
     ) -> Result<Option<(crate::disk::DeleteRecoveryJournal, bool, bool)>> {
         self.track_disk_health_mutation(
             "settle_delete_transaction",
@@ -2411,7 +2465,7 @@ impl LocalDiskWrapper {
             || {
                 Box::pin(async {
                     self.disk
-                        .try_publish_prepared_delete_local(volume, object, transaction, namespace_owner)
+                        .try_publish_prepared_delete_local_with_budget(volume, object, transaction, namespace_owner, work_budget)
                         .await
                 })
             },
@@ -2427,6 +2481,7 @@ impl LocalDiskWrapper {
         transaction: uuid::Uuid,
         journal: &crate::disk::DeleteRecoveryJournal,
         namespace_owner: Option<Arc<dyn Send + Sync>>,
+        work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
     ) -> Result<bool> {
         self.track_disk_health_mutation(
             "settle_delete_transaction",
@@ -2434,7 +2489,14 @@ impl LocalDiskWrapper {
             || {
                 Box::pin(async {
                     self.disk
-                        .try_finish_and_finalize_prepared_delete_local(volume, object, transaction, journal, namespace_owner)
+                        .try_finish_and_finalize_prepared_delete_local_with_budget(
+                            volume,
+                            object,
+                            transaction,
+                            journal,
+                            namespace_owner,
+                            work_budget,
+                        )
                         .await
                 })
             },
@@ -2634,6 +2696,13 @@ impl LocalDiskWrapper {
             }
             return result;
         }
+        #[cfg(test)]
+        let timeout_duration = self
+            .metrics
+            .operation_timeout_override
+            .lock()
+            .expect("test timeout lock")
+            .map_or(timeout_duration, |injected| timeout_duration.min(injected));
         // Execute the operation with timeout
         let result = tokio::time::timeout(timeout_duration, operation).await;
 
@@ -2682,6 +2751,18 @@ impl LocalDiskWrapper {
 
 #[async_trait::async_trait]
 impl DiskAPI for LocalDiskWrapper {
+    async fn list_multipart_index_roots(
+        &self,
+        bucket: &str,
+        marker: Option<&str>,
+    ) -> Result<crate::disk::multipart_index::MultipartIndexPage> {
+        if self.health.is_faulty() {
+            return Err(DiskError::FaultyDisk);
+        }
+        self.check_disk_stale().await?;
+        self.disk.list_multipart_index_roots(bucket, marker).await
+    }
+
     async fn supports_guarded_delete_versions(&self) -> Result<()> {
         // The local implementation shares this binary's capability. Availability
         // and disk identity remain mutation checks and write-quorum inputs.
@@ -2735,6 +2816,62 @@ impl DiskAPI for LocalDiskWrapper {
         )
         .await
     }
+    async fn discover_delete_transactions_bounded(
+        &self,
+        volume: &str,
+        object: &str,
+        max_entries: u32,
+    ) -> Result<crate::disk::BoundedDeleteRecoveryDiscovery> {
+        self.track_disk_health_with_op(
+            "discover_delete_transactions",
+            || async {
+                self.disk
+                    .discover_delete_transactions_bounded(volume, object, max_entries)
+                    .await
+            },
+            get_max_timeout_duration(),
+        )
+        .await
+    }
+
+    async fn inspect_delete_transaction_bounded(
+        &self,
+        volume: &str,
+        object: &str,
+        transaction: uuid::Uuid,
+        max_entries: u32,
+    ) -> Result<crate::disk::BoundedDeleteRecoveryInspection> {
+        self.track_disk_health_with_op(
+            "inspect_delete_transaction",
+            || async {
+                self.disk
+                    .inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
+                    .await
+            },
+            get_max_timeout_duration(),
+        )
+        .await
+    }
+
+    async fn settle_delete_transaction_bounded(
+        &self,
+        volume: &str,
+        request: crate::disk::BoundedDeleteRecoverySettlement,
+        namespace_owner: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<crate::disk::BoundedDeleteRecoverySettlementReply> {
+        self.track_disk_health_mutation(
+            "settle_delete_transaction",
+            DiskMetricMutation::Delete,
+            || async {
+                self.disk
+                    .settle_delete_transaction_bounded(volume, request, namespace_owner)
+                    .await
+            },
+            crate::disk::get_delete_recovery_timeout()?,
+        )
+        .await
+    }
+
     async fn discover_delete_transactions(
         &self,
         volume: &str,
@@ -3703,7 +3840,11 @@ mod tests {
         assert!(futures::poll!(&mut operation).is_pending());
         time::advance(SLOW_DISK_OPERATION_THRESHOLD - Duration::from_millis(1)).await;
         assert!(futures::poll!(&mut operation).is_pending());
-        assert!(logs.contents().is_empty(), "normal disk latency should not warn");
+        assert!(
+            !logs.contents().contains("disk_operation_slow") && !logs.contents().contains("WARN"),
+            "normal disk latency should not warn: {}",
+            logs.contents()
+        );
         time::advance(Duration::from_millis(1)).await;
         assert!(futures::poll!(&mut operation).is_pending());
         let warning = logs.contents();
@@ -6040,6 +6181,67 @@ mod tests {
         assert!(health.offline_duration().is_none());
     }
 
+    #[tokio::test]
+    async fn scoped_operation_timeout_is_per_disk_and_preserves_caller_policy() {
+        let directories = [
+            tempfile::tempdir().expect("faulted drive"),
+            tempfile::tempdir().expect("healthy drive"),
+        ];
+        let mut wrappers = Vec::new();
+        for directory in &directories {
+            let endpoint = Endpoint::try_from(directory.path().to_str().expect("UTF-8 drive path")).expect("drive endpoint");
+            let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local drive"));
+            wrappers.push(LocalDiskWrapper::new(disk, false));
+        }
+        let reconnected = LocalDiskWrapper::new_with_reconnect_state(
+            Arc::clone(&wrappers[0].disk),
+            false,
+            Some(wrappers[0].health_tracker_epoch_for_reconnect()),
+        );
+        let scope = wrappers[0].override_operation_timeout_for_test(Duration::from_secs(1));
+        tokio::time::pause();
+        let operation = || async {
+            tokio::time::sleep(Duration::from_secs(2)).await;
+            Ok(())
+        };
+        let (faulted, healthy) = tokio::join!(
+            reconnected.track_disk_health_with_op_and_timeout_action(
+                "read_metadata",
+                operation,
+                Duration::from_secs(3),
+                TimeoutHealthAction::IgnoreFailure,
+            ),
+            wrappers[1].track_disk_health_with_op_and_timeout_action(
+                "read_metadata",
+                operation,
+                Duration::from_secs(3),
+                TimeoutHealthAction::IgnoreFailure,
+            ),
+        );
+        assert_eq!(faulted.expect_err("only the selected drive must time out"), DiskError::Timeout);
+        healthy.expect("healthy drive keeps its caller's deadline");
+        let shorter_started = tokio::time::Instant::now();
+        let shorter = wrappers[0]
+            .track_disk_health_with_op_and_timeout_action(
+                "read_metadata",
+                operation,
+                Duration::from_millis(500),
+                TimeoutHealthAction::IgnoreFailure,
+            )
+            .await;
+        assert_eq!(shorter.expect_err("fault injection cannot extend a caller deadline"), DiskError::Timeout);
+        assert!(shorter_started.elapsed() < Duration::from_secs(1));
+        wrappers[0]
+            .track_disk_health_with_op("read_metadata", operation, Duration::ZERO)
+            .await
+            .expect("explicit unbounded physical ownership cannot acquire a test deadline");
+        drop(scope);
+        reconnected
+            .track_disk_health_with_op("read_metadata", operation, Duration::from_secs(3))
+            .await
+            .expect("dropping the scope restores reconnected callers' deadlines");
+    }
+
     #[tokio::test]
     async fn ignored_timeout_does_not_mark_drive_failure() {
         let dir = tempfile::tempdir().expect("temp dir should be created");

crates/ecstore/src/disk/error.rs

@@ -172,6 +172,8 @@ pub enum DiskError {
 
     #[error("guarded batch deletion is not supported by this disk")]
     GuardedDeleteUnsupported,
+    #[error("foreground delete recovery work limit reached")]
+    DeleteRecoveryWorkLimit,
 
     #[error("erasure write quorum")]
     ErasureWriteQuorum,
@@ -770,6 +772,7 @@ impl Clone for DiskError {
             DiskError::MethodNotAllowed => DiskError::MethodNotAllowed,
             DiskError::ConditionalDeleteConflict => DiskError::ConditionalDeleteConflict,
             DiskError::GuardedDeleteUnsupported => DiskError::GuardedDeleteUnsupported,
+            DiskError::DeleteRecoveryWorkLimit => DiskError::DeleteRecoveryWorkLimit,
             DiskError::ErasureWriteQuorum => DiskError::ErasureWriteQuorum,
             DiskError::ErasureReadQuorum => DiskError::ErasureReadQuorum,
             DiskError::ShortWrite => DiskError::ShortWrite,
@@ -829,6 +832,7 @@ impl DiskError {
             DiskError::RemoteClientUnavailable(_) => 0x2B,
             DiskError::ConditionalDeleteConflict => 0x2C,
             DiskError::GuardedDeleteUnsupported => 0x2D,
+            DiskError::DeleteRecoveryWorkLimit => 0x2E,
         }
     }
 
@@ -879,6 +883,7 @@ impl DiskError {
             0x2B => Some(DiskError::RemoteClientUnavailable(String::new())),
             0x2C => Some(DiskError::ConditionalDeleteConflict),
             0x2D => Some(DiskError::GuardedDeleteUnsupported),
+            0x2E => Some(DiskError::DeleteRecoveryWorkLimit),
             _ => None,
         }
     }
@@ -1662,6 +1667,7 @@ mod tests {
         for (error, code) in [
             (DiskError::ConditionalDeleteConflict, 0x2C),
             (DiskError::GuardedDeleteUnsupported, 0x2D),
+            (DiskError::DeleteRecoveryWorkLimit, 0x2E),
         ] {
             assert_eq!(error.to_u32(), code);
             assert_eq!(DiskError::from_u32(code), Some(error.clone()));

crates/ecstore/src/disk/fd_cache_tests.rs

@@ -5,6 +5,7 @@ use super::{FdCache, FdCacheEntry, FdKey};
 use std::fs::File;
 use std::os::unix::fs::FileExt;
 use std::sync::Arc;
+use std::sync::atomic::AtomicBool;
 
 fn key(volume: &str, path: &str, direct: bool) -> FdKey {
     FdKey {
@@ -52,7 +53,11 @@ async fn uring_fd_cache_exact_invalidation_removes_both_modes_and_preserves_othe
 async fn uring_fd_cache_exact_invalidation_bumps_one_generation_and_rejects_old_epoch_inserts() {
     let (_directory, file) = fixture();
     let cache = FdCache::new();
-    let entry = Arc::new(FdCacheEntry { file, len: 3 });
+    let entry = Arc::new(FdCacheEntry {
+        file,
+        len: 3,
+        invalidated: AtomicBool::new(false),
+    });
     let old_generation = cache.generation();
     cache.invalidate_exact("v", "part").await;
     let current_generation = cache.generation();
@@ -125,7 +130,7 @@ async fn uring_fd_cache_volume_and_clear_invalidation_cover_both_modes() {
         cache.insert(key("v", "part", direct), Arc::clone(&file)).await;
         assert!(
             cache.get(&key("v", "part", direct)).await.is_some(),
-            "new entries must survive the earlier volume predicate"
+            "new entries must survive the earlier volume invalidation"
         );
     }
     cache.clear();
@@ -164,6 +169,7 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
     let replacement = Arc::new(FdCacheEntry {
         file: Arc::new(File::open(directory.path().join("part")).expect("open replacement inode")),
         len: 11,
+        invalidated: AtomicBool::new(false),
     });
     for direct in [false, true] {
         let key = key("v", "part", direct);
@@ -177,3 +183,459 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
         assert_eq!(fresh.len, 11);
     }
 }
+
+#[tokio::test]
+async fn fd_cache_tracker_skips_unmatched_invalidations_and_handles_deep_prefixes() {
+    let (_directory, file) = fixture();
+    let mut cache = FdCache::new();
+
+    // Unmatched invalidations on an empty cache must not spawn the maintenance task.
+    cache.invalidate_under("v", "missing/prefix");
+    cache.invalidate_volume("v");
+    cache.invalidate_exact("v", "missing/prefix/part.1").await;
+    cache.clear();
+    assert!(
+        cache.maintenance_task.get().is_none(),
+        "empty-cache invalidations must not spawn maintenance"
+    );
+
+    // Populate a normal entry and verify matched vs unmatched filter gates directly,
+    // and ensure unmatched invalidations do not enqueue pending removals.
+    let cached_key = key("v", "obj-1/0198a3f0/part.1", false);
+    cache.insert(cached_key.clone(), Arc::clone(&file)).await;
+    assert_eq!(cache.tracker.live_entries(), 1);
+    assert!(cache.tracker.may_match_volume("v"));
+    assert!(!cache.tracker.may_match_volume("other-volume"));
+    assert!(cache.tracker.may_match_prefix("v", "obj-1"));
+    assert!(cache.tracker.may_match_prefix("v", "obj-1/0198a3f0"));
+    assert!(!cache.tracker.may_match_prefix("v", "obj-2/0198a3f0"));
+    assert!(cache.tracker.may_match_exact("v", "obj-1/0198a3f0/part.1"));
+    assert!(!cache.tracker.may_match_exact("v", "obj-1/0198a3f0/xl.meta"));
+
+    cache.invalidate_volume("other-volume");
+    cache.invalidate_under("v", "obj-2/0198a3f0");
+    cache.invalidate_exact("v", "obj-1/0198a3f0/xl.meta").await;
+    assert!(
+        !cache
+            .tracker
+            .has_pending_invalidations
+            .load(std::sync::atomic::Ordering::SeqCst),
+        "unmatched invalidations must not enqueue maintenance removals"
+    );
+    assert!(cache.get(&cached_key).await.is_some());
+
+    // Deeply nested path (> FD_CACHE_MAX_INDEXED_PREFIXES slashes) must fall back
+    // to per-volume unindexed_counts without degrading other volumes.
+    let other_vol_key = key("v2", "shallow/part.1", false);
+    cache.insert(other_vol_key.clone(), Arc::clone(&file)).await;
+    let deep_path = "c1/c2/c3/c4/c5/c6/c7/c8/c9/c10/c11/c12/c13/c14/part.1";
+    let deep_key = key("v", deep_path, false);
+    cache.insert(deep_key.clone(), Arc::clone(&file)).await;
+    assert_eq!(
+        cache
+            .tracker
+            .unindexed_counts
+            .iter()
+            .map(|c| c.load(std::sync::atomic::Ordering::Acquire))
+            .sum::<u32>(),
+        1
+    );
+    assert!(
+        !cache.tracker.may_match_prefix("v2", "unmatched/prefix"),
+        "deep key in volume v must not disable prefix filtering in volume v2"
+    );
+    cache.invalidate_under("v", "c1/c2/c3/c4/c5/c6/c7/c8/c9/c10/c11/c12/c13");
+    assert!(cache.get(&deep_key).await.is_none());
+    cache.invalidate_volume("v2");
+    assert!(cache.get(&other_vol_key).await.is_none());
+
+    // Replacing an invalidated key before maintenance drains pending_invalidations
+    // must release the old descriptor immediately and keep the replacement live
+    // even when a strong reference to the old CachedFdEntry shell survives until
+    // maintenance runs its Arc::ptr_eq check.
+    let old_file = Arc::new(tempfile::tempfile().expect("create replaced descriptor"));
+    let old_weak = Arc::downgrade(&old_file);
+    cache.insert(cached_key.clone(), old_file).await;
+    let held_old_shell = cache.cache.get(&cached_key).expect("cached shell must exist");
+    cache.invalidate_under("v", "obj-1/0198a3f0");
+    assert!(cache.get(&cached_key).await.is_none());
+    cache.insert(cached_key.clone(), Arc::clone(&file)).await;
+    assert!(
+        old_weak.upgrade().is_none(),
+        "replacing an entry must retire and drop the previous descriptor immediately"
+    );
+    tokio::time::timeout(std::time::Duration::from_secs(5), async {
+        while cache
+            .tracker
+            .has_pending_invalidations
+            .load(std::sync::atomic::Ordering::SeqCst)
+        {
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("maintenance should drain pending invalidations");
+    drop(held_old_shell);
+    assert!(
+        cache.get(&cached_key).await.is_some(),
+        "stale pending invalidation must not evict a newer replacement"
+    );
+    assert_eq!(
+        cache
+            .tracker
+            .unindexed_counts
+            .iter()
+            .map(|c| c.load(std::sync::atomic::Ordering::Acquire))
+            .sum::<u32>(),
+        0
+    );
+
+    // Dropping a cache while a live non-invalidated entry is present (even if a
+    // cloned Moka handle outlives FdCache and the maintenance task is aborted)
+    // must retire the descriptor and release FdCacheTracker without cycles.
+    let live_drop_file = Arc::new(tempfile::tempfile().expect("create live drop descriptor"));
+    let live_drop_weak = Arc::downgrade(&live_drop_file);
+    let shutdown_file = Arc::new(tempfile::tempfile().expect("create shutdown descriptor"));
+    let shutdown_weak = Arc::downgrade(&shutdown_file);
+    let tracker_weak = Arc::downgrade(&cache.tracker);
+    let cloned_moka = cache.cache.clone();
+    cache.insert(key("v", "live-on-drop/part.1", false), live_drop_file).await;
+    cache.insert(key("v", "shutdown/part.1", false), shutdown_file).await;
+    cache.invalidate_under("v", "shutdown");
+    if let Some(handle) = cache.maintenance_task.take() {
+        handle.abort();
+        let _ = handle.await;
+    }
+    drop(cache);
+    assert!(
+        live_drop_weak.upgrade().is_none(),
+        "Drop for FdCache must retire live descriptors even if a cloned Moka handle survives"
+    );
+    assert!(
+        shutdown_weak.upgrade().is_none(),
+        "dropping cache when maintenance is aborted must still release descriptors"
+    );
+    assert!(
+        tracker_weak.upgrade().is_none(),
+        "dropping cache must release FdCacheTracker without reference cycles"
+    );
+    drop(cloned_moka);
+
+    // Short-TTL expired entries (including mixed expired and fresh entries) must
+    // be retired by invalidate_under and clear() even though Moka's iterator skips
+    // expired entries.
+    let ttl_cache = FdCache::with_ttl(std::time::Duration::from_millis(15));
+    let expired_file = Arc::new(tempfile::tempfile().expect("create short-ttl descriptor"));
+    let expired_weak = Arc::downgrade(&expired_file);
+    ttl_cache.insert(key("v", "ttl-obj/part.1", false), expired_file).await;
+    tokio::time::sleep(std::time::Duration::from_millis(30)).await;
+    let fresh_file = Arc::new(tempfile::tempfile().expect("create fresh descriptor"));
+    let fresh_weak = Arc::downgrade(&fresh_file);
+    ttl_cache.insert(key("v", "ttl-obj/part.2", false), fresh_file).await;
+    ttl_cache.invalidate_under("v", "ttl-obj");
+    tokio::time::timeout(std::time::Duration::from_secs(5), async {
+        while expired_weak.upgrade().is_some() || fresh_weak.upgrade().is_some() {
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("invalidate_under must retire both expired and fresh matching descriptors");
+    assert_eq!(ttl_cache.tracker.live_entries(), 0);
+
+    let clear_expired_file = Arc::new(tempfile::tempfile().expect("create clear short-ttl descriptor"));
+    let clear_expired_weak = Arc::downgrade(&clear_expired_file);
+    ttl_cache
+        .insert(key("v", "ttl-clear/part.1", false), clear_expired_file)
+        .await;
+    tokio::time::sleep(std::time::Duration::from_millis(30)).await;
+    let clear_live_file = Arc::new(tempfile::tempfile().expect("create clear live descriptor"));
+    let clear_live_weak = Arc::downgrade(&clear_live_file);
+    ttl_cache.insert(key("v", "ttl-clear/part.2", false), clear_live_file).await;
+    ttl_cache.clear();
+    assert!(
+        clear_expired_weak.upgrade().is_none() && clear_live_weak.upgrade().is_none(),
+        "clear() must synchronously retire both TTL-expired and live descriptors"
+    );
+    assert_eq!(ttl_cache.tracker.live_entries(), 0);
+}
+
+#[tokio::test]
+async fn fd_cache_lockstep_oracle_equivalence_and_counter_conservation() {
+    use std::collections::HashMap;
+    use std::sync::atomic::Ordering;
+
+    let (_directory, file) = fixture();
+    let cache = FdCache::with_ttl(std::time::Duration::from_secs(3600));
+    let mut oracle: HashMap<FdKey, Arc<FdCacheEntry>> = HashMap::new();
+
+    let volumes = ["vol-a", "vol-b", "vol-c", ".rustfs.sys/multipart"];
+    let paths = [
+        "a/b",
+        "a/b/part.1",
+        "a/b/dir/part.2",
+        "a/bc",
+        "a/bc/part.1",
+        "prefix/obj-0001.bin/0198a3f0/part.1",
+        "prefix/obj-0002.bin/0198a3f0/part.1",
+        "d1/d2/d3/d4/d5/d6/d7/d8/d9/d10/d11/d12/d13/part.1",
+    ];
+    let prefixes = [
+        "a",
+        "a/",
+        "a/b",
+        "a/b/",
+        "a/b/dir",
+        "a/bc",
+        "prefix",
+        "prefix/obj-0001.bin",
+        "prefix/obj-0001.bin/0198a3f0",
+        "d1/d2/d3/d4/d5/c_missing",
+        "d1/d2/d3/d4/d5/d6/d7/d8/d9/d10/d11/d12/d13",
+        "nonexistent/prefix",
+    ];
+
+    let mut candidate_keys = Vec::new();
+    for v in volumes {
+        for p in paths {
+            for direct in [false, true] {
+                candidate_keys.push(key(v, p, direct));
+            }
+        }
+    }
+
+    let mut rng = 0x9e37_79b9_7f4a_7c15u64;
+    let mut next_u64 = || {
+        rng ^= rng << 13;
+        rng ^= rng >> 7;
+        rng ^= rng << 17;
+        rng
+    };
+
+    for step in 0..1000 {
+        let op = next_u64() % 100;
+        if op < 45 {
+            // Insert or replace (either fresh or stale generation)
+            let k = candidate_keys[(next_u64() as usize) % candidate_keys.len()].clone();
+            let entry = Arc::new(FdCacheEntry {
+                file: Arc::clone(&file),
+                len: step,
+                invalidated: AtomicBool::new(false),
+            });
+            let cur_gen = cache.generation();
+            let stale = (next_u64() % 5) == 0 && cur_gen > 0;
+            let gen_to_use = if stale { cur_gen - 1 } else { cur_gen };
+            cache.insert_if_fresh(k.clone(), Arc::clone(&entry), gen_to_use).await;
+            if !stale {
+                oracle.insert(k, entry);
+            }
+        } else if op < 65 {
+            // Exact invalidation
+            let v = volumes[(next_u64() as usize) % volumes.len()];
+            let p = paths[(next_u64() as usize) % paths.len()];
+            cache.invalidate_exact(v, p).await;
+            oracle.remove(&key(v, p, false));
+            oracle.remove(&key(v, p, true));
+        } else if op < 88 {
+            // Prefix invalidation
+            let v = volumes[(next_u64() as usize) % volumes.len()];
+            let raw_prefix = prefixes[(next_u64() as usize) % prefixes.len()];
+            cache.invalidate_under(v, raw_prefix);
+            let trimmed = raw_prefix.trim_end_matches('/');
+            oracle.retain(|k, _| {
+                !(k.volume == v && (k.path == trimmed || k.path.strip_prefix(trimmed).is_some_and(|r| r.starts_with('/'))))
+            });
+        } else if op < 97 {
+            // Volume invalidation
+            let v = volumes[(next_u64() as usize) % volumes.len()];
+            cache.invalidate_volume(v);
+            oracle.retain(|k, _| k.volume != v);
+        } else {
+            // Full clear
+            cache.clear();
+            oracle.clear();
+        }
+
+        // Verify lockstep equivalence against the reference oracle after every operation.
+        for k in &candidate_keys {
+            let got = cache.get(k).await;
+            let expected = oracle.get(k);
+            match (got, expected) {
+                (Some(g), Some(e)) => assert!(
+                    Arc::ptr_eq(&g, e),
+                    "step {step}: pointer mismatch for key ({}, {}, {})",
+                    k.volume,
+                    k.path,
+                    k.direct
+                ),
+                (None, None) => {}
+                (g, e) => panic!(
+                    "step {step}: oracle mismatch for key ({}, {}, {}): got={}, expected={}",
+                    k.volume,
+                    k.path,
+                    k.direct,
+                    g.is_some(),
+                    e.is_some()
+                ),
+            }
+        }
+    }
+
+    // Clear and wait for maintenance to drain all entries, then verify exact counter conservation.
+    cache.clear();
+    tokio::time::timeout(std::time::Duration::from_secs(5), async {
+        while cache.tracker.live_entries() != 0 || cache.tracker.has_pending_invalidations.load(Ordering::SeqCst) {
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("clear must drain all live entries and pending invalidations");
+
+    assert_eq!(cache.tracker.live_entries(), 0);
+    assert!(cache.tracker.unindexed_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.volume_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.exact_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.prefix_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.prefix_shards.iter().all(|s| s.lock().is_empty()));
+    assert!(cache.tracker.entry_shards.iter().all(|s| s.lock().is_empty()));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn fd_cache_concurrent_insert_and_invalidation_stress() {
+    use std::sync::atomic::{AtomicU64, Ordering};
+
+    let (_directory, file) = fixture();
+    let cache = Arc::new(FdCache::new());
+
+    // Deterministic Dekker-partner interleaving check: pause `CachedFdEntry::new`
+    // at `entry_shards[shard].lock()` after it has passed the initial generation
+    // check and incremented `volume_counts`, bump generation via an unmatched
+    // invalidation, and verify the post-`new()` recheck refuses publication and
+    // preserves the existing entry.
+    let seam_key = key("v-seam", "obj-1/part.1", false);
+    cache.insert(seam_key.clone(), Arc::clone(&file)).await;
+    let initial_entry = cache.get(&seam_key).await.expect("initial entry must exist");
+    let (vol_hash, vol_slot) = super::fd_cache_volume_hash_and_slot("v-seam");
+    let entry_shard = (vol_hash as usize) & (super::FD_CACHE_PREFIX_SHARDS - 1);
+    let stale_gen = cache.generation();
+    let stale_entry = Arc::new(FdCacheEntry {
+        file: Arc::clone(&file),
+        len: 999_999,
+        invalidated: AtomicBool::new(false),
+    });
+    let blocked_insert = {
+        let hold = cache.tracker.entry_shards[entry_shard].lock();
+        let cache_clone = Arc::clone(&cache);
+        let seam_key_clone = seam_key.clone();
+        let task = tokio::spawn(async move {
+            cache_clone.insert_if_fresh(seam_key_clone, stale_entry, stale_gen).await;
+        });
+        let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
+        while cache.tracker.volume_counts[usize::from(vol_slot)].load(Ordering::SeqCst) < 2 {
+            assert!(
+                std::time::Instant::now() < deadline,
+                "insert never reached CachedFdEntry::new with the previous entry still live"
+            );
+            std::thread::yield_now();
+        }
+        cache.invalidate_under("v-seam", "unmatched-prefix");
+        drop(hold);
+        task
+    };
+    blocked_insert.await.expect("blocked insert task should complete");
+    let preserved = cache
+        .get(&seam_key)
+        .await
+        .expect("refused stale insert must not retire a valid previous entry");
+    assert!(Arc::ptr_eq(&preserved, &initial_entry));
+    cache.invalidate_volume("v-seam");
+
+    let floor_v1 = Arc::new(AtomicU64::new(0));
+    let floor_v2 = Arc::new(AtomicU64::new(0));
+    let mut handles = Vec::new();
+
+    for t in 0..8usize {
+        let cache = Arc::clone(&cache);
+        let file = Arc::clone(&file);
+        let floor_v1 = Arc::clone(&floor_v1);
+        let floor_v2 = Arc::clone(&floor_v2);
+        handles.push(tokio::spawn(async move {
+            for i in 0..250u64 {
+                let is_v1 = (t + i as usize).is_multiple_of(2);
+                let vol = if is_v1 { "v1" } else { "v2" };
+                let path = format!("dir-{}/obj-{}/part.1", t % 4, i % 16);
+                let k = key(vol, &path, i.is_multiple_of(2));
+                let g = cache.generation();
+                let entry = Arc::new(FdCacheEntry {
+                    file: Arc::clone(&file),
+                    len: g,
+                    invalidated: AtomicBool::new(false),
+                });
+                cache.insert_if_fresh(k.clone(), entry, g).await;
+                let min_gen = if is_v1 {
+                    floor_v1.load(Ordering::SeqCst)
+                } else {
+                    floor_v2.load(Ordering::SeqCst)
+                };
+                if let Some(hit) = cache.get(&k).await {
+                    assert!(
+                        hit.len >= min_gen,
+                        "stale descriptor served after volume invalidation: opened_gen={}, floor_gen={min_gen}",
+                        hit.len
+                    );
+                }
+            }
+        }));
+    }
+
+    for t in 0..4usize {
+        let cache = Arc::clone(&cache);
+        let floor_v1 = Arc::clone(&floor_v1);
+        let floor_v2 = Arc::clone(&floor_v2);
+        handles.push(tokio::spawn(async move {
+            for i in 0..250u64 {
+                let is_v1 = (t + i as usize).is_multiple_of(2);
+                let vol = if is_v1 { "v1" } else { "v2" };
+                match i % 4 {
+                    0 => cache.invalidate_under(vol, &format!("dir-{}/obj-{}", t % 4, i % 16)),
+                    1 => {
+                        let p = format!("dir-{}/obj-{}/part.1", t % 4, i % 16);
+                        cache.invalidate_exact(vol, &p).await;
+                    }
+                    2 => {
+                        let before = cache.generation();
+                        cache.invalidate_volume(vol);
+                        let floor = before + 1;
+                        if is_v1 {
+                            floor_v1.fetch_max(floor, Ordering::SeqCst);
+                        } else {
+                            floor_v2.fetch_max(floor, Ordering::SeqCst);
+                        }
+                    }
+                    _ => cache.invalidate_under("uncached-vol", "uncached/prefix"),
+                }
+                tokio::task::yield_now().await;
+            }
+        }));
+    }
+
+    for h in handles {
+        h.await.expect("worker task should succeed");
+    }
+
+    cache.clear();
+    tokio::time::timeout(std::time::Duration::from_secs(5), async {
+        while cache.tracker.live_entries() != 0 || cache.tracker.has_pending_invalidations.load(Ordering::SeqCst) {
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("clear after concurrent stress must drain all live entries");
+
+    assert_eq!(cache.tracker.live_entries(), 0);
+    assert!(cache.tracker.unindexed_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.volume_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.exact_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.prefix_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.prefix_shards.iter().all(|s| s.lock().is_empty()));
+    assert!(cache.tracker.entry_shards.iter().all(|s| s.lock().is_empty()));
+}

crates/ecstore/src/disk/fs.rs

@@ -148,47 +148,58 @@ pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
     native_io(move || create_dir_all_std(&path)).await
 }
 
-fn is_dir_error(e: &io::Error) -> bool {
+fn is_dir_error(path: &Path, e: &io::Error) -> bool {
     e.raw_os_error() == Some(libc::EISDIR)
         || e.kind() == io::ErrorKind::IsADirectory
         // macOS: remove_file on a directory returns EPERM
         || (cfg!(target_os = "macos") && e.raw_os_error() == Some(libc::EPERM))
+        || is_windows_dir_access_denied(path, e)
+}
+
+/// Windows reports `remove_file` on a directory (or directory symlink) as
+/// ERROR_ACCESS_DENIED, which is indistinguishable from a real permission
+/// failure without inspecting the entry itself.
+#[cfg(windows)]
+fn is_windows_dir_access_denied(path: &Path, e: &io::Error) -> bool {
+    use std::os::windows::fs::FileTypeExt;
+
+    e.kind() == io::ErrorKind::PermissionDenied
+        && std::fs::symlink_metadata(path).is_ok_and(|metadata| {
+            let file_type = metadata.file_type();
+            file_type.is_dir() || file_type.is_symlink_dir()
+        })
+}
+
+#[cfg(not(windows))]
+fn is_windows_dir_access_denied(_path: &Path, _e: &io::Error) -> bool {
+    false
 }
 
 #[tracing::instrument(level = "debug", skip_all)]
 pub async fn remove(path: impl AsRef<Path>) -> io::Result<()> {
-    // Try remove_file first; fall back to remove_dir if it's a directory
+    let path = path.as_ref().to_path_buf();
-    let file_path = path.as_ref().to_path_buf();
+    native_io(move || remove_std(path)).await
-    match native_io(move || std::fs::remove_file(file_path)).await {
-        Ok(()) => Ok(()),
-        Err(e) if is_dir_error(&e) => {
-            let path = path.as_ref().to_path_buf();
-            native_io(move || std::fs::remove_dir(path)).await
-        }
-        Err(e) => Err(e),
-    }
 }
 
 #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
 pub async fn remove_all(path: impl AsRef<Path>) -> io::Result<()> {
     // Try remove_file first; fall back to remove_dir_all if it's a directory
-    let file_path = path.as_ref().to_path_buf();
+    let path = path.as_ref().to_path_buf();
-    match native_io(move || std::fs::remove_file(file_path)).await {
+    native_io(move || match std::fs::remove_file(&path) {
         Ok(()) => Ok(()),
-        Err(e) if is_dir_error(&e) => {
+        Err(e) if is_dir_error(&path, &e) => std::fs::remove_dir_all(&path),
-            let path = path.as_ref().to_path_buf();
-            native_io(move || std::fs::remove_dir_all(path)).await
-        }
         Err(e) => Err(e),
-    }
+    })
+    .await
 }
 
 #[tracing::instrument(level = "debug", skip_all)]
 pub fn remove_std(path: impl AsRef<Path>) -> io::Result<()> {
     // Try remove_file first; fall back to remove_dir if it's a directory
-    match std::fs::remove_file(path.as_ref()) {
+    let path = path.as_ref();
+    match std::fs::remove_file(path) {
         Ok(()) => Ok(()),
-        Err(e) if is_dir_error(&e) => std::fs::remove_dir(path.as_ref()),
+        Err(e) if is_dir_error(path, &e) => std::fs::remove_dir(path),
         Err(e) => Err(e),
     }
 }

crates/ecstore/src/disk/local/commit.rs

@@ -175,17 +175,30 @@ pub(super) fn create_local_inline_rollback_backup(
     dst_file_path: &Path,
     staging_file_path: &Path,
     old_metadata: &[u8],
+    sync: bool,
 ) -> std::io::Result<PathBuf> {
     let Some(staging_parent) = staging_file_path.parent() else {
         return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing staging metadata parent"));
     };
     let backup_path = staging_parent.join(STORAGE_FORMAT_FILE_BACKUP);
     remove_file_if_exists(&backup_path)?;
-    if (should_fail_local_inline_rollback_hardlink(dst_file_path) || std::fs::hard_link(dst_file_path, &backup_path).is_err())
+    if should_fail_local_inline_rollback_hardlink(dst_file_path) || std::fs::hard_link(dst_file_path, &backup_path).is_err() {
-        && let Err(err) = std::fs::write(&backup_path, old_metadata)
+        let write_res = (|| -> std::io::Result<()> {
-    {
+            #[cfg(test)]
-        let _ = remove_file_if_exists(&backup_path);
+            run_owned_file_before_open(&backup_path);
-        return Err(err);
+            let mut file = std::fs::File::create(&backup_path)?;
+            std::io::Write::write_all(&mut file, old_metadata)?;
+            if sync && !old_metadata.is_empty() {
+                file.sync_data()?;
+            }
+            Ok(())
+        })();
+        if let Err(err) = write_res {
+            let _ = remove_file_if_exists(&backup_path);
+            return Err(err);
+        }
+    } else if sync && !old_metadata.is_empty() {
+        os::open_file_for_sync_std(&backup_path)?.sync_data()?;
     }
     Ok(backup_path)
 }
@@ -356,9 +369,12 @@ fn prepare_rename_commit_and_destination_metadata_std(
 
 #[cfg(target_os = "linux")]
 enum FusedCommitAttempt {
-    Committed(RenameDataResp),
+    Committed(Box<RenameDataResp>),
     CommitErr(DiskError),
-    Fallback(os::RenameCommitGuard, Option<Bytes>, FileInfo),
+    RestoreDataSource(os::RenameCommitGuard, DiskError),
+    Fallback(os::RenameCommitGuard, Option<Bytes>, Box<FileInfo>),
+    #[cfg(test)]
+    ResumePanic(Box<dyn std::any::Any + Send + 'static>),
 }
 
 async fn prepare_rename_commit_and_destination_metadata(
@@ -470,6 +486,9 @@ impl LocalDisk {
             state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
         })?;
         crate::hp_guard!("LocalDisk::rename_data");
+        if dst_volume == super::super::RUSTFS_META_MULTIPART_BUCKET {
+            self.multipart_index.record_info(dst_path, &fi).await?;
+        }
         #[cfg(feature = "e2e-test-hooks")]
         if fi.is_healing() {
             wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?;
@@ -540,7 +559,9 @@ impl LocalDisk {
             None
         };
         let multipart_publication = if let Some(staging) = multipart_staging.as_ref() {
-            Some(pin_multipart_future(|| self.prepare_native_multipart_publication(src_path, &mut fi, staging)).await?)
+            Some(Arc::new(
+                pin_multipart_future(|| self.prepare_native_multipart_publication_deferred(src_path, &mut fi, staging)).await?,
+            ))
         } else {
             None
         };
@@ -741,11 +762,6 @@ impl LocalDisk {
             && !fi_healing
             && !os::dst_dir_fsync_group_commit_enabled()
             && !os::file_fdatasync_group_commit_enabled()
-            && !super::has_rename_data_test_failpoints(dst_path)
-            && !crash_inject::is_armed(CrashPoint::RenameAfterDataRename, dst_path)
-            && !crash_inject::is_armed(CrashPoint::RenameAfterBackupBeforeMetaCommit, dst_path)
-            && !crash_inject::is_armed(CrashPoint::RenameAfterMetaCommit, dst_path)
-            && !super::has_delete_version_test_failpoints()
         {
             #[cfg(test)]
             if !no_inline {
@@ -785,10 +801,11 @@ impl LocalDisk {
                     &dst_volume_dir_owned,
                     &publication_root_owned,
                 )?;
-                // An existing xl.meta can share versions or require rollback
+                // An existing xl.meta that replaces an existing version slot
-                // restoration when a post-rename directory sync fails.
+                // requires a rollback directory; appending a new version keeps
-                if is_cancelled() || (sync_commit_metadata && has_dst_buf.is_some()) {
+                // an inline rollback backup for post-rename directory sync errors.
-                    return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, fi_moved));
+                if is_cancelled() {
+                    return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
                 }
                 let mut xlmeta = FileMeta::new();
                 let mut dst_meta_unparsable = false;
@@ -811,7 +828,19 @@ impl LocalDisk {
                     }
                 });
                 if rollback_data_dir.is_some() {
-                    return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, fi_moved));
+                    return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
+                }
+                #[cfg(test)]
+                let probe_active = os::file_sync_probe::is_active();
+                #[cfg(not(test))]
+                let probe_active = false;
+                if no_inline
+                    && sync_data_shards
+                    && let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref()
+                    && (probe_active || !super::is_presynced_tmp_data_dir(src_data_path))
+                    && os::regular_files(src_data_path).is_ok_and(|files| files.len() >= os::PARALLEL_FILE_SYNC_THRESHOLD)
+                {
+                    return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
                 }
                 let (old_current_size, old_current_source_checked, old_current_source) = observe_old_current(
                     has_dst_buf.is_some(),
@@ -830,13 +859,31 @@ impl LocalDisk {
                     skip_parent = dst_parent_owned.clone();
                 }
 
+                #[cfg(test)]
+                run_owned_file_before_open(&src_file_path_owned);
                 let mut prepared_metadata_source =
                     os::create_prepared_rename_source_with_commit_guard(&src_file_path_owned, &dst_file_path_owned, &guard)?;
-                prepared_metadata_source.write_all(&new_dst_buf, sync_commit_metadata)?;
+                prepared_metadata_source.write_all(&new_dst_buf, false)?;
+                if !no_inline {
+                    run_inline_preparation_before_backup(&dst_path_str);
+                }
+
+                let mut staged_rollback_path = None;
+                if let Some(ref old_metadata) = has_dst_buf
+                    && (sync_commit_metadata || cfg!(test))
+                {
+                    let backup_path = create_local_inline_rollback_backup(
+                        &dst_file_path_owned,
+                        &src_file_path_owned,
+                        old_metadata,
+                        sync_commit_metadata,
+                    )?;
+                    staged_rollback_path = Some(StagedRollbackBackup(backup_path));
+                }
 
                 if no_inline {
                     if sync_data_shards && let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref() {
-                        let sync_result = if super::take_presynced_tmp_data_dir(src_data_path) {
+                        let sync_result = if !probe_active && super::take_presynced_tmp_data_dir(src_data_path) {
                             // The payload was synced, but its part.1 directory entry
                             // must also be persisted before moving the data directory.
                             os::fsync_dir_std(src_data_path)
@@ -846,49 +893,157 @@ impl LocalDisk {
                         if let Err(err) = sync_result
                             && err.kind() != ErrorKind::NotFound
                         {
+                            if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
+                                let _ = remove_file_if_exists(backup_path);
+                            }
                             return Err(err);
                         }
                     }
+                    if sync_commit_metadata && let Err(err) = prepared_metadata_source.sync_data() {
+                        if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
+                            let _ = remove_file_if_exists(backup_path);
+                        }
+                        return Err(err);
+                    }
+                } else if sync_commit_metadata && let Err(err) = prepared_metadata_source.sync_data() {
+                    if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
+                        let _ = remove_file_if_exists(backup_path);
+                    }
+                    return Err(err);
+                }
+
+                if is_cancelled() {
+                    drop(prepared_metadata_source);
+                    if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
+                        let _ = remove_file_if_exists(backup_path);
+                    }
+                    return Err(std::io::Error::new(ErrorKind::Interrupted, "commit cancelled"));
+                }
+
+                let guard = match super::remove_dst_base_before_commit_std(
+                    &dst_path_str,
+                    guard,
+                    &src_parent_owned,
+                    &dst_parent_owned,
+                    &dst_volume_dir_owned,
+                    &publication_root_owned,
+                ) {
+                    Ok(guard) => guard,
+                    Err(err) => {
+                        if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
+                            let _ = remove_file_if_exists(backup_path);
+                        }
+                        return Ok(FusedCommitAttempt::CommitErr(err));
+                    }
+                };
+                if should_remove_staged_meta_before_commit(&dst_path_str) {
+                    drop(prepared_metadata_source);
+                    if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
+                        let _ = remove_file_if_exists(backup_path);
+                    }
+                    std::fs::remove_file(&src_file_path_owned)?;
+                    return Ok(FusedCommitAttempt::CommitErr(DiskError::FileNotFound));
+                }
+
+                if no_inline {
                     if let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref()
                         && let Err(err) = os::rename_all_with_commit_guard_on_thread(src_data_path, dst_data_path, &guard)
                     {
-                        return Ok(FusedCommitAttempt::CommitErr(err));
+                        if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
+                            let _ = remove_file_if_exists(backup_path);
+                        }
+                        return Ok(FusedCommitAttempt::RestoreDataSource(guard, err));
                     }
                     #[cfg(test)]
-                    if has_data_dir_path_owned.is_some() {
+                    if has_data_dir_path_owned.is_some()
-                        run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
+                        && let Err(panic_payload) = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+                            run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
+                        }))
+                    {
+                        return Ok(FusedCommitAttempt::ResumePanic(panic_payload));
+                    }
+                    if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, &dst_path_str) {
+                        return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
+                    }
+                    if should_fail_before_old_metadata_backup(&dst_path_str) {
+                        if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
+                            let _ = remove_file_if_exists(backup_path);
+                        }
+                        return Ok(FusedCommitAttempt::RestoreDataSource(guard, DiskError::Unexpected));
+                    }
+                    if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, &dst_path_str) {
+                        return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
                     }
                 }
 
-                if let Err(err) = os::rename_prepared_source_on_thread(
+                let rename_res = if should_fail_commit_rename(&dst_path_str) {
-                    prepared_metadata_source,
+                    drop(prepared_metadata_source);
-                    &src_file_path_owned,
+                    Err(DiskError::other("test fail during metadata commit rename"))
-                    &dst_file_path_owned,
+                } else {
-                    &skip_parent,
+                    os::rename_prepared_source_on_thread(
-                    &publication_root_owned,
+                        prepared_metadata_source,
-                    &guard,
+                        &src_file_path_owned,
-                ) {
+                        &dst_file_path_owned,
-                    if no_inline && let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
+                        &skip_parent,
-                        let _ = std::fs::rename(dst_data_path, src_data_path);
+                        &publication_root_owned,
+                        &guard,
+                    )
+                };
+                if let Err(err) = rename_res {
+                    if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
+                        let _ = remove_file_if_exists(backup_path);
+                    }
+                    if no_inline {
+                        return Ok(FusedCommitAttempt::RestoreDataSource(guard, err));
                     }
                     return Ok(FusedCommitAttempt::CommitErr(err));
                 }
 
-                if sync_commit_metadata
+                if should_fail_after_metadata_commit(&dst_path_str) {
-                    && let Err(err) =
+                    if no_inline {
-                        fsync_inline_commit_std(&dst_file_path_owned, &dst_volume_dir_owned, destination_was_absent, None, &guard)
+                        rollback_inline_metadata_std(&dst_file_path_owned, None, staged_rollback_path.as_ref())?;
-                {
+                        if let Some((_, new_data_path)) = has_data_dir_path_owned.as_ref() {
-                    if no_inline && let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
+                            remove_dir_all_if_exists(new_data_path)?;
-                        let _ = std::fs::rename(dst_data_path, src_data_path);
+                        }
+                        return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
                     }
-                    return Err(err);
+                    let err = rollback_inline_metadata_std(&dst_file_path_owned, None, staged_rollback_path.as_ref())
+                        .err()
+                        .unwrap_or_else(|| std::io::Error::other("test fail after metadata commit"));
+                    return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
+                }
+
+                if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, &dst_path_str) {
+                    return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
+                }
+
+                if sync_commit_metadata
+                    && let Err(err) = fsync_inline_commit_std(
+                        &dst_file_path_owned,
+                        &dst_volume_dir_owned,
+                        destination_was_absent,
+                        staged_rollback_path.as_ref(),
+                        &guard,
+                    )
+                {
+                    if no_inline {
+                        if let Some((_, new_data_path)) = has_data_dir_path_owned.as_ref() {
+                            remove_dir_all_if_exists(new_data_path)?;
+                        }
+                        return Ok(FusedCommitAttempt::CommitErr(to_file_error(err).into()));
+                    }
+                    return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
+                }
+
+                if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
+                    let _ = remove_file_if_exists(backup_path);
                 }
 
                 if let Some(parent) = src_file_path_owned.parent() {
                     let _ = std::fs::remove_dir(parent);
                 }
 
-                Ok(FusedCommitAttempt::Committed(RenameDataResp {
+                Ok(FusedCommitAttempt::Committed(Box::new(RenameDataResp {
                     multipart_source: None,
                     old_data_dir: None,
                     rollback_data_dir: None,
@@ -897,7 +1052,7 @@ impl LocalDisk {
                     old_current_size,
                     old_current_source_checked,
                     old_current_source,
-                }))
+                })))
             };
 
             let attempt_res = if let Some(admission) = file_sync_admission.as_ref() {
@@ -917,7 +1072,23 @@ impl LocalDisk {
                     if let Some(receipt_mutation) = receipt_mutation.as_mut() {
                         receipt_mutation.complete();
                     }
-                    return Ok(resp);
+                    return Ok(*resp);
+                }
+                Ok(FusedCommitAttempt::RestoreDataSource(directory_guard, err)) => {
+                    let commit = RenameCommitContext {
+                        directory_guard,
+                        publication_root: &self.publication_root,
+                        mutation_lease,
+                        source_metadata: &src_file_path,
+                        destination_metadata: &dst_file_path,
+                        source_volume_dir: &src_volume_dir,
+                        data_paths: has_data_dir_path.as_ref(),
+                    };
+                    commit.restore_published_data_source().await?;
+                    for part_path in &invalidate_part_paths {
+                        self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
+                    }
+                    return Err(err);
                 }
                 Ok(FusedCommitAttempt::CommitErr(err)) | Err(err) => {
                     for part_path in &invalidate_part_paths {
@@ -925,11 +1096,269 @@ impl LocalDisk {
                     }
                     return Err(err);
                 }
+                #[cfg(test)]
+                Ok(FusedCommitAttempt::ResumePanic(panic_payload)) => {
+                    std::panic::resume_unwind(panic_payload);
+                }
                 Ok(FusedCommitAttempt::Fallback(guard, dst_buf, restored_fi)) => {
-                    fi = restored_fi;
+                    fi = *restored_fi;
                     (guard, dst_buf)
                 }
             }
+        } else if let Some(publication) = multipart_publication.as_ref()
+            && no_inline
+            && !preserve_upload_metadata
+            && !publication.legacy
+            && !fi_healing
+            && !os::dst_dir_fsync_group_commit_enabled()
+        {
+            enum MultipartFusedCommitOutcome {
+                Committed(Box<RenameDataResp>),
+                AbortBeforeDataRename(DiskError),
+                RestoreDataSource(os::RenameCommitGuard, DiskError),
+                RestoreTargetAfterError(DiskError, bool),
+            }
+            let publication_owned = Arc::clone(publication);
+            let src_parent_owned = src_file_parent.to_path_buf();
+            let dst_parent_owned = dst_file_parent.to_path_buf();
+            let dst_file_path_owned = dst_file_path.clone();
+            let src_file_path_owned = src_file_path.clone();
+            let dst_volume_dir_owned = dst_volume_dir.clone();
+            let publication_root_owned = self.publication_root.clone();
+            let has_data_dir_path_owned = has_data_dir_path.clone();
+            let sync_data_shards = durability.syncs_data_shards();
+            let sync_commit_metadata = durability.syncs_commit_metadata();
+            let staging_cleanup = multipart_staging.as_ref().map(|s| s.cleanup_handle());
+            let dst_volume_str = dst_volume.to_string();
+            let dst_path_str = dst_path.to_string();
+            let fi_moved = fi;
+            #[cfg(test)]
+            let root_owned = self.root.clone();
+
+            let outcome = os::run_blocking_namespace_operation_with_sync(
+                mutation_lease.clone(),
+                true,
+                move || -> std::io::Result<MultipartFusedCommitOutcome> {
+                    let (directory_guard_owned, has_dst_buf_owned) = match prepare_rename_commit_and_destination_metadata_std(
+                        false,
+                        &src_parent_owned,
+                        &dst_parent_owned,
+                        &dst_file_path_owned,
+                        &dst_volume_dir_owned,
+                        &publication_root_owned,
+                    ) {
+                        Ok(res) => res,
+                        Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into())),
+                    };
+                    let mut xlmeta = FileMeta::new();
+                    let mut dst_meta_unparsable = false;
+                    if let Some(dst_buf) = has_dst_buf_owned.as_ref() {
+                        if FileMeta::is_xl2_v1_format(dst_buf)
+                            && let Ok(nmeta) = FileMeta::load(dst_buf)
+                        {
+                            xlmeta = nmeta;
+                        } else {
+                            dst_meta_unparsable = true;
+                        }
+                    }
+                    let skip_parent_owned = if has_dst_buf_owned.is_some() {
+                        dst_parent_owned.clone()
+                    } else {
+                        dst_volume_dir_owned.clone()
+                    };
+                    let version_id = fi_moved.version_id.unwrap_or_default();
+                    let (old_current_size, old_current_source_checked, old_current_source) = observe_old_current(
+                        has_dst_buf_owned.is_some(),
+                        dst_meta_unparsable,
+                        &xlmeta,
+                        &dst_volume_str,
+                        &dst_path_str,
+                        version_id,
+                    );
+                    let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id));
+                    if let Some(old_data_dir) = has_old_data_dir.as_ref() {
+                        let _ = xlmeta.data.remove_two(version_id, *old_data_dir);
+                    }
+                    if let Err(err) = xlmeta.add_version(fi_moved) {
+                        return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err.into()));
+                    }
+                    let version_signature = rename_data_versions_signature(&xlmeta);
+                    let new_dst_buf_owned = match xlmeta.marshal_msg() {
+                        Ok(buf) => buf,
+                        Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err.into())),
+                    };
+
+                    let prepared_metadata_source = match publication_owned.prepare_and_install_head_on_thread(
+                        &new_dst_buf_owned,
+                        has_dst_buf_owned.as_deref(),
+                        &dst_file_path_owned,
+                        &directory_guard_owned,
+                    ) {
+                        Ok(source) => source,
+                        Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err)),
+                    };
+                    #[cfg(test)]
+                    let probe_active = os::file_sync_probe::is_active();
+                    #[cfg(not(test))]
+                    let probe_active = false;
+                    // When presynced, shard payloads were synced during UploadPart and
+                    // prepare_and_install_head_on_thread just fsynced src_data_path after
+                    // writing RETIREMENT_ANCHOR.
+                    if sync_data_shards
+                        && let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref()
+                        && (probe_active || !super::take_presynced_tmp_data_dir(src_data_path))
+                        && let Err(err) = os::sync_dir_files_std(src_data_path)
+                        && err.kind() != ErrorKind::NotFound
+                    {
+                        return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into()));
+                    }
+                    let directory_guard_owned = match super::remove_dst_base_before_commit_std(
+                        &dst_path_str,
+                        directory_guard_owned,
+                        &src_parent_owned,
+                        &dst_parent_owned,
+                        &dst_volume_dir_owned,
+                        &publication_root_owned,
+                    ) {
+                        Ok(guard) => guard,
+                        Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err)),
+                    };
+                    if should_remove_staged_meta_before_commit(&dst_path_str) {
+                        drop(prepared_metadata_source);
+                        if let Err(err) = std::fs::remove_file(&src_file_path_owned) {
+                            return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into()));
+                        }
+                        return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::FileNotFound));
+                    }
+                    if let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
+                        let Some(src_parent) = src_data_path.parent() else {
+                            return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::InvalidPath));
+                        };
+                        let payload_guard = match os::multipart_payload_commit_guard_std(&directory_guard_owned, src_parent) {
+                            Ok(guard) => guard,
+                            Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into())),
+                        };
+                        if let Err(err) = os::rename_all_with_commit_guard_on_thread(src_data_path, dst_data_path, &payload_guard)
+                        {
+                            return Ok(MultipartFusedCommitOutcome::RestoreDataSource(directory_guard_owned, err));
+                        }
+                        #[cfg(test)]
+                        run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
+                        if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, &dst_path_str) {
+                            return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
+                        }
+                        if should_fail_before_old_metadata_backup(&dst_path_str) {
+                            return Ok(MultipartFusedCommitOutcome::RestoreDataSource(
+                                directory_guard_owned,
+                                DiskError::Unexpected,
+                            ));
+                        }
+                        if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, &dst_path_str) {
+                            return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
+                        }
+                    }
+                    let rename_res = if should_fail_commit_rename(&dst_path_str) {
+                        drop(prepared_metadata_source);
+                        Err(DiskError::other("test fail during metadata commit rename"))
+                    } else {
+                        os::rename_prepared_source_on_thread(
+                            prepared_metadata_source,
+                            &src_file_path_owned,
+                            &dst_file_path_owned,
+                            &skip_parent_owned,
+                            &publication_root_owned,
+                            &directory_guard_owned,
+                        )
+                    };
+                    if let Err(err) = rename_res {
+                        return Ok(MultipartFusedCommitOutcome::RestoreDataSource(directory_guard_owned, err));
+                    }
+                    if should_fail_after_metadata_commit(&dst_path_str) {
+                        return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(DiskError::Unexpected, false));
+                    }
+                    if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, &dst_path_str) {
+                        return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
+                    }
+                    if sync_commit_metadata {
+                        if let Some(parent) = dst_file_path_owned.parent()
+                            && let Err(err) = os::fsync_commit_directory_limited_std(parent, &directory_guard_owned)
+                        {
+                            return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(to_file_error(err).into(), true));
+                        }
+                        if has_dst_buf_owned.is_none() {
+                            let mut ancestor = dst_file_path_owned.parent().and_then(|object_dir| object_dir.parent());
+                            while let Some(dir) = ancestor {
+                                if !dir.starts_with(&dst_volume_dir_owned) {
+                                    break;
+                                }
+                                if let Err(err) = os::fsync_dir_limited_std(dir) {
+                                    return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(
+                                        to_file_error(err).into(),
+                                        true,
+                                    ));
+                                }
+                                if dir == dst_volume_dir_owned.as_path() {
+                                    break;
+                                }
+                                ancestor = dir.parent();
+                            }
+                        }
+                    }
+                    match publication_owned.finish_on_thread(sync_commit_metadata, staging_cleanup) {
+                        Ok(result) => Ok(MultipartFusedCommitOutcome::Committed(Box::new(RenameDataResp {
+                            multipart_source: Some(Box::new(result)),
+                            old_data_dir: has_old_data_dir,
+                            rollback_data_dir: None,
+                            cleanup_data_dir: has_old_data_dir,
+                            sign: version_signature,
+                            old_current_size,
+                            old_current_source_checked,
+                            old_current_source,
+                        }))),
+                        Err(err) => Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(err, false)),
+                    }
+                },
+            )
+            .await
+            .map_err(to_file_error)
+            .map_err(DiskError::from)?;
+
+            match outcome {
+                MultipartFusedCommitOutcome::Committed(resp) => {
+                    for part_path in &invalidate_part_paths {
+                        self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
+                    }
+                    return Ok(*resp);
+                }
+                MultipartFusedCommitOutcome::AbortBeforeDataRename(err) => return Err(err),
+                MultipartFusedCommitOutcome::RestoreDataSource(directory_guard, err) => {
+                    let commit = RenameCommitContext {
+                        directory_guard,
+                        publication_root: &self.publication_root,
+                        mutation_lease,
+                        source_metadata: &src_file_path,
+                        destination_metadata: &dst_file_path,
+                        source_volume_dir: &src_volume_dir,
+                        data_paths: has_data_dir_path.as_ref(),
+                    };
+                    commit.restore_published_data_source().await?;
+                    return Err(err);
+                }
+                MultipartFusedCommitOutcome::RestoreTargetAfterError(err, invalidate_before_return) => {
+                    if !invalidate_before_return {
+                        for part_path in &invalidate_part_paths {
+                            self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
+                        }
+                    }
+                    pin_multipart_future(|| publication.restore_target_after_error(mutation_lease.clone())).await?;
+                    if invalidate_before_return {
+                        for part_path in &invalidate_part_paths {
+                            self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
+                        }
+                    }
+                    return Err(err);
+                }
+            }
         } else {
             prepare_rename_commit_and_destination_metadata(
                 !no_inline && !preserve_upload_metadata,
@@ -1053,71 +1482,81 @@ impl LocalDisk {
             // after the commit rename, and a crash before the rename means the
             // PUT was never acknowledged. A metadata commit: relaxed tiers
             // leave it to the page cache.
-            if let Some(publication) = multipart_publication.as_ref() {
+            #[cfg(all(test, target_os = "linux"))]
-                pin_multipart_future(|| publication.prepare_metadata(&new_dst_buf, has_dst_buf.as_deref())).await?;
+            let probe_active = os::file_sync_probe::is_active();
-            }
+            #[cfg(all(not(test), target_os = "linux"))]
-            let tmp_meta_sync = if durability.syncs_commit_metadata() || multipart_publication.is_some() {
+            let probe_active = false;
-                SyncMode::FileOnly
-            } else {
-                SyncMode::None
-            };
-            // The tmp xl.meta write and the shard-file fdatasync are independent
-            // (disjoint paths) and both only need to be durable before the commit
-            // renames below, so run them concurrently to drop a blocking
-            // round-trip from the PUT commit critical path (rustfs/backlog#922
-            // step 2). The "contents durable -> rename -> dst dir fsync" ordering
-            // is unchanged — both futures complete before any rename — which the
-            // rename_data crash-consistency harness (backlog#935) exercises.
-            //
-            // Shard durability: once rename_data succeeds the write is
-            // acknowledged, so data must not live only in the page cache.
-            // Multipart parts were already synced during rename_part, so their
-            // fdatasync here is a cheap no-op. A missing source dir is left for the
-            // rename below to report through the existing rollback path. Payload
-            // durability is kept by both strict and relaxed.
-            let tmp_meta_write = {
-                let src_file_path = multipart_publication
-                    .as_ref()
-                    .map_or_else(|| commit.source_metadata.to_path_buf(), |publication| publication.prepared_path.clone());
-                let dst_file_path = dst_file_path.clone();
-                let owned_directory_guard = commit.directory_guard.clone();
-                let owned_mutation_lease = commit.mutation_lease.clone();
-                async move {
-                    os::run_blocking_namespace_operation(owned_mutation_lease, move || {
-                        #[cfg(test)]
-                        run_owned_file_before_open(&src_file_path);
-                        let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard(
-                            &src_file_path,
-                            &dst_file_path,
-                            &owned_directory_guard,
-                        )?;
-                        prepared_metadata_source.write_all(&new_dst_buf, tmp_meta_sync != SyncMode::None)?;
-                        Ok(prepared_metadata_source)
-                    })
-                    .await
-                    .map_err(to_file_error)
-                    .map_err(DiskError::from)
-                }
-            };
             let shard_sync = async {
                 if durability.syncs_data_shards()
                     && let Some((src_data_path, _)) = has_data_dir_path.as_ref()
-                    && let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await
-                    && err.kind() != ErrorKind::NotFound
                 {
-                    return Err::<(), DiskError>(to_file_error(err).into());
+                    #[cfg(target_os = "linux")]
+                    if !probe_active && super::take_presynced_tmp_data_dir(src_data_path) {
+                        if multipart_publication.is_none()
+                            && let Err(err) = os::fsync_dir_with_owner(src_data_path, Some(commit.mutation_lease.clone())).await
+                            && err.kind() != ErrorKind::NotFound
+                        {
+                            return Err::<(), DiskError>(to_file_error(err).into());
+                        }
+                        return Ok(());
+                    }
+                    if let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await
+                        && err.kind() != ErrorKind::NotFound
+                    {
+                        return Err::<(), DiskError>(to_file_error(err).into());
+                    }
                 }
                 Ok(())
             };
-            let (tmp_meta_res, shard_sync_res) = tokio::join!(tmp_meta_write, shard_sync);
+            let prepared_metadata_source = if let Some(publication) = multipart_publication.as_ref() {
-            // Surface a tmp-meta failure first (its prior serial position), then a
+                let dst_file_path_ref = &dst_file_path;
-            // shard-sync failure; either aborts before any rename, exactly as the
+                let dir_guard_ref = &commit.directory_guard;
-            // sequential version did.
+                let lease_clone = commit.mutation_lease.clone();
-            let prepared_metadata_source = tmp_meta_res?;
+                let prepare_fut = pin_multipart_future(|| {
-            shard_sync_res?;
+                    publication.prepare_and_install_head(
-            let prepared_metadata_source = match multipart_publication.as_ref() {
+                        &new_dst_buf,
-                Some(publication) => pin_multipart_future(|| publication.install_head(prepared_metadata_source)).await?,
+                        has_dst_buf.as_deref(),
-                None => prepared_metadata_source,
+                        dst_file_path_ref,
+                        dir_guard_ref,
+                        lease_clone,
+                    )
+                });
+                let (tmp_meta_res, shard_sync_res) = tokio::join!(prepare_fut, shard_sync);
+                let prepared_metadata_source = tmp_meta_res?;
+                shard_sync_res?;
+                prepared_metadata_source
+            } else {
+                let tmp_meta_sync = if durability.syncs_commit_metadata() {
+                    SyncMode::FileOnly
+                } else {
+                    SyncMode::None
+                };
+                let tmp_meta_write = {
+                    let src_file_path = commit.source_metadata.to_path_buf();
+                    let dst_file_path = dst_file_path.clone();
+                    let owned_directory_guard = commit.directory_guard.clone();
+                    let owned_mutation_lease = commit.mutation_lease.clone();
+                    async move {
+                        os::run_blocking_namespace_operation(owned_mutation_lease, move || {
+                            #[cfg(test)]
+                            run_owned_file_before_open(&src_file_path);
+                            let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard(
+                                &src_file_path,
+                                &dst_file_path,
+                                &owned_directory_guard,
+                            )?;
+                            prepared_metadata_source.write_all(&new_dst_buf, tmp_meta_sync != SyncMode::None)?;
+                            Ok(prepared_metadata_source)
+                        })
+                        .await
+                        .map_err(to_file_error)
+                        .map_err(DiskError::from)
+                    }
+                };
+                let (tmp_meta_res, shard_sync_res) = tokio::join!(tmp_meta_write, shard_sync);
+                let prepared_metadata_source = tmp_meta_res?;
+                shard_sync_res?;
+                prepared_metadata_source
             };
             commit.directory_guard = remove_dst_base_before_commit(
                 dst_path,
@@ -1158,12 +1597,13 @@ impl LocalDisk {
                 );
             }
             if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref() {
-                #[cfg(target_os = "linux")]
+                #[cfg(any(target_os = "linux", windows))]
                 let payload_guard = if multipart_publication.is_some() {
                     Some(
                         os::multipart_payload_commit_guard(
                             &commit.directory_guard,
                             src_data_path.parent().ok_or(DiskError::InvalidPath)?,
+                            commit.publication_root,
                             commit.mutation_lease.clone(),
                         )
                         .await
@@ -1172,9 +1612,9 @@ impl LocalDisk {
                 } else {
                     None
                 };
-                #[cfg(target_os = "linux")]
+                #[cfg(any(target_os = "linux", windows))]
                 let directory_guard = payload_guard.as_ref().unwrap_or(&commit.directory_guard);
-                #[cfg(not(target_os = "linux"))]
+                #[cfg(not(any(target_os = "linux", windows)))]
                 let directory_guard = &commit.directory_guard;
                 if let Err(err) = os::rename_all_with_commit_guard(
                     src_data_path,
@@ -1457,13 +1897,22 @@ impl LocalDisk {
             }
 
             let multipart_source = match multipart_publication.as_ref() {
-                Some(publication) => match pin_multipart_future(|| publication.finish(commit.mutation_lease.clone())).await {
+                Some(publication) => {
-                    Ok(result) => Some(Box::new(result)),
+                    let target_already_synced = durability.syncs_commit_metadata();
-                    Err(error) => {
+                    let staging_cleanup = multipart_staging.as_ref().map(|s| s.cleanup_handle());
-                        pin_multipart_future(|| publication.restore_target_after_error(commit.mutation_lease.clone())).await?;
+                    match pin_multipart_future(|| {
-                        return Err(error);
+                        publication.finish(commit.mutation_lease.clone(), target_already_synced, staging_cleanup)
+                    })
+                    .await
+                    {
+                        Ok(result) => Some(Box::new(result)),
+                        Err(error) => {
+                            pin_multipart_future(|| publication.restore_target_after_error(commit.mutation_lease.clone()))
+                                .await?;
+                            return Err(error);
+                        }
                     }
-                },
+                }
                 None => None,
             };
             Ok(RenameDataResp {
@@ -1663,13 +2112,8 @@ impl LocalDisk {
                     #[cfg(not(windows))]
                     let backup_path = {
                         let _backup = os::StageTimer::new(native_timing, "pool_meta_native_backup_create");
-                        create_local_inline_rollback_backup(&dst, &src, old_metadata)?
+                        create_local_inline_rollback_backup(&dst, &src, old_metadata, sync)?
                     };
-                    #[cfg(not(windows))]
-                    if sync {
-                        let _backup_sync = os::StageTimer::new(native_timing, "pool_meta_native_backup_file_sync");
-                        std::fs::File::open(&backup_path)?.sync_data()?;
-                    }
                     staged_rollback_path = Some(StagedRollbackBackup(backup_path));
                 }
 

crates/ecstore/src/disk/local/multipart_source_abort.rs

@@ -421,6 +421,7 @@ impl LocalDisk {
                     return Err(DiskError::ConditionalDeleteConflict);
                 }
                 let permit = AbortDeletePermit::new(&store, &expected, certificate.clone())?;
+                super::multipart_source_state::invalidate_active_multipart_source_head(&store.source);
                 let destination = self.io_get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, &Uuid::new_v4().to_string())?;
                 let source = store.source.clone();
                 let base = store.base.clone();
@@ -618,6 +619,9 @@ pub(super) struct OrdinaryMultipartDeleteCheck {
     root: os::PublicationRoot,
 }
 impl OrdinaryMultipartDeleteCheck {
+    pub(super) fn source(&self) -> &Path {
+        &self.source
+    }
     pub(super) fn validate(&self) -> Result<()> {
         let pinned = match os::pin_multipart_source_directory(&self.source, &self.base, &self.root) {
             Ok(pinned) => pinned,

crates/ecstore/src/disk/local/multipart_source_state.rs

@@ -117,6 +117,7 @@ pub(super) struct LoadedSource {
 }
 
 pub(super) struct SourceStore {
+    multipart_index: Arc<super::multipart_index::IndexOwner>,
     pub disk_id: Uuid,
     pub physical_slot: usize,
     pub disk_ids: Vec<Uuid>,
@@ -186,6 +187,7 @@ impl LocalDisk {
         let logical = self.get_object_path(RUSTFS_META_MULTIPART_BUCKET, path)?;
         let lease = os::acquire_namespace_mutation_lease_with_owner(&logical, owner.clone()).await;
         Ok(SourceStore {
+            multipart_index: self.multipart_index.clone(),
             disk_id,
             physical_slot,
             disk_ids,
@@ -238,9 +240,177 @@ pub(super) fn read_optional_in_pinned(
         Err(error) => Err(error.into()),
     }
 }
+#[derive(Clone, Copy, PartialEq, Eq)]
+struct FileStamp {
+    len: u64,
+    modified: Option<std::time::SystemTime>,
+    #[cfg(unix)]
+    ino: u64,
+}
+
+impl FileStamp {
+    fn from_path(path: &Path) -> Option<Self> {
+        let meta = std::fs::symlink_metadata(path).ok()?;
+        Some(Self {
+            len: meta.len(),
+            modified: meta.modified().ok(),
+            #[cfg(unix)]
+            ino: {
+                use std::os::unix::fs::MetadataExt;
+                meta.ino()
+            },
+        })
+    }
+}
+
+#[derive(Clone)]
+struct ActiveMultipartSourceEntry {
+    raw: Bytes,
+    fi: FileInfo,
+    head_stamp: FileStamp,
+    unsettled_parts: std::collections::HashSet<usize>,
+    parts: std::collections::HashMap<usize, (FileStamp, rustfs_filemeta::ObjectPartInfo)>,
+}
+
+const ACTIVE_MULTIPART_SOURCE_HEADS_PER_SHARD: usize = 64;
+
+static ACTIVE_MULTIPART_SOURCE_HEADS: std::sync::LazyLock<os::ShardedPathCache<ActiveMultipartSourceEntry>> =
+    std::sync::LazyLock::new(|| os::ShardedPathCache::new(ACTIVE_MULTIPART_SOURCE_HEADS_PER_SHARD));
+
+pub(super) fn put_active_multipart_source_head(source: impl AsRef<Path>, raw: Bytes, fi: FileInfo) {
+    let source = source.as_ref();
+    let Some(head_stamp) = FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) else {
+        return;
+    };
+    ACTIVE_MULTIPART_SOURCE_HEADS.insert(
+        source,
+        ActiveMultipartSourceEntry {
+            raw,
+            fi,
+            head_stamp,
+            unsettled_parts: std::collections::HashSet::new(),
+            parts: std::collections::HashMap::new(),
+        },
+    );
+}
+
+fn stamp_matches_active_disk(source: &Path, expected_head_stamp: FileStamp) -> bool {
+    FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) == Some(expected_head_stamp)
+        && !source.join(STATE).exists()
+        && !source.join(crate::disk::MULTIPART_RECOVERY_DIR).exists()
+        && !source.join(".mpu-completions").exists()
+}
+
+pub(super) fn get_active_multipart_source_head(source: &Path) -> Option<(Bytes, FileInfo)> {
+    let (raw, fi, head_stamp) =
+        ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| (entry.raw.clone(), entry.fi.clone(), entry.head_stamp))?;
+    if !stamp_matches_active_disk(source, head_stamp) {
+        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
+        return None;
+    }
+    Some((raw, fi))
+}
+
+pub(super) fn has_active_multipart_source_head(source: &Path) -> bool {
+    let Some(head_stamp) = ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| entry.head_stamp) else {
+        return false;
+    };
+    if !stamp_matches_active_disk(source, head_stamp) {
+        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
+        return false;
+    }
+    true
+}
+
+pub(super) fn has_clean_active_multipart_source(source: &Path) -> bool {
+    let Some((head_stamp, clean)) =
+        ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| (entry.head_stamp, entry.unsettled_parts.is_empty()))
+    else {
+        return false;
+    };
+    if !stamp_matches_active_disk(source, head_stamp) {
+        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
+        return false;
+    }
+    clean
+}
+
+pub(super) fn begin_active_multipart_part_transaction(source: &Path, part_number: usize) {
+    ACTIVE_MULTIPART_SOURCE_HEADS.with_get_mut(source, |entry| {
+        entry.unsettled_parts.insert(part_number);
+        entry.parts.remove(&part_number);
+    });
+}
+
+pub(super) fn record_active_multipart_part_committed(source: &Path, part: rustfs_filemeta::ObjectPartInfo) {
+    let Some(data_dir) = ACTIVE_MULTIPART_SOURCE_HEADS
+        .with_get(source, |entry| entry.fi.data_dir)
+        .flatten()
+        .filter(|dir| !dir.is_nil())
+    else {
+        return;
+    };
+    let part_meta_path = source.join(data_dir.to_string()).join(format!("part.{}.meta", part.number));
+    let Some(part_stamp) = FileStamp::from_path(&part_meta_path) else {
+        return;
+    };
+    ACTIVE_MULTIPART_SOURCE_HEADS.with_get_mut(source, |entry| {
+        entry.parts.insert(part.number, (part_stamp, part));
+    });
+}
+
+pub(super) fn finish_active_multipart_part_transaction(
+    source: &Path,
+    part_number: Option<usize>,
+    committed: bool,
+    retired_existing_txn: bool,
+) {
+    let Some(part_number) = part_number.filter(|_| committed) else {
+        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
+        return;
+    };
+    if !retired_existing_txn {
+        return;
+    }
+    ACTIVE_MULTIPART_SOURCE_HEADS.remove_if_mut(source, |entry| !entry.unsettled_parts.remove(&part_number));
+}
+
+pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize) -> Option<rustfs_filemeta::ObjectPartInfo> {
+    let (head_stamp, data_dir, expected_stamp, part) = ACTIVE_MULTIPART_SOURCE_HEADS
+        .with_get(source, |entry| {
+            if !entry.unsettled_parts.is_empty() {
+                return None;
+            }
+            let data_dir = entry.fi.data_dir.filter(|dir| !dir.is_nil())?;
+            let (expected_stamp, part) = entry.parts.get(&part_number)?;
+            Some((entry.head_stamp, data_dir, *expected_stamp, part.clone()))
+        })
+        .flatten()?;
+    if !stamp_matches_active_disk(source, head_stamp) {
+        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
+        return None;
+    }
+    let part_meta_path = source.join(data_dir.to_string()).join(format!("part.{part_number}.meta"));
+    if FileStamp::from_path(&part_meta_path) != Some(expected_stamp) {
+        return None;
+    }
+    Some(part)
+}
+
+pub(super) fn invalidate_active_multipart_source_head(source: &Path) {
+    ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
+}
+
+pub(super) fn invalidate_active_multipart_sources_under(prefix: &Path) {
+    ACTIVE_MULTIPART_SOURCE_HEADS.retain_outside_prefix(prefix);
+}
+
 /// Canonical source mutations are admitted under SourceRW at the syscall,
 /// not on the earlier async caller that may have timed out.
 pub(super) fn validate_source_mutation(source: &Path, base: &Path, root: &os::PublicationRoot, initial: bool) -> Result<()> {
+    if !initial && has_active_multipart_source_head(source) {
+        return Ok(());
+    }
     let pinned = match os::pin_multipart_source_directory(source, base, root) {
         Ok(pinned) => pinned,
         Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
@@ -440,6 +610,7 @@ impl SourceStore {
         terminal: Option<&Path>,
         sync_source_before: bool,
     ) -> std::io::Result<()> {
+        invalidate_active_multipart_source_head(source);
         #[cfg(unix)]
         let mut pinned = os::pin_multipart_source_directory(source, base, root)?;
         #[cfg(not(unix))]
@@ -470,7 +641,7 @@ impl SourceStore {
         let terminal = terminal_record_path(&source, state);
         let bytes = bytes.to_vec();
         os::run_blocking_namespace_operation_with_sync(self.lease.clone(), true, move || {
-            Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes)?;
+            Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes, false)?;
             Self::persist_state_std(&source, &base, &root, &raw, terminal.as_deref(), false)
         })
         .await
@@ -539,7 +710,7 @@ impl SourceStore {
         let root = self.publication_root.clone();
         let bytes = bytes.to_vec();
         os::run_blocking_namespace_operation(self.lease.clone(), move || {
-            Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes)
+            Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes, true)
         })
         .await
         .map_err(DiskError::from)
@@ -551,6 +722,7 @@ impl SourceStore {
         owner: Uuid,
         target: bool,
         bytes: &[u8],
+        sync_source_root: bool,
     ) -> std::io::Result<()> {
         let path = if target {
             target_backup_path(source, owner)
@@ -582,9 +754,13 @@ impl SourceStore {
         if !existed {
             os::replace_multipart_source_metadata(&path, bytes, base, root)?;
         }
-        os::sync_multipart_owned_record_parents(source, parent, existed, base, root)
+        os::sync_multipart_owned_record_parents_options(source, parent, existed, sync_source_root, base, root)
     }
     pub async fn replace_head(&self, bytes: &[u8]) -> Result<()> {
+        self.multipart_index
+            .record_head(&format!("{}/xl.meta", self.path), bytes)
+            .await?;
+        invalidate_active_multipart_source_head(&self.source);
         let path = self.source.join(STORAGE_FORMAT_FILE);
         let base = self.base.clone();
         let root = self.publication_root.clone();
@@ -1097,6 +1273,82 @@ pub(super) fn load_source(
     })
 }
 
+fn load_uninitialized_source(
+    raw: Bytes,
+    path: &str,
+    disk_id: Uuid,
+    physical_slot: usize,
+    disk_ids: &[Uuid],
+) -> Result<LoadedSource> {
+    let sha = raw_sha(&raw);
+    let length = raw.len() as u64;
+    let current = get_file_info(
+        &raw,
+        RUSTFS_META_MULTIPART_BUCKET,
+        path,
+        "",
+        FileInfoOpts {
+            data: false,
+            include_free_versions: false,
+            include_part_checksums: true,
+        },
+    )
+    .map_err(Error::other)?;
+    native_source_protocol(&current)?;
+    if source_status(&current)?.is_some()
+        || publication_identity(&current)?.is_some()
+        || SourceConditionV1::take(&mut current.clone())?.is_some()
+    {
+        return Err(DiskError::FileCorrupt);
+    }
+    let bucket = rustfs_utils::http::metadata_compat::get_consistent_str(&current.metadata, "multipart-bucket")
+        .ok_or(DiskError::FileCorrupt)?;
+    let object = rustfs_utils::http::metadata_compat::get_consistent_str(&current.metadata, "multipart-object")
+        .ok_or(DiskError::FileCorrupt)?;
+    let condition = capture_source_condition(bucket, object, path, &current, current.data_dir.ok_or(DiskError::FileCorrupt)?)?;
+    if condition.generation.epoch != 0 {
+        return Err(DiskError::FileCorrupt);
+    }
+    let slot = condition
+        .lineage
+        .geometry
+        .distribution
+        .iter()
+        .position(|index| *index == current.erasure.index)
+        .ok_or(DiskError::FileCorrupt)?;
+    if slot != physical_slot || condition.lineage.geometry.disks() != disk_ids.len() {
+        return Err(DiskError::ConditionalDeleteConflict);
+    }
+    let inventory = raw_sha(&encode(&(None::<[u8; 32]>, Some(sha), sha, None::<[u8; 32]>, None::<[u8; 32]>))?);
+    let snapshot = Some(MultipartSourceSnapshot {
+        disk_id,
+        physical_slot: slot,
+        lineage: condition.lineage,
+        generation: condition.generation,
+        revision: 0,
+        phase: MultipartSourcePhase::Active,
+        state_sha256: [0; 32],
+        head_sha256: Some(sha),
+        inventory_sha256: inventory,
+        capability_sha256: None,
+        publication: None,
+    });
+    Ok(LoadedSource {
+        root_missing: false,
+        state: None,
+        head: Some(SourceHead {
+            sha256: sha,
+            length,
+            raw: Some(raw.clone()),
+        }),
+        original: Some(raw),
+        info: Some(current),
+        snapshot,
+        head_matches_state: true,
+        head_matches_activation: false,
+    })
+}
+
 pub(super) const RETIREMENT_ANCHOR: &str = ".mpu-source-retirement";
 
 #[derive(Serialize, Deserialize)]
@@ -1265,7 +1517,7 @@ impl LocalDisk {
                     }
                     Err(error) => return Err(error),
                 };
-                let _directory = os::pin_multipart_source_directory(&directory, base, root)?;
+                let directory_pin = os::pin_multipart_source_directory(&directory, base, root)?;
                 if !parts.is_empty() {
                     let anchor_path = directory.join(RETIREMENT_ANCHOR);
                     let bytes = os::read_multipart_source_metadata(&anchor_path, base, root)?;
@@ -1298,7 +1550,13 @@ impl LocalDisk {
                 }
                 // An anchor-less empty DD is only rmdir'ed. A new child makes
                 // this fail closed; no recursive removal can consume its bytes.
+                // A Windows pin refuses DELETE sharing, so the leaf pin is
+                // released first; the object pin still holds the parent.
+                #[cfg(windows)]
+                drop(directory_pin);
                 std::fs::remove_dir(&directory)?;
+                #[cfg(not(windows))]
+                drop(directory_pin);
                 if sync {
                     os::fsync_dir_std(&object_path)?;
                 }
@@ -1323,41 +1581,6 @@ impl LocalDisk {
 }
 
 impl SourceStore {
-    /// Only native source validation may create provenance. Ordinary TMP/heal
-    /// publication can retain a valid old local anchor, never issue/rebind one.
-    pub async fn write_retirement_anchor(
-        &self,
-        state: &SourceState,
-        publication: &PublicationIdentity,
-        directory: &Path,
-        lease: Arc<os::NamespaceMutationLease>,
-    ) -> Result<()> {
-        if state.lineage != publication.lineage
-            || state.generation != publication.source_generation
-            || state.disk_id != self.disk_id
-            || state.physical_slot != self.physical_slot
-        {
-            return Err(DiskError::ConditionalDeleteConflict);
-        }
-        let record = NativeRetirementAnchor {
-            protocol_version: MULTIPART_SOURCE_PROTOCOL_VERSION,
-            disk_id: self.disk_id,
-            physical_slot: self.physical_slot,
-            publication: publication.clone(),
-            source_original_sha256: state.original_sha256,
-            source_validation_receipt_id: state.backup_owner,
-        };
-        let bytes = encode(&record)?;
-        let directory = directory.to_path_buf();
-        let base = self.base.clone();
-        let root = self.publication_root.clone();
-        os::run_blocking_namespace_operation(lease, move || {
-            let _parent = os::mkdir_all_below_existing_base_std(&directory, &base, &root)?;
-            os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &bytes, &base, &root)
-        })
-        .await
-        .map_err(DiskError::from)
-    }
     /// This caller already owns SourceRW and destination metadata ownership.
     /// An Aborting survivor can confirm a historical target only with its
     /// independent, same-disk publication anchor. Closure never creates one.
@@ -1456,6 +1679,8 @@ struct ValidatedPreparedPublication {
     length: u64,
 }
 
+type DeferredInitialPublication = (Vec<u8>, PathBuf, Vec<u8>);
+
 pub(super) struct NativeMultipartPublication {
     pub store: Arc<SourceStore>,
     pub publication: PublicationIdentity,
@@ -1463,22 +1688,49 @@ pub(super) struct NativeMultipartPublication {
     state: parking_lot::Mutex<SourceState>,
     capability: parking_lot::Mutex<Option<MultipartUndoCapability>>,
     validated_prepared: parking_lot::Mutex<Option<ValidatedPreparedPublication>>,
+    deferred_initial: parking_lot::Mutex<Option<DeferredInitialPublication>>,
     pub legacy: bool,
 }
 
 impl LocalDisk {
+    #[cfg(test)]
     pub(super) async fn prepare_native_multipart_publication(
         &self,
         path: &str,
         incoming: &mut FileInfo,
         staging: &MultipartCommitStaging,
+    ) -> Result<NativeMultipartPublication> {
+        self.prepare_native_multipart_publication_inner(path, incoming, staging, false)
+            .await
+    }
+
+    pub(super) async fn prepare_native_multipart_publication_deferred(
+        &self,
+        path: &str,
+        incoming: &mut FileInfo,
+        staging: &MultipartCommitStaging,
+    ) -> Result<NativeMultipartPublication> {
+        self.prepare_native_multipart_publication_inner(path, incoming, staging, true)
+            .await
+    }
+
+    async fn prepare_native_multipart_publication_inner(
+        &self,
+        path: &str,
+        incoming: &mut FileInfo,
+        staging: &MultipartCommitStaging,
+        defer_persist: bool,
     ) -> Result<NativeMultipartPublication> {
         // Real Complete is parts-backed, including a zero-byte last part.
         if incoming.data.is_some() || incoming.inline_data() {
             return Err(DiskError::UnsupportedDisk);
         }
         let store = Arc::new(self.multipart_store_from_staging(path, staging).await?);
-        let loaded = store.load().await?;
+        let loaded = if defer_persist && let Some(raw) = staging.fresh_source_head() {
+            load_uninitialized_source(raw, &store.path, store.disk_id, store.physical_slot, &store.disk_ids)?
+        } else {
+            store.load().await?
+        };
         let snapshot = loaded.snapshot.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
         let original = loaded
             .head
@@ -1569,7 +1821,44 @@ impl LocalDisk {
         };
         // Copy the small source FI to an independent immutable inode. Its
         // canonical inode is never truncated to build the destination proxy.
-        store.write_owned_backup_and_persist(owner, false, original, &state).await?;
+        let directory = staging.private_directory(publication.final_data_dir);
+        let anchor_record = NativeRetirementAnchor {
+            protocol_version: MULTIPART_SOURCE_PROTOCOL_VERSION,
+            disk_id: store.disk_id,
+            physical_slot: store.physical_slot,
+            publication: publication.clone(),
+            source_original_sha256: state.original_sha256,
+            source_validation_receipt_id: state.backup_owner,
+        };
+        let anchor_bytes = encode(&anchor_record)?;
+        state.validate(store.disk_id, &store.path)?;
+        let original_bytes = original.to_vec();
+        let deferred_initial = if defer_persist {
+            Some((original_bytes, directory, anchor_bytes))
+        } else {
+            let state_raw = encode(&state)?;
+            let source_dir = store.source.clone();
+            let base_dir = store.base.clone();
+            let pub_root = store.publication_root.clone();
+            let terminal = terminal_record_path(&source_dir, &state);
+            #[cfg(target_os = "linux")]
+            let can_mark_presynced = has_active_multipart_source_head(&source_dir)
+                && super::effective_durability(RUSTFS_META_MULTIPART_BUCKET).syncs_data_shards();
+            os::run_blocking_namespace_operation_with_sync(staging.lease.clone(), true, move || {
+                SourceStore::write_owned_backup_std(&source_dir, &base_dir, &pub_root, owner, false, &original_bytes, false)?;
+                SourceStore::persist_state_std(&source_dir, &base_dir, &pub_root, &state_raw, terminal.as_deref(), false)?;
+                let _parent = os::mkdir_all_below_existing_base_std(&directory, &base_dir, &pub_root)?;
+                os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, &base_dir, &pub_root)?;
+                #[cfg(target_os = "linux")]
+                if can_mark_presynced {
+                    super::mark_presynced_tmp_data_dir(directory);
+                }
+                Ok(())
+            })
+            .await
+            .map_err(DiskError::from)?;
+            None
+        };
         let prepared_path = store.source.join(format!(".mpu-prepared-{owner}"));
         let context = NativeMultipartPublication {
             store,
@@ -1578,14 +1867,9 @@ impl LocalDisk {
             state: parking_lot::Mutex::new(state),
             capability: parking_lot::Mutex::new(Some(capability)),
             validated_prepared: parking_lot::Mutex::new(None),
+            deferred_initial: parking_lot::Mutex::new(deferred_initial),
             legacy,
         };
-        let directory = staging.private_directory(context.publication.final_data_dir);
-        let reserved = context.state.lock().clone();
-        context
-            .store
-            .write_retirement_anchor(&reserved, &context.publication, &directory, staging.lease.clone())
-            .await?;
         Ok(context)
     }
 }
@@ -1594,18 +1878,15 @@ impl NativeMultipartPublication {
     pub fn canonical_head(&self) -> PathBuf {
         self.store.source.join(STORAGE_FORMAT_FILE)
     }
+    #[cfg(test)]
     pub async fn prepare_metadata(&self, new_metadata: &[u8], old_metadata: Option<&[u8]>) -> Result<()> {
         let mut state = self.state.lock().clone();
-        let current = self.store.load().await?;
-        let current_state = current.state.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
-        if encode(current_state)? != encode(&state)? || !current.head_matches_state {
-            return Err(DiskError::ConditionalDeleteConflict);
-        }
         // Validate the actual persisted representation once, after FileMeta's
         // canonicalization. No unvalidated input can install a source proxy.
         if !self.store.target_matches(&self.publication, new_metadata)? {
             return Err(DiskError::ConditionalDeleteConflict);
         }
+        let expected_state_raw = encode(&state)?;
         let validated = ValidatedPreparedPublication {
             publication: self.publication.clone(),
             sha256: raw_sha(new_metadata),
@@ -1618,17 +1899,298 @@ impl NativeMultipartPublication {
         state.prepared_sha256 = Some(validated.sha256);
         state.prepared_length = Some(validated.length);
         state.advance(MultipartSourcePhase::Prepared)?;
-        if let Some(previous) = old_metadata {
+        state.validate(self.store.disk_id, &self.store.path)?;
-            self.store
+        let next_state_raw = encode(&state)?;
-                .write_owned_backup_and_persist(state.backup_owner, true, previous, &state)
+        let source = self.store.source.clone();
-                .await?;
+        let base = self.store.base.clone();
-        } else {
+        let root = self.store.publication_root.clone();
-            self.store.persist(&state).await?;
+        let path = self.store.path.clone();
-        }
+        let disk_id = self.store.disk_id;
+        let physical_slot = self.store.physical_slot;
+        let disk_ids = self.store.disk_ids.clone();
+        let backup_owner = state.backup_owner;
+        let old_bytes = old_metadata.map(|b| b.to_vec());
+        let terminal = terminal_record_path(&source, &state);
+        os::run_blocking_namespace_operation_with_sync(self.store.lease.clone(), true, move || {
+            let current = load_source(&source, &base, &root, &path, disk_id, physical_slot, &disk_ids).map_err(IoError::other)?;
+            let current_state = current
+                .state
+                .as_ref()
+                .ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
+            let current_raw = encode(current_state).map_err(IoError::other)?;
+            if current_raw != expected_state_raw || !current.head_matches_state {
+                return Err(IoError::other(DiskError::ConditionalDeleteConflict));
+            }
+            if let Some(previous) = old_bytes.as_deref() {
+                SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
+            }
+            SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)
+        })
+        .await
+        .map_err(DiskError::from)?;
         *self.state.lock() = state;
         *self.validated_prepared.lock() = Some(validated);
         Ok(())
     }
+    #[cfg(target_os = "linux")]
+    pub(super) fn prepare_and_install_head_on_thread(
+        &self,
+        new_metadata: &[u8],
+        old_metadata: Option<&[u8]>,
+        dst_file_path: &Path,
+        directory_guard: &os::RenameCommitGuard,
+    ) -> Result<os::PreparedRenameSource> {
+        let mut state = self.state.lock().clone();
+        if !self.store.target_matches(&self.publication, new_metadata)? {
+            return Err(DiskError::ConditionalDeleteConflict);
+        }
+        let deferred_initial = self.deferred_initial.lock().take();
+        let expected_state_raw = encode(&state)?;
+        let expected_lineage = state.lineage.clone();
+        let expected_generation = state.generation;
+        let expected_revision = state.revision;
+        let validated = ValidatedPreparedPublication {
+            publication: self.publication.clone(),
+            sha256: raw_sha(new_metadata),
+            length: new_metadata.len() as u64,
+        };
+        if let Some(previous) = old_metadata {
+            state.previous_target_sha256 = Some(raw_sha(previous));
+            state.previous_target_length = Some(previous.len() as u64);
+        }
+        state.prepared_sha256 = Some(validated.sha256);
+        state.prepared_length = Some(validated.length);
+        state.advance(MultipartSourcePhase::Prepared)?;
+        state.validate(self.store.disk_id, &self.store.path)?;
+        let next_state_raw = encode(&state)?;
+        let source = &self.store.source;
+        let base = &self.store.base;
+        let root = &self.store.publication_root;
+        let path = &self.store.path;
+        let disk_id = self.store.disk_id;
+        let physical_slot = self.store.physical_slot;
+        let disk_ids = &self.store.disk_ids;
+        let backup_owner = state.backup_owner;
+        #[cfg(target_os = "linux")]
+        let can_mark_presynced = has_active_multipart_source_head(source)
+            && super::effective_durability(RUSTFS_META_MULTIPART_BUCKET).syncs_data_shards();
+        let terminal = terminal_record_path(source, &state);
+        let prepared_path = &self.prepared_path;
+        let canonical = self.canonical_head();
+        let prepared = (|| -> std::io::Result<os::PreparedRenameSource> {
+            if let Some((ref original_bytes, _, _)) = deferred_initial {
+                let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
+                let snapshot = current
+                    .snapshot
+                    .as_ref()
+                    .ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
+                let current_head = current
+                    .head
+                    .as_ref()
+                    .and_then(SourceHead::raw)
+                    .ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
+                if !current.head_matches_state
+                    || snapshot.phase != MultipartSourcePhase::Active
+                    || snapshot.capability_sha256.is_some()
+                    || snapshot.lineage != expected_lineage
+                    || snapshot.generation != expected_generation
+                    || snapshot.revision.checked_add(1) != Some(expected_revision)
+                    || current_head != original_bytes.as_slice()
+                {
+                    return Err(IoError::other(DiskError::ConditionalDeleteConflict));
+                }
+            } else {
+                let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
+                let current_state = current
+                    .state
+                    .as_ref()
+                    .ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
+                let current_raw = encode(current_state).map_err(IoError::other)?;
+                if current_raw != expected_state_raw || !current.head_matches_state {
+                    return Err(IoError::other(DiskError::ConditionalDeleteConflict));
+                }
+            }
+            #[cfg(test)]
+            super::run_owned_file_before_open(prepared_path);
+            let mut prepared_metadata_source =
+                os::create_prepared_rename_source_with_commit_guard(prepared_path, dst_file_path, directory_guard)?;
+            if let Err(err) = prepared_metadata_source.write_all(new_metadata, false) {
+                drop(prepared_metadata_source);
+                let _ = std::fs::remove_file(prepared_path);
+                return Err(err);
+            }
+            let setup_res: std::io::Result<()> = (|| {
+                if let Some((original_bytes, directory, anchor_bytes)) = deferred_initial {
+                    SourceStore::write_owned_backup_std(source, base, root, backup_owner, false, &original_bytes, false)?;
+                    if let Some(previous) = old_metadata {
+                        SourceStore::write_owned_backup_std(source, base, root, backup_owner, true, previous, false)?;
+                    }
+                    SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), false)?;
+                    let _parent = os::mkdir_all_below_existing_base_std(&directory, base, root)?;
+                    os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, base, root)?;
+                    #[cfg(target_os = "linux")]
+                    if can_mark_presynced {
+                        super::mark_presynced_tmp_data_dir(directory);
+                    }
+                } else {
+                    if let Some(previous) = old_metadata {
+                        SourceStore::write_owned_backup_std(source, base, root, backup_owner, true, previous, false)?;
+                    }
+                    SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), false)?;
+                }
+                Ok(())
+            })();
+            if let Err(err) = setup_res {
+                drop(prepared_metadata_source);
+                let _ = std::fs::remove_file(prepared_path);
+                return Err(err);
+            }
+            if let Err(err) = prepared_metadata_source.sync_data() {
+                drop(prepared_metadata_source);
+                let _ = std::fs::remove_file(prepared_path);
+                return Err(err);
+            }
+            os::install_multipart_prepared_head(prepared_metadata_source, &canonical, base, root)
+        })()
+        .map_err(DiskError::from)?;
+        *self.state.lock() = state;
+        *self.validated_prepared.lock() = Some(validated);
+        Ok(prepared)
+    }
+
+    #[cfg(not(target_os = "linux"))]
+    pub async fn prepare_and_install_head(
+        &self,
+        new_metadata: &[u8],
+        old_metadata: Option<&[u8]>,
+        dst_file_path: &Path,
+        directory_guard: &os::RenameCommitGuard,
+        mutation_lease: Arc<os::NamespaceMutationLease>,
+    ) -> Result<os::PreparedRenameSource> {
+        let mut state = self.state.lock().clone();
+        if !self.store.target_matches(&self.publication, new_metadata)? {
+            return Err(DiskError::ConditionalDeleteConflict);
+        }
+        let deferred_initial = self.deferred_initial.lock().take();
+        let expected_state_raw = encode(&state)?;
+        let expected_lineage = state.lineage.clone();
+        let expected_generation = state.generation;
+        let expected_revision = state.revision;
+        let validated = ValidatedPreparedPublication {
+            publication: self.publication.clone(),
+            sha256: raw_sha(new_metadata),
+            length: new_metadata.len() as u64,
+        };
+        if let Some(previous) = old_metadata {
+            state.previous_target_sha256 = Some(raw_sha(previous));
+            state.previous_target_length = Some(previous.len() as u64);
+        }
+        state.prepared_sha256 = Some(validated.sha256);
+        state.prepared_length = Some(validated.length);
+        state.advance(MultipartSourcePhase::Prepared)?;
+        state.validate(self.store.disk_id, &self.store.path)?;
+        let next_state_raw = encode(&state)?;
+        let source = self.store.source.clone();
+        let base = self.store.base.clone();
+        let root = self.store.publication_root.clone();
+        let path = self.store.path.clone();
+        let disk_id = self.store.disk_id;
+        let physical_slot = self.store.physical_slot;
+        let disk_ids = self.store.disk_ids.clone();
+        let backup_owner = state.backup_owner;
+        #[cfg(target_os = "linux")]
+        let can_mark_presynced = has_active_multipart_source_head(&source)
+            && super::effective_durability(RUSTFS_META_MULTIPART_BUCKET).syncs_data_shards();
+        let old_bytes = old_metadata.map(|b| b.to_vec());
+        let new_bytes = new_metadata.to_vec();
+        let terminal = terminal_record_path(&source, &state);
+        let prepared_path = self.prepared_path.clone();
+        let canonical = self.canonical_head();
+        let dst_file_path = dst_file_path.to_path_buf();
+        let owned_directory_guard = directory_guard.clone();
+        let prepared = os::run_blocking_namespace_operation_with_sync(mutation_lease, true, move || {
+            let current = load_source(&source, &base, &root, &path, disk_id, physical_slot, &disk_ids).map_err(IoError::other)?;
+            if let Some((original_bytes, directory, anchor_bytes)) = deferred_initial {
+                let snapshot = current
+                    .snapshot
+                    .as_ref()
+                    .ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
+                let current_head = current
+                    .head
+                    .as_ref()
+                    .and_then(SourceHead::raw)
+                    .ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
+                if !current.head_matches_state
+                    || snapshot.phase != MultipartSourcePhase::Active
+                    || snapshot.capability_sha256.is_some()
+                    || snapshot.lineage != expected_lineage
+                    || snapshot.generation != expected_generation
+                    || snapshot.revision.checked_add(1) != Some(expected_revision)
+                    || current_head != original_bytes.as_slice()
+                {
+                    return Err(IoError::other(DiskError::ConditionalDeleteConflict));
+                }
+                SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, false, &original_bytes, false)?;
+                if let Some(previous) = old_bytes.as_deref() {
+                    SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
+                }
+                SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)?;
+                let _parent = os::mkdir_all_below_existing_base_std(&directory, &base, &root)?;
+                os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, &base, &root)?;
+                #[cfg(target_os = "linux")]
+                if can_mark_presynced {
+                    super::mark_presynced_tmp_data_dir(directory);
+                }
+            } else {
+                let current_state = current
+                    .state
+                    .as_ref()
+                    .ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
+                let current_raw = encode(current_state).map_err(IoError::other)?;
+                if current_raw != expected_state_raw || !current.head_matches_state {
+                    return Err(IoError::other(DiskError::ConditionalDeleteConflict));
+                }
+                if let Some(previous) = old_bytes.as_deref() {
+                    SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
+                }
+                SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)?;
+            }
+            #[cfg(test)]
+            super::run_owned_file_before_open(&prepared_path);
+            let mut prepared_metadata_source =
+                os::create_prepared_rename_source_with_commit_guard(&prepared_path, &dst_file_path, &owned_directory_guard)?;
+            prepared_metadata_source.write_all(&new_bytes, true)?;
+            os::install_multipart_prepared_head(prepared_metadata_source, &canonical, &base, &root)
+        })
+        .await
+        .map_err(DiskError::from)?;
+        *self.state.lock() = state;
+        *self.validated_prepared.lock() = Some(validated);
+        Ok(prepared)
+    }
+
+    #[cfg(target_os = "linux")]
+    pub async fn prepare_and_install_head(
+        self: &Arc<Self>,
+        new_metadata: &[u8],
+        old_metadata: Option<&[u8]>,
+        dst_file_path: &Path,
+        directory_guard: &os::RenameCommitGuard,
+        mutation_lease: Arc<os::NamespaceMutationLease>,
+    ) -> Result<os::PreparedRenameSource> {
+        let this = Arc::clone(self);
+        let new_bytes = new_metadata.to_vec();
+        let old_bytes = old_metadata.map(|b| b.to_vec());
+        let dst_file_path = dst_file_path.to_path_buf();
+        let owned_directory_guard = directory_guard.clone();
+        os::run_blocking_namespace_operation_with_sync(mutation_lease, true, move || {
+            this.prepare_and_install_head_on_thread(&new_bytes, old_bytes.as_deref(), &dst_file_path, &owned_directory_guard)
+                .map_err(IoError::other)
+        })
+        .await
+        .map_err(DiskError::from)
+    }
+    #[cfg(test)]
     pub async fn install_head(&self, prepared: os::PreparedRenameSource) -> Result<os::PreparedRenameSource> {
         {
             let guard = self.validated_prepared.lock();
@@ -1699,13 +2261,13 @@ impl NativeMultipartPublication {
         // return an error after consuming the last claimable source heads.
         self.store.restore_exact_target_with_lease(&state, lease).await
     }
-    pub async fn finish(&self, target_lease: Arc<os::NamespaceMutationLease>) -> Result<MultipartPublicationResult> {
+    pub(super) fn finish_on_thread(
+        &self,
+        target_already_synced: bool,
+        staging_cleanup: Option<(PathBuf, Arc<std::sync::atomic::AtomicBool>)>,
+    ) -> Result<MultipartPublicationResult> {
         let mut state = self.state.lock().clone();
-        let current = self.store.load().await?;
+        let expected_state_raw = encode(&state)?;
-        let current_state = current.state.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
-        if current.head.is_some() || encode(current_state)? != encode(&state)? {
-            return Err(DiskError::ConditionalDeleteConflict);
-        }
         let (expected_sha256, expected_length) = {
             let guard = self.validated_prepared.lock();
             let validated = guard.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
@@ -1717,16 +2279,88 @@ impl NativeMultipartPublication {
             }
             (validated.sha256, validated.length)
         };
-        self.store
+        let target_dir = self.store.target_path(&self.publication)?;
-            .read_and_sync_target_with_lease(&self.publication, target_lease, move |target| {
-                if target.len() as u64 != expected_length || raw_sha(target) != expected_sha256 {
-                    return Err(DiskError::ConditionalDeleteConflict);
-                }
-                Ok(())
-            })
-            .await?;
         state.advance(MultipartSourcePhase::ConsumedUnsealed)?;
-        let snapshot = self.store.persist_with_source_sync(&state, true).await?;
+        state.validate(self.store.disk_id, &self.store.path)?;
+        let next_state_raw = encode(&state)?;
+        let snapshot = snapshot_for_state(self.store.disk_id, &state, &next_state_raw, None)?;
+        let source = &self.store.source;
+        let base = &self.store.base;
+        let root = &self.store.publication_root;
+        let path = &self.store.path;
+        let disk_id = self.store.disk_id;
+        let physical_slot = self.store.physical_slot;
+        let disk_ids = &self.store.disk_ids;
+        let terminal = terminal_record_path(source, &state);
+        (|| -> std::io::Result<()> {
+            let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
+            let current_state = current
+                .state
+                .as_ref()
+                .ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
+            let current_raw = encode(current_state).map_err(IoError::other)?;
+            if current.head.is_some() || current_raw != expected_state_raw {
+                return Err(IoError::other(DiskError::ConditionalDeleteConflict));
+            }
+            if target_already_synced {
+                let pinned = os::pin_multipart_source_directory(&target_dir, base, root).map_err(|error| {
+                    if error.kind() == std::io::ErrorKind::NotFound {
+                        IoError::other(DiskError::ConditionalDeleteConflict)
+                    } else {
+                        error
+                    }
+                })?;
+                let target = os::read_multipart_source_metadata_in_pinned(
+                    &pinned,
+                    &target_dir,
+                    std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
+                    base,
+                    root,
+                )
+                .map_err(|error| {
+                    if error.kind() == std::io::ErrorKind::NotFound {
+                        IoError::other(DiskError::ConditionalDeleteConflict)
+                    } else {
+                        error
+                    }
+                })?;
+                if target.len() as u64 != expected_length || raw_sha(&target) != expected_sha256 {
+                    return Err(IoError::other(DiskError::ConditionalDeleteConflict));
+                }
+            } else {
+                os::read_and_sync_multipart_target_metadata(
+                    &target_dir,
+                    std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
+                    base,
+                    root,
+                    |target| {
+                        if target.len() as u64 != expected_length || raw_sha(target) != expected_sha256 {
+                            return Err(IoError::other(DiskError::ConditionalDeleteConflict));
+                        }
+                        Ok(())
+                    },
+                )
+                .map_err(|error| {
+                    if error.kind() == std::io::ErrorKind::NotFound {
+                        IoError::other(DiskError::ConditionalDeleteConflict)
+                    } else {
+                        error
+                    }
+                })?;
+            }
+            SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), true)?;
+            if let Some((io_path, cleaned)) = staging_cleanup {
+                match std::fs::remove_dir(&io_path) {
+                    Ok(()) => cleaned.store(true, std::sync::atomic::Ordering::Release),
+                    Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
+                        cleaned.store(true, std::sync::atomic::Ordering::Release);
+                    }
+                    Err(_) => {}
+                }
+            }
+            Ok(())
+        })()
+        .map_err(DiskError::from)?;
         *self.state.lock() = state;
         let proof = self
             .store
@@ -1738,6 +2372,21 @@ impl NativeMultipartPublication {
             undo_capability: capability,
         })
     }
+
+    pub async fn finish(
+        self: &Arc<Self>,
+        target_lease: Arc<os::NamespaceMutationLease>,
+        target_already_synced: bool,
+        staging_cleanup: Option<(PathBuf, Arc<std::sync::atomic::AtomicBool>)>,
+    ) -> Result<MultipartPublicationResult> {
+        let this = Arc::clone(self);
+        os::run_blocking_namespace_operation_with_sync(target_lease, true, move || {
+            this.finish_on_thread(target_already_synced, staging_cleanup)
+                .map_err(IoError::other)
+        })
+        .await
+        .map_err(DiskError::from)
+    }
 }
 
 impl SourceStore {

crates/ecstore/src/disk/local/multipart_staging.rs

@@ -207,6 +207,12 @@ pub(super) struct MultipartSourceAdmission {
     initial: bool,
 }
 impl MultipartSourceAdmission {
+    pub(super) fn source(&self) -> &Path {
+        &self.source
+    }
+    pub(super) fn is_initial(&self) -> bool {
+        self.initial
+    }
     pub(super) fn precondition(&self) -> Arc<dyn Fn() -> std::io::Result<()> + Send + Sync> {
         let admission = self.clone();
         Arc::new(move || admission.validate())
@@ -250,6 +256,8 @@ pub(super) struct MultipartCommitStaging {
     logical_path: PathBuf,
     io_path: PathBuf,
     owner: Option<Arc<dyn Send + Sync>>,
+    cleaned: Arc<std::sync::atomic::AtomicBool>,
+    fresh_source_head: Option<bytes::Bytes>,
 }
 
 impl MultipartCommitStaging {
@@ -259,10 +267,19 @@ impl MultipartCommitStaging {
     pub(super) fn private_directory(&self, directory: Uuid) -> PathBuf {
         self.io_path.join(directory.to_string())
     }
+    pub(super) fn cleanup_handle(&self) -> (PathBuf, Arc<std::sync::atomic::AtomicBool>) {
+        (self.io_path.clone(), Arc::clone(&self.cleaned))
+    }
+    pub(super) fn fresh_source_head(&self) -> Option<bytes::Bytes> {
+        self.fresh_source_head.clone()
+    }
 }
 
 impl Drop for MultipartCommitStaging {
     fn drop(&mut self) {
+        if self.cleaned.load(std::sync::atomic::Ordering::Acquire) {
+            return;
+        }
         let logical_path = self.logical_path.clone();
         let io_path = self.io_path.clone();
         let owner = self.owner.clone();
@@ -285,6 +302,14 @@ impl Drop for MultipartCommitStaging {
 }
 
 impl LocalDisk {
+    pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
+        let root = upload_scope_root(upload_or_part_path, part_path);
+        let Ok(source) = self.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, root) else {
+            return false;
+        };
+        super::multipart_source_state::has_clean_active_multipart_source(&source)
+    }
+
     pub(super) fn multipart_source_admission(
         &self,
         volume: &str,
@@ -356,12 +381,14 @@ impl LocalDisk {
         let io_path = self.io_get_object_path(RUSTFS_META_TMP_BUCKET, &attempt)?;
         // The attempt root is exactly the namespace used by the tmp sweeper.
         let lease = os::acquire_namespace_mutation_lease_with_owner(&logical_path, owner.clone()).await;
-        let staging = MultipartCommitStaging {
+        let mut staging = MultipartCommitStaging {
             path: attempt,
             lease,
             logical_path,
             io_path,
             owner,
+            cleaned: Arc::new(std::sync::atomic::AtomicBool::new(false)),
+            fresh_source_head: None,
         };
         let canonical_metadata = self.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, &format!("{source_path}/xl.meta"))?;
         let source_object = self.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, source_path)?;
@@ -371,31 +398,37 @@ impl LocalDisk {
         let source_path = source_path.to_owned();
         let incoming = incoming.clone();
         let sync_directory = effective_durability(destination_volume).syncs_data_shards();
-        os::run_cancellable_blocking_namespace_operation(Arc::clone(&staging.lease), move |cancelled| {
+        let fresh_head = os::run_cancellable_blocking_namespace_operation(Arc::clone(&staging.lease), move |cancelled| {
             if cancelled() {
                 return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "multipart staging cancelled"));
             }
-            let raw = os::read_multipart_source_metadata(&canonical_metadata, &base, &publication_root).map_err(|error| {
+            let cached_head = super::multipart_source_state::get_active_multipart_source_head(&source_object);
-                // A retired source has no authority for this delayed completion.
+            let (raw, source, from_active_cache) = if let Some((raw, source)) = cached_head {
-                // Preserve confinement and other I/O failures as their real errors.
+                (raw, source, true)
-                if error.kind() == std::io::ErrorKind::NotFound {
+            } else {
-                    DiskError::FileNotFound.into()
+                let raw = os::read_multipart_source_metadata(&canonical_metadata, &base, &publication_root).map_err(|error| {
-                } else {
+                    // A retired source has no authority for this delayed completion.
-                    error
+                    // Preserve confinement and other I/O failures as their real errors.
-                }
+                    if error.kind() == std::io::ErrorKind::NotFound {
-            })?;
+                        DiskError::FileNotFound.into()
-            let source = get_file_info(
+                    } else {
-                &raw,
+                        error
-                RUSTFS_META_MULTIPART_BUCKET,
+                    }
-                &source_path,
+                })?;
-                "",
+                let source = get_file_info(
-                FileInfoOpts {
+                    &raw,
-                    data: false,
+                    RUSTFS_META_MULTIPART_BUCKET,
-                    include_free_versions: false,
+                    &source_path,
-                    include_part_checksums: true,
+                    "",
-                },
+                    FileInfoOpts {
-            )
+                        data: false,
-            .map_err(std::io::Error::other)?;
+                        include_free_versions: false,
+                        include_part_checksums: true,
+                    },
+                )
+                .map_err(std::io::Error::other)?;
+                (bytes::Bytes::from(raw), source, false)
+            };
             let source_dir = source
                 .data_dir
                 .filter(|dir| !dir.is_nil())
@@ -412,23 +445,31 @@ impl LocalDisk {
             let _attempt = os::mkdir_all_below_existing_base_std(&private_object, &base, &publication_root)?;
             if incoming.data.is_none() && !incoming.is_remote() {
                 let source_data_dir = source_object.join(source_dir.to_string());
-                let pinned_source_data = (!incoming.parts.is_empty())
+                let mut pinned_source_data = None;
-                    .then(|| os::pin_multipart_source_directory(&source_data_dir, &base, &publication_root))
-                    .transpose()?;
                 let mut names = Vec::with_capacity(incoming.parts.len().saturating_mul(3));
                 for part in &incoming.parts {
                     if cancelled() {
                         return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "multipart staging cancelled"));
                     }
                     let meta_name = format!("part.{}.meta", part.number);
-                    let bytes = os::read_multipart_source_metadata_in_pinned(
+                    let actual = if let Some(cached_part) =
-                        pinned_source_data.as_ref().expect("pinned source data directory"),
+                        super::multipart_source_state::get_active_multipart_source_part(&source_object, part.number)
-                        &source_data_dir,
+                    {
-                        std::ffi::OsStr::new(&meta_name),
+                        cached_part
-                        &base,
+                    } else {
-                        &publication_root,
+                        if pinned_source_data.is_none() {
-                    )?;
+                            pinned_source_data =
-                    let actual = rustfs_filemeta::ObjectPartInfo::unmarshal(&bytes).map_err(std::io::Error::other)?;
+                                Some(os::pin_multipart_source_directory(&source_data_dir, &base, &publication_root)?);
+                        }
+                        let bytes = os::read_multipart_source_metadata_in_pinned(
+                            pinned_source_data.as_ref().expect("pinned source data directory"),
+                            &source_data_dir,
+                            std::ffi::OsStr::new(&meta_name),
+                            &base,
+                            &publication_root,
+                        )?;
+                        rustfs_filemeta::ObjectPartInfo::unmarshal(&bytes).map_err(std::io::Error::other)?
+                    };
                     let normalized = i64::try_from(actual.size)
                         .map_err(std::io::Error::other)?
                         .max(actual.actual_size);
@@ -464,10 +505,15 @@ impl LocalDisk {
                     cancelled,
                 )?;
             }
-            Ok(())
+            let has_state = !from_active_cache
+                && (source_object.join(".mpu-source-state").exists()
+                    || source_object.join(".mpu-recovery").exists()
+                    || source_object.join(".mpu-completions").exists());
+            Ok((!has_state).then_some(raw))
         })
         .await
         .map_err(DiskError::from)?;
+        staging.fresh_source_head = fresh_head;
         Ok(staging)
     }
 }
@@ -1219,4 +1265,211 @@ mod tests {
             .expect("non-part empty hash cleanup must remain supported");
         assert!(!directory.exists(), "a per-upload safety guard must not strand empty hash directories");
     }
+
+    #[tokio::test]
+    async fn multipart_source_active_head_cache_populates_and_invalidates_across_part_and_delete_paths() {
+        use crate::disk::{DiskAPI as _, PartTransactionAction, ReadOptions};
+        let (_root, disk, completed, _) = fixture().await;
+        std::fs::create_dir_all(disk.io_get_bucket_path(RUSTFS_META_TMP_BUCKET).expect("TMP path")).expect("TMP bucket");
+        let upload = format!("{}/{}", "c".repeat(64), Uuid::new_v4());
+        let data_dir = Uuid::new_v4();
+        let mut init_fi = FileInfo::new(&upload, 2, 2);
+        init_fi.erasure.index = 1;
+        init_fi.data_dir = Some(data_dir);
+        init_fi.fresh = true;
+        init_fi.mod_time = Some(time::OffsetDateTime::now_utc());
+
+        disk.write_metadata("", RUSTFS_META_MULTIPART_BUCKET, &upload, init_fi.clone())
+            .await
+            .expect("fresh multipart upload metadata");
+        assert!(disk.has_clean_active_multipart_source(&upload, false));
+
+        let read_fi = disk
+            .read_version("", RUSTFS_META_MULTIPART_BUCKET, &upload, "", &ReadOptions::default())
+            .await
+            .expect("cached read_version");
+        assert_eq!(read_fi.data_dir, Some(data_dir));
+
+        let destination = format!("{upload}/{data_dir}/part.1");
+        let part_meta = bytes::Bytes::from(completed.parts[0].marshal_msg().expect("part meta"));
+        disk.write_all(
+            RUSTFS_META_TMP_BUCKET,
+            "cache-test-part/part.1",
+            bytes::Bytes::from_static(b"immutable shard"),
+        )
+        .await
+        .expect("stage tmp part");
+        disk.prepare_part_transaction(
+            RUSTFS_META_TMP_BUCKET,
+            "cache-test-part/part.1",
+            RUSTFS_META_MULTIPART_BUCKET,
+            &destination,
+            part_meta.clone(),
+        )
+        .await
+        .expect("prepare part transaction");
+        assert!(
+            !disk.has_clean_active_multipart_source(&upload, false),
+            "in-flight part transaction must mark active source as having unsettled part txn"
+        );
+        disk.rename_part(
+            RUSTFS_META_TMP_BUCKET,
+            "cache-test-part/part.1",
+            RUSTFS_META_MULTIPART_BUCKET,
+            &destination,
+            part_meta,
+        )
+        .await
+        .expect("rename part");
+        disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &destination, PartTransactionAction::Commit)
+            .await
+            .expect("settle part transaction");
+        assert!(
+            disk.has_clean_active_multipart_source(&upload, false),
+            "settled commit must restore clean active multipart source status"
+        );
+        let read_parts = disk
+            .read_parts(RUSTFS_META_MULTIPART_BUCKET, &[format!("{destination}.meta")])
+            .await
+            .expect("cached read_parts");
+        assert_eq!(read_parts.len(), 1);
+        assert_eq!(read_parts[0].number, 1);
+
+        // Prepare part 2, then replay Commit for already-settled part 1: part 2's
+        // pending transaction must keep the upload marked as having unsettled part txns.
+        let destination2 = format!("{upload}/{data_dir}/part.2");
+        let mut part2_info = completed.parts[0].clone();
+        part2_info.number = 2;
+        let part2_meta = bytes::Bytes::from(part2_info.marshal_msg().expect("part 2 meta"));
+        disk.write_all(
+            RUSTFS_META_TMP_BUCKET,
+            "cache-test-part/part.2",
+            bytes::Bytes::from_static(b"immutable shard"),
+        )
+        .await
+        .expect("stage tmp part 2");
+        disk.prepare_part_transaction(
+            RUSTFS_META_TMP_BUCKET,
+            "cache-test-part/part.2",
+            RUSTFS_META_MULTIPART_BUCKET,
+            &destination2,
+            part2_meta.clone(),
+        )
+        .await
+        .expect("prepare part 2 transaction");
+        assert!(!disk.has_clean_active_multipart_source(&upload, false));
+        disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &destination, PartTransactionAction::Commit)
+            .await
+            .expect("replayed part 1 commit settlement");
+        assert!(
+            !disk.has_clean_active_multipart_source(&upload, false),
+            "replayed settlement for part 1 must not clear part 2's pending transaction"
+        );
+        disk.rename_part(
+            RUSTFS_META_TMP_BUCKET,
+            "cache-test-part/part.2",
+            RUSTFS_META_MULTIPART_BUCKET,
+            &destination2,
+            part2_meta,
+        )
+        .await
+        .expect("rename part 2");
+        disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &destination2, PartTransactionAction::Commit)
+            .await
+            .expect("settle part 2 transaction");
+        assert!(disk.has_clean_active_multipart_source(&upload, false));
+
+        // Deleting {upload}/xl.meta through delete_paths must invalidate the canonical {upload} cache key.
+        disk.delete_paths(RUSTFS_META_MULTIPART_BUCKET, &[format!("{upload}/xl.meta")])
+            .await
+            .expect("delete xl.meta via delete_paths");
+        assert!(
+            !disk.has_clean_active_multipart_source(&upload, false),
+            "per-file head deletion must invalidate the canonical upload cache key"
+        );
+        assert!(matches!(
+            disk.read_version("", RUSTFS_META_MULTIPART_BUCKET, &upload, "", &ReadOptions::default())
+                .await,
+            Err(DiskError::FileNotFound)
+        ));
+    }
+
+    #[cfg(not(windows))]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn multipart_source_cache_publication_holds_source_lock_against_concurrent_retirement() {
+        use crate::disk::{DiskAPI as _, disk_store::LocalDiskWrapper, os::prepared_publication_test_hooks as hooks};
+
+        let (_root, disk, _, _) = fixture().await;
+        std::fs::create_dir_all(disk.io_get_bucket_path(RUSTFS_META_TMP_BUCKET).expect("TMP path")).expect("TMP bucket");
+        let disk = Arc::new(disk);
+        let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false);
+        let upload = format!("{}/{}", "d".repeat(64), Uuid::new_v4());
+        let data_dir = Uuid::new_v4();
+        let mut init_fi = FileInfo::new(&upload, 2, 2);
+        init_fi.erasure.index = 1;
+        init_fi.data_dir = Some(data_dir);
+        init_fi.fresh = true;
+        init_fi.mod_time = Some(time::OffsetDateTime::now_utc());
+
+        let source_dir = disk
+            .io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, &upload)
+            .expect("source dir");
+        let base_dir = disk
+            .io_get_bucket_path(RUSTFS_META_MULTIPART_BUCKET)
+            .expect("multipart bucket path");
+        let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
+        let (release_tx, release_rx) = std::sync::mpsc::channel();
+        let _hook = hooks::install_at(hooks::Stage::DirFsync, &source_dir, move || {
+            let _ = entered_tx.send(());
+            let _ = release_rx.recv();
+        });
+
+        let init_wrapper = wrapper.clone();
+        let init_upload = upload.clone();
+        let init_task = tokio::spawn(async move {
+            init_wrapper
+                .write_metadata("", RUSTFS_META_MULTIPART_BUCKET, &init_upload, init_fi)
+                .await
+        });
+
+        tokio::time::timeout(std::time::Duration::from_secs(5), entered_rx)
+            .await
+            .expect("initial write_metadata reaches parent dir fsync")
+            .expect("entered channel");
+
+        let retire_wrapper = wrapper.clone();
+        let retire_path = format!("{upload}/xl.meta");
+        let mut retire_fut = Box::pin(async move {
+            retire_wrapper
+                .delete_paths(RUSTFS_META_MULTIPART_BUCKET, &[retire_path])
+                .await
+        });
+        assert!(
+            futures::poll!(tokio::task::unconstrained(retire_fut.as_mut())).is_pending(),
+            "concurrent retirement must block on SourceRW while initial head publication is in flight"
+        );
+
+        let _ = release_tx.send(());
+        init_task.await.expect("init join").expect("initial write_metadata succeeds");
+        retire_fut
+            .await
+            .expect("concurrent retirement completes after SourceRW release");
+
+        assert!(
+            !super::super::multipart_source_state::has_active_multipart_source_head(&source_dir),
+            "retired source must not remain active in cache after interleaved initial publication"
+        );
+        assert!(
+            matches!(
+                super::super::multipart_source_state::validate_source_mutation(
+                    &source_dir,
+                    &base_dir,
+                    &disk.publication_root,
+                    false
+                ),
+                Err(DiskError::FileNotFound)
+            ),
+            "validate_source_mutation must reject non-initial mutation on retired source"
+        );
+    }
 }

crates/ecstore/src/disk/mod.rs

@@ -27,6 +27,7 @@ mod io_admission;
 #[cfg(test)]
 pub(crate) mod lifecycle_test_io;
 pub mod local;
+pub mod multipart_index;
 pub mod multipart_source;
 pub mod os;
 
@@ -52,6 +53,10 @@ pub const RUSTFS_META_MULTIPART_BUCKET: &str = ".rustfs.sys/multipart";
 pub const RUSTFS_META_TMP_BUCKET: &str = ".rustfs.sys/tmp";
 pub const RUSTFS_META_TMP_DELETED_BUCKET: &str = ".rustfs.sys/tmp/.trash";
 pub const BUCKET_META_PREFIX: &str = "buckets";
+/// Directory under `BUCKET_META_PREFIX` that holds drive-replacement heal
+/// intents, seals and recovery proofs. It shares the bucket-metadata root with
+/// a user bucket of the same name, so bucket deletion must leave it in place.
+pub const HEAL_REPLACEMENT_RECOVERY_DIR: &str = "ahm-replacement";
 pub const FORMAT_CONFIG_FILE: &str = "format.json";
 /// Per-disk marker present while an erasure-set heal is rebuilding this disk.
 /// `LocalDisk::disk_info` reports `healing = true` while the file exists.
@@ -401,16 +406,24 @@ impl Disk {
         }
     }
 
+    pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
+        match self {
+            Self::Local(disk) => disk.has_clean_active_multipart_source(upload_or_part_path, part_path),
+            Self::Remote(_) => false,
+        }
+    }
+
     pub(crate) async fn try_publish_prepared_delete_local(
         &self,
         volume: &str,
         object: &str,
         transaction: Uuid,
         namespace_owner: Option<Arc<dyn Send + Sync>>,
+        work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
     ) -> Result<Option<(DeleteRecoveryJournal, bool, bool)>> {
         match self {
             Self::Local(disk) => {
-                disk.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner)
+                disk.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner, work_budget)
                     .await
             }
             Self::Remote(_) => Ok(None),
@@ -424,11 +437,19 @@ impl Disk {
         transaction: Uuid,
         journal: &DeleteRecoveryJournal,
         namespace_owner: Option<Arc<dyn Send + Sync>>,
+        work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
     ) -> Result<bool> {
         match self {
             Self::Local(disk) => {
-                disk.try_finish_and_finalize_prepared_delete_local(volume, object, transaction, journal, namespace_owner)
+                disk.try_finish_and_finalize_prepared_delete_local(
-                    .await
+                    volume,
+                    object,
+                    transaction,
+                    journal,
+                    namespace_owner,
+                    work_budget,
+                )
+                .await
             }
             Self::Remote(_) => Ok(false),
         }
@@ -505,6 +526,17 @@ impl Disk {
 
 #[async_trait::async_trait]
 impl DiskAPI for Disk {
+    async fn list_multipart_index_roots(
+        &self,
+        bucket: &str,
+        marker: Option<&str>,
+    ) -> Result<multipart_index::MultipartIndexPage> {
+        match self {
+            Self::Local(disk) => disk.list_multipart_index_roots(bucket, marker).await,
+            Self::Remote(disk) => disk.list_multipart_index_roots(bucket, marker).await,
+        }
+    }
+
     async fn supports_guarded_delete_versions(&self) -> Result<()> {
         match self {
             Self::Local(disk) => disk.supports_guarded_delete_versions().await,
@@ -548,6 +580,17 @@ impl DiskAPI for Disk {
             Self::Remote(disk) => disk.has_delete_transaction_evidence(volume, object).await,
         }
     }
+    async fn discover_delete_transactions_bounded(
+        &self,
+        volume: &str,
+        object: &str,
+        max_entries: u32,
+    ) -> Result<BoundedDeleteRecoveryDiscovery> {
+        match self {
+            Self::Local(disk) => disk.discover_delete_transactions_bounded(volume, object, max_entries).await,
+            Self::Remote(disk) => disk.discover_delete_transactions_bounded(volume, object, max_entries).await,
+        }
+    }
     async fn discover_delete_transactions(
         &self,
         volume: &str,
@@ -560,6 +603,35 @@ impl DiskAPI for Disk {
             Self::Remote(disk) => disk.discover_delete_transactions(volume, object, cursor, limit).await,
         }
     }
+    async fn settle_delete_transaction_bounded(
+        &self,
+        volume: &str,
+        request: BoundedDeleteRecoverySettlement,
+        namespace_owner: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<BoundedDeleteRecoverySettlementReply> {
+        match self {
+            Self::Local(disk) => disk.settle_delete_transaction_bounded(volume, request, namespace_owner).await,
+            Self::Remote(disk) => disk.settle_delete_transaction_bounded(volume, request, namespace_owner).await,
+        }
+    }
+    async fn inspect_delete_transaction_bounded(
+        &self,
+        volume: &str,
+        object: &str,
+        transaction: Uuid,
+        max_entries: u32,
+    ) -> Result<BoundedDeleteRecoveryInspection> {
+        match self {
+            Self::Local(disk) => {
+                disk.inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
+                    .await
+            }
+            Self::Remote(disk) => {
+                disk.inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
+                    .await
+            }
+        }
+    }
     async fn inspect_delete_transaction(
         &self,
         volume: &str,
@@ -1506,6 +1578,15 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
     fn host_name(&self) -> String;
     fn endpoint(&self) -> Endpoint;
     async fn close(&self) -> Result<()>;
+    /// Return candidates only while this disk maintains complete boot-local coverage.
+    /// Unsupported or unready disks must never return an authoritative empty page.
+    async fn list_multipart_index_roots(
+        &self,
+        _bucket: &str,
+        _marker: Option<&str>,
+    ) -> Result<multipart_index::MultipartIndexPage> {
+        Err(DiskError::MethodNotAllowed)
+    }
     async fn get_disk_id(&self) -> Result<Option<Uuid>>;
     async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()>;
 
@@ -1683,6 +1764,35 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
     ) -> Result<DeleteRecoveryPage> {
         Err(DiskError::MethodNotAllowed)
     }
+    /// Bounded foreground inventory owns all traversal state until return or cancellation.
+    /// An implementation without the bounded protocol is an unsupported participant
+    /// (the same verdict a remote peer without it receives); callers must never fall
+    /// back to paginated discovery.
+    async fn discover_delete_transactions_bounded(
+        &self,
+        _volume: &str,
+        _object: &str,
+        _max_entries: u32,
+    ) -> Result<BoundedDeleteRecoveryDiscovery> {
+        Err(DiskError::UnsupportedDisk)
+    }
+    async fn inspect_delete_transaction_bounded(
+        &self,
+        _volume: &str,
+        _object: &str,
+        _transaction: Uuid,
+        _max_entries: u32,
+    ) -> Result<BoundedDeleteRecoveryInspection> {
+        Err(DiskError::UnsupportedDisk)
+    }
+    async fn settle_delete_transaction_bounded(
+        &self,
+        _volume: &str,
+        _request: BoundedDeleteRecoverySettlement,
+        _namespace_owner: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<BoundedDeleteRecoverySettlementReply> {
+        Err(DiskError::UnsupportedDisk)
+    }
     /// A read-only capability check; unsupported implementations must fail closed.
     async fn supports_guarded_delete_versions(&self) -> Result<()> {
         Err(DiskError::GuardedDeleteUnsupported)
@@ -2068,6 +2178,13 @@ pub struct WalkDirOptions {
     #[serde(default)]
     pub skip_hidden_prefix_check: bool,
 
+    /// Bound hidden metadata read while proving a non-recursive prefix visible.
+    #[serde(default)]
+    pub hidden_entry_budget: usize,
+    /// A prefix whose visibility was not established before the preceding cut.
+    #[serde(default)]
+    pub pending_prefix: Option<String>,
+
     // ReportNotFound will return errFileNotFound if all disks reports the BaseDir cannot be found.
     pub report_notfound: bool,
 
@@ -2400,9 +2517,62 @@ pub enum DeleteRecoveryAction {
     AdoptCommit { authority: Box<DeleteRecoveryJournal> },
 }
 
+/// A complete inventory or a work-limit refusal. No public discovery cursor is allocated.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct BoundedDeleteRecoveryDiscovery {
+    pub transactions: Vec<DeleteTransactionId>,
+    pub visited: u32,
+    pub complete: bool,
+}
+
+pub const DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES: u32 = 4096;
+pub const DELETE_RECOVERY_MAX_INSPECTION_ENTRIES: u32 = 65536;
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct BoundedDeleteRecoverySettlement {
+    pub object: String,
+    pub transaction: Uuid,
+    pub expected_fingerprint: DeleteRecoveryDigest,
+    pub action: DeleteRecoveryAction,
+    pub max_entries: u32,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub enum DeleteRecoverySettlementOutcome {
+    Inspected(Box<DeleteRecoveryInspection>),
+    ReceiptPublishedInspectionDeferred,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct BoundedDeleteRecoverySettlementReply {
+    pub outcome: DeleteRecoverySettlementOutcome,
+    pub visited: u32,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct BoundedDeleteRecoveryInspection {
+    pub inspection: DeleteRecoveryInspection,
+    pub visited: u32,
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
 pub enum DeleteRecoveryRpcOperation {
+    SettleBounded(BoundedDeleteRecoverySettlement),
+    InspectBounded {
+        object: String,
+        transaction: Uuid,
+        max_entries: u32,
+    },
+    DiscoverBounded {
+        object: String,
+        max_entries: u32,
+    },
     InspectOrphanDataDir {
         path: String,
     },
@@ -2440,7 +2610,11 @@ impl DeleteRecoveryRpcOperation {
     /// Full metadata application and adoption share the existing mutation envelope.
     pub fn max_wire_payload_bytes(&self) -> usize {
         match self {
-            Self::ApplyVersion { .. }
+            Self::SettleBounded(BoundedDeleteRecoverySettlement {
+                action: DeleteRecoveryAction::AdoptCommit { .. },
+                ..
+            })
+            | Self::ApplyVersion { .. }
             | Self::ApplyVersions { .. }
             | Self::Settle {
                 action: DeleteRecoveryAction::AdoptCommit { .. },
@@ -2456,8 +2630,11 @@ impl DeleteRecoveryRpcOperation {
 pub enum DeleteRecoveryRpcReply {
     OrphanDataDir { plain: bool },
     Discovered(DeleteRecoveryPage),
+    DiscoveredBounded(BoundedDeleteRecoveryDiscovery),
     Inspected(DeleteRecoveryInspection),
+    InspectedBounded(BoundedDeleteRecoveryInspection),
     Settled(DeleteRecoveryInspection),
+    SettledBounded(BoundedDeleteRecoverySettlementReply),
     AppliedVersion,
     AppliedVersions(Vec<Option<u32>>),
     Evidence { present: bool },
@@ -2935,6 +3112,8 @@ mod tests {
             recursive: true,
             incl_deleted: false,
             skip_hidden_prefix_check: false,
+            hidden_entry_budget: 0,
+            pending_prefix: None,
             report_notfound: false,
             filter_prefix: Some("prefix_".to_string()),
             forward_to: Some("object/path".to_string()),

crates/ecstore/src/disk/multipart_index.rs

@@ -0,0 +1,359 @@
+// Copyright 2026 RustFS Team
+// SPDX-License-Identifier: Apache-2.0
+
+//! A candidate index never replaces multipart metadata or quorum validation.
+
+use serde::{Deserialize, Serialize};
+use uuid::Uuid;
+
+pub(crate) const RUSTFS_MULTIPART_BUCKET_KEY: &str = "x-rustfs-internal-multipart-bucket";
+pub(crate) const RUSTFS_MULTIPART_OBJECT_KEY: &str = "x-rustfs-internal-multipart-object";
+
+pub(crate) const MULTIPART_INDEX_PAGE_SIZE: usize = 256;
+/// Pages one listing accepts from one disk. Pruned pages may carry few roots, so
+/// the cursor alone no longer bounds a peer's answer; a disk that keeps
+/// continuing past this many pages contributes no coverage.
+pub(crate) const MULTIPART_INDEX_MAX_PAGES_PER_DISK: usize = 4096;
+
+/// A bounded page from one disk's continuously maintained, boot-local index.
+/// `epoch` changes whenever coverage must be rebuilt; callers must not combine
+/// pages from different epochs or physical disk identities.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct MultipartIndexPage {
+    pub bucket: String,
+    pub disk_id: Uuid,
+    pub epoch: Uuid,
+    pub roots: Vec<String>,
+    pub next_marker: Option<String>,
+}
+
+pub const MULTIPART_INDEX_RPC_MAX_BYTES: usize = 64 * 1024;
+pub const MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES: usize = 4096;
+
+/// Additive operation inside the legacy read-only DiskInfo JSON envelope.
+/// Old peers may ignore this field, but their ordinary response cannot decode
+/// as the required signed, challenge-bound MultipartIndexRpcReply.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct MultipartIndexRpcRequest {
+    pub disk_id: String,
+    pub metrics: bool,
+    pub noop: bool,
+    pub multipart_index: MultipartIndexQuery,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct MultipartIndexQuery {
+    pub operation: String,
+    pub version: u32,
+    pub challenge: Uuid,
+    pub bucket: String,
+    pub marker: Option<String>,
+    pub page_cap: usize,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct MultipartIndexRpcReply {
+    pub operation: String,
+    pub version: u32,
+    pub challenge: Uuid,
+    pub page: MultipartIndexPage,
+    pub proof: Vec<u8>,
+}
+
+impl MultipartIndexRpcRequest {
+    pub fn new(bucket: &str, marker: Option<&str>) -> Self {
+        Self {
+            disk_id: String::new(),
+            metrics: false,
+            noop: true,
+            multipart_index: MultipartIndexQuery {
+                operation: "list_multipart_index_roots".into(),
+                version: 1,
+                challenge: Uuid::new_v4(),
+                bucket: bucket.to_owned(),
+                marker: marker.map(str::to_owned),
+                page_cap: MULTIPART_INDEX_PAGE_SIZE,
+            },
+        }
+    }
+
+    pub fn validate(&self) -> crate::disk::error::Result<()> {
+        let query = &self.multipart_index;
+        if !self.disk_id.is_empty()
+            || self.metrics
+            || !self.noop
+            || query.operation != "list_multipart_index_roots"
+            || query.version != 1
+            || query.challenge.is_nil()
+            || query.bucket.is_empty()
+            || query.bucket.len() > 255
+            || query.bucket.contains('/')
+            || matches!(query.bucket.as_str(), "." | "..")
+            || query.page_cap != MULTIPART_INDEX_PAGE_SIZE
+            || query.marker.as_ref().is_some_and(|marker| !valid_root(marker))
+        {
+            return Err(crate::disk::error::DiskError::MethodNotAllowed);
+        }
+        Ok(())
+    }
+
+    pub fn canonical_request(&self, disk: &str) -> crate::disk::error::Result<Vec<u8>> {
+        self.validate()?;
+        if disk.len() > MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES {
+            return Err(crate::disk::error::DiskError::FileCorrupt);
+        }
+        Ok(serde_json::to_vec(&("rustfs-multipart-index-request-v1", disk, self))?)
+    }
+
+    pub fn canonical_response(&self, disk: &str, page: &MultipartIndexPage) -> crate::disk::error::Result<Vec<u8>> {
+        self.validate()?;
+        Ok(serde_json::to_vec(&("rustfs-multipart-index-response-v1", disk, self, page))?)
+    }
+}
+
+fn valid_root(root: &str) -> bool {
+    root.len() == 64 && root.bytes().all(|byte| byte.is_ascii_hexdigit())
+}
+
+impl MultipartIndexPage {
+    pub fn validate(&self, bucket: &str, marker: Option<&str>, expected_disk: Uuid) -> crate::disk::error::Result<()> {
+        if expected_disk.is_nil()
+            || self.disk_id != expected_disk
+            || self.epoch.is_nil()
+            || self.bucket != bucket
+            || self.roots.len() > MULTIPART_INDEX_PAGE_SIZE
+            || self
+                .roots
+                .iter()
+                .any(|root| !valid_root(root) || marker.is_some_and(|marker| root.as_str() <= marker))
+            || self.roots.windows(2).any(|pair| pair[0] >= pair[1])
+            // The cursor names the last examined marker; pruning may leave fewer
+            // live roots than examined names, so it only has to bound them.
+            || self.next_marker.as_ref().is_some_and(|next| {
+                !valid_root(next)
+                    || marker.is_some_and(|marker| next.as_str() <= marker)
+                    || self.roots.last().is_some_and(|last| last > next)
+            })
+        {
+            return Err(crate::disk::error::DiskError::FileCorrupt);
+        }
+        Ok(())
+    }
+}
+
+impl MultipartIndexRpcReply {
+    /// This is structural validation only; the caller must also verify `proof`.
+    pub fn validate(&self, request: &MultipartIndexRpcRequest, expected_disk: Uuid) -> crate::disk::error::Result<()> {
+        let query = &request.multipart_index;
+        request.validate()?;
+        if self.operation != query.operation
+            || self.version != query.version
+            || self.challenge != query.challenge
+            || self.proof.len() != 32
+        {
+            return Err(crate::disk::error::DiskError::FileCorrupt);
+        }
+        self.page.validate(&query.bucket, query.marker.as_deref(), expected_disk)
+    }
+}
+
+/// A disk can attest coverage only after every page belongs to the same physical
+/// identity and index generation. An interrupted/changed stream contributes no
+/// partial candidates to the erasure set's coverage count.
+pub(crate) async fn collect_pages<F, Fut>(bucket: &str, identity: Uuid, mut fetch: F) -> crate::disk::error::Result<Vec<String>>
+where
+    F: FnMut(Option<String>) -> Fut,
+    Fut: std::future::Future<Output = crate::disk::error::Result<MultipartIndexPage>>,
+{
+    let mut epoch = None;
+    let mut marker = None;
+    let mut roots = Vec::new();
+    let mut pages = 0_usize;
+    loop {
+        if pages >= MULTIPART_INDEX_MAX_PAGES_PER_DISK {
+            return Err(crate::disk::error::DiskError::FileCorrupt);
+        }
+        pages += 1;
+        let page = fetch(marker.clone()).await?;
+        page.validate(bucket, marker.as_deref(), identity)?;
+        if epoch.is_some_and(|epoch| epoch != page.epoch) {
+            return Err(crate::disk::error::DiskError::DiskNotFound);
+        }
+        epoch = Some(page.epoch);
+        roots.extend(page.roots);
+        marker = page.next_marker;
+        if marker.is_none() {
+            return Ok(roots);
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn multipart_index_reply_rejects_challenge_disk_cursor_and_unbounded_pages() {
+        let request = MultipartIndexRpcRequest::new("bucket", None);
+        let id = Uuid::new_v4();
+        let mut reply = MultipartIndexRpcReply {
+            operation: request.multipart_index.operation.clone(),
+            version: 1,
+            challenge: request.multipart_index.challenge,
+            page: MultipartIndexPage {
+                bucket: "bucket".into(),
+                disk_id: id,
+                epoch: Uuid::new_v4(),
+                roots: vec!["a".repeat(64)],
+                next_marker: None,
+            },
+            proof: vec![0; 32],
+        };
+        reply.validate(&request, id).expect("valid structural envelope");
+        assert!(
+            reply.validate(&MultipartIndexRpcRequest::new("bucket", None), id).is_err(),
+            "old challenge cannot replay"
+        );
+        assert!(reply.validate(&request, Uuid::new_v4()).is_err(), "physical disk identity cannot change");
+        reply.page.next_marker = Some("9".repeat(64));
+        assert!(reply.validate(&request, id).is_err(), "a cursor below the last root would re-list it");
+        reply.page.next_marker = Some("a".repeat(64));
+        reply
+            .validate(&request, id)
+            .expect("a pruned short page continues from its last examined name");
+        reply.page.next_marker = None;
+        reply.page.roots = vec!["a".repeat(64); MULTIPART_INDEX_PAGE_SIZE + 1];
+        assert!(reply.validate(&request, id).is_err(), "peer cannot enlarge the fixed page cap");
+    }
+
+    #[test]
+    fn multipart_index_response_proof_binds_the_complete_query_and_page() {
+        crate::runtime::sources::ensure_test_rpc_secret();
+        let mut request = MultipartIndexRpcRequest::new("bucket", None);
+        let mut page = MultipartIndexPage {
+            bucket: "bucket".into(),
+            disk_id: Uuid::new_v4(),
+            epoch: Uuid::new_v4(),
+            roots: vec![],
+            next_marker: None,
+        };
+        let canonical = request.canonical_response("disk", &page).expect("canonical response");
+        let proof = crate::cluster::rpc::sign_tonic_rpc_response_proof(&canonical).expect("signed response");
+        let verify = |request: &MultipartIndexRpcRequest, page: &MultipartIndexPage| {
+            crate::cluster::rpc::verify_tonic_rpc_response_proof(
+                &request.canonical_response("disk", page).expect("canonical"),
+                &proof,
+            )
+        };
+        verify(&request, &page).expect("exact response");
+        request.multipart_index.marker = Some("0".repeat(64));
+        assert!(verify(&request, &page).is_err(), "cursor is signed");
+        request.multipart_index.marker = None;
+        page.epoch = Uuid::new_v4();
+        assert!(verify(&request, &page).is_err(), "boot-local coverage epoch is signed");
+    }
+    #[test]
+    fn multipart_index_page_cursor_bounds_roots_without_requiring_a_full_page() {
+        let disk = Uuid::new_v4();
+        let page = |roots: Vec<String>, next_marker: Option<String>| MultipartIndexPage {
+            bucket: "bucket".into(),
+            disk_id: disk,
+            epoch: Uuid::new_v4(),
+            roots,
+            next_marker,
+        };
+        // Pruning leaves fewer live roots than examined names; the cursor still advances.
+        page(vec![format!("{:064x}", 1)], Some(format!("{:064x}", 5)))
+            .validate("bucket", None, disk)
+            .expect("short page with a cursor");
+        page(vec![], Some(format!("{:064x}", 5)))
+            .validate("bucket", Some(&format!("{:064x}", 2)), disk)
+            .expect("all examined names pruned");
+        assert!(
+            page(vec![format!("{:064x}", 6)], Some(format!("{:064x}", 5)))
+                .validate("bucket", None, disk)
+                .is_err(),
+            "a root beyond the cursor would be re-listed"
+        );
+        assert!(
+            page(vec![], Some(format!("{:064x}", 2)))
+                .validate("bucket", Some(&format!("{:064x}", 2)), disk)
+                .is_err(),
+            "the cursor must advance past the request marker"
+        );
+        assert!(
+            page(vec![], Some("not-a-sha".to_owned()))
+                .validate("bucket", None, disk)
+                .is_err(),
+            "the cursor is a marker name"
+        );
+    }
+
+    #[tokio::test]
+    async fn multipart_index_page_collection_bounds_the_pages_one_disk_may_answer() {
+        let id = Uuid::new_v4();
+        let epoch = Uuid::new_v4();
+        let mut served = 0_usize;
+        let result = collect_pages("bucket", id, |marker| {
+            served += 1;
+            let next = marker.map_or(0_u64, |marker| u64::from_str_radix(&marker[48..], 16).expect("hex cursor") + 1);
+            std::future::ready(Ok(MultipartIndexPage {
+                bucket: "bucket".into(),
+                disk_id: id,
+                epoch,
+                roots: vec![],
+                next_marker: Some(format!("{next:064x}")),
+            }))
+        })
+        .await;
+        assert!(matches!(result, Err(crate::disk::error::DiskError::FileCorrupt)), "{result:?}");
+        assert_eq!(served, MULTIPART_INDEX_MAX_PAGES_PER_DISK, "the peer is cut off at the page cap");
+    }
+
+    #[tokio::test]
+    async fn multipart_index_page_collection_rejects_replaced_disk_or_restarted_epoch() {
+        let id = Uuid::new_v4();
+        let epoch = Uuid::new_v4();
+        for replace_disk in [false, true] {
+            let first = MultipartIndexPage {
+                bucket: "bucket".into(),
+                disk_id: id,
+                epoch,
+                roots: (0..MULTIPART_INDEX_PAGE_SIZE).map(|index| format!("{index:064x}")).collect(),
+                next_marker: Some(format!("{:064x}", MULTIPART_INDEX_PAGE_SIZE - 1)),
+            };
+            let second = MultipartIndexPage {
+                bucket: "bucket".into(),
+                disk_id: if replace_disk { Uuid::new_v4() } else { id },
+                epoch: if replace_disk { epoch } else { Uuid::new_v4() },
+                roots: vec![],
+                next_marker: None,
+            };
+            let mut replies = std::collections::VecDeque::from([first, second]);
+            let result = collect_pages("bucket", id, |marker| {
+                let page = replies.pop_front().expect("bounded fixture pages");
+                if marker.is_some() {
+                    assert!(page.roots.is_empty());
+                }
+                std::future::ready(Ok(page))
+            })
+            .await;
+            // A replaced disk fails the page's identity check; a restarted
+            // epoch fails the cross-page continuity check.
+            let expected_corrupt = replace_disk;
+            assert!(
+                match &result {
+                    Err(crate::disk::error::DiskError::FileCorrupt) => expected_corrupt,
+                    Err(crate::disk::error::DiskError::DiskNotFound) => !expected_corrupt,
+                    _ => false,
+                },
+                "changed physical source cannot contribute partial coverage: {result:?}"
+            );
+            assert!(replies.is_empty(), "exercise the continuation response");
+        }
+    }
+}

crates/ecstore/src/error/mod.rs

@@ -269,6 +269,11 @@ pub enum StorageError {
     /// retained for operator inspection.
     #[error("delete recovery found an ambiguous legacy residue; operator inspection is required")]
     DeleteRecoveryLegacyAmbiguous,
+    /// Delete recovery exhausted the per-request work budget before it could
+    /// classify every residue. Retrying later makes progress, so callers treat
+    /// it like `SlowDown`; it stays distinct so the disk error round-trips.
+    #[error("delete recovery work limit reached; please retry")]
+    DeleteRecoveryWorkLimit,
     #[error("Decommission not started")]
     DecommissionNotStarted,
     #[error("Decommission already running")]
@@ -480,6 +485,7 @@ impl From<DiskError> for StorageError {
             DiskError::InvalidPath => StorageError::InvalidPath,
             DiskError::ConditionalDeleteConflict => StorageError::ConditionalDeleteConflict,
             DiskError::GuardedDeleteUnsupported => StorageError::GuardedDeleteUnsupported,
+            DiskError::DeleteRecoveryWorkLimit => StorageError::DeleteRecoveryWorkLimit,
             DiskError::RemoteClientUnavailable(detail) => StorageError::RemoteClientUnavailable(detail),
         }
     }
@@ -507,6 +513,7 @@ impl StorageError {
             StorageError::MethodNotAllowed => DiskError::MethodNotAllowed,
             StorageError::StorageFull => DiskError::DiskFull,
             StorageError::SlowDown => DiskError::TooManyOpenFiles,
+            StorageError::DeleteRecoveryWorkLimit => DiskError::DeleteRecoveryWorkLimit,
             StorageError::MaxVersionsExceeded => DiskError::MaxVersionsExceeded,
             StorageError::InconsistentDisk => DiskError::InconsistentDisk,
             StorageError::UnsupportedDisk => DiskError::UnsupportedDisk,
@@ -710,6 +717,7 @@ impl Clone for StorageError {
             StorageError::StorageFull => StorageError::StorageFull,
             StorageError::SlowDown => StorageError::SlowDown,
             StorageError::DeleteRecoveryLegacyAmbiguous => StorageError::DeleteRecoveryLegacyAmbiguous,
+            StorageError::DeleteRecoveryWorkLimit => StorageError::DeleteRecoveryWorkLimit,
             StorageError::PrefixAccessDenied(a, b) => StorageError::PrefixAccessDenied(a.clone(), b.clone()),
             StorageError::InvalidUploadIDKeyCombination(a, b) => {
                 StorageError::InvalidUploadIDKeyCombination(a.clone(), b.clone())
@@ -852,6 +860,7 @@ impl StorageError {
             StorageError::StorageFull => StorageErrorCode::StorageFull,
             StorageError::SlowDown => StorageErrorCode::SlowDown,
             StorageError::DeleteRecoveryLegacyAmbiguous => StorageErrorCode::DeleteRecoveryLegacyAmbiguous,
+            StorageError::DeleteRecoveryWorkLimit => StorageErrorCode::DeleteRecoveryWorkLimit,
             StorageError::PrefixAccessDenied(_, _) => StorageErrorCode::PrefixAccessDenied,
             StorageError::InvalidUploadIDKeyCombination(_, _) => StorageErrorCode::InvalidUploadIDKeyCombination,
             StorageError::MalformedUploadID(_) => StorageErrorCode::MalformedUploadID,
@@ -958,6 +967,7 @@ impl StorageError {
             StorageErrorCode::StorageFull => Some(StorageError::StorageFull),
             StorageErrorCode::SlowDown => Some(StorageError::SlowDown),
             StorageErrorCode::DeleteRecoveryLegacyAmbiguous => Some(StorageError::DeleteRecoveryLegacyAmbiguous),
+            StorageErrorCode::DeleteRecoveryWorkLimit => Some(StorageError::DeleteRecoveryWorkLimit),
             StorageErrorCode::PrefixAccessDenied => {
                 Some(StorageError::PrefixAccessDenied(Default::default(), Default::default()))
             }
@@ -1700,8 +1710,8 @@ mod tests {
         // covers every variant and picks up newly appended ones automatically.
         let all_variants: Vec<DiskError> = (1u32..).map_while(DiskError::from_u32).collect();
         assert!(
-            all_variants.len() >= 42,
+            all_variants.len() >= 46,
-            "DiskError variant enumeration shrank: got {}, expected at least 42",
+            "DiskError variant enumeration shrank: got {}, expected at least 46",
             all_variants.len()
         );
 

crates/ecstore/src/io_support/bitrot.rs

@@ -828,13 +828,27 @@ pub async fn create_bitrot_writer(
     length: i64,
     shard_size: usize,
     checksum_algo: HashAlgorithm,
+) -> disk::error::Result<BitrotWriterWrapper> {
+    create_bitrot_writer_with_orig_volume(is_inline_buffer, disk, "", volume, path, length, shard_size, checksum_algo).await
+}
+
+#[allow(clippy::too_many_arguments)]
+pub async fn create_bitrot_writer_with_orig_volume(
+    is_inline_buffer: bool,
+    disk: Option<&DiskStore>,
+    origvolume: &str,
+    volume: &str,
+    path: &str,
+    length: i64,
+    shard_size: usize,
+    checksum_algo: HashAlgorithm,
 ) -> disk::error::Result<BitrotWriterWrapper> {
     let writer = if is_inline_buffer {
         CustomWriter::new_inline_buffer()
     } else if let Some(disk) = disk {
         let length = bitrot_create_file_size(length, shard_size, &checksum_algo);
 
-        let file = disk.create_file("", volume, path, length).await?;
+        let file = disk.create_file(origvolume, volume, path, length).await?;
         #[cfg(feature = "hotpath")]
         let file = instrument_raw_shard_writer(file, disk.is_local());
         CustomWriter::new_tokio_writer(file)

crates/ecstore/src/io_support/rio.rs

@@ -79,6 +79,14 @@ pub fn compression_scheme_to_read_plan(scheme: &str) -> std::io::Result<(Compres
     Ok((CompressionAlgorithm::from_str(scheme)?, ReadCompressionBackend::Legacy))
 }
 
+/// Validate that replicated compression metadata can be read by this build.
+///
+/// # Errors
+/// Returns an error for an unsupported compression scheme.
+pub fn validate_compression_scheme(scheme: &str) -> std::io::Result<()> {
+    compression_scheme_to_read_plan(scheme).map(|_| ())
+}
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum ReadEncryptionBackend {
     Legacy,
@@ -429,6 +437,7 @@ impl WritePlan {
         let checksum = reader.content_hash().clone();
         let trailer = reader.get_trailer().cloned();
 
+        let delegate_inner_etag = self.compression.is_none() && reader.is_etag_reader();
         let encrypted = self.encryption.is_some();
         if let Some(algorithm) = self.compression {
             reader = HashReader::from_reader(
@@ -443,12 +452,15 @@ impl WritePlan {
 
         if let Some(encryption) = self.encryption {
             let etag_nonce = match encryption.mode {
-                WriteEncryptionMode::Singlepart { base_nonce } | WriteEncryptionMode::MultipartLegacy { base_nonce, .. } => {
+                WriteEncryptionMode::Singlepart { base_nonce } => base_nonce,
-                    base_nonce
+                WriteEncryptionMode::SinglepartObjectKey => [0u8; 12],
-                }
+                // Each part write needs an independent fingerprint, including a
-                WriteEncryptionMode::SinglepartObjectKey | WriteEncryptionMode::MultipartObjectKey { .. } => [0u8; 12],
+                // retry at the same part number. This salt affects only the ETag
+                // MAC; encryption framing and its data nonce stay unchanged.
+                WriteEncryptionMode::MultipartLegacy { .. } | WriteEncryptionMode::MultipartObjectKey { .. } => rand::random(),
             };
-            let protected_reader = rustfs_rio::EtagReader::new_encrypted(reader, encryption.key_bytes, etag_nonce)?;
+            let protected_reader = rustfs_rio::EtagReader::new_encrypted(reader, encryption.key_bytes, etag_nonce)?
+                .with_inner_etag_delegation(delegate_inner_etag);
             reader = match encryption.mode {
                 WriteEncryptionMode::SinglepartObjectKey => HashReader::from_reader(
                     #[cfg(feature = "rio-v2")]
@@ -556,6 +568,54 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn multipart_etags_are_unique_per_write_without_changing_data_encryption() {
+        let plaintext = b"same confidential part";
+        let size = i64::try_from(plaintext.len()).expect("test length");
+        for object_key_mode in [false, true] {
+            let mut tags = std::collections::HashSet::new();
+            for part_number in [1, 2, 1] {
+                let encryption = if object_key_mode {
+                    WriteEncryption::multipart_object_key([7; 32], part_number)
+                } else {
+                    WriteEncryption::multipart([7; 32], [9; 12], part_number as usize)
+                };
+                let source = HashReader::from_stream(Cursor::new(plaintext), size, size, None, None, false).expect("plaintext");
+                let mut reader = WritePlan::new()
+                    .with_encryption(encryption)
+                    .apply(source, size)
+                    .expect("part plan");
+                let mut ciphertext = Vec::new();
+                reader.read_to_end(&mut ciphertext).await.expect("encrypt part");
+                let tag = reader.try_resolve_etag().expect("part fingerprint");
+                assert_eq!(reader.try_resolve_etag().as_ref(), Some(&tag), "reading a stored result stays stable");
+                assert!(
+                    tags.insert(tag),
+                    "different part numbers and retries must not identify matching plaintext"
+                );
+                #[cfg(feature = "rio-v2")]
+                let mut decrypt = if object_key_mode {
+                    DecryptReader::new_multipart_with_object_key(Cursor::new(ciphertext), [7; 32], vec![part_number as usize])
+                } else {
+                    DecryptReader::new_multipart(Cursor::new(ciphertext), [7; 32], [9; 12], vec![part_number as usize])
+                };
+                #[cfg(not(feature = "rio-v2"))]
+                let mut decrypt = DecryptReader::new_multipart(
+                    Cursor::new(ciphertext),
+                    [7; 32],
+                    if object_key_mode { [0; 12] } else { [9; 12] },
+                    vec![part_number as usize],
+                );
+                let mut decoded = Vec::new();
+                decrypt
+                    .read_to_end(&mut decoded)
+                    .await
+                    .expect("existing data nonce still decrypts");
+                assert_eq!(decoded, plaintext);
+            }
+        }
+    }
+
     async fn assert_non_trailing_checksum_survives(plan: WritePlan) {
         let plaintext = b"checksum-context-through-write-plan".repeat(256);
         let actual_size = plaintext.len() as i64;

crates/ecstore/src/io_support/shard_integrity.rs

@@ -333,6 +333,34 @@ struct ProofCache {
     preferred: usize,
 }
 
+type VerifiedStripeKey = (uuid::Uuid, [u8; 32], u32);
+
+static VERIFIED_STRIPE_RECORDS: std::sync::LazyLock<parking_lot::Mutex<std::collections::HashMap<VerifiedStripeKey, Bytes>>> =
+    std::sync::LazyLock::new(|| parking_lot::Mutex::new(std::collections::HashMap::new()));
+
+fn get_verified_stripe_record(part: &PartIntegrity, stripe: u32) -> Option<Bytes> {
+    VERIFIED_STRIPE_RECORDS
+        .lock()
+        .get(&(part.generation, part.root, stripe))
+        .cloned()
+}
+
+fn put_verified_stripe_record(part: &PartIntegrity, stripe: u32, record: Bytes) {
+    let mut map = VERIFIED_STRIPE_RECORDS.lock();
+    if map.len() >= 4096
+        && let Some(evict_key) = map.keys().next().copied()
+    {
+        map.remove(&evict_key);
+    }
+    map.insert((part.generation, part.root, stripe), record);
+}
+
+pub(crate) fn invalidate_verified_stripe_records(generation: uuid::Uuid) {
+    VERIFIED_STRIPE_RECORDS
+        .lock()
+        .retain(|(stored_gen, _, _), _| *stored_gen != generation);
+}
+
 impl PartProofReader {
     pub(crate) async fn reconstruction_proof(
         &self,
@@ -410,6 +438,9 @@ impl PartProofReader {
 
     pub(crate) async fn record(&self, stripe: u32) -> io::Result<Bytes> {
         let offset = self.part.record_offset(stripe).map_err(|_| corrupt())?;
+        if let Some(bytes) = get_verified_stripe_record(&self.part, stripe) {
+            return Ok(bytes);
+        }
         let mut cached = self.cached.lock().await;
         if let Some((_, bytes)) = cached.records.iter().find(|(number, _)| *number == stripe) {
             return Ok(bytes.clone());
@@ -465,6 +496,9 @@ impl PartProofReader {
             bytes
         };
         self.part.verify_record(stripe, &record).map_err(|_| corrupt())?;
+        if self.part.stripes <= 4 {
+            put_verified_stripe_record(&self.part, stripe, record.clone());
+        }
         if cached.records.len() == 2 {
             cached.records.pop_front();
         }
@@ -863,4 +897,41 @@ mod tests {
             prepared.part.verify_shard(stripe, 15, b"x", &record).expect("spilled proof");
         }
     }
+
+    #[tokio::test]
+    async fn verified_stripe_record_cache_serves_verified_record_and_still_verifies_payload() {
+        let layout = IntegrityLayout::new(2, 2, 8, false).expect("layout");
+        let mut builder = IntegrityBuilder::new(layout, 1).expect("builder");
+        let shards = [b"ABCD".as_slice(), b"EFGH", b"IJKL", b"MNOP"];
+        builder.push(shards.into_iter()).await.expect("stripe");
+        let prepared = builder.finish(8).await.expect("index");
+        let inline_bytes = prepared.inline_bytes().expect("inline bytes");
+        let part = prepared.part;
+
+        let warm_reader = PartProofReader {
+            part: part.clone(),
+            sources: Vec::new(),
+            volume: "bucket".to_owned(),
+            inline: Some(inline_bytes),
+            cached: Mutex::new(ProofCache::default()),
+        };
+        warm_reader.verify(0, 0, shards[0]).await.expect("warm verify");
+        assert!(
+            get_verified_stripe_record(&part, 0).is_some(),
+            "small verified stripe record should populate the verified stripe cache"
+        );
+
+        let cached_reader = Arc::new(PartProofReader {
+            part,
+            sources: Vec::new(),
+            volume: "bucket".to_owned(),
+            inline: None,
+            cached: Mutex::new(ProofCache::default()),
+        });
+        cached_reader.verify(0, 0, shards[0]).await.expect("cached verify");
+        assert!(
+            cached_reader.verify(0, 0, shards[1]).await.is_err(),
+            "cached record must still reject corrupt or mismatched shard payloads"
+        );
+    }
 }

crates/ecstore/src/object_api/types.rs

@@ -949,8 +949,82 @@ pub enum ShardIntegrityWriteMode {
     Protected,
 }
 
+/// Shared foreground recovery allowance. Clones of ObjectOptions retain one request budget.
+/// Credits cover directory entries (including a minimum charge for an empty probe) and
+/// transaction inspections. Each bounded call is admitted while credits remain and is
+/// charged for the entries it visited when it completes.
+#[derive(Debug)]
+pub struct DeleteRecoveryWorkBudget {
+    remaining: std::sync::atomic::AtomicU32,
+}
+
+/// Entries one disk may visit per recovery step (discovery plus inspection).
+/// Every disk of every set a request fans out to is admitted up to this much
+/// and charged on completion, so a budget must cover that many disks.
+const DELETE_RECOVERY_RESERVATION_PER_DISK: u32 = 4096 + 65_536;
+
+impl Default for DeleteRecoveryWorkBudget {
+    /// Covers a full 16-drive set with headroom; `for_disks` scales it when a
+    /// request spans more drives than that.
+    fn default() -> Self {
+        Self::new(16 * 1024 * 1024)
+    }
+}
+
+impl DeleteRecoveryWorkBudget {
+    /// A per-request budget sized for `disks` concurrently admitted recovery steps.
+    pub(crate) fn for_disks(disks: usize) -> Self {
+        let per_disk = u64::from(DELETE_RECOVERY_RESERVATION_PER_DISK) * 2;
+        let scaled = u64::try_from(disks).unwrap_or(u64::MAX).saturating_mul(per_disk);
+        let default = u64::from(Self::default().remaining());
+        Self::new(u32::try_from(scaled.max(default)).unwrap_or(u32::MAX))
+    }
+
+    pub(crate) fn new(entries: u32) -> Self {
+        Self {
+            remaining: std::sync::atomic::AtomicU32::new(entries),
+        }
+    }
+
+    pub(crate) fn reserve(&self, maximum: u32) -> crate::disk::error::Result<u32> {
+        use std::sync::atomic::Ordering;
+        self.remaining
+            .try_update(Ordering::AcqRel, Ordering::Acquire, |remaining| {
+                (remaining > 0).then(|| remaining.saturating_sub(maximum))
+            })
+            .map(|remaining| remaining.min(maximum))
+            .map_err(|_| crate::disk::error::DiskError::DeleteRecoveryWorkLimit)
+    }
+
+    pub(crate) fn remaining(&self) -> u32 {
+        self.remaining.load(std::sync::atomic::Ordering::Acquire)
+    }
+
+    /// Admit one bounded disk call: refuse once the budget is spent, otherwise
+    /// cap the call at `maximum` or what remains. The work actually performed is
+    /// charged on completion with `charge`, so concurrent calls never exhaust
+    /// the budget by reservation alone (a bulk delete fans out dozens of
+    /// inspections per key); each call is still bounded by its own cap.
+    pub(crate) fn admit(&self, maximum: u32) -> crate::disk::error::Result<u32> {
+        match self.remaining() {
+            0 => Err(crate::disk::error::DiskError::DeleteRecoveryWorkLimit),
+            remaining => Ok(remaining.min(maximum)),
+        }
+    }
+
+    /// Record the entries a bounded call visited (at least one per call).
+    pub(crate) fn charge(&self, visited: u32) {
+        use std::sync::atomic::Ordering;
+        let _ = self.remaining.try_update(Ordering::AcqRel, Ordering::Acquire, |remaining| {
+            Some(remaining.saturating_sub(visited.max(1)))
+        });
+    }
+}
+
 #[derive(Default, Clone)]
 pub struct ObjectOptions {
+    #[doc(hidden)]
+    pub delete_recovery_work_budget: Option<Arc<DeleteRecoveryWorkBudget>>,
     // Use the maximum parity (N/2), used when saving server configuration files
     pub max_parity: bool,
     pub mod_time: Option<OffsetDateTime>,
@@ -1102,6 +1176,12 @@ pub struct ObjectOptions {
     pub expiration: ExpirationOptions,
     pub lifecycle_audit_event: LcAuditEvent,
 
+    /// In-process request gate evaluated on authoritative completion metadata.
+    /// Not persisted or sent over RPC. The trait deliberately has no Debug bound:
+    /// implementations may own customer encryption keys.
+    #[doc(hidden)]
+    pub multipart_completion_authorization:
+        Option<Arc<dyn crate::storage_api_contracts::multipart::MultipartCompletionAuthorization<Error = Error>>>,
     pub eval_metadata: Option<HashMap<String, String>>,
     /// Internal compare-and-set condition for replication workers publishing
     /// terminal status after remote I/O. Storage validates it while holding
@@ -1115,6 +1195,10 @@ pub struct ObjectOptions {
     /// Authoritative bucket Object Lock snapshot installed inside `ECStore`
     /// before a destructive commit reaches the set layer.
     pub object_lock_config_snapshot: Option<Arc<ObjectLockConfigSnapshot>>,
+    /// ECStore single PUTs acquire their destructive-write configuration guards
+    /// only after staging the client body. External lock owners supply a snapshot.
+    #[doc(hidden)]
+    pub defer_put_object_lock_snapshot: bool,
 
     pub want_checksum: Option<Checksum>,
     pub skip_verify_bitrot: bool,
@@ -1146,6 +1230,11 @@ pub enum ReplicationStatusWritebackMode {
 }
 
 impl ObjectOptions {
+    pub(crate) fn ensure_delete_recovery_work_budget(&mut self) {
+        self.delete_recovery_work_budget
+            .get_or_insert_with(|| Arc::new(DeleteRecoveryWorkBudget::default()));
+    }
+
     pub(crate) fn shard_integrity_write_enabled(&self) -> bool {
         match self.shard_integrity_write_mode {
             Some(ShardIntegrityWriteMode::Protected) => true,
@@ -1543,6 +1632,8 @@ pub struct ObjectInfo {
     /// published by the same upload. Callers must not repeat quota, replication,
     /// lifecycle, or object-created side effects for that response.
     pub multipart_completion_replayed: bool,
+    /// Previous current version's size observed during `rename_data` commit.
+    pub backfilled_old_current_size: Option<crate::disk::OldCurrentSize>,
 }
 
 impl Default for ObjectInfo {
@@ -1585,6 +1676,7 @@ impl Default for ObjectInfo {
             replication_decision: String::new(),
             checksum: None,
             multipart_completion_replayed: false,
+            backfilled_old_current_size: None,
         }
     }
 }
@@ -1628,6 +1720,7 @@ impl Clone for ObjectInfo {
             replication_decision: self.replication_decision.clone(),
             checksum: self.checksum.clone(),
             multipart_completion_replayed: self.multipart_completion_replayed,
+            backfilled_old_current_size: self.backfilled_old_current_size,
             expires: self.expires,
         }
     }

crates/ecstore/src/runtime/instance.rs

@@ -941,6 +941,32 @@ impl InstanceContext {
             .clone()
     }
 
+    /// Remove the `BucketMetadataSys` when its backing `ECStore` is gone.
+    ///
+    /// Called from `ECStore::drop`. When test fixtures share the bootstrap
+    /// context, the `BucketMetadataSys` stores a `Weak<ECStore>` to prevent a
+    /// reference cycle; once the store drops the weak is dangling and any
+    /// `object_store()` call panics. Clearing the slot here lets the next
+    /// fixture install a fresh `BucketMetadataSys` without hitting the stale one.
+    pub fn clear_bucket_metadata_sys_if_stale(&self) {
+        let mut guard = self.bucket_metadata_sys.lock().expect("bucket_metadata_sys lock poisoned");
+        if let Some(sys) = guard.as_ref() {
+            // `read()` may block briefly; the async RwLock is acceptable here
+            // because Drop is sync and the lock is held only long enough to
+            // check the weak reference.
+            let is_stale = {
+                let s = sys.try_read();
+                match s {
+                    Ok(s) => s.is_store_stale(),
+                    Err(_) => true, // contended: treat as stale-safe to clear
+                }
+            };
+            if is_stale {
+                *guard = None;
+            }
+        }
+    }
+
     /// Set this instance's background-services cancellation token (once).
     pub fn init_background_cancel_token(&self, token: CancellationToken) -> Result<(), CancellationToken> {
         self.background_cancel_token.set(token)

crates/ecstore/src/services/rebalance/entry.rs

@@ -283,6 +283,17 @@ impl ECStore {
             arrived.notify_one();
             release.notified().await;
         }
+        #[cfg(feature = "e2e-test-hooks")]
+        tokio::select! {
+            // Stop must drain this pause before taking the activation writer.
+            _ = cancel.cancelled() => return Err(Error::OperationCanceled),
+            result = data_movement::e2e_barrier::wait(
+                "rebalance",
+                bucket.as_str(),
+                entry.name.as_str(),
+                data_movement::e2e_barrier::STAGE_AFTER_ENTRY_ADMISSION,
+            ) => result.map_err(Error::from)?,
+        }
         let lock_lost_signal = run_guard.lock_lost_signal();
         #[cfg(test)]
         let _run_signal_test_fence = lock_lost_signal
@@ -1899,6 +1910,10 @@ mod tests {
                 &ObjectOptions {
                     version_id: Some(version_id.to_string()),
                     mod_time: earlier(version.mod_time),
+                    // Without the object lock the rename waits for every
+                    // drive, so no early-ACK tail is still writing the copies
+                    // removed below.
+                    no_lock: true,
                     ..versioned
                 },
             ))