fix(select): admit legacy and linked LZ4 working memory

* refactor(table-catalog): retire durable-strong backing

* fix(table-catalog): avoid backup locks during S3 authorization

* fix(table-catalog): preserve directory and maintenance behavior

* fix(table-catalog): back up retained v1 snapshot history

* test(table-catalog): cover retained v1 backup recovery

Exercise backup inventory sealing and restore after a v1-to-v2 upgrade, including missing or corrupt references, nonzero inline snapshot sequences, v2 manifests in retained v1 history, and unchanged commit admission rules.

Clarify when direct maintenance can replace reports and discard retry budgets, and require quiescing scheduler admission and workers before invocation.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

---------

Co-authored-by: Chris An <anzhengchao@gmail.com>
Co-authored-by: Hauser Zhi <housemecn@gmail.com>
Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: zhi22915 <qiuzgang@gmail.com>

.config/e2e-distributed-selection.txt

@@ -1,2 +1,2 @@
-sha256-linux=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
-sha256-darwin=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
+sha256-linux=348fa5eed9ae58962c06b33283ff4f15122cff97bce9b4eb1e3ffcb227264b0b
+sha256-darwin=348fa5eed9ae58962c06b33283ff4f15122cff97bce9b4eb1e3ffcb227264b0b

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=7c705942d3a9e583ebaa85845daa21c4f81fdbf36def24db931aaef38636c53e
-sha256-linux=5c33dc395233671cb77452f8fb12900ecdbeb368632222b96835d310bce1aab8
+sha256-darwin=219ac11e720b3f54dfee4d5d5e95171785ba159b2fe01db3037a8f5061c1ad32
+sha256-linux=2d34698cc8ec52d5b6c37ac612919a1d21f36cd56a1af539a7522a4eea518e26

.config/e2e-nightly-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=3814cdde8cacf4187e18188d342ed70931cc32e268e42e5bb438d565bb4d0f81
-sha256-linux=663025a80273579f95ae3f49af02c0afcbfce94df09701c360a13e607b7637db
+sha256-darwin=e84e04208f19dc0596b10d4d1b799ce43d6cc431415902f9c5458aa8232bc2d8
+sha256-linux=1f9c7add72c44d1c12ef4ac050bb7917431be657c0c90b29f70c3df56f8bbae3

.config/e2e-smoke-selection.txt

@@ -1 +1 @@
-sha256=00f8a56b75990b487a594b97c36610b94476a6944ba00473e37187ea654d42bf
+sha256=5bdd4c2eadbc41625ae3184fff34062e000d328108436259b2b7c4fe2531ecaf

.config/make/tests.mak

@@ -28,6 +28,7 @@ script-tests: ## Run shell script tests
 	./scripts/test_build_rustfs_options.sh
 	./scripts/test_docker_runtime_timezone.sh
 	./scripts/test_entrypoint_credentials.sh
+	./scripts/test_entrypoint_directories.sh
 	./scripts/test_internode_grpc_ab_bench.sh
 	./scripts/test_object_batch_bench_enhanced.sh
 	./scripts/test_package_service_scripts.sh
@@ -70,6 +71,7 @@ script-tests: ## Run shell script tests
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain_health.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_repro_rejoin_readiness_docker.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_validation_registry.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_validation_runner.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_retain_validation_evidence.py

.config/nextest.toml

@@ -110,7 +110,7 @@ threads-required = "num-test-threads"
 # Reserve capacity so sibling disk-heavy tests cannot consume its 2s
 # registration budget; retain the queued backup and deadlock assertions.
 [[profile.default.overrides]]
-filter = 'package(rustfs) & test(=admin::handlers::table_catalog::tests::cold_strong_registration_completes_while_other_bucket_backup_waits)'
+filter = 'package(rustfs) & test(=admin::handlers::table_catalog::tests::cold_object_registration_completes_while_other_bucket_backup_waits)'
 threads-required = "num-test-threads"
 
 [[profile.default.overrides]]
@@ -410,7 +410,7 @@ threads-required = "num-test-threads"
 # Reserve capacity so sibling disk-heavy tests cannot consume its 2s
 # registration budget; retain the queued backup and deadlock assertions.
 [[profile.ci.overrides]]
-filter = 'package(rustfs) & test(=admin::handlers::table_catalog::tests::cold_strong_registration_completes_while_other_bucket_backup_waits)'
+filter = 'package(rustfs) & test(=admin::handlers::table_catalog::tests::cold_object_registration_completes_while_other_bucket_backup_waits)'
 threads-required = "num-test-threads"
 
 [[profile.ci.overrides]]
@@ -701,7 +701,7 @@ test-group = 'ecstore-serial-flaky'
 [profile.e2e-smoke]
 default-filter = """
     package(e2e_test) & (
-        test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
+        test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|multipart_upload_id|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
         | test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
         | test(/^reliant::lifecycle::/)
         | test(/^conditional_delete_objects_test::/)
@@ -827,7 +827,7 @@ slow-timeout = { period = "120s", terminate-after = 12, grace-period = "10s" }
 # ---------------------------------------------------------------------------
 # e2e-distributed profile — 4-node 4-disk Actions suite
 # ---------------------------------------------------------------------------
-# Storage-sensitive PR / nightly / dispatch lane owned by
+# Nightly and manual-dispatch lane owned by
 # .github/workflows/e2e-distributed.yml.
 # Each case starts four rustfs processes (and for site replication, two
 # clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.

.docker/README.md

@@ -22,9 +22,9 @@ The following files in the project root are essential for Docker operations:
 | File | Description | Usage |
 | :--- | :--- | :--- |
 | **`docker-buildx.sh`** | **Multi-Arch Build Script**<br>Automates building and pushing Docker images for `amd64` and `arm64`. Supports release and dev channels. | `./docker-buildx.sh --push` |
-| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads pre-built binaries from GitHub Releases. | `docker build -t rustfs:latest .` |
-| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Useful if you need specific dynamic libraries. | `docker build -f Dockerfile.glibc .` |
-| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code. Includes build tools. Ideal for local development and CI. | `docker build -f Dockerfile.source .` |
+| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads verified RustFS and `rc` binaries from GitHub Releases and bundles `/etc/rustfs/bootstrap`. | `docker build -t rustfs:latest .` |
+| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Includes the bundled bootstrap configuration. | `docker build -f Dockerfile.glibc .` |
+| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code and bundles `rc` and the default bootstrap configuration. | `docker build -f Dockerfile.source .` |
 
 ### Docker Compose Configurations
 
@@ -110,6 +110,36 @@ docker compose up -d
 
 ---
 
+## Docker Compose environment values
+
+Use map syntax when quoting environment values in YAML:
+
+```yaml
+services:
+  rustfs:
+    image: rustfs/rustfs:latest
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    volumes:
+      - ./data:/data
+      - ./logs:/var/log/rustfs
+    environment:
+      RUSTFS_ADDRESS: ":9000"
+      RUSTFS_CONSOLE_ADDRESS: ":9001"
+      RUSTFS_CONSOLE_ENABLE: "true"
+      RUSTFS_OBS_LOGGER_LEVEL: error
+      RUSTFS_OBS_LOG_DIRECTORY: "/var/log/rustfs/"
+      RUSTFS_ACCESS_KEY: "${RUSTFS_ACCESS_KEY:?Set RUSTFS_ACCESS_KEY}"
+      RUSTFS_SECRET_KEY: "${RUSTFS_SECRET_KEY:?Set RUSTFS_SECRET_KEY}"
+```
+
+Create the host directories and grant the container user write access before starting this example; official images run as UID/GID `10001:10001`. Set both credential variables to your own values, then run `docker compose config` to inspect the resolved configuration locally before `docker compose up -d`. The resolved output includes credentials, so do not share it publicly.
+
+With list syntax, use `- RUSTFS_OBS_LOG_DIRECTORY=/var/log/rustfs/` or quote the whole item as `- "RUSTFS_OBS_LOG_DIRECTORY=/var/log/rustfs/"`. Writing `- RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"` passes the inner quotes literally, making this a relative path beginning with `"`, which can fail with `mkdir: cannot create directory '"': Permission denied`. This also applies to addresses and credentials: fix every affected list item, rather than only the log directory. The entrypoint preserves configured values, including literal quotes in valid paths and secrets.
+
+---
+
 ## 🔧 Common Operations
 
 ### Cleaning Up

.docker/observability/otel-collector-config.yaml

@@ -28,6 +28,15 @@ processors:
     check_interval: 1s
     limit_mib: 1024
     spike_limit_mib: 256
+  transform/metrics:
+    error_mode: ignore
+    metric_statements:
+      # RustFS process metrics already expose process_pid as a metric label.
+      # Remove only the duplicate datapoint label and keep the shared resource
+      # process.pid available to sibling metrics and lock histograms.
+      - context: datapoint
+        statements:
+          - delete_key(attributes, "process_pid") where IsMatch(metric.name, "^rustfs_system_process_")
   transform/logs:
     error_mode: ignore
     log_statements:
@@ -112,7 +121,7 @@ service:
       exporters: [ otlp/tempo, otlp/jaeger ]
     metrics:
       receivers: [ otlp ]
-      processors: [ batch ]
+      processors: [ transform/metrics, memory_limiter, batch ]
       exporters: [ prometheus ]
     logs:
       receivers: [ otlp ]

.docker/rejoin-readiness.Dockerfile

@@ -0,0 +1,40 @@
+# syntax=docker/dockerfile:1.6
+# Copyright 2026 RustFS Team
+# Licensed under the Apache License, Version 2.0.
+
+# A compatible native runtime with Python 3.11+, getent and stat is required by
+# the isolated fixture's staging/provenance checks. This is a test image.
+ARG RUNTIME_BASE=python:3.13-slim-trixie
+FROM ${RUNTIME_BASE}
+
+ARG SOURCE_REVISION
+ARG BINARY_SHA256
+ARG BUILD_PROFILE=dev
+COPY rustfs /usr/bin/rustfs
+RUN python3 - "$SOURCE_REVISION" "$BINARY_SHA256" "$BUILD_PROFILE" <<'PY'
+import hashlib
+import os
+import re
+import shutil
+import sys
+
+source, digest, profile = sys.argv[1:]
+if not re.fullmatch(r"[0-9a-f]{40}", source):
+    raise SystemExit("invalid source revision")
+if not re.fullmatch(r"[0-9a-f]{64}", digest):
+    raise SystemExit("invalid binary digest")
+if profile not in ("dev", "release"):
+    raise SystemExit("invalid build profile")
+with open("/usr/bin/rustfs", "rb") as binary:
+    if hashlib.file_digest(binary, "sha256").hexdigest() != digest:
+        raise SystemExit("binary digest mismatch")
+for tool in ("getent", "stat"):
+    if shutil.which(tool) is None:
+        raise SystemExit(f"missing runtime tool: {tool}")
+os.chmod("/usr/bin/rustfs", 0o755)
+PY
+LABEL io.rustfs.repro.source_revision="${SOURCE_REVISION}" \
+      io.rustfs.repro.binary_sha256="${BINARY_SHA256}" \
+      io.rustfs.repro.build_profile="${BUILD_PROFILE}"
+USER 10001:10001
+ENTRYPOINT ["/usr/bin/rustfs"]

.github/dependabot.yml

@@ -32,6 +32,8 @@ updates:
       - "overtrue"
       - "majinghe"
     ignore:
+      - dependency-name: "pulsar"
+        versions: [ "6.9.x" ]
       - dependency-name: "object_store"
         versions: [ "0.13.x" ]
       - dependency-name: "libunftp"

.github/scheduled-validations.json

@@ -769,12 +769,10 @@
       "workflow": ".github/workflows/e2e-distributed.yml",
       "name": "e2e-distributed",
       "roles": [
-        "pr",
         "periodic",
         "manual"
       ],
       "events": [
-        "pull_request",
         "workflow_dispatch",
         "schedule"
       ],
@@ -849,12 +847,11 @@
       "workflow": ".github/workflows/e2e-upgrade.yml",
       "name": "Upgrade Compatibility",
       "roles": [
-        "pr",
         "periodic",
-        "manual"
+        "manual",
+        "release"
       ],
       "events": [
-        "pull_request",
         "push",
         "schedule",
         "workflow_dispatch"

.github/workflows/ci.yml

@@ -113,7 +113,7 @@ jobs:
             printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
             echo "Base CI policy unavailable; running the full matrix."
           fi
-          if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=2$' "$GITHUB_OUTPUT"; then
+          if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=3$' "$GITHUB_OUTPUT"; then
             printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
             echo "Base policy does not cover this workflow contract; retaining the full matrix."
           fi
@@ -215,7 +215,8 @@ jobs:
 
       - name: Archive workspace test binaries
         env:
-          CARGO_BUILD_JOBS: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && '3' || '2' }}
+          # Workspace test compilation has high per-process memory use.
+          CARGO_BUILD_JOBS: "2"
         run: |
           ./scripts/ci/resource_sampler.sh start workspace-archive
           trap './scripts/ci/resource_sampler.sh stop' EXIT
@@ -497,6 +498,63 @@ jobs:
       - name: Check offline enrollment E2E root boundary
         run: ./scripts/check_offline_enrollment_e2e.sh
 
+  fips-provider:
+    name: FIPS provider boundaries
+    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
+    needs: [ quick-checks, classify-changes ]
+    # This builds the RustFS library test target, which exceeds hosted-runner disk capacity.
+    runs-on: sm-standard-4
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        mode: [default, fips]
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+      CARGO_BUILD_JOBS: "2"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-fips-provider
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'
+
+      - name: Install FIPS build tools
+        if: matrix.mode == 'fips'
+        run: sudo apt-get install -y cmake golang clang libclang-dev
+
+      - name: Verify provider installation
+        shell: bash
+        run: |
+          args=()
+          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
+            args+=(--features fips)
+          fi
+          cargo nextest run --locked -p rustfs-tls-runtime --lib "${args[@]}" --no-tests fail -E 'test(provider_policy_isolated)'
+          cargo nextest run --locked -p rustfs-s3-client --lib "${args[@]}" --no-tests fail -E 'test(overflowing_closed_range_is_rejected_before_network_io)'
+          cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(validate_pulsar)'
+          cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(webhook_provider_policy_isolated)'
+          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
+            cargo nextest run --locked -p rustfs-protocols --lib --features ftps,webdav,fips --no-tests fail -E 'test(protocols_reject_incompatible_host_provider)'
+            cargo nextest run --locked -p rustfs-targets --lib --features fips --no-tests fail -E 'test(fips_postgres_rejects_incompatible_host_provider)'
+            cargo nextest run --locked -p rustfs-s3-client --lib --features fips --no-tests fail -E 'test(fips_transition_rejects_incompatible_host_provider)'
+          fi
+          tests=(provider_installation_isolated prepared_generation_shares_crypto_and_serves_sni_with_mtls)
+          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
+            tests+=(fips_provider_builds_fips_tls_configs fips_provider_tls_handshake_accepts_and_rejects_algorithms)
+          fi
+          for test_name in "${tests[@]}"; do
+            cargo nextest run --locked -p rustfs --lib "${args[@]}" --no-tests fail -E "test($test_name)"
+          done
+          cargo clippy --locked -p rustfs --lib "${args[@]}" -- -D warnings
+
   test-and-lint-rio-v2:
     name: Test and Lint (rio-v2)
     # Specialized boundary, hardware and alternate-feature lanes run only on
@@ -1397,6 +1455,7 @@ jobs:
       - workspace-shard-evidence
       - core-contract
       - connect-feature-boundaries
+      - fips-provider
       - test-and-lint-rio-v2
       - test-and-lint-protocols
       - build-rustfs-debug-binary
@@ -1424,6 +1483,7 @@ jobs:
     needs:
       - classify-changes
       - connect-feature-boundaries
+      - fips-provider
       - required-checks
       - typos
       - quick-checks

.github/workflows/cla.yml

@@ -95,7 +95,7 @@ jobs:
 
       - name: Run CLA Bot
         if: github.event_name != 'merge_group'
-        uses: overtrue/cla-bot@5e5929846a60cfb42b16d0d38e135ef67d94fc88 # v0.0.9
+        uses: overtrue/cla-bot@4bd1387c19691168c42fcf5edfa17b15819b2ab3 # v0.1.1
         with:
           github-token: ${{ github.token }}
           registry-token: ${{ steps.registry-token.outputs.token }}

.github/workflows/e2e-distributed.yml

@@ -17,8 +17,8 @@
 # Each selected test starts a real localhost cluster via
 # `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
 # case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
-# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
-# nightly runs, and manual dispatches all execute the same fail-closed suite.
+# `[profile.e2e-distributed]` in `.config/nextest.toml`. The nightly schedule
+# and manual dispatches execute the same fail-closed suite.
 # Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
 #
 # Isolated pool filesystems: expand/decommission/rebalance cases require
@@ -33,32 +33,6 @@
 name: e2e-distributed
 
 on:
-  pull_request:
-    paths:
-      - "Cargo.lock"
-      - "Cargo.toml"
-      - ".config/nextest.toml"
-      - ".config/e2e-distributed-selection.txt"
-      - ".github/workflows/e2e-distributed.yml"
-      - "crates/audit/**"
-      - "crates/common/**"
-      - "crates/config/**"
-      - "crates/e2e_test/**"
-      - "crates/ecstore/**"
-      - "crates/filemeta/**"
-      - "crates/heal/**"
-      - "crates/iam/**"
-      - "crates/lock/**"
-      - "crates/madmin/**"
-      - "crates/notify/**"
-      - "crates/replication/**"
-      - "crates/s3-client/**"
-      - "crates/s3-ops/**"
-      - "crates/s3-types/**"
-      - "crates/scanner/**"
-      - "crates/storage-api/**"
-      - "crates/utils/**"
-      - "rustfs/**"
   workflow_dispatch:
     inputs:
       filter:

.github/workflows/e2e-upgrade.yml

@@ -12,30 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Single-node upgrade and rollback lane.
+#
+# The weekly Monday schedule, version tags, and workflow_dispatch run this
+# suite. Distributed upgrade cases stay on the nightly e2e-distributed lane.
+
 name: Upgrade Compatibility
 
 on:
-  pull_request:
-    paths:
-      - ".github/workflows/e2e-upgrade.yml"
-      - "crates/e2e_test/src/common.rs"
-      - "crates/e2e_test/src/distributed/**"
-      - "crates/e2e_test/src/fake_s3_target/**"
-      - "crates/e2e_test/src/lib.rs"
-      - "crates/e2e_test/src/replication_extension_test.rs"
-      - "crates/e2e_test/src/rpc_fault_proxy.rs"
-      - "crates/e2e_test/src/upgrade_compatibility_test.rs"
-      - "crates/ecstore/**"
-      - "crates/filemeta/**"
-      - "crates/heal/**"
-      - "crates/kms/**"
-      - "crates/storage-api/**"
-      - "rustfs/**"
-      - "Cargo.lock"
   push:
     tags:
       - "[0-9]*.[0-9]*.[0-9]*"
   schedule:
+    # 03:17 UTC every Monday.
     - cron: "17 3 * * 1"
   workflow_dispatch:
 
@@ -44,7 +33,7 @@ permissions:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+  cancel-in-progress: false
 
 env:
   CARGO_TERM_COLOR: always

.github/workflows/nightly-gnu.yml

@@ -87,18 +87,206 @@ jobs:
           install-build-packaging-tools: 'false'
           install-test-tools: 'false'
 
+      - name: Download static console assets
+        shell: bash
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          # Packaging tools are off on this lane, so setup does not install unzip.
+          if ! command -v unzip >/dev/null 2>&1; then
+            sudo apt-get update -qq
+            sudo apt-get install -y -qq unzip
+          fi
+          if ! command -v unzip >/dev/null 2>&1; then
+            echo "unzip is required to extract console assets" >&2
+            exit 1
+          fi
+
+          mkdir -p ./rustfs/static
+
+          verify_sha256() {
+            local expected="$1"
+            local file="$2"
+            local actual=""
+            local actual_line=""
+
+            if command -v sha256sum >/dev/null 2>&1; then
+              actual_line=$(sha256sum -- "$file") || return 1
+            elif command -v shasum >/dev/null 2>&1; then
+              actual_line=$(shasum -a 256 -- "$file") || return 1
+            else
+              echo "No SHA-256 verification tool found" >&2
+              return 1
+            fi
+            actual="${actual_line%%[[:space:]]*}"
+
+            if [ "$actual" = "$expected" ]; then
+              echo "SHA256 verified OK: $actual"
+              return 0
+            else
+              echo "SHA256 mismatch: expected=$expected actual=$actual" >&2
+              return 1
+            fi
+          }
+
+          download_console_assets() {
+            local curl_bin="curl"
+            local python_bin="python3"
+            local console_api="https://api.github.com/repos/rustfs/console/releases/latest"
+            local console_json="console-release.json"
+            local console_url
+            local console_sha256
+            local curl_auth_args=()
+            console_tag=""
+
+            chmod +w ./rustfs/static/LICENSE || true
+
+            if ! command -v "$python_bin" >/dev/null 2>&1; then
+              python_bin="python"
+            fi
+            if ! command -v "$python_bin" >/dev/null 2>&1; then
+              echo "No Python interpreter found for release metadata parsing" >&2
+              return 1
+            fi
+
+            if [[ -n "${GITHUB_TOKEN:-}" ]]; then
+              curl_auth_args=(-H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28")
+            fi
+
+            "$curl_bin" "${curl_auth_args[@]}" --fail -L "$console_api" \
+              -o "$console_json" --retry 3 --retry-delay 5 --max-time 300 || return 1
+
+            read -r console_tag console_url console_sha256 < <("$python_bin" - "$console_json" <<'PY'
+          import json
+          import re
+          import sys
+
+          with open(sys.argv[1], encoding="utf-8") as handle:
+              release = json.load(handle)
+
+          tag = release.get("tag_name", "") or "unknown"
+
+          for asset in release.get("assets", []):
+              name = asset.get("name", "")
+              digest = asset.get("digest", "")
+              url = asset.get("browser_download_url", "")
+              if name.startswith("rustfs-console-") and name.endswith(".zip") and digest.startswith("sha256:") and url:
+                  sha256 = digest.split(":", 1)[1]
+                  if not re.fullmatch(r"[0-9a-fA-F]{64}", sha256):
+                      raise SystemExit(f"console zip asset has invalid sha256 digest: {sha256}")
+                  sys.stdout.buffer.write(f"{tag} {url} {sha256}\n".encode("utf-8"))
+                  break
+          else:
+              raise SystemExit("no console zip asset with sha256 digest found")
+          PY
+            ) || return 1
+
+            if [[ -z "$console_url" || -z "$console_sha256" ]]; then
+              echo "Console release metadata is missing URL or SHA-256 digest" >&2
+              return 1
+            fi
+
+            echo "Console release: ${console_tag}"
+            echo "Downloading console asset: ${console_url}"
+            "$curl_bin" --fail -L "$console_url" -o console.zip --retry 3 --retry-delay 5 --max-time 300 || return 1
+            verify_sha256 "$console_sha256" console.zip || return 2
+            unzip -o console.zip -d ./rustfs/static || return 2
+          }
+
+          if download_console_assets; then
+            rm -f console.zip console-release.json
+          else
+            status=$?
+            rm -f console.zip console-release.json
+            if [[ "$status" -eq 2 ]]; then
+              echo "Console asset integrity verification failed" >&2
+            fi
+            echo "Failed to download verified console assets" >&2
+            exit 1
+          fi
+
+          if [[ ! -s ./rustfs/static/index.html ]]; then
+            echo "Console asset archive is missing static/index.html" >&2
+            exit 1
+          fi
+
+          asset_count=$(find ./rustfs/static -type f | wc -l | tr -d '[:space:]')
+          echo "Console assets ready: version=${console_tag:-unknown}, ${asset_count} files extracted to ./rustfs/static"
+
       - name: Build RustFS
         # Pin the glibc baseline (`<target>.<glibc>` for cargo-zigbuild) so
         # nightlies run on older distros such as RHEL/Rocky 9 (glibc 2.34); #8062.
-        run: cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bins
+        shell: bash
+        run: |
+          # Force rebuild by touching build.rs so a restored cache cannot
+          # reuse a rustfs artifact compiled before these console assets existed.
+          touch rustfs/build.rs
+          cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bin rustfs
 
       - name: Verify glibc symbol baseline
         run: |
           python3 scripts/release/check_glibc.py 2.31 \
-            target/x86_64-unknown-linux-gnu/release/rustfs \
-            target/x86_64-unknown-linux-gnu/release/rustfs-cli
+            target/x86_64-unknown-linux-gnu/release/rustfs
           target/x86_64-unknown-linux-gnu/release/rustfs --version | grep -Fx 'build target : x86_64-unknown-linux-gnu'
 
+      - name: Verify console
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          find_free_port() {
+            python3 - <<'PY'
+          import socket
+
+          with socket.socket() as sock:
+              sock.bind(("127.0.0.1", 0))
+              print(sock.getsockname()[1])
+          PY
+          }
+
+          binary="target/x86_64-unknown-linux-gnu/release/rustfs"
+          data_dir="$(mktemp -d)"
+          log_file="${RUNNER_TEMP}/rustfs-console-smoke.log"
+          server_pid=""
+          trap '
+            if [[ -n "${server_pid:-}" ]]; then
+              kill "$server_pid" 2>/dev/null || true
+              wait "$server_pid" 2>/dev/null || true
+            fi
+            rm -rf "$data_dir"
+          ' EXIT
+
+          if [[ ! -x "$binary" ]]; then
+            echo "rustfs binary not found: ${binary}" >&2
+            exit 1
+          fi
+
+          api_port="$(find_free_port)"
+          console_port="$(find_free_port)"
+          console_url="http://127.0.0.1:${console_port}/rustfs/console/"
+
+          "$binary" server \
+            --address "127.0.0.1:${api_port}" \
+            --console-enable \
+            --console-address "127.0.0.1:${console_port}" \
+            --access-key console-smoke \
+            --secret-key console-smoke-secret \
+            "$data_dir" >"$log_file" 2>&1 &
+          server_pid=$!
+
+          response=""
+          for _ in {1..40}; do
+            response="$(curl --silent --output /dev/null --write-out '%{http_code} %{content_type}' "$console_url" || true)"
+            if [[ "$response" == 200\ text/html* ]]; then
+              exit 0
+            fi
+            sleep 0.25
+          done
+
+          echo "Console endpoint did not return 200 text/html: $response" >&2
+          cat "$log_file"
+          exit 1
+
       - name: Build DEB package
         id: deb
         shell: bash
@@ -346,12 +534,12 @@ jobs:
           path: ${{ steps.publish.outputs.candidate_file }}
           if-no-files-found: error
 
-      # Publish the deb/rpm pair to the auto-testing repo's `assets` branch so
-      # engineers can download and install the nightly directly. The branch is
-      # a single-commit orphan rewritten on every build, which keeps the repo
-      # small while the latest files stay reachable at stable raw URLs.
       # Publish the deb/rpm pair as assets of the rolling `nightly` release on
       # rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
+      # The dispatch below runs only after this upload and the immutable R2
+      # candidate both succeed. It does not replace rustfs-functional-chain.yml,
+      # which still starts from this workflow's workflow_run completion.
+      # kms-vault-lane and kms-vault-ha-failover stay parallel with this job.
       - name: Publish packages to auto-testing release assets
         env:
           ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
@@ -360,6 +548,90 @@ jobs:
           DEB_DATE: ${{ steps.deb.outputs.deb_date }}
           BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
         run: bash scripts/release/publish_nightly_assets.sh
+
+      # Cross-repo repository_dispatch cannot use GITHUB_TOKEN. The secret is a
+      # PAT or GitHub App token for rustfs/auto-testing: classic `repo` scope,
+      # or fine-grained Contents write (the permission GitHub checks for
+      # POST /repos/{owner}/{repo}/dispatches). A dispatch failure fails this
+      # step, and therefore this job, so the scheduled failure alert fires.
+      # GitHub allows at most 10 top-level client_payload properties, so the
+      # schema 2 candidate object is nested under `candidate` and the release
+      # fields sit beside it.
+      - name: Dispatch nightly candidate to auto-testing
+        if: ${{ steps.publish.outputs.candidate_file != '' }}
+        env:
+          DISPATCH_TOKEN: ${{ secrets.AUTO_TESTING_DISPATCH_TOKEN }}
+          CANDIDATE_FILE: ${{ steps.publish.outputs.candidate_file }}
+          DEB_DATE: ${{ steps.deb.outputs.deb_date }}
+          SOURCE_REPO: ${{ github.repository }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          : "${DISPATCH_TOKEN:?AUTO_TESTING_DISPATCH_TOKEN is required to dispatch rustfs/auto-testing}"
+          : "${CANDIDATE_FILE:?nightly candidate manifest is required}"
+          : "${DEB_DATE:?DEB_DATE is required}"
+          [[ "${DEB_DATE}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]
+          [[ "${SOURCE_REPO}" == "rustfs/rustfs" ]]
+
+          payload="${RUNNER_TEMP}/nightly-candidate-dispatch-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
+          jq -n --slurpfile candidate "${CANDIDATE_FILE}" \
+            --arg event_type "nightly-candidate-ready" \
+            --arg source_repo "${SOURCE_REPO}" \
+            --arg latest_alias_url "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb" \
+            --arg auto_testing_release "https://github.com/rustfs/auto-testing/releases/tag/nightly" \
+            --arg channel "nightly" \
+            --arg deb_date "${DEB_DATE}" \
+            --arg nightly_workflow ".github/workflows/nightly-gnu.yml" \
+            '{
+              event_type: $event_type,
+              source_repo: $source_repo,
+              latest_alias_url: $latest_alias_url,
+              auto_testing_release: $auto_testing_release,
+              channel: $channel,
+              deb_date: $deb_date,
+              nightly_workflow: $nightly_workflow,
+              candidate: $candidate[0]
+            }' > "${payload}"
+          if ! jq -e '
+            .event_type == "nightly-candidate-ready"
+            and .source_repo == "rustfs/rustfs"
+            and .channel == "nightly"
+            and .deb_date == env.DEB_DATE
+            and .nightly_workflow == ".github/workflows/nightly-gnu.yml"
+            and .latest_alias_url == "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb"
+            and .auto_testing_release == "https://github.com/rustfs/auto-testing/releases/tag/nightly"
+            and .candidate.schema == 2
+            and ((.candidate | keys | sort) == ["build_run_attempt","build_run_id","package_sha256","package_url","schema","source_ref","source_sha","workflow_sha"])
+            and ((keys | sort) == ["auto_testing_release","candidate","channel","deb_date","event_type","latest_alias_url","nightly_workflow","source_repo"])
+          ' "${payload}" >/dev/null; then
+            echo "nightly dispatch payload does not match the nightly-candidate-ready contract" >&2
+            exit 1
+          fi
+
+          body="${RUNNER_TEMP}/nightly-candidate-dispatch-body-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
+          jq -n --slurpfile client_payload "${payload}" \
+            '{event_type: "nightly-candidate-ready", client_payload: $client_payload[0]}' > "${body}"
+          response="${RUNNER_TEMP}/nightly-candidate-dispatch-response-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.txt"
+          status="$(curl -sS --retry 3 --connect-timeout 15 --max-time 60 \
+            -o "${response}" -w '%{http_code}' \
+            -X POST \
+            -H "Authorization: Bearer ${DISPATCH_TOKEN}" \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            -H "Content-Type: application/json" \
+            --data-binary @"${body}" \
+            https://api.github.com/repos/rustfs/auto-testing/dispatches)" || {
+            echo "repository_dispatch request to rustfs/auto-testing failed before an HTTP status" >&2
+            exit 1
+          }
+          if [[ "${status}" != "204" ]]; then
+            echo "repository_dispatch to rustfs/auto-testing failed: HTTP ${status}" >&2
+            if [[ -s "${response}" ]]; then
+              cat "${response}" >&2
+            fi
+            exit 1
+          fi
+          echo "dispatched nightly-candidate-ready to rustfs/auto-testing (deb_date=${DEB_DATE}, run=${GITHUB_RUN_ID}, attempt=${GITHUB_RUN_ATTEMPT})"
   # Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
   #
   # RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and

.github/workflows/windows-filesystem.yml

@@ -107,6 +107,32 @@ jobs:
         shell: pwsh
         run: cargo test -p rustfs-ecstore --lib multipart_owned_record_parent_barriers_ -- --nocapture
 
+      # Native multipart completion publishes its payload from a different
+      # parent than its metadata, and abort renames under a precondition; both
+      # need their own Windows handle discipline.
+      - name: Test native multipart complete and abort publication
+        shell: pwsh
+        run: >-
+          cargo test -p rustfs-ecstore --lib --
+          native_abort_missing_state_with_actual_owned_records_never_becomes_legacy_or_missing
+          native_abort_pending_intent_is_not_a_delete_certificate_and_raw_cleanup_preserves_exact_evidence
+          --nocapture
+
+      # Owned-undo reclaim removes a directory it pinned, copy heal persists
+      # fault bytes through a handle, and metadata reads meet a directory.
+      - name: Test native multipart reclaim, copy heal, and directory reads
+        shell: pwsh
+        run: >-
+          cargo test -p rustfs-ecstore --lib --
+          native_multipart_owned_undo
+          native_multipart_copy_heal
+          orphan_multipart_cleanup_rejects_metadata_directory_on_one_disk
+          --nocapture
+
+      - name: Test directory removal fallback
+        shell: pwsh
+        run: "cargo test -p rustfs-ecstore --lib disk::fs::tests::test_remove -- --nocapture"
+
       - name: Test Windows path conversion
         shell: pwsh
         run: "cargo test -p rustfs-utils --no-default-features --features path --lib dunce:: -- --nocapture"

CHANGELOG.md

@@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - IAM snapshot/deletion replay, target-assigned delete-marker purges, timestamp ordering, and best-effort peer broadcast now close the control-plane gaps found by the R6 review ([#7195](https://github.com/rustfs/rustfs/pull/7195)).
 - A `set-remote-target?update=true&creds=true` request on a site-replication peer's bucket target is now rejected with `400 InvalidRequest`, naming the target and why it counts as peer-owned. It used to return success while keeping the stored endpoint, credentials, and TLS settings, so an endpoint edit from the Console's bucket replication form appeared to apply but did not. Change a peer's address with a site-replication edit instead; see "Changing a site's address" in `docs/operations/site-replication-operations.md`. The `sync`, `bandwidth`, `proxy`, and `path` update groups are unchanged.
 - Site-replication edits that omit the site name, such as `mc admin replicate update --deployment-id <ID> --endpoint <URL>`, now keep the stored name on the editing site and on every receiver instead of renaming the site after the new endpoint host. No persisted format changes.
+- The persisted replication MRF journal is now replayed every 5 minutes while the server runs, not only at startup. Delete-marker purges whose targets were unreachable, and replications that overflowed the worker queues, now converge within one replay interval after the target returns instead of waiting for a restart. A target that stays down is re-attempted once per retained entry per interval across the cluster: the start time of the last pass is shared through a small `config/replication/mrf.bin.replay` object, so nodes do not each replay once per interval. Older versions ignore that object; existing persisted formats are unchanged.
 - Upgrade and rollback: upgrade every node in one site consecutively and verify reconciliation before moving to the next site; do not intentionally run a site mixed-version. Target-version ledger keys are harmless on rollback, although old code cannot use their routing. Before rolling back past [#7307](https://github.com/rustfs/rustfs/pull/7307), drain or repair every pending version purge: older code can free a retained version's data directory before its remote purge is acknowledged. See `docs/operations/site-replication-operations.md`.
 
 ### Security

Cargo.lock

@@ -40,9 +40,9 @@ dependencies = [
 
 [[package]]
 name = "aes"
-version = "0.9.3"
+version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35f0f96ce78e38c3dc6d8948aa8163d06385be74000f3c7a95bf1eef35d3ea32"
+checksum = "fc0fc36d7440d015b45468ba08a21e80e056ebbd014c611c6298bb7503e04bd1"
 dependencies = [
  "cipher 0.5.2",
  "cpubits",
@@ -57,7 +57,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f2b8006a0c83f52b62ba44a97b58bf76fe2f70a329e588f67f89691d93d498f"
 dependencies = [
  "aead",
- "aes 0.9.3",
+ "aes 0.9.4",
  "cipher 0.5.2",
  "ctr",
  "ctutils",
@@ -370,17 +370,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f04e0a89cc175eaf3740282537851e6fd9d7276f97730b9d0ea2859c11a85df"
 dependencies = [
  "arrow-arith",
- "arrow-array",
- "arrow-buffer",
+ "arrow-array 59.3.2",
+ "arrow-buffer 59.3.2",
  "arrow-cast",
  "arrow-csv",
- "arrow-data",
- "arrow-ipc",
+ "arrow-data 59.3.2",
+ "arrow-ipc 59.3.2",
  "arrow-json",
  "arrow-ord",
  "arrow-row",
- "arrow-schema",
- "arrow-select",
+ "arrow-schema 59.3.2",
+ "arrow-select 59.3.2",
  "arrow-string",
 ]
 
@@ -390,10 +390,10 @@ version = "59.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "157943c720919e8f5b6731407ce5cd4c9a5ef47a75bf10edaf24b7081b5763ef"
 dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-data",
- "arrow-schema",
+ "arrow-array 59.3.2",
+ "arrow-buffer 59.3.2",
+ "arrow-data 59.3.2",
+ "arrow-schema 59.3.2",
  "chrono",
  "num-traits",
 ]
@@ -405,9 +405,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "72b5b1682924f11ecf6929689c16cbc69efbb6dd141b06ad98ee1cb10d858c43"
 dependencies = [
  "ahash",
- "arrow-buffer",
- "arrow-data",
- "arrow-schema",
+ "arrow-buffer 59.3.2",
+ "arrow-data 59.3.2",
+ "arrow-schema 59.3.2",
  "chrono",
  "chrono-tz",
  "half",
@@ -417,6 +417,24 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "arrow-array"
+version = "60.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f3d22a41d3df78b71df0b685af219ae3a94188ab98d6fb29c958eb55c3247f4"
+dependencies = [
+ "ahash",
+ "arrow-buffer 60.1.0",
+ "arrow-data 60.1.0",
+ "arrow-schema 60.1.0",
+ "chrono",
+ "half",
+ "hashbrown 0.17.1",
+ "num-complex",
+ "num-integer",
+ "num-traits",
+]
+
 [[package]]
 name = "arrow-buffer"
 version = "59.3.2"
@@ -429,18 +447,30 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "arrow-buffer"
+version = "60.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7bd46a530bc1a8e763102fc36c1dc0f51f3ffa150c78e7f5658ccb54b1458ade"
+dependencies = [
+ "bytes",
+ "half",
+ "num-bigint 0.5.1",
+ "num-traits",
+]
+
 [[package]]
 name = "arrow-cast"
 version = "59.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f23c3379feede42135ef27ca60e67f6e643cd4343ae74e4c8b16c83f55afd85"
 dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-data",
+ "arrow-array 59.3.2",
+ "arrow-buffer 59.3.2",
+ "arrow-data 59.3.2",
  "arrow-ord",
- "arrow-schema",
- "arrow-select",
+ "arrow-schema 59.3.2",
+ "arrow-select 59.3.2",
  "atoi 2.0.0",
  "base64 0.23.1",
  "chrono",
@@ -451,15 +481,26 @@ dependencies = [
  "ryu",
 ]
 
+[[package]]
+name = "arrow-cmp"
+version = "60.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1913e57a64fbf2c4502fa6d54b547d6a1ae59e3146cbc15321d47468c937abd1"
+dependencies = [
+ "arrow-array 60.1.0",
+ "arrow-buffer 60.1.0",
+ "arrow-schema 60.1.0",
+]
+
 [[package]]
 name = "arrow-csv"
 version = "59.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "85b1984628b091b8319b3c305fce21d04f9e9b2578bb733b0d0108462ae1d698"
 dependencies = [
- "arrow-array",
+ "arrow-array 59.3.2",
  "arrow-cast",
- "arrow-schema",
+ "arrow-schema 59.3.2",
  "chrono",
  "csv",
  "csv-core",
@@ -472,8 +513,21 @@ version = "59.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bb0a031c7ff5e2bdb443117054fb466af1d4df5b7f0ecbe197713dc8b113b1ba"
 dependencies = [
- "arrow-buffer",
- "arrow-schema",
+ "arrow-buffer 59.3.2",
+ "arrow-schema 59.3.2",
+ "half",
+ "num-integer",
+ "num-traits",
+]
+
+[[package]]
+name = "arrow-data"
+version = "60.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "415b7fbc4cf56b080bb5677a4ae74211b5c79e571ed37a1d547633ec07f3b43b"
+dependencies = [
+ "arrow-buffer 60.1.0",
+ "arrow-schema 60.1.0",
  "half",
  "num-integer",
  "num-traits",
@@ -485,28 +539,43 @@ version = "59.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f5dd1afadda2eb821aaf851bb4c0adef60c30cc0f1a29bd02fb99dd07fb9f58d"
 dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-data",
- "arrow-schema",
- "arrow-select",
+ "arrow-array 59.3.2",
+ "arrow-buffer 59.3.2",
+ "arrow-data 59.3.2",
+ "arrow-schema 59.3.2",
+ "arrow-select 59.3.2",
  "flatbuffers",
  "lz4_flex",
  "zstd 0.13.4",
 ]
 
+[[package]]
+name = "arrow-ipc"
+version = "60.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "45f3e766bccf5a0cb57a5de52a4cba450dea5171547c2cb5939b486d08887cba"
+dependencies = [
+ "arrow-array 60.1.0",
+ "arrow-buffer 60.1.0",
+ "arrow-data 60.1.0",
+ "arrow-schema 60.1.0",
+ "arrow-select 60.1.0",
+ "flatbuffers",
+ "zstd 0.14.3",
+]
+
 [[package]]
 name = "arrow-json"
 version = "59.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e2bc4d56a0dad84cc51c5812ccff48b642047a1f3be80349cd84318983e79edb"
 dependencies = [
- "arrow-array",
- "arrow-buffer",
+ "arrow-array 59.3.2",
+ "arrow-buffer 59.3.2",
  "arrow-cast",
  "arrow-ord",
- "arrow-schema",
- "arrow-select",
+ "arrow-schema 59.3.2",
+ "arrow-select 59.3.2",
  "chrono",
  "half",
  "indexmap 2.14.2",
@@ -526,11 +595,11 @@ version = "59.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "561b8209faa2cd489764b7fd07d7cb9e8d274d78048c38de59f77c41140cb834"
 dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-data",
- "arrow-schema",
- "arrow-select",
+ "arrow-array 59.3.2",
+ "arrow-buffer 59.3.2",
+ "arrow-data 59.3.2",
+ "arrow-schema 59.3.2",
+ "arrow-select 59.3.2",
 ]
 
 [[package]]
@@ -539,10 +608,10 @@ version = "59.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0d8c68b87a569f92b90c46cb0a09d86bd9609ddc9b1bf1fc5a503c8c6907c14e"
 dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-data",
- "arrow-schema",
+ "arrow-array 59.3.2",
+ "arrow-buffer 59.3.2",
+ "arrow-data 59.3.2",
+ "arrow-schema 59.3.2",
  "half",
 ]
 
@@ -556,6 +625,12 @@ dependencies = [
  "serde_json",
 ]
 
+[[package]]
+name = "arrow-schema"
+version = "60.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "de2c8e714b466404cdc1189e89f892426efea614d938e16c15518c340cd43ba3"
+
 [[package]]
 name = "arrow-select"
 version = "59.3.2"
@@ -563,10 +638,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c844141958d805b47d1208e0d12aac53fcf388ce8011639fe82aa492adde0bb4"
 dependencies = [
  "ahash",
- "arrow-array",
- "arrow-buffer",
- "arrow-data",
- "arrow-schema",
+ "arrow-array 59.3.2",
+ "arrow-buffer 59.3.2",
+ "arrow-data 59.3.2",
+ "arrow-schema 59.3.2",
+ "num-traits",
+]
+
+[[package]]
+name = "arrow-select"
+version = "60.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a5af3f6656c5f2c6944f93d9a357a7f204c93d5f349385a44cf46c4a26fbcf30"
+dependencies = [
+ "ahash",
+ "arrow-array 60.1.0",
+ "arrow-buffer 60.1.0",
+ "arrow-cmp",
+ "arrow-data 60.1.0",
+ "arrow-schema 60.1.0",
  "num-traits",
 ]
 
@@ -576,11 +666,11 @@ version = "59.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bc130892b1a9552da77971924b390e06cf375d1f7e4344a7b7b40ca4426fa470"
 dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-data",
- "arrow-schema",
- "arrow-select",
+ "arrow-array 59.3.2",
+ "arrow-buffer 59.3.2",
+ "arrow-data 59.3.2",
+ "arrow-schema 59.3.2",
+ "arrow-select 59.3.2",
  "memchr",
  "num-traits",
  "regex",
@@ -669,9 +759,9 @@ dependencies = [
 
 [[package]]
 name = "async-compression"
-version = "0.4.51"
+version = "0.4.52"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d5755640120b24961f8231e05892e4a4f7e0d04e8aeca161508c9c397946970"
+checksum = "6ad47a740fb906fffdd891e8b6d5fde2511600f7bfaebd68939fa08a14999459"
 dependencies = [
  "compression-codecs",
  "compression-core",
@@ -682,9 +772,9 @@ dependencies = [
 
 [[package]]
 name = "async-executor"
-version = "1.14.0"
+version = "1.14.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c96bf972d85afc50bf5ab8fe2d54d1586b4e0b46c97c50a0c9e71e2f7bcd812a"
+checksum = "f426a31126c735126297bc5ea7a521263c9faff4f63ca415267ee625651c4f37"
 dependencies = [
  "async-task",
  "concurrent-queue",
@@ -710,9 +800,9 @@ dependencies = [
 
 [[package]]
 name = "async-lock"
-version = "3.4.2"
+version = "3.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311"
+checksum = "0876700b0d007ad25d5508865a4259c3af9af6d296bf663a0d586d0bc3369f6f"
 dependencies = [
  "event-listener",
  "event-listener-strategy",
@@ -1993,9 +2083,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.10.0"
+version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0"
+checksum = "375989decec1a0fa424442bed520fa12ede43a48475ce8c7417d2c69b32b5f6b"
 dependencies = [
  "find-msvc-tools",
  "jobserver",
@@ -2251,9 +2341,9 @@ dependencies = [
 
 [[package]]
 name = "compression-codecs"
-version = "0.4.46"
+version = "0.4.47"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cae36683101b2899f48bd31f8db9839437356188e3b1ff23096e73dab09ba8fe"
+checksum = "84f840cff84d10085260e460023934d0af699e71212b09b2a41452e8bd9cda43"
 dependencies = [
  "brotli 9.1.1",
  "bzip2",
@@ -2490,18 +2580,18 @@ dependencies = [
 
 [[package]]
 name = "crc32c"
-version = "0.6.8"
+version = "0.6.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47"
+checksum = "7f132a176423ee852e8f0a35d43088d1850966b50745be818a88b76f64b1c6cb"
 dependencies = [
  "rustc_version",
 ]
 
 [[package]]
 name = "crc32fast"
-version = "1.5.3"
+version = "1.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055"
+checksum = "5f97d3416d5337d15c4b0d78c8c91df922ca143f57ec20e2d1c188333047ef04"
 dependencies = [
  "cfg-if",
 ]
@@ -2676,9 +2766,9 @@ dependencies = [
 
 [[package]]
 name = "ctutils"
-version = "0.4.7"
+version = "0.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2376ffd6f96d8a9db59875b4b6b0237d6900b5baf64279714e05493ce64828a"
+checksum = "261af4dc7a06ea8a527a8717d8cbba3417530d6ea14e1edb3de6b67806be474f"
 dependencies = [
  "cmov",
  "subtle",
@@ -2891,7 +2981,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4cd5317db77ba6bca25c761a8091b9d294a9f4188e4f9dcd0243e6a57de862ac"
 dependencies = [
  "arrow",
- "arrow-schema",
+ "arrow-schema 59.3.2",
  "async-trait",
  "chrono",
  "datafusion-catalog",
@@ -2925,7 +3015,7 @@ dependencies = [
  "log",
  "object_store",
  "parking_lot 0.12.5",
- "parquet",
+ "parquet 59.3.2",
  "sqlparser",
  "tempfile",
  "tokio",
@@ -2989,8 +3079,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d26626e12b224b83351239465dcd00f72d47b02fb5e5b65a643e2dc8ed37f687"
 dependencies = [
  "arrow",
- "arrow-ipc",
- "arrow-schema",
+ "arrow-ipc 59.3.2",
+ "arrow-schema 59.3.2",
  "chrono",
  "foldhash 0.2.0",
  "half",
@@ -3001,7 +3091,7 @@ dependencies = [
  "log",
  "num-traits",
  "object_store",
- "parquet",
+ "parquet 59.3.2",
  "recursive",
  "sqlparser",
  "tokio",
@@ -3057,7 +3147,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "65b1b041688cad707fa103c3e7f4c6d89b3134ebf67da0129070dad3381851eb"
 dependencies = [
  "arrow",
- "arrow-ipc",
+ "arrow-ipc 59.3.2",
  "async-trait",
  "bytes",
  "datafusion-common",
@@ -3127,7 +3217,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7c521ae91b6d899d7190812881f45fe9463c222c11d90149f5332a2e56555904"
 dependencies = [
  "arrow",
- "arrow-schema",
+ "arrow-schema 59.3.2",
  "async-trait",
  "bytes",
  "datafusion-common",
@@ -3148,7 +3238,7 @@ dependencies = [
  "log",
  "object_store",
  "parking_lot 0.12.5",
- "parquet",
+ "parquet 59.3.2",
  "tokio",
 ]
 
@@ -3165,7 +3255,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39cfdd5ac0769d1e74a885450e49fde8f63ca7ab41eb479a068564aaea80b837"
 dependencies = [
  "arrow",
- "arrow-buffer",
+ "arrow-buffer 59.3.2",
  "async-trait",
  "bytes",
  "dashmap",
@@ -3191,7 +3281,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "86560fd7df4dda924f5474b588c56e7ae8e96dcf158c7327c9cfe9daff448ec5"
 dependencies = [
  "arrow",
- "arrow-schema",
+ "arrow-schema 59.3.2",
  "async-trait",
  "chrono",
  "datafusion-common",
@@ -3227,7 +3317,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "939ae2771da8a3fdc23ab4de610ae8fef9515476626b49d96d0d5c782b105771"
 dependencies = [
  "arrow",
- "arrow-buffer",
+ "arrow-buffer 59.3.2",
  "base64 0.23.1",
  "chrono",
  "chrono-tz",
@@ -3387,7 +3477,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9f002cef849e10c0597a217ec7891c2796c0f8b42d739f5d24abc24496373406"
 dependencies = [
  "arrow",
- "arrow-schema",
+ "arrow-schema 59.3.2",
  "datafusion-common",
  "datafusion-expr",
  "datafusion-expr-common",
@@ -3462,10 +3552,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b1451d6817e5ac0dd21819b19949d1c420ab51d748f55dd9a96ecb5418a24c97"
 dependencies = [
  "arrow",
- "arrow-data",
- "arrow-ipc",
+ "arrow-data 59.3.2",
+ "arrow-ipc 59.3.2",
  "arrow-ord",
- "arrow-schema",
+ "arrow-schema 59.3.2",
  "async-trait",
  "bytes",
  "datafusion-common",
@@ -3513,7 +3603,7 @@ version = "55.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "62dc5ce37a5ce1ff5362949776829aded2529a11dbaa13d24b0ffa6a059e106a"
 dependencies = [
- "arrow-schema",
+ "arrow-schema 59.3.2",
  "async-trait",
  "datafusion-common",
  "datafusion-execution",
@@ -3816,9 +3906,9 @@ dependencies = [
 
 [[package]]
 name = "dial9"
-version = "0.5.6"
+version = "0.5.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "11185c62245c5ff61c753ec8ec3d56341a13fe4195344ecc100ce94f780533cc"
+checksum = "2f3298a54fd82af243c9f122595762f5b0e48bd3d1ea8fe3f3eb5e95c130e881"
 dependencies = [
  "bon",
  "dial9-core",
@@ -3833,9 +3923,9 @@ dependencies = [
 
 [[package]]
 name = "dial9-core"
-version = "0.5.6"
+version = "0.5.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "54f1dcb5d20bdc85b9beb7c2864540526c1394c00783e9f3544feed1cfe529e8"
+checksum = "ebc7907b149a394404e3075f57bfbc751c7292ef115027f13f4995ff3a8b360c"
 dependencies = [
  "ahash",
  "arc-swap",
@@ -3867,9 +3957,9 @@ dependencies = [
 
 [[package]]
 name = "dial9-metrique"
-version = "0.5.6"
+version = "0.5.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56645773b4185f51a9528625a5aedcd4bbc5b5919ba68e7a6108cf076f635efc"
+checksum = "35f30e80d69214ed7c684d81afd8e52315b4acbfa5be9ebb84ccee02ebd06f44"
 dependencies = [
  "dial9-core",
  "dial9-trace-format",
@@ -3881,9 +3971,9 @@ dependencies = [
 
 [[package]]
 name = "dial9-perf-self-profile"
-version = "0.5.6"
+version = "0.5.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "00c8c96d18b28ec76adde2107f7d2d5f8bb53b0670fba31bc012841703bc49ca"
+checksum = "17f09434b7623ef8b9b107062941a8e471e6044d6c946fa4af899130223b7a28"
 dependencies = [
  "blazesym",
  "bon",
@@ -3900,9 +3990,9 @@ dependencies = [
 
 [[package]]
 name = "dial9-tokio-telemetry"
-version = "0.5.6"
+version = "0.5.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ccc79f3683219471a8758b8ce459589167c55d72a649d27b89068a1a4cfffe9"
+checksum = "0428cba60fbcf37ab82666f3a7ef3b6d0671e90dfb6a352376813b471319f061"
 dependencies = [
  "bon",
  "bytes",
@@ -3925,9 +4015,9 @@ dependencies = [
 
 [[package]]
 name = "dial9-trace-format"
-version = "0.5.3"
+version = "0.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44026339a4c6cbd4914c3eb85e653e741d5739f1e401695a039127ba49c19731"
+checksum = "af753ba562fab9b99024b74234fe3a93b01c32d6474db8ae4f2c3bb562958f8f"
 dependencies = [
  "ahash",
  "dial9-trace-format-derive",
@@ -3937,9 +4027,9 @@ dependencies = [
 
 [[package]]
 name = "dial9-trace-format-derive"
-version = "0.5.3"
+version = "0.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbdbf5c76eb6824b0576e6891811caae93fc0bc96c23003556e3f752d069a84d"
+checksum = "e00786284c5d77b60aac60ee96797b8ac020306da8f02fecb0e86c5e82610aae"
 dependencies = [
  "proc-macro-crate",
  "proc-macro2",
@@ -4641,7 +4731,6 @@ version = "1.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "337d46834ee672ab3e48caca2cb0c78cc174fb12b3a68d0d88f99a0519a5e36e"
 dependencies = [
- "generic-array 0.14.7",
  "rustversion",
  "typenum",
 ]
@@ -5805,9 +5894,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
 
 [[package]]
 name = "jiff"
-version = "0.2.38"
+version = "0.2.39"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383"
+checksum = "3ac703058abac89e5c39b0b879dce2b6b14a4ba36600c7e7e3d0c60a6883ef3c"
 dependencies = [
  "defmt 1.2.0",
  "jiff-core",
@@ -5822,9 +5911,9 @@ dependencies = [
 
 [[package]]
 name = "jiff-core"
-version = "0.1.1"
+version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c"
+checksum = "aee43309eb53bafb8c36d35d6e444769c8f8e6206a9fbf7546f5db9d7d4a0e93"
 dependencies = [
  "defmt 1.2.0",
  "log",
@@ -5832,9 +5921,9 @@ dependencies = [
 
 [[package]]
 name = "jiff-static"
-version = "0.2.38"
+version = "0.2.39"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c"
+checksum = "6dc32470c67eb7d0e9ccb74303fa925313dc04cecb1aa048beaa6f391e36a93d"
 dependencies = [
  "jiff-core",
  "proc-macro2",
@@ -6323,9 +6412,9 @@ dependencies = [
 
 [[package]]
 name = "lzma-rust2"
-version = "0.23.0"
+version = "0.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1c46f3fcd2e888afdb01a07efc071cb21300e07c68d32c3e8b01b5c8cb1fcaa"
+checksum = "490750560fa7851fbbf5d3ab1a8750f09350ed361ed8aed85b1e1e40587812ee"
 dependencies = [
  "sha2 0.11.1",
 ]
@@ -7591,12 +7680,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8eb260d566f2fbdabe356d26aaa5d13403aac29b5b4e44463b8f9bd7b2b12680"
 dependencies = [
  "ahash",
- "arrow-array",
- "arrow-buffer",
- "arrow-data",
- "arrow-ipc",
- "arrow-schema",
- "arrow-select",
+ "arrow-array 59.3.2",
+ "arrow-buffer 59.3.2",
+ "arrow-data 59.3.2",
+ "arrow-ipc 59.3.2",
+ "arrow-schema 59.3.2",
+ "arrow-select 59.3.2",
  "base64 0.23.1",
  "brotli 8.0.4",
  "bytes",
@@ -7618,6 +7707,32 @@ dependencies = [
  "zstd 0.13.4",
 ]
 
+[[package]]
+name = "parquet"
+version = "60.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "806eaa0f1c0436dcb43bee8e6a82bf2efff90521a60172f8281cb9ef577b43a4"
+dependencies = [
+ "ahash",
+ "arrow-array 60.1.0",
+ "arrow-buffer 60.1.0",
+ "arrow-data 60.1.0",
+ "arrow-ipc 60.1.0",
+ "arrow-schema 60.1.0",
+ "arrow-select 60.1.0",
+ "base64 0.23.1",
+ "bytes",
+ "chrono",
+ "half",
+ "hashbrown 0.17.1",
+ "num-bigint 0.5.1",
+ "num-integer",
+ "num-traits",
+ "seq-macro",
+ "twox-hash",
+ "zstd 0.14.3",
+]
+
 [[package]]
 name = "password-hash"
 version = "0.6.1"
@@ -7924,7 +8039,7 @@ version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "63d440a804ec8d6fafbb6b84471e013286658d373248927692ab3366686220ca"
 dependencies = [
- "aes 0.9.3",
+ "aes 0.9.4",
  "aes-gcm",
  "cbc 0.2.1",
  "der 0.8.3",
@@ -8976,9 +9091,9 @@ dependencies = [
 
 [[package]]
 name = "reflink-copy"
-version = "0.1.31"
+version = "0.1.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3dd31f458a66eeda05a9e837fc5f0effbaacbe8565868ecae5b37bf0544c19f5"
+checksum = "1ad64f61357bcf16105a0f353f9783733409c8bd889bc326ad857959605ea927"
 dependencies = [
  "cfg-if",
  "libc",
@@ -9233,11 +9348,11 @@ dependencies = [
 
 [[package]]
 name = "russh"
-version = "0.65.0"
+version = "0.65.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3bb77a4b259312b2abbc541bca1c13cb4e9b61c560a7fd894068c3b92e0c9dc3"
+checksum = "94be3d86c71b9ed220a675c405e35c7a02565a009bf8da2e9cc13a700320a5c5"
 dependencies = [
- "aes 0.9.3",
+ "aes 0.9.4",
  "aws-lc-rs",
  "bitflags 2.13.2",
  "block-padding 0.4.2",
@@ -9497,6 +9612,7 @@ dependencies = [
  "rustfs-log-analyzer",
  "rustfs-madmin",
  "rustfs-mimalloc",
+ "rustfs-mimalloc-sys",
  "rustfs-notify",
  "rustfs-object-capacity",
  "rustfs-object-data-cache",
@@ -9747,6 +9863,7 @@ dependencies = [
  "path-absolutize",
  "pin-project-lite",
  "proptest",
+ "quick-xml",
  "rand 0.10.3",
  "ratelimit",
  "rcgen",
@@ -9779,7 +9896,9 @@ dependencies = [
  "rustfs-s3-ops",
  "rustfs-s3-types",
  "rustfs-scanner-metrics",
+ "rustfs-signer",
  "rustfs-storage-api",
+ "rustfs-tls-runtime",
  "rustfs-uring",
  "rustfs-utils",
  "rustix",
@@ -9975,6 +10094,7 @@ version = "1.0.2"
 dependencies = [
  "bytes",
  "hotpath",
+ "metrics",
  "rustfs-io-metrics",
  "thiserror 2.0.21",
  "tokio",
@@ -9999,9 +10119,9 @@ dependencies = [
 
 [[package]]
 name = "rustfs-kafka"
-version = "1.4.0"
+version = "1.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd2f8b46ec4dd462c2235948d1efa2103957230c171e118c6ff8e5d1a845ab48"
+checksum = "82dc39ba1a5a9f6b2e49decb0b54c23ebaebdfb08a899307c266694bfd269bf9"
 dependencies = [
  "base64 0.23.1",
  "bytes",
@@ -10024,9 +10144,9 @@ dependencies = [
 
 [[package]]
 name = "rustfs-kafka-async"
-version = "1.4.0"
+version = "1.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b3b98705f6f9f5071b8793d29935de764c31668c6e9877553b229eb67127af6"
+checksum = "684f5e78ea1421bbf915e4cd0515bdefd587cfa281da000e41ae4a434f90195a"
 dependencies = [
  "base64 0.23.1",
  "bytes",
@@ -10161,6 +10281,7 @@ dependencies = [
  "hotpath",
  "parking_lot 0.13.0",
  "rand 0.10.3",
+ "rustfs-common",
  "rustfs-io-metrics",
  "rustfs-utils",
  "serde",
@@ -10214,18 +10335,18 @@ dependencies = [
 
 [[package]]
 name = "rustfs-mimalloc"
-version = "0.7.2"
+version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8301a578772fc9823d68e083e42c3b414613d588a0eac594c1ca6bc0bbe0653f"
+checksum = "216ecfc9341b5d6e2506d47ce57c9232ae1b8020bb10264ea05286fb234286e7"
 dependencies = [
  "rustfs-mimalloc-sys",
 ]
 
 [[package]]
 name = "rustfs-mimalloc-sys"
-version = "0.7.2"
+version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6eb4082853043cb29556b4b776b05f28c3146706f43c53869dbf1ef262e00e91"
+checksum = "fb16ad68f96883407a75cdb88252b43dc2ea592428f0fc54ff7885756de4b814"
 dependencies = [
  "cc",
 ]
@@ -10419,6 +10540,7 @@ dependencies = [
  "md-5 0.11.0",
  "percent-encoding",
  "proptest",
+ "rcgen",
  "regex",
  "russh",
  "russh-sftp",
@@ -10637,6 +10759,7 @@ dependencies = [
  "arc-swap",
  "async-compression",
  "async-trait",
+ "brotli 9.1.1",
  "bytes",
  "chrono",
  "crc-fast",
@@ -10646,6 +10769,8 @@ dependencies = [
  "futures-core",
  "hotpath",
  "http 1.5.0",
+ "lz4",
+ "lz4_flex",
  "memchr",
  "metrics",
  "parking_lot 0.13.0",
@@ -10672,8 +10797,10 @@ version = "1.0.2"
 dependencies = [
  "async-trait",
  "bytes",
+ "csv-core",
  "datafusion",
  "derive_builder",
+ "flate2",
  "futures",
  "hotpath",
  "parking_lot 0.13.0",
@@ -10776,6 +10903,7 @@ dependencies = [
  "time",
  "tracing",
  "tracing-subscriber",
+ "url",
 ]
 
 [[package]]
@@ -10908,6 +11036,7 @@ version = "1.0.2"
 dependencies = [
  "async-trait",
  "axum",
+ "futures",
  "hotpath",
  "http 1.5.0",
  "ipnetwork",
@@ -10942,6 +11071,7 @@ dependencies = [
 name = "rustfs-utils"
 version = "1.0.2"
 dependencies = [
+ "aws-lc-rs",
  "base64-simd",
  "blake2",
  "brotli 9.1.1",
@@ -11082,15 +11212,14 @@ dependencies = [
 
 [[package]]
 name = "rustls-connector"
-version = "0.23.9"
+version = "0.23.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "524994fd6957827c775abc941f41a146ce351bbd0a9b8cf632ee37f43a13c205"
+checksum = "f799ef81c26fda57609b9456ae1e6750e2ca3dafc2dc5478232c0705f1458fbf"
 dependencies = [
  "futures-io",
  "futures-rustls",
  "log",
  "rustls",
- "rustls-pki-types",
  "rustls-platform-verifier",
  "rustls-webpki",
 ]
@@ -12028,7 +12157,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d801accda99469cde6d73da741422610fdf6508a72d9a69d1b55cb241c720597"
 dependencies = [
  "aead",
- "aes 0.9.3",
+ "aes 0.9.4",
  "aes-gcm",
  "chacha20",
  "cipher 0.5.2",
@@ -12420,18 +12549,18 @@ dependencies = [
 
 [[package]]
 name = "test-case"
-version = "3.4.0"
+version = "3.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "124953e7f67cb0b2fcfb87e899e4ae5a64fb68e2e160767933cc67b646fb0042"
+checksum = "ae6e80608d2037173d7d59a09089d249e43352c182f6dde6b00d138bcb7cfdbe"
 dependencies = [
  "test-case-macros",
 ]
 
 [[package]]
 name = "test-case-core"
-version = "3.4.0"
+version = "3.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd097615b407247e102046ba34426bb594923ab3554cf0f3cdfed050f1a5a3e8"
+checksum = "29b8e1f65819001927b8cdcbb5560f2428212f2e5ab5684e3c6815a4d80e72db"
 dependencies = [
  "cfg-if",
  "proc-macro2",
@@ -12441,9 +12570,9 @@ dependencies = [
 
 [[package]]
 name = "test-case-macros"
-version = "3.4.0"
+version = "3.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4ce35ece947bccac166e1ded639133827a953f69a48ba134d17f91108333082"
+checksum = "802d6165a14df1598b7290a8c546df402940d3cfbe82d15c0fd35de176771693"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -12741,9 +12870,9 @@ dependencies = [
 
 [[package]]
 name = "toml"
-version = "1.1.8+spec-1.1.0"
+version = "1.2.0+spec-1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "20489e00e4d8741d6be680764cc12e270655e375a20d1011e844a9c3379e678d"
+checksum = "c5ffcdf8f50436dbbfd675c1fe1b8e0854f42b7defea8584bdebe1aee900e561"
 dependencies = [
  "indexmap 2.14.2",
  "serde_core",
@@ -12765,9 +12894,9 @@ dependencies = [
 
 [[package]]
 name = "toml_edit"
-version = "0.25.17+spec-1.1.0"
+version = "0.25.18+spec-1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3641d5bbb5349a79e1020a242d251efbc546ad8048d133958323ce9c40a9c9c"
+checksum = "1ec6e78bd70db9420d8d212fa9dc97f116448a1ceff5d00eabb991d407e21396"
 dependencies = [
  "indexmap 2.14.2",
  "toml_datetime",
@@ -12786,9 +12915,9 @@ dependencies = [
 
 [[package]]
 name = "toml_writer"
-version = "1.1.3+spec-1.1.0"
+version = "1.2.0+spec-1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06bdbd8cfc056b8d2e2e85f29b56a3bdbecb527cef81eb39e3e7b98af4652770"
+checksum = "009dbb7700f11c55ef83aebbe4c0e9cf1f1ac0844bd8d6f00dc38f302026e13a"
 
 [[package]]
 name = "tonic"
@@ -14082,11 +14211,11 @@ dependencies = [
 
 [[package]]
 name = "zip"
-version = "9.2.1"
+version = "9.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "848dad21639597378c3be94c59fd4de683f0f6d8a5387575896e1245737d4f37"
+checksum = "eaeb7f16d2605b49aef586996ded29e65642fab7259a3077e537351df6d80426"
 dependencies = [
- "aes 0.9.3",
+ "aes 0.9.4",
  "bzip2",
  "constant_time_eq",
  "crc32fast",
@@ -14095,6 +14224,7 @@ dependencies = [
  "getrandom 0.4.3",
  "hmac 0.13.0",
  "indexmap 2.14.2",
+ "libc",
  "lzma-rust2",
  "memchr",
  "nix 0.31.3",
@@ -14138,12 +14268,12 @@ name = "zstd"
 version = "0.13.4"
 dependencies = [
  "apache-avro",
- "arrow-array",
- "arrow-ipc",
- "arrow-schema",
+ "arrow-array 60.1.0",
+ "arrow-ipc 60.1.0",
+ "arrow-schema 60.1.0",
  "bytes",
  "kafka-protocol",
- "parquet",
+ "parquet 60.1.0",
  "semver",
  "toml",
  "zstd 0.14.3",

Cargo.toml

@@ -147,7 +147,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
 async-channel = "2.5.0"
 async_zip = { default-features = false, version = "0.0.19" }
 mysql_async = { default-features = false, version = "0.37.1" }
-async-compression = { version = "0.4.51" }
+async-compression = { version = "0.4.52" }
 async-trait = "0.1.92"
 async-nats = { version = "0.51.0", default-features = false }
 axum = "0.8.9"
@@ -168,7 +168,7 @@ http-body-util = "0.1.5"
 memchr = "2.8.3"
 minlz = "1.2.3"
 reqwest = "0.13.5"
-rustfs-kafka-async = { version = "1.4.0" }
+rustfs-kafka-async = { version = "1.4.1" }
 socket2 = { version = "0.6.5" }
 tokio = { version = "1.54.1" }
 tokio-rustls = { default-features = false, version = "0.26.6" }
@@ -191,6 +191,7 @@ quinn = { version = "0.11.12", default-features = false, features = [
 # Serialization and Data Formats
 apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
 bytes = { version = "1.12.1" }
+csv-core = "0.1.13"
 bytesize = "2.7.0"
 byteorder = "1.5.0"
 flatbuffers = "25.12.19"
@@ -212,6 +213,7 @@ serde_urlencoded = "0.7.1"
 # releases.
 aes-gcm = { version = "0.11.1" }
 argon2 = { version = "0.6.0" }
+aws-lc-rs = { version = "1.19.0", default-features = false }
 blake2 = "0.11.0"
 chacha20poly1305 = { version = "0.11.0" }
 crc-fast = "1.10.0"
@@ -234,7 +236,7 @@ proptest = "1"
 # Time and Date
 chrono = { version = "0.4.45" }
 humantime = "2.4.0"
-jiff = { version = "0.2.38" }
+jiff = { version = "0.2.39" }
 time = { version = "0.3.55" }
 
 # Database
@@ -294,6 +296,7 @@ local-ip-address = "0.6.13"
 log = "0.4"
 memmap2 = "0.9.12"
 lz4 = "1.28.1"
+lz4_flex = { version = "=0.14.0", default-features = false, features = ["std", "frame"] }
 matchit = "0.9.2"
 md-5 = "0.11.0"
 mime_guess = "2.0.5"
@@ -337,7 +340,7 @@ strum = { version = "0.28.0" }
 sysinfo = "0.40.1"
 temp-env = "0.3.6"
 tempfile = "3.27.0"
-test-case = "3.4.0"
+test-case = "3.4.1"
 thiserror = "2.0.21"
 tracing = { version = "0.1.44" }
 tracing-appender = "0.2.5"
@@ -356,13 +359,13 @@ winapi-util = "0.1.11"
 windows = { version = "0.62.2" }
 windows-sys = "0.61.2"
 xxhash-rust = { version = "0.8.19" }
-zip = "9.2.1"
+zip = "9.3.0"
 zstd = "0.14.3"
 
 # Observability and Metrics
 metrics = "0.24.6"
 metrics-util = "0.20"
-dial9 = { version = "0.5.6", default-features = false }
+dial9 = { version = "0.5.7", default-features = false }
 opentelemetry = { version = "0.33.1" }
 opentelemetry-appender-tracing = { version = "0.33.1" }
 opentelemetry-otlp = { version = "0.33.1" }
@@ -377,14 +380,14 @@ libunftp = { version = "0.24.0" }
 unftp-core = "0.1.0"
 suppaftp = { version = "12.2.1" }
 rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
-russh = { version = "0.65.0" }
+russh = { version = "0.65.1" }
 russh-sftp = "3.0.1"
 
 # WebDAV
 dav-server = "0.11.0"
 
 # Performance Analysis and Memory Profiling
-rustfs-mimalloc = { version = "0.7.2" }
+rustfs-mimalloc = { version = "0.7.3" }
 # Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
 hotpath = { version = "0.30.3", default-features = false }
 # Snapshot testing for output format regression detection

Dockerfile

@@ -16,6 +16,7 @@ FROM alpine:3.24.2 AS build
 
 ARG TARGETARCH
 ARG RELEASE=latest
+ARG RC_RELEASE=latest
 
 RUN apk add --no-cache ca-certificates curl jq unzip
 WORKDIR /build
@@ -69,6 +70,9 @@ RUN set -eux; \
     chmod +x /build/rustfs; \
     rm -rf rustfs.zip /build/.tmp || true
 
+COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
+RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /build/rc
+
 
 FROM alpine:3.24.2
 
@@ -95,14 +99,18 @@ RUN apk upgrade --no-cache && \
     ca-certificates \
     coreutils \
     curl \
+    jq \
     tzdata \
     && test "$(TZ=Asia/Kolkata date +%z)" = "+0530"
 
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=build /build/rustfs /usr/bin/rustfs
-COPY entrypoint.sh /entrypoint.sh
+COPY --from=build /build/rc /usr/bin/rc
+COPY entrypoint.sh /etc/rustfs/entrypoint.sh
+COPY bootstrap/ /etc/rustfs/bootstrap/
 
-RUN chmod +x /usr/bin/rustfs /entrypoint.sh
+RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
+    ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
 
 RUN addgroup -g 10001 -S rustfs && \
     adduser -u 10001 -G rustfs -S rustfs -D && \
@@ -124,6 +132,6 @@ VOLUME ["/data"]
 
 USER rustfs
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
 
 CMD ["rustfs"]

Dockerfile.decommission-local

@@ -12,7 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM rust:1.97.1-trixie
+FROM rust:1.99.1-trixie
+
+ARG TARGETARCH
+ARG RC_RELEASE=latest
 
 RUN set -eux; \
     export DEBIAN_FRONTEND=noninteractive; \
@@ -21,6 +24,7 @@ RUN set -eux; \
     build-essential \
     ca-certificates \
     curl \
+    jq \
     git \
     pkg-config \
     libssl-dev \
@@ -36,6 +40,7 @@ COPY . .
 RUN ./scripts/static.sh
 RUN cargo run --bin gproto
 RUN cargo build --release --locked --bin rustfs
+RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
 
 RUN set -eux; \
     groupadd -g 10001 rustfs; \
@@ -50,8 +55,12 @@ RUN set -eux; \
     chown -R rustfs:rustfs /data /logs /app; \
     chmod 0750 /data /logs
 
-COPY entrypoint.sh /entrypoint.sh
-RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && chmod +x /entrypoint.sh
+COPY entrypoint.sh /etc/rustfs/entrypoint.sh
+COPY bootstrap/ /etc/rustfs/bootstrap/
+RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && \
+    install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
+    chmod +x /etc/rustfs/entrypoint.sh && \
+    ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
 
 ENV RUSTFS_VOLUMES="/data" \
     RUST_LOG="warn" \
@@ -63,5 +72,5 @@ ENV RUSTFS_VOLUMES="/data" \
 
 EXPOSE 9000 9001
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
 CMD ["/usr/bin/rustfs"]

Dockerfile.glibc

@@ -16,6 +16,7 @@ FROM ubuntu:26.04 AS build
 
 ARG TARGETARCH
 ARG RELEASE=latest
+ARG RC_RELEASE=latest
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     ca-certificates \
@@ -76,6 +77,9 @@ RUN set -eux; \
     chmod +x /build/rustfs; \
     rm -rf rustfs.zip /build/.tmp || true
 
+COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
+RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" gnu "$RC_RELEASE" /build/rc
+
 FROM ubuntu:26.04
 
 ARG RELEASE=latest
@@ -99,14 +103,18 @@ RUN apt-get update && apt-get upgrade -y \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
     ca-certificates \
     curl \
+    jq \
     tzdata \
     && test "$(TZ=Asia/Kolkata date +%z)" = "+0530" \
     && rm -rf /var/lib/apt/lists/*
 
 COPY --from=build /build/rustfs /usr/bin/rustfs
-COPY entrypoint.sh /entrypoint.sh
+COPY --from=build /build/rc /usr/bin/rc
+COPY entrypoint.sh /etc/rustfs/entrypoint.sh
+COPY bootstrap/ /etc/rustfs/bootstrap/
 
-RUN chmod +x /usr/bin/rustfs /entrypoint.sh
+RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
+    ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
 
 RUN groupadd -g 10001 rustfs && \
     useradd -u 10001 -g rustfs -m -s /sbin/nologin rustfs && \
@@ -131,6 +139,6 @@ VOLUME ["/data"]
 
 USER rustfs
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
 
 CMD ["rustfs"]

Dockerfile.source

@@ -28,17 +28,19 @@ ARG TARGETPLATFORM
 ARG BUILDPLATFORM
 ARG TARGETARCH
 ARG RUSTFS_BUILD_FEATURES=""
+ARG RC_RELEASE=latest
 
 # -----------------------------
 # Build stage
 # -----------------------------
-FROM rust:1.97.1-trixie AS builder
+FROM rust:1.99.1-trixie AS builder
 
 # Re-declare args after FROM
 ARG TARGETPLATFORM
 ARG BUILDPLATFORM
 ARG TARGETARCH
 ARG RUSTFS_BUILD_FEATURES
+ARG RC_RELEASE
 
 # Debug: print platforms
 RUN echo "Build info -> BUILDPLATFORM=${BUILDPLATFORM}, TARGETPLATFORM=${TARGETPLATFORM}"
@@ -52,6 +54,7 @@ RUN set -eux; \
     build-essential \
     ca-certificates \
     curl \
+    jq \
     git \
     pkg-config \
     libssl-dev \
@@ -166,6 +169,8 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
     ;; \
     esac
 
+RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
+
 # -----------------------------
 # Development stage (keeps toolchain)
 # -----------------------------
@@ -199,10 +204,13 @@ ENV RUSTFS_VOLUMES="/data" \
 # We rely on cargo run to build and run
 EXPOSE 9000 9001
 
-COPY entrypoint.sh /entrypoint.sh
-RUN chmod +x /entrypoint.sh
+COPY entrypoint.sh /etc/rustfs/entrypoint.sh
+COPY bootstrap/ /etc/rustfs/bootstrap/
+RUN install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
+    chmod +x /etc/rustfs/entrypoint.sh && \
+    ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
 CMD ["cargo", "run", "--bin", "rustfs", "--"]
 
 # -----------------------------
@@ -227,6 +235,7 @@ RUN set -eux; \
     apt-get install -y --no-install-recommends \
     ca-certificates \
     curl \
+    jq \
     tzdata \
     coreutils; \
     rm -rf /var/lib/apt/lists/*
@@ -248,8 +257,11 @@ RUN set -eux; \
 
 # Copy the freshly built binary and the entrypoint
 COPY --from=builder /usr/local/bin/rustfs /usr/bin/rustfs
-COPY entrypoint.sh /entrypoint.sh
-RUN chmod +x /usr/bin/rustfs /entrypoint.sh
+COPY --from=builder /usr/local/bin/rc /usr/bin/rc
+COPY entrypoint.sh /etc/rustfs/entrypoint.sh
+COPY bootstrap/ /etc/rustfs/bootstrap/
+RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
+    ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
 
 # Default environment (override in docker run/compose as needed)
 ENV RUSTFS_VOLUMES="/data" \
@@ -263,5 +275,5 @@ EXPOSE 9000
 VOLUME ["/data"]
 
 # Keep root here; entrypoint will drop privileges using chroot --userspec
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
 CMD ["/usr/bin/rustfs"]

bootstrap/README.md

@@ -0,0 +1,21 @@
+# RustFS image bootstrap
+
+The official image bundles this directory at `/etc/rustfs/bootstrap`. The default `buckets.json` is an empty JSON array, so the image starts RustFS directly and does not require a bootstrap endpoint or bootstrap credentials.
+
+To create buckets, replace `buckets.json` in a derived image with a JSON array of bucket names. The entrypoint creates each bucket if it does not exist and applies `public-access-block.json`, whose default blocks all four public access paths. Edit that file in the derived image only when public access is an explicit requirement.
+
+Optional per-bucket files use the bucket name as the filename without an extension:
+
+| Directory | File format | Applied operation |
+| --- | --- | --- |
+| `policies/` | S3 bucket policy JSON | `rc bucket anonymous set-json` |
+| `versioning/` | JSON boolean (`true` or `false`) | Enable or suspend bucket versioning |
+| `lifecycle/` | `rc bucket lifecycle rule import` JSON | Replace lifecycle rules |
+| `cors/` | JSON or XML accepted by `rc bucket cors set` | Replace CORS rules |
+| `notifications/` | JSON array of `{ "arn": "...", "events": ["..."] }` | Add or replace event rules by ARN (prefix/suffix filters are not currently accepted) |
+
+Every per-bucket file must correspond to a bucket listed in `buckets.json`. Applying these files is repeatable: bucket creation ignores existing buckets, while each configuration operation replaces or reconciles that bucket's requested state.
+
+When `buckets.json` is non-empty, set `RUSTFS_BOOTSTRAP_ENDPOINT` to the RustFS S3 endpoint reachable from the container, for example `http://rustfs:9000` for a Compose service or the internal service DNS name in Kubernetes. The entrypoint waits on that endpoint's `/health/ready`; it never assumes `localhost`. `RUSTFS_BOOTSTRAP_TIMEOUT` controls the readiness wait in seconds (default `120`, maximum `3600`). `RUSTFS_BOOTSTRAP_DIR` can point to a different in-image configuration directory; it defaults to `/etc/rustfs/bootstrap`. Set `RUSTFS_ACCESS_KEY` and `RUSTFS_SECRET_KEY`, or their `_FILE` variants, for bootstrap authentication. The endpoint and credentials are read at runtime and are not baked into the image.
+
+RustFS built-in IAM policies remain unchanged; the default bootstrap installs no custom bucket policies, lifecycle rules, versioning, CORS rules, or notification rules. The bundled RustFS CLI is resolved from a published release and its release checksum is verified. Set the Docker build argument `RC_RELEASE` to a version to pin it. Build the RustFS image after a CLI release containing the `bucket public-access-block` commands is available. For Kubernetes, the configured service must route traffic to the starting pod before its readiness probe succeeds (for example, with `publishNotReadyAddresses: true`); otherwise the service endpoint cannot reach the pod while the entrypoint waits for readiness.

bootstrap/buckets.json

@@ -0,0 +1 @@
+[]

bootstrap/cors/.gitkeep

@@ -0,0 +1 @@
+

bootstrap/lifecycle/.gitkeep

@@ -0,0 +1 @@
+

bootstrap/notifications/.gitkeep

@@ -0,0 +1 @@
+

bootstrap/policies/.gitkeep

@@ -0,0 +1 @@
+

bootstrap/public-access-block.json

@@ -0,0 +1,6 @@
+{
+  "block_public_acls": true,
+  "ignore_public_acls": true,
+  "block_public_policy": true,
+  "restrict_public_buckets": true
+}

bootstrap/versioning/.gitkeep

@@ -0,0 +1 @@
+

crates/config/src/constants/object.rs

@@ -32,8 +32,9 @@ pub const ENV_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD: &str = "RUSTFS_OBJECT_MEDIUM_
 
 /// Environment variable name for maximum concurrent disk reads for object operations.
 /// - Purpose: Limit the number of concurrent disk read operations for object reads to prevent I/O saturation.
-/// - Unit: request count (usize).
-/// - Semantics: Throttling disk reads helps maintain overall system responsiveness under load.
+/// - Unit: request count (usize) for GET; estimated drive fanout for native LIST.
+/// - Semantics: GET bodies and native LIST walks use separate per-instance budgets of this size.
+///   A LIST consumes up to one budget, including its disk-producer cleanup. `0` disables both caps.
 /// - Example: `export RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=16`
 /// - Note: This setting may interact with OS-level I/O scheduling and should be tuned based on hardware capabilities.
 pub const ENV_OBJECT_MAX_CONCURRENT_DISK_READS: &str = "RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS";

crates/config/src/constants/proxy.rs

@@ -30,10 +30,10 @@ pub const ENV_TRUSTED_PROXY_VALIDATION_MODE: &str = "RUSTFS_TRUSTED_PROXY_VALIDA
 /// Default validation mode is "hop_by_hop".
 pub const DEFAULT_TRUSTED_PROXY_VALIDATION_MODE: &str = "hop_by_hop";
 
-/// Environment variable to enable RFC 7239 "Forwarded" header support.
+/// Select RFC 7239 "Forwarded" instead of "X-Forwarded-For" in legacy mode.
 pub const ENV_TRUSTED_PROXY_ENABLE_RFC7239: &str = "RUSTFS_TRUSTED_PROXY_ENABLE_RFC7239";
-/// RFC 7239 support is enabled by default.
-pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = true;
+/// Legacy mode uses X-Forwarded-For unless RFC 7239 is explicitly selected.
+pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = false;
 
 /// Environment variable for the maximum allowed proxy hops.
 pub const ENV_TRUSTED_PROXY_MAX_HOPS: &str = "RUSTFS_TRUSTED_PROXY_MAX_HOPS";

crates/config/src/constants/targets.rs

@@ -146,3 +146,9 @@ pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";
 
 /// Queue-store compression is enabled by default to reduce disk footprint.
 pub const DEFAULT_TARGET_STORE_COMPRESS: bool = true;
+
+/// Maximum on-disk bytes in each target queue, including failed entries and pending writes.
+pub const ENV_TARGET_STORE_MAX_BYTES: &str = "RUSTFS_TARGET_STORE_MAX_BYTES";
+
+/// Queue writes fail at one GiB unless the operator sets another positive byte budget.
+pub const DEFAULT_TARGET_STORE_MAX_BYTES: u64 = 1024 * 1024 * 1024;

crates/config/src/constants/tls.rs

@@ -175,12 +175,17 @@ pub const ENV_HTTP_SOCKET_RECV_BUFFER_BYTES: &str = "RUSTFS_HTTP_SOCKET_RECV_BUF
 pub const DEFAULT_HTTP_SOCKET_RECV_BUFFER_BYTES: usize = 0;
 
 /// Environment variable for the S3 request-body inter-chunk read timeout
-/// (seconds). Default: 300. Set to 0 to disable.
+/// (seconds). Default: 300. Set to 0 to disable the inactivity timeout.
 ///
 /// This bounds how long a `PutObject`/upload may wait for the *next* body chunk
 /// while more bytes are still expected. It resets on every chunk, so it does not
 /// penalize slow-but-progressing uploads — it only fires when a peer sends a
 /// partial body and then goes silent *without* closing the connection (no EOF).
+/// Independently of this knob, `PutObject` and `UploadPart` require at least
+/// 64 KiB of body progress per 60 seconds of server wait (the minimum-progress
+/// floor, logged as `upload_body_progress_stalled`); it bounds trickle uploads
+/// that would otherwise pin upload admission indefinitely, and it stays in
+/// force when this timeout is set to 0.
 /// A reverse proxy or CDN that forwards a truncated body this way would
 /// otherwise make RustFS wait forever for bytes that never arrive; with this
 /// timeout the stalled read is aborted and logged with the received/expected

crates/data-usage/src/data_usage.rs

@@ -675,6 +675,10 @@ impl DataUsageInfo {
 /// cycle. The timestamp fallback preserves ordering for legacy snapshots that
 /// predate those fields.
 pub fn data_usage_snapshot_is_newer(candidate: &DataUsageInfo, baseline: &DataUsageInfo) -> bool {
+    data_usage_identity_is_newer(candidate.snapshot_identity(), baseline.snapshot_identity())
+}
+
+fn data_usage_identity_is_newer(candidate: DataUsageSnapshotIdentity, baseline: DataUsageSnapshotIdentity) -> bool {
     match (
         candidate.scanner_epoch.zip(candidate.scanner_cycle),
         baseline.scanner_epoch.zip(baseline.scanner_cycle),
@@ -697,10 +701,18 @@ pub fn data_usage_snapshot_is_newer(candidate: &DataUsageInfo, baseline: &DataUs
 /// already advance the authoritative timestamp when deleting a bucket, so a
 /// rollback delete/recreate fences the previous bucket incarnation too.
 pub fn observed_data_usage_is_newer(observed: &DataUsageInfo, authoritative: &DataUsageInfo) -> bool {
+    observed_data_usage_is_newer_than_identity(observed, authoritative.snapshot_identity())
+}
+
+/// Return whether a nonconverged observation may supersede `authoritative`.
+///
+/// This applies the structural, exact-baseline, and generation checks of
+/// [`observed_data_usage_is_newer`] without retaining the baseline's bucket maps.
+pub fn observed_data_usage_is_newer_than_identity(observed: &DataUsageInfo, authoritative: DataUsageSnapshotIdentity) -> bool {
     observed.usage_snapshot_converged == Some(false)
         && (observed.is_complete_bucket_usage_snapshot() || observed.is_valid_partial_snapshot())
-        && observed.usage_snapshot_authoritative_baseline.as_ref() == Some(&authoritative.snapshot_identity())
-        && data_usage_snapshot_is_newer(observed, authoritative)
+        && observed.usage_snapshot_authoritative_baseline == Some(authoritative)
+        && data_usage_identity_is_newer(observed.snapshot_identity(), authoritative)
 }
 
 /// Metadata describing the status of a disk-level data usage snapshot.
@@ -3140,12 +3152,17 @@ mod tests {
         };
 
         assert!(observed_data_usage_is_newer(&observed, &authoritative));
+        assert!(observed_data_usage_is_newer_than_identity(&observed, authoritative.snapshot_identity()));
 
         authoritative.last_update = Some(SystemTime::UNIX_EPOCH + Duration::from_secs(601));
         assert!(
             !observed_data_usage_is_newer(&observed, &authoritative),
             "an old-binary namespace mutation must fence the prior bucket incarnation regardless of clock skew"
         );
+        assert!(
+            !observed_data_usage_is_newer_than_identity(&observed, authoritative.snapshot_identity()),
+            "a captured identity must preserve the exact namespace fence"
+        );
     }
 
     #[test]
@@ -3172,6 +3189,18 @@ mod tests {
         assert!(!observed_data_usage_is_newer(&candidate(2, 9, Some(false), true), &authoritative));
         assert!(!observed_data_usage_is_newer(&candidate(2, 11, Some(true), true), &authoritative));
         assert!(!observed_data_usage_is_newer(&candidate(2, 11, Some(false), false), &authoritative));
+        for (observed, expected) in [
+            (candidate(2, 11, Some(false), true), true),
+            (candidate(2, 9, Some(false), true), false),
+            (candidate(2, 11, Some(true), true), false),
+            (candidate(2, 11, Some(false), false), false),
+        ] {
+            assert_eq!(
+                observed_data_usage_is_newer_than_identity(&observed, authoritative.snapshot_identity()),
+                expected,
+                "observation identity selection must preserve convergence, completeness, and generation gates"
+            );
+        }
 
         let mut partial = candidate(2, 11, Some(false), false);
         partial.usage_snapshot_partial = true;
@@ -3186,6 +3215,30 @@ mod tests {
             segment_invalidation_proof: None,
         }];
         assert!(observed_data_usage_is_newer(&partial, &authoritative));
+        assert!(observed_data_usage_is_newer_than_identity(&partial, authoritative.snapshot_identity()));
+    }
+
+    #[test]
+    fn observation_identity_selection_keeps_legacy_timestamp_ordering() {
+        let baseline = DataUsageSnapshotIdentity {
+            last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(10)),
+            // An epoch without a cycle is not a coordinated position.
+            scanner_epoch: Some(2),
+            scanner_cycle: None,
+        };
+        let mut observed = DataUsageInfo {
+            last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(11)),
+            scanner_epoch: Some(1),
+            usage_snapshot_complete: true,
+            usage_snapshot_converged: Some(false),
+            usage_snapshot_authoritative_baseline: Some(baseline),
+            ..Default::default()
+        };
+        assert!(observed_data_usage_is_newer_than_identity(&observed, baseline));
+        observed.last_update = baseline.last_update;
+        assert!(!observed_data_usage_is_newer_than_identity(&observed, baseline));
+        observed.last_update = None;
+        assert!(!observed_data_usage_is_newer_than_identity(&observed, baseline));
     }
 
     #[test]

crates/e2e_test/README.md

@@ -26,7 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
 | **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
 | **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
 | **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
-| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
+| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
 | **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
 | **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
 
@@ -194,9 +194,9 @@ the same profile for membership and execution with one nightly worker.
 | `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
 | ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
 | KMS suite | `e2e-full` job, merge queue + main | **Active** |
-| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
+| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, release tags + weekly schedule + manual dispatch | **Active** |
 | Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
-| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
+| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (nightly / dispatch) |
 | Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
 | Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
 | Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |

crates/e2e_test/src/conditional_write_policy_test.rs

@@ -260,7 +260,7 @@ async fn iam_policy_requires_if_match(env: &RustFSTestEnvironment, admin: &Clien
     let key = "guarded.txt";
 
     admin_create_user(env, &user, user_secret).await?;
-    let policy = serde_json::json!({
+    let mut policy = serde_json::json!({
         "Version": "2012-10-17",
         "Statement": [{
             "Effect": "Allow",
@@ -268,9 +268,8 @@ async fn iam_policy_requires_if_match(env: &RustFSTestEnvironment, admin: &Clien
             "Resource": [format!("arn:aws:s3:::{bucket}/*")],
             "Condition": { "Null": { "s3:if-match": "false" } }
         }]
-    })
-    .to_string();
-    admin_add_and_attach_policy(env, &policy_name, &policy, &user).await?;
+    });
+    admin_add_and_attach_policy(env, &policy_name, &policy.to_string(), &user).await?;
 
     admin.create_bucket().bucket(&bucket).send().await?;
     let original = admin
@@ -293,6 +292,46 @@ async fn iam_policy_requires_if_match(env: &RustFSTestEnvironment, admin: &Clien
         .expect_err("PutObject without If-Match must be denied");
     assert_error(&denied, "AccessDenied", 403, "unconditional PutObject");
 
+    let denied = client
+        .put_object()
+        .bucket(&bucket)
+        .key(key)
+        .if_match(&etag)
+        .body(ByteStream::from_static(b"missing-read-permission"))
+        .send()
+        .await
+        .expect_err("If-Match without GetObject must be denied");
+    assert_error(&denied, "AccessDenied", 403, "conditional PutObject without GetObject");
+
+    // S3 requires GetObject as well as PutObject for an ETag-based write.
+    policy["Statement"]
+        .as_array_mut()
+        .expect("policy statements")
+        .push(serde_json::json!({
+            "Effect": "Allow",
+            "Action": ["s3:GetObject"],
+            "Resource": [format!("arn:aws:s3:::{bucket}/{key}")]
+        }));
+    admin_add_canned_policy_via(
+        AdminTransport::Signed,
+        &env.url,
+        &env.access_key,
+        &env.secret_key,
+        &policy_name,
+        &policy.to_string(),
+    )
+    .await?;
+
+    let denied = client
+        .put_object()
+        .bucket(&bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"blind-overwrite"))
+        .send()
+        .await
+        .expect_err("GetObject permission must not allow an unconditional PutObject");
+    assert_error(&denied, "AccessDenied", 403, "unconditional PutObject with GetObject");
+
     client
         .put_object()
         .bucket(&bucket)
@@ -301,6 +340,8 @@ async fn iam_policy_requires_if_match(env: &RustFSTestEnvironment, admin: &Clien
         .body(ByteStream::from_static(b"v2"))
         .send()
         .await?;
+    let written = client.get_object().bucket(&bucket).key(key).send().await?;
+    assert_eq!(written.body.collect().await?.into_bytes().as_ref(), b"v2");
 
     let _ = admin.delete_object().bucket(&bucket).key(key).send().await;
     let _ = admin.delete_bucket().bucket(&bucket).send().await;

crates/e2e_test/src/create_bucket_region_test.rs

@@ -502,10 +502,163 @@ mod tests {
                 assert!(response.contains(if expected == reqwest::StatusCode::FORBIDDEN {
                     "<Code>AccessDenied</Code>"
                 } else {
-                    "<Code>BucketAlreadyExists</Code>"
+                    "<Code>BucketAlreadyOwnedByYou</Code>"
                 }));
             }
         }
         Ok(())
     }
+
+    /// A deployment owns a single bucket namespace, so an IAM user that recreates
+    /// an existing bucket is the owner, not another account. In the default region
+    /// every such request returns the legacy idempotent `200 OK` (AWS CreateBucket
+    /// semantics), including buckets created by root and by another IAM user.
+    #[tokio::test]
+    async fn test_create_bucket_duplicate_in_default_region_is_idempotent_for_iam_user()
+    -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec!["--region", "us-east-1"]).await?;
+        let root = env.create_s3_client();
+
+        admin_create_user(&env, "duplicate-iam", "duplicate-iam-secret").await?;
+        admin_attach_user_policy_via(
+            AdminTransport::Signed,
+            &env.url,
+            &env.access_key,
+            &env.secret_key,
+            "readwrite",
+            "duplicate-iam",
+        )
+        .await?;
+        let iam = Client::from_conf(build_test_s3_config(
+            &env.url,
+            "duplicate-iam",
+            "duplicate-iam-secret",
+            None,
+            "duplicate-create",
+        ));
+
+        root.create_bucket()
+            .bucket("duplicate-root-owned")
+            .send()
+            .await
+            .expect("root creates the first bucket");
+        iam.create_bucket()
+            .bucket("duplicate-root-owned")
+            .send()
+            .await
+            .expect("an IAM user recreating a root-owned bucket in us-east-1 gets 200 OK");
+
+        iam.create_bucket()
+            .bucket("duplicate-iam-owned")
+            .send()
+            .await
+            .expect("the IAM user creates its own bucket");
+        iam.create_bucket()
+            .bucket("duplicate-iam-owned")
+            .send()
+            .await
+            .expect("the same IAM user recreating its bucket gets 200 OK");
+        root.create_bucket()
+            .bucket("duplicate-iam-owned")
+            .send()
+            .await
+            .expect("root recreating an IAM-created bucket gets 200 OK");
+
+        Ok(())
+    }
+
+    /// Outside the default region the same ownership answer becomes
+    /// `409 BucketAlreadyOwnedByYou` for every authorized requester. Requests that
+    /// lack `s3:CreateBucket` still fail authorization before the duplicate check.
+    #[tokio::test]
+    async fn test_create_bucket_duplicate_outside_default_region_reports_owned_by_you() -> Result<(), Box<dyn Error + Send + Sync>>
+    {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec!["--region", "eu-west-1"]).await?;
+        let region = "eu-west-1";
+        let root = Client::from_conf(
+            env.create_s3_client()
+                .config()
+                .to_builder()
+                .region(Region::new(region))
+                .build(),
+        );
+
+        admin_create_user(&env, "duplicate-eu-iam", "duplicate-eu-iam-secret").await?;
+        admin_attach_user_policy_via(
+            AdminTransport::Signed,
+            &env.url,
+            &env.access_key,
+            &env.secret_key,
+            "readwrite",
+            "duplicate-eu-iam",
+        )
+        .await?;
+        let iam = Client::from_conf(
+            build_test_s3_config(&env.url, "duplicate-eu-iam", "duplicate-eu-iam-secret", None, "duplicate-create")
+                .to_builder()
+                .region(Region::new(region))
+                .build(),
+        );
+
+        admin_create_user(&env, "duplicate-eu-readonly", "duplicate-eu-readonly-secret").await?;
+        admin_attach_user_policy_via(
+            AdminTransport::Signed,
+            &env.url,
+            &env.access_key,
+            &env.secret_key,
+            "readonly",
+            "duplicate-eu-readonly",
+        )
+        .await?;
+        let readonly = Client::from_conf(
+            build_test_s3_config(
+                &env.url,
+                "duplicate-eu-readonly",
+                "duplicate-eu-readonly-secret",
+                None,
+                "duplicate-create",
+            )
+            .to_builder()
+            .region(Region::new(region))
+            .build(),
+        );
+
+        for (client, bucket) in [(&root, "duplicate-eu-root"), (&iam, "duplicate-eu-iam")] {
+            client
+                .create_bucket()
+                .bucket(bucket)
+                .send()
+                .await
+                .unwrap_or_else(|error| panic!("{bucket} must be created: {error:?}"));
+            // Both the creator and the other authenticated identity own the namespace.
+            for creator in [&root, &iam] {
+                let error = creator
+                    .create_bucket()
+                    .bucket(bucket)
+                    .send()
+                    .await
+                    .expect_err("a duplicate create outside us-east-1 must report ownership");
+                assert_eq!(
+                    error.as_service_error().and_then(ProvideErrorMetadata::code),
+                    Some("BucketAlreadyOwnedByYou"),
+                    "{bucket}"
+                );
+                assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(409), "{bucket}");
+            }
+        }
+
+        let denied = readonly
+            .create_bucket()
+            .bucket("duplicate-eu-root")
+            .send()
+            .await
+            .expect_err("a requester without s3:CreateBucket must be denied before the duplicate check");
+        assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
+
+        Ok(())
+    }
 }

crates/e2e_test/src/distributed/concurrency_stability_test.rs

@@ -12,11 +12,249 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
+use super::harness::{DistCluster, DistLayout, TestResult, payload_for, s3_probe_client, sha256_hex, unique_bucket};
 use crate::common::init_logging;
+use aws_sdk_s3::{
+    Client,
+    error::{ProvideErrorMetadata, SdkError},
+    primitives::ByteStream,
+};
+use std::any::TypeId;
 use std::collections::BTreeSet;
+use std::fmt::Debug;
+use std::future::Future;
 use std::sync::Arc;
+use std::time::Duration;
 use tokio::sync::Barrier;
+use tokio::time::Instant;
+
+const LOCK_ADMISSION_BUDGET: Duration = Duration::from_secs(45);
+
+async fn after_lock_contention<T, E, F, Fut>(operation: &str, mut call: F) -> TestResult<Result<T, SdkError<E>>>
+where
+    E: Debug + ProvideErrorMetadata + 'static,
+    F: FnMut() -> Fut,
+    Fut: Future<Output = Result<T, SdkError<E>>>,
+{
+    let started = Instant::now();
+    let mut attempts = 0;
+    // The workload uses unique unversioned keys. Repeat only explicitly
+    // advertised lock backpressure; every step still needs its real result.
+    let outcome = tokio::time::timeout(LOCK_ADMISSION_BUDGET, async {
+        loop {
+            attempts += 1;
+            let result = call().await;
+            let retry = result.as_ref().err().is_some_and(|error| {
+                let Some(response) = error.raw_response() else {
+                    return false;
+                };
+                let Some(service_error) = error.as_service_error() else {
+                    return false;
+                };
+                // HEAD has no XML error body, so its SDK service error can
+                // lack a code even when the server supplies retry headers.
+                let namespace_code = service_error.code() == Some("ServiceUnavailable")
+                    || (service_error.code().is_none()
+                        && TypeId::of::<E>() == TypeId::of::<aws_sdk_s3::operation::head_object::HeadObjectError>());
+                response.status().as_u16() == 503
+                    && namespace_code
+                    && response.headers().get("x-rustfs-error-cause") == Some("namespace_lock_timeout")
+                    && response.headers().get("retry-after") == Some("1")
+            });
+            if !retry {
+                return result;
+            }
+            tokio::time::sleep(Duration::from_secs(1)).await;
+        }
+    })
+    .await;
+    if attempts > 1 || !matches!(&outcome, Ok(Ok(_))) {
+        eprintln!(
+            "mixed workload request: operation={operation}, attempts={attempts}, elapsed={:?}",
+            started.elapsed()
+        );
+    }
+    outcome.map_err(|_| {
+        format!("{operation} exceeded its {LOCK_ADMISSION_BUDGET:?} request budget after {attempts} attempts").into()
+    })
+}
+
+async fn assert_workload_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
+    let output = after_lock_contention(&format!("GET {key}"), || client.get_object().bucket(bucket).key(key).send()).await??;
+    let actual = output.body.collect().await?.into_bytes();
+    if actual.as_ref() != expected {
+        return Err(format!(
+            "object {key} bytes mismatch: expected {} bytes sha256={} got {} bytes sha256={}",
+            expected.len(),
+            sha256_hex(expected),
+            actual.len(),
+            sha256_hex(&actual)
+        )
+        .into());
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use aws_smithy_http_client::test_util::infallible_client_fn;
+    use aws_smithy_types::body::SdkBody;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+
+    type Reply = (u16, &'static str, Option<&'static str>, Option<&'static str>, &'static str);
+
+    fn client(respond: impl Fn(usize) -> Reply + Send + Sync + 'static) -> (Client, Arc<AtomicUsize>) {
+        let requests = Arc::new(AtomicUsize::new(0));
+        let seen = Arc::clone(&requests);
+        let transport = infallible_client_fn(move |_| {
+            let (status, code, cause, retry_after, body) = respond(seen.fetch_add(1, Ordering::SeqCst));
+            let body = if code.is_empty() {
+                body.to_string()
+            } else {
+                format!("<Error><Code>{code}</Code></Error>")
+            };
+            let mut response = http::Response::builder()
+                .status(status)
+                .header("content-type", "application/xml")
+                .header("content-length", body.len());
+            if let Some(cause) = cause {
+                response = response.header("x-rustfs-error-cause", cause);
+            }
+            if let Some(retry_after) = retry_after {
+                response = response.header("retry-after", retry_after);
+            }
+            response.body(SdkBody::from(body)).expect("scripted S3 response")
+        });
+        let config = crate::common::build_test_s3_config(
+            "http://contention.example.invalid",
+            "synthetic-access",
+            "synthetic-secret",
+            None,
+            "contention-test",
+        );
+        let client = Client::from_conf(config.to_builder().http_client(transport).build());
+        (s3_probe_client(&client), requests)
+    }
+
+    fn contention() -> Reply {
+        (503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("1"), "")
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn advertised_contention_recovers_after_standard_sdk_attempts_are_exhausted() -> TestResult {
+        let (client, requests) = client(|attempt| {
+            if attempt < 3 {
+                contention()
+            } else {
+                (200, "", None, None, "")
+            }
+        });
+        let standard = Client::from_conf(
+            client
+                .config()
+                .to_builder()
+                .retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(3))
+                .build(),
+        );
+        standard
+            .put_object()
+            .bucket("bucket")
+            .key("key")
+            .body(ByteStream::from_static(b"body"))
+            .send()
+            .await
+            .expect_err("three advertised lock responses exhaust the standard SDK policy");
+        assert_eq!(requests.load(Ordering::SeqCst), 3);
+        requests.store(0, Ordering::SeqCst);
+        let started = Instant::now();
+        after_lock_contention("PUT recovery", || {
+            client
+                .put_object()
+                .bucket("bucket")
+                .key("key")
+                .body(ByteStream::from_static(b"body"))
+                .send()
+        })
+        .await??;
+        assert_eq!(requests.load(Ordering::SeqCst), 4);
+        assert!(started.elapsed() >= Duration::from_secs(3), "honor each Retry-After response");
+        Ok(())
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn bodyless_head_contention_uses_explicit_retry_headers() -> TestResult {
+        let (client, requests) = client(|attempt| {
+            if attempt == 0 {
+                (503, "", Some("namespace_lock_timeout"), Some("1"), "")
+            } else {
+                (200, "", None, None, "")
+            }
+        });
+        after_lock_contention("HEAD recovery", || client.head_object().bucket("bucket").key("key").send()).await??;
+        assert_eq!(requests.load(Ordering::SeqCst), 2);
+        Ok(())
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn unrelated_or_malformed_responses_are_not_retried() -> TestResult {
+        for reply in [
+            (500, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
+            (403, "AccessDenied", None, None, ""),
+            (404, "NoSuchKey", None, None, ""),
+            (503, "ServiceUnavailable", Some("storage_write_quorum"), None, ""),
+            (503, "ServiceUnavailable", Some("namespace_lock_unavailable"), Some("1"), ""),
+            (503, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
+            (503, "", Some("namespace_lock_timeout"), Some("1"), ""),
+            (503, "ServiceUnavailable", None, Some("1"), ""),
+            (503, "ServiceUnavailable", Some("namespace_lock_timeout"), None, ""),
+            (503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("2"), ""),
+        ] {
+            let (client, requests) = client(move |_| reply);
+            let result = after_lock_contention("PUT fatal response", || {
+                client
+                    .put_object()
+                    .bucket("bucket")
+                    .key("key")
+                    .body(ByteStream::from_static(b"body"))
+                    .send()
+            })
+            .await?;
+            assert!(result.is_err(), "fatal response cannot become success: {reply:?}");
+            assert_eq!(requests.load(Ordering::SeqCst), 1, "fatal response must remain visible: {reply:?}");
+        }
+        Ok(())
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn persistent_lock_contention_exhausts_the_request_budget() {
+        let (client, requests) = client(|_| contention());
+        let started = Instant::now();
+        let result = after_lock_contention("PUT unavailable", || {
+            client
+                .put_object()
+                .bucket("bucket")
+                .key("key")
+                .body(ByteStream::from_static(b"body"))
+                .send()
+        })
+        .await;
+        assert!(result.is_err(), "persistent contention cannot acknowledge an operation");
+        assert!(started.elapsed() >= LOCK_ADMISSION_BUDGET);
+        assert!(requests.load(Ordering::SeqCst) > 1);
+        assert!(requests.load(Ordering::SeqCst) <= LOCK_ADMISSION_BUDGET.as_secs() as usize + 1);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn acknowledged_wrong_bytes_fail_without_retry() {
+        let (client, requests) = client(|_| (200, "", None, None, "wrong bytes"));
+        let error = assert_workload_bytes(&client, "bucket", "key", b"expected bytes")
+            .await
+            .expect_err("a successful but incorrect read is a consistency failure");
+        assert!(error.to_string().contains("bytes mismatch"));
+        assert_eq!(requests.load(Ordering::SeqCst), 1);
+    }
+}
 
 #[tokio::test]
 async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
@@ -24,7 +262,9 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
     let dist = DistCluster::start(DistLayout::FourByFour).await?;
     let bucket = unique_bucket("conc");
     dist.create_bucket(&bucket).await?;
-    let clients = Arc::new(dist.clients()?);
+    // This consistency probe owns its admission policy; unrelated responses
+    // must not be hidden by the SDK's general-purpose retries.
+    let clients = Arc::new(dist.clients()?.iter().map(s3_probe_client).collect::<Vec<_>>());
     let worker_count = 24;
     let rounds = 4;
     let barrier = Arc::new(Barrier::new(worker_count));
@@ -44,31 +284,46 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
                 let key = format!("source/worker-{idx:02}-round-{round}.bin");
                 let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
                 let body = payload_for(&key, 64 * 1024);
-                put_object(writer, &bucket, &key, body.clone()).await?;
+                let context = format!("worker={idx} round={round}");
+                after_lock_contention(&format!("{context} PUT source"), || {
+                    writer
+                        .put_object()
+                        .bucket(&bucket)
+                        .key(&key)
+                        .body(ByteStream::from(body.clone()))
+                        .send()
+                })
+                .await??;
 
-                let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
+                let head = after_lock_contention(&format!("{context} HEAD source"), || {
+                    reader.head_object().bucket(&bucket).key(&key).send()
+                })
+                .await??;
                 if head.content_length() != Some(body.len() as i64) {
                     return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
                 }
-                assert_object_bytes(reader, &bucket, &key, &body).await?;
+                assert_workload_bytes(reader, &bucket, &key, &body).await?;
 
-                copier
-                    .copy_object()
-                    .bucket(&bucket)
-                    .key(&copy_key)
-                    .copy_source(format!("{bucket}/{key}"))
-                    .send()
-                    .await?;
-                assert_object_bytes(writer, &bucket, &copy_key, &body).await?;
+                after_lock_contention(&format!("{context} COPY retained"), || {
+                    copier
+                        .copy_object()
+                        .bucket(&bucket)
+                        .key(&copy_key)
+                        .copy_source(format!("{bucket}/{key}"))
+                        .send()
+                })
+                .await??;
+                assert_workload_bytes(writer, &bucket, &copy_key, &body).await?;
 
-                writer.delete_object().bucket(&bucket).key(&key).send().await?;
-                let missing = reader
-                    .head_object()
-                    .bucket(&bucket)
-                    .key(&key)
-                    .send()
-                    .await
-                    .expect_err("deleted source key must not remain visible");
+                after_lock_contention(&format!("{context} DELETE source"), || {
+                    writer.delete_object().bucket(&bucket).key(&key).send()
+                })
+                .await??;
+                let missing = after_lock_contention(&format!("{context} HEAD deleted source"), || {
+                    reader.head_object().bucket(&bucket).key(&key).send()
+                })
+                .await?
+                .expect_err("deleted source key must not remain visible");
                 if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
                     return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
                 }
@@ -82,14 +337,22 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
     for handle in handles {
         inventory.extend(handle.await??);
     }
+    assert_eq!(
+        inventory.len(),
+        worker_count * rounds,
+        "every worker round must retain its acknowledged copy"
+    );
 
     let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
     for (node_idx, client) in clients.iter().enumerate() {
-        let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
+        let listed = after_lock_contention(&format!("node={node_idx} LIST retained"), || {
+            client.list_objects_v2().bucket(&bucket).prefix("retained/").send()
+        })
+        .await??;
         let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
         assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
         for (key, body) in &inventory {
-            assert_object_bytes(client, &bucket, key, body)
+            assert_workload_bytes(client, &bucket, key, body)
                 .await
                 .map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
         }

crates/e2e_test/src/distributed/scanner_heal_topology_fixture.rs

@@ -25,7 +25,8 @@
 
 use super::harness::{
     DistCluster, DistLayout, TestResult, cluster_admin, cluster_admin_ok, decommission_status_json, get_object_bytes,
-    payload_for, put_object, s3_probe_client, s3_probe_error_code, sha256_hex, unique_bucket, wait_until,
+    payload_for, put_object, rebalance_complete, rebalance_status_json, s3_probe_client, s3_probe_error_code, sha256_hex,
+    unique_bucket, wait_until,
 };
 use crate::chaos::{VersionShardCensus, census_object_version_on_disk};
 use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment};
@@ -47,6 +48,7 @@ use tokio::time::{Instant, sleep};
 use uuid::Uuid;
 
 pub(crate) const MOVEMENT_BARRIER_DIR_ENV: &str = "RUSTFS_E2E_DATA_MOVEMENT_BARRIER_DIR";
+pub(crate) const STAGE_AFTER_ENTRY_ADMISSION: &str = "after_entry_admission";
 pub(crate) const STAGE_BEFORE_TARGET_COMMIT: &str = "before_target_commit";
 pub(crate) const STAGE_BEFORE_SOURCE_CLEANUP: &str = "before_source_cleanup";
 pub(crate) const STAGE_AFTER_RESPONSIBILITY_PERSISTED: &str = "after_responsibility_persisted";
@@ -1166,8 +1168,21 @@ pub(crate) async fn clear_decommission(cluster: &RustFSTestClusterEnvironment, p
     .await
 }
 
-pub(crate) async fn stop_rebalance(cluster: &RustFSTestClusterEnvironment) -> TestResult<String> {
-    cluster_admin_ok(cluster, Method::POST, "/rustfs/admin/v3/rebalance/stop", None).await
+pub(crate) async fn stop_rebalance(cluster: &RustFSTestClusterEnvironment, expected_id: &str) -> TestResult<String> {
+    let path = "/rustfs/admin/v3/rebalance/stop";
+    let (status, response) = cluster_admin(cluster, Method::POST, path, None).await?;
+    if status.is_success() {
+        return Ok(response);
+    }
+    // The run can finish between the active observation and the stop request.
+    // Only completion of that same run makes a not-started response acceptable.
+    if status == StatusCode::NOT_FOUND
+        && response.contains("<Code>NoSuchResource</Code>")
+        && rebalance_complete(&rebalance_status_json(cluster).await?, expected_id)?
+    {
+        return Ok(response);
+    }
+    Err(format!("POST {path} failed: {status} {response}").into())
 }
 
 pub(crate) fn decommission_pool_entry(status: &Value, pool_id: usize) -> TestResult<Value> {
@@ -1585,6 +1600,129 @@ pub(crate) async fn assert_every_node_serves(dist: &DistCluster, bucket: &str, l
 mod tests {
     use super::*;
 
+    const REBALANCE_NOT_STARTED: &str =
+        "<Error><Code>NoSuchResource</Code><Message>pool rebalance is not started</Message></Error>";
+
+    async fn mock_rebalance_stop(
+        stop_status: StatusCode,
+        stop_body: &str,
+        run_status: Value,
+    ) -> TestResult<(TestResult<String>, Vec<String>)> {
+        use bytes::Bytes;
+        use http_body_util::Full;
+        use hyper::service::service_fn;
+        use hyper_util::rt::TokioIo;
+        use tokio::net::TcpListener;
+        use tokio::sync::mpsc;
+        use tokio::task::JoinSet;
+
+        let listener = TcpListener::bind("127.0.0.1:0").await?;
+        let mut cluster = RustFSTestClusterEnvironment::new(1).await?;
+        cluster.nodes[0].url = format!("http://{}", listener.local_addr()?);
+        let stop_body = stop_body.to_string();
+        let (requests_tx, mut requests_rx) = mpsc::unbounded_channel();
+        let mut server = JoinSet::new();
+        server.spawn(async move {
+            loop {
+                let (stream, _) = match listener.accept().await {
+                    Ok(connection) => connection,
+                    Err(error) => return Err::<(), Box<dyn std::error::Error + Send + Sync>>(error.into()),
+                };
+                let stop_body = stop_body.clone();
+                let run_status = run_status.clone();
+                let requests_tx = requests_tx.clone();
+                let handler = service_fn(move |request: hyper::Request<hyper::body::Incoming>| {
+                    let route = format!("{} {}", request.method(), request.uri().path());
+                    requests_tx.send(route.clone()).expect("record mock admin request");
+                    let (status, body) = match route.as_str() {
+                        "POST /rustfs/admin/v3/rebalance/stop" => (stop_status, stop_body.clone()),
+                        "GET /rustfs/admin/v3/rebalance/status" => (StatusCode::OK, run_status.to_string()),
+                        _ => (StatusCode::BAD_REQUEST, "unexpected admin request".to_string()),
+                    };
+                    async move { http::Response::builder().status(status).body(Full::new(Bytes::from(body))) }
+                });
+                hyper::server::conn::http1::Builder::new()
+                    .keep_alive(false)
+                    .serve_connection(TokioIo::new(stream), handler)
+                    .await?;
+            }
+        });
+        let outcome = tokio::time::timeout(Duration::from_secs(5), stop_rebalance(&cluster, "run-1")).await?;
+        server.abort_all();
+        let requests = std::iter::from_fn(|| requests_rx.try_recv().ok()).collect();
+        Ok((outcome, requests))
+    }
+
+    fn completed_rebalance() -> Value {
+        serde_json::json!({
+            "id": "run-1",
+            "pools": [
+                { "id": 0, "status": "Completed", "progress": { "versions": 113 } },
+                { "id": 1, "status": "None" }
+            ]
+        })
+    }
+
+    #[tokio::test]
+    async fn rebalance_stop_success_does_not_query_completion() -> TestResult {
+        let (outcome, requests) = mock_rebalance_stop(StatusCode::OK, "", Value::Null).await?;
+        assert_eq!(outcome?, "");
+        assert_eq!(requests, ["POST /rustfs/admin/v3/rebalance/stop"]);
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn rebalance_stop_accepts_completed_run_after_not_started_response() -> TestResult {
+        let (outcome, requests) =
+            mock_rebalance_stop(StatusCode::NOT_FOUND, REBALANCE_NOT_STARTED, completed_rebalance()).await?;
+        outcome?;
+        assert_eq!(
+            requests,
+            [
+                "POST /rustfs/admin/v3/rebalance/stop",
+                "GET /rustfs/admin/v3/rebalance/status"
+            ]
+        );
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn rebalance_stop_rejects_unproven_completion_and_other_errors() -> TestResult {
+        let complete = completed_rebalance();
+        let mut invalid = vec![Value::Null];
+        for (pointer, value) in [
+            ("/id", serde_json::json!("run-2")),
+            ("/pools/0/status", serde_json::json!("Started")),
+            ("/pools/0/status", serde_json::json!("Failed")),
+            ("/pools/0/status", serde_json::json!("Stopped")),
+            ("/pools/0/progress/versions", serde_json::json!(0)),
+        ] {
+            let mut status = complete.clone();
+            *status.pointer_mut(pointer).expect("completion fixture field") = value;
+            invalid.push(status);
+        }
+        let mut pending = complete.clone();
+        pending["stopPropagation"] = serde_json::json!({ "pendingTerminalReload": true });
+        invalid.push(pending);
+        let mut failed = complete.clone();
+        failed["pools"][0]["lastError"] = serde_json::json!("migration failed");
+        invalid.push(failed);
+        for status in invalid {
+            let (outcome, _) = mock_rebalance_stop(StatusCode::NOT_FOUND, REBALANCE_NOT_STARTED, status.clone()).await?;
+            assert!(outcome.is_err(), "unproven completion accepted: {status}");
+        }
+        for (code, body) in [
+            (StatusCode::FORBIDDEN, REBALANCE_NOT_STARTED),
+            (StatusCode::INTERNAL_SERVER_ERROR, REBALANCE_NOT_STARTED),
+            (StatusCode::NOT_FOUND, "<Error><Code>NoSuchBucket</Code></Error>"),
+        ] {
+            let (outcome, requests) = mock_rebalance_stop(code, body, complete.clone()).await?;
+            assert!(outcome.is_err(), "unrelated error accepted: {code} {body}");
+            assert_eq!(requests, ["POST /rustfs/admin/v3/rebalance/stop"]);
+        }
+        Ok(())
+    }
+
     #[test]
     fn version_ids_normalize_null_spellings() {
         for spelling in [None, Some(""), Some("null"), Some("00000000-0000-0000-0000-000000000000")] {

crates/e2e_test/src/distributed/scanner_heal_topology_test.rs

@@ -28,7 +28,7 @@ use super::harness::{
     wait_for_rebalance_active, wait_for_rebalance_complete,
 };
 use super::scanner_heal_topology_fixture::{
-    LiveWorkload, MOVEMENT_ACTIVE_TIMEOUT, MOVEMENT_COMPLETE_TIMEOUT, MovementOperation, SEED_POOL,
+    LiveWorkload, MOVEMENT_ACTIVE_TIMEOUT, MOVEMENT_COMPLETE_TIMEOUT, MovementOperation, SEED_POOL, STAGE_AFTER_ENTRY_ADMISSION,
     STAGE_AFTER_RESPONSIBILITY_PERSISTED, STAGE_BEFORE_SOURCE_CLEANUP, STAGE_BEFORE_TARGET_COMMIT, TopologyManifest, TopologyRun,
     assert_bucket_durability_strict, assert_every_node_serves, assert_probe_objects, assert_process_durability_strict,
     assert_seed_pool_census_unchanged, assert_single_pool_placement, cancel_decommission, capture_seed_pool_census,
@@ -274,6 +274,7 @@ async fn wait_for_rebalance_stopped_or_completed(run: &TopologyRun, rebalance_id
 #[tokio::test]
 async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart() -> TestResult {
     init_logging();
+    require_test_hooks();
     let mut run = TopologyRun::start("rebalance").await?;
     let manifest = seed_manifest(&run).await?;
     assert_bucket_durability_strict(&run.dist.cluster, &manifest.buckets()).await?;
@@ -288,23 +289,52 @@ async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart()
         .map(|entry| (entry.key.clone(), entry.sha256.clone()))
         .collect();
 
-    let first_id = start_rebalance(&run.dist.cluster).await?;
-    wait_for_rebalance_active(&run.dist.cluster, &first_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
-    let during_rebalance = put_probe_objects(&run.dist, &scratch, "during-rebalance", 4).await?;
-    let workload = tokio::spawn(run_live_workload(LiveWorkload {
-        clients: run.dist.clients()?,
-        bucket: scratch.clone(),
-        prefix: "live".to_string(),
-        writers: LIVE_WRITERS,
-        rounds: LIVE_ROUNDS,
-        baseline_bucket: manifest.plain_bucket.clone(),
-        baseline: baseline.clone(),
-        pause: Duration::from_millis(100),
-    }));
-    sleep(Duration::from_secs(2)).await;
-    stop_rebalance(&run.dist.cluster).await?;
-    let (first_outcome, first_status) = wait_for_rebalance_stopped_or_completed(&run, &first_id).await?;
-    let ops = workload.await??;
+    // Keep an admitted entry on the source until stop cancels admission. Start
+    // propagation may await that entry, so observe and stop before awaiting ACK.
+    let first_key = manifest.plain_key("external", 0)?;
+    run.movement_barrier
+        .arm_movement("rebalance", &manifest.plain_bucket, &first_key, STAGE_AFTER_ENTRY_ADMISSION)?;
+    let observe_first = async {
+        let (observed_id, reached) = tokio::time::timeout(MOVEMENT_ACTIVE_TIMEOUT, async {
+            let reached = run.movement_barrier.wait_reached(MOVEMENT_ACTIVE_TIMEOUT).await?;
+            let status = rebalance_status_json(&run.dist.cluster).await?;
+            let observed_id = status
+                .get("id")
+                .and_then(Value::as_str)
+                .ok_or("rebalance status omitted id")?;
+            wait_for_rebalance_active(&run.dist.cluster, observed_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
+            Ok::<_, Box<dyn std::error::Error + Send + Sync>>((observed_id.to_string(), reached))
+        })
+        .await??;
+        let probes = put_probe_objects(&run.dist, &scratch, "during-rebalance", 4).await?;
+        let workload = run_live_workload(LiveWorkload {
+            clients: run.dist.clients()?,
+            bucket: scratch.clone(),
+            prefix: "live".to_string(),
+            writers: LIVE_WRITERS,
+            rounds: LIVE_ROUNDS,
+            baseline_bucket: manifest.plain_bucket.clone(),
+            baseline: baseline.clone(),
+            pause: Duration::from_millis(100),
+        });
+        let stop = async {
+            sleep(Duration::from_secs(2)).await;
+            stop_rebalance(&run.dist.cluster, &observed_id).await?;
+            let (outcome, status) = wait_for_rebalance_stopped_or_completed(&run, &observed_id).await?;
+            if outcome != "stopped" {
+                return Err("rebalance completed before stop; mid-run stop was not exercised".into());
+            }
+            Ok::<_, Box<dyn std::error::Error + Send + Sync>>((outcome, status))
+        };
+        let (ops, (outcome, status)) = tokio::try_join!(workload, stop)?;
+        Ok::<_, Box<dyn std::error::Error + Send + Sync>>((observed_id, reached, probes, ops, outcome, status))
+    };
+    let (first_id, (observed_id, first_reached, during_rebalance, ops, first_outcome, first_status)) =
+        tokio::try_join!(start_rebalance(&run.dist.cluster), observe_first)?;
+    if first_id != observed_id {
+        return Err(format!("rebalance start acknowledged {first_id}, but the stopped run was {observed_id}").into());
+    }
+    run.movement_barrier.disarm()?;
     let live_state = expected_live_state(&ops);
 
     for node_idx in 0..run.dist.cluster.nodes.len() {
@@ -333,8 +363,40 @@ async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart()
     verify_from_every_node(&run.dist, &manifest, "after restart following the stop").await?;
     verify_live_state(&run.dist.client(0)?, &scratch, &live_state, "after restart following the stop").await?;
 
-    let second_id = start_rebalance(&run.dist.cluster).await?;
-    wait_for_rebalance_active(&run.dist.cluster, &second_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
+    // A short second run can finish before the start response reaches us.
+    // Hold one remaining source entry until its active state is observed.
+    let mut second_key = None;
+    for entry in manifest.plain_entries() {
+        if pools_holding(&run.dist, &manifest.plain_bucket, &entry.key, None)?.contains(&SEED_POOL) {
+            second_key = Some(entry.key.clone());
+            break;
+        }
+    }
+    let second_key = second_key.ok_or("no seed-pool object remains for the second rebalance")?;
+    let second_barrier = run.movement_barrier.next_phase();
+    second_barrier.arm_movement("rebalance", &manifest.plain_bucket, &second_key, STAGE_BEFORE_SOURCE_CLEANUP)?;
+    let observe_second = async {
+        let reached = second_barrier.wait_reached(MOVEMENT_ACTIVE_TIMEOUT).await?;
+        let status = rebalance_status_json(&run.dist.cluster).await?;
+        let observed_id = status
+            .get("id")
+            .and_then(Value::as_str)
+            .ok_or("rebalance status omitted id")?;
+        if observed_id == first_id {
+            return Err("the second rebalance reused the stopped run id".into());
+        }
+        wait_for_rebalance_active(&run.dist.cluster, observed_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
+        second_barrier.disarm()?;
+        second_barrier.release_movement()?;
+        Ok::<_, Box<dyn std::error::Error + Send + Sync>>((observed_id.to_string(), reached))
+    };
+    // Worker propagation can hold the start response until movement drains.
+    let (second_id, (observed_id, second_reached)) = tokio::try_join!(start_rebalance(&run.dist.cluster), async {
+        tokio::time::timeout(MOVEMENT_ACTIVE_TIMEOUT, observe_second).await?
+    })?;
+    if second_id != observed_id {
+        return Err(format!("rebalance start acknowledged {second_id}, but the active run was {observed_id}").into());
+    }
     wait_for_rebalance_complete(&run.dist.cluster, &second_id, MOVEMENT_COMPLETE_TIMEOUT).await?;
     verify_from_every_node(&run.dist, &manifest, "after the second rebalance completed").await?;
     verify_live_state(&run.dist.client(3)?, &scratch, &live_state, "after the second rebalance completed").await?;
@@ -351,8 +413,8 @@ async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart()
         &json!({
             "gate": "G-REB",
             "manifest": manifest_summary(&manifest),
-            "first_run": { "id": first_id, "outcome": first_outcome, "status": first_status, "versions_held_by_two_pools_after_stop": retained_after_stop },
-            "second_run": { "id": second_id, "versions_off_seed_pool": moved },
+            "first_run": { "id": first_id, "outcome": first_outcome, "status": first_status, "versions_held_by_two_pools_after_stop": retained_after_stop, "barrier_object": first_key, "barrier": first_reached },
+            "second_run": { "id": second_id, "versions_off_seed_pool": moved, "barrier_object": second_key, "barrier": second_reached },
             "live_ops": ops,
             "probe_placement_during_rebalance": during_placement,
             "process_durability_strict_nodes": strict_nodes,

crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs

@@ -33,7 +33,7 @@ mod tests {
     use sha2::{Digest, Sha256};
     use std::collections::HashSet;
     use std::error::Error;
-    use std::io::{Read, Write};
+    use std::io::{Read, Seek, SeekFrom, Write};
     use std::net::SocketAddr;
     use std::path::{Path, PathBuf};
     use std::process::Command;
@@ -1654,6 +1654,89 @@ mod tests {
         TargetEndpointBlackhole,
     }
 
+    async fn release_heal_commit_barrier_after_shutdown(
+        log: &Path,
+        log_offset: u64,
+        barrier: &Path,
+    ) -> Result<serde_json::Value, Box<dyn Error + Send + Sync>> {
+        // Observe within the existing graceful process-stop budget.
+        timeout(Duration::from_secs(45), async {
+            loop {
+                let mut file = std::fs::File::open(log)?;
+                file.seek(SeekFrom::Start(log_offset))?;
+                let mut tail = String::new();
+                file.read_to_string(&mut tail)?;
+                if let Some(event) = tail
+                    .lines()
+                    .filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
+                    .find(|event| {
+                        event["event"] == "server_shutdown_state"
+                            && event["component"] == "main"
+                            && event["subsystem"] == "startup"
+                            && event["state"] == "stopping"
+                    })
+                {
+                    // Local heal has stopped and HTTP shutdown has been signalled.
+                    // Let admitted peer RPCs finish; keeping this synthetic barrier
+                    // until process exit would make the fixture block its own drain.
+                    std::fs::remove_file(barrier)?;
+                    return Ok(event);
+                }
+                sleep(Duration::from_millis(10)).await;
+            }
+        })
+        .await?
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn heal_commit_barrier_waits_for_new_shutdown_event() -> Result<(), Box<dyn Error + Send + Sync>> {
+        let directory = tempfile::tempdir()?;
+        let log = directory.path().join("server.log");
+        let barrier = directory.path().join("barrier");
+        let event = serde_json::json!({
+            "event": "server_shutdown_state", "component": "main", "subsystem": "startup", "state": "stopping"
+        });
+        std::fs::write(&log, format!("{event}\n"))?;
+        std::fs::write(&barrier, "bucket/prefix/")?;
+        let log_offset = log.metadata()?.len();
+        let mut writer = std::fs::OpenOptions::new().append(true).open(&log)?;
+        writeln!(writer, "{}", serde_json::json!({"event": "shutdown_signal_received"}))?;
+        writeln!(writer, "{}", serde_json::json!({"event": "server_shutdown_state", "state": "stopping"}))?;
+
+        let release = release_heal_commit_barrier_after_shutdown(&log, log_offset, &barrier);
+        tokio::pin!(release);
+        assert!(
+            futures::poll!(release.as_mut()).is_pending(),
+            "old or unrelated events must not release the barrier"
+        );
+        assert!(barrier.is_file(), "the partial-rebuild barrier must remain armed before shutdown");
+        tokio::time::advance(Duration::from_secs(6)).await;
+        assert!(
+            futures::poll!(release.as_mut()).is_pending(),
+            "background shutdown may take longer than five seconds within the process-stop budget"
+        );
+        writeln!(writer, "{event}")?;
+        tokio::time::advance(Duration::from_millis(10)).await;
+        assert_eq!(release.await?, event, "retain the actual new shutdown witness");
+        assert!(!barrier.exists(), "shutdown must release the synthetic hold");
+        Ok(())
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn heal_commit_barrier_missing_shutdown_fails_without_releasing() -> Result<(), Box<dyn Error + Send + Sync>> {
+        let directory = tempfile::tempdir()?;
+        let log = directory.path().join("server.log");
+        let barrier = directory.path().join("barrier");
+        std::fs::write(&log, "")?;
+        std::fs::write(&barrier, "bucket/prefix/")?;
+        let error = release_heal_commit_barrier_after_shutdown(&log, 0, &barrier)
+            .await
+            .expect_err("missing shutdown evidence must fail the fixture");
+        assert!(error.is::<tokio::time::error::Elapsed>());
+        assert!(barrier.is_file(), "a missing shutdown witness must not release repairs");
+        Ok(())
+    }
+
     async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
         let server_binary = rustfs_binary_path();
         let evidence_case = match scenario {
@@ -1791,6 +1874,7 @@ mod tests {
         let mut created_online_objects = Vec::with_capacity(online_object_count);
         let mut expected_manifests = Vec::with_capacity(online_object_count);
         let mut unclean_shutdown_marker_observed = None;
+        let mut shutdown_barrier_release = None;
         let mut attempt_count = 0usize;
         let max_online_attempts = online_object_count.saturating_mul(topology.total_drives().max(1));
         while expected_manifests.len() < online_object_count {
@@ -2373,7 +2457,21 @@ mod tests {
                     | InterruptionScenario::BackgroundTargetRestartEc84MultiSet
                     | InterruptionScenario::BackgroundCoordinatorRestart
             ) {
-                cluster.stop_node_gracefully(interruption_node).await?;
+                if interruption_node == 1
+                    && let Some(barrier) = &commit_barrier
+                {
+                    let target_log = PathBuf::from(format!("{log_dir}/node1.log"));
+                    let log_offset = target_log.metadata()?.len();
+                    let (stopped, released) = tokio::join!(
+                        cluster.stop_node_gracefully(interruption_node),
+                        release_heal_commit_barrier_after_shutdown(&target_log, log_offset, barrier),
+                    );
+                    // Await both branches even on error so the child is reaped.
+                    stopped?;
+                    shutdown_barrier_release = Some(released?);
+                } else {
+                    cluster.stop_node_gracefully(interruption_node).await?;
+                }
             } else {
                 cluster.stop_node(interruption_node)?;
             }
@@ -2415,7 +2513,9 @@ mod tests {
                     }
                 }
             }
-            if let Some(barrier) = &commit_barrier {
+            if shutdown_barrier_release.is_none()
+                && let Some(barrier) = &commit_barrier
+            {
                 std::fs::remove_file(barrier)?;
             }
             cluster.start_node_from_binary(interruption_node, &server_binary).await?;
@@ -2746,6 +2846,7 @@ mod tests {
                 "all_peers_bound_to_generation_window": true,
                 "pid_before": target_pid, "pid_after": restarted_pid,
                 "unclean_shutdown_marker": unclean_shutdown_marker_observed.unwrap_or(false),
+                "graceful_barrier_release": shutdown_barrier_release,
                 "objects": evidence_objects, "node_listings": node_listings,
             });
             if let Some(outage_write) = outage_write_diagnostic {

crates/e2e_test/src/lib.rs

@@ -283,6 +283,9 @@ mod ssec_copy_test;
 #[cfg(test)]
 mod multipart_storage_class_test;
 
+#[cfg(test)]
+mod multipart_upload_id_test;
+
 #[cfg(test)]
 mod storage_class_capability_test;
 

crates/e2e_test/src/multipart_upload_id_test.rs

@@ -0,0 +1,175 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Multipart upload ID regression tests.
+//!
+//! Every operation that addresses an existing multipart upload must report an
+//! upload ID that does not name a live upload as `404 NoSuchUpload`, whether
+//! the ID is malformed or is a well-formed ID of an aborted upload.
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging};
+    use aws_sdk_s3::Client;
+    use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
+    use aws_sdk_s3::primitives::ByteStream;
+    use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
+
+    type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
+
+    const BUCKET: &str = "multipart-upload-id";
+    const KEY: &str = "target.bin";
+    const COPY_SOURCE_KEY: &str = "copy-source.bin";
+    // RustFS upload IDs are URL-safe base64; '.' is outside that alphabet, so
+    // the object layer rejects this ID as malformed instead of looking it up.
+    const MALFORMED_UPLOAD_ID: &str = "malformed.upload.id";
+    const PART_NUMBER: i32 = 1;
+    const PART_BODY: &[u8] = b"multipart upload id regression part";
+    const PART_ETAG: &str = "\"00000000000000000000000000000000\"";
+    const NO_SUCH_UPLOAD_STATUS: u16 = 404;
+    const NO_SUCH_UPLOAD_CODE: &str = "NoSuchUpload";
+    const OBJECT_NOT_FOUND_STATUS: u16 = 404;
+
+    /// Describes how `result` differs from `404 NoSuchUpload`, or returns `None` when it matches.
+    fn no_such_upload_mismatch<T, E>(operation: &str, result: Result<T, SdkError<E>>) -> Option<String>
+    where
+        E: ProvideErrorMetadata + std::fmt::Debug,
+    {
+        let Err(error) = result else {
+            return Some(format!("{operation}: succeeded"));
+        };
+        let status = error.raw_response().map(|response| response.status().as_u16());
+        let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
+        (status != Some(NO_SUCH_UPLOAD_STATUS) || code != Some(NO_SUCH_UPLOAD_CODE))
+            .then(|| format!("{operation}: status {status:?}, code {code:?}: {error:?}"))
+    }
+
+    async fn start_server_with_copy_source() -> Result<(RustFSTestEnvironment, Client), Box<dyn std::error::Error + Send + Sync>>
+    {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(Vec::new()).await?;
+        env.create_test_bucket(BUCKET).await?;
+        let client = env.create_s3_client();
+        client
+            .put_object()
+            .bucket(BUCKET)
+            .key(COPY_SOURCE_KEY)
+            .body(ByteStream::from_static(PART_BODY))
+            .send()
+            .await?;
+        Ok((env, client))
+    }
+
+    /// Sends every operation that addresses an existing upload and asserts that all of them report `404 NoSuchUpload`.
+    async fn assert_every_upload_operation_reports_no_such_upload(client: &Client, upload_id: &str) {
+        let upload_part = client
+            .upload_part()
+            .bucket(BUCKET)
+            .key(KEY)
+            .upload_id(upload_id)
+            .part_number(PART_NUMBER)
+            .body(ByteStream::from_static(PART_BODY))
+            .send()
+            .await;
+        let upload_part_copy = client
+            .upload_part_copy()
+            .bucket(BUCKET)
+            .key(KEY)
+            .upload_id(upload_id)
+            .part_number(PART_NUMBER)
+            .copy_source(format!("/{BUCKET}/{COPY_SOURCE_KEY}"))
+            .send()
+            .await;
+        let list_parts = client.list_parts().bucket(BUCKET).key(KEY).upload_id(upload_id).send().await;
+        let complete = client
+            .complete_multipart_upload()
+            .bucket(BUCKET)
+            .key(KEY)
+            .upload_id(upload_id)
+            .multipart_upload(
+                CompletedMultipartUpload::builder()
+                    .parts(CompletedPart::builder().part_number(PART_NUMBER).e_tag(PART_ETAG).build())
+                    .build(),
+            )
+            .send()
+            .await;
+        let abort = client
+            .abort_multipart_upload()
+            .bucket(BUCKET)
+            .key(KEY)
+            .upload_id(upload_id)
+            .send()
+            .await;
+
+        let mismatches: Vec<String> = [
+            no_such_upload_mismatch("UploadPart", upload_part),
+            no_such_upload_mismatch("UploadPartCopy", upload_part_copy),
+            no_such_upload_mismatch("ListParts", list_parts),
+            no_such_upload_mismatch("CompleteMultipartUpload", complete),
+            no_such_upload_mismatch("AbortMultipartUpload", abort),
+        ]
+        .into_iter()
+        .flatten()
+        .collect();
+        assert!(
+            mismatches.is_empty(),
+            "upload ID {upload_id:?}: operations not reporting {NO_SUCH_UPLOAD_STATUS} {NO_SUCH_UPLOAD_CODE}:\n{}",
+            mismatches.join("\n")
+        );
+    }
+
+    /// Asserts that the rejected operations left neither a multipart upload nor an object under `KEY`.
+    async fn assert_no_upload_or_object_published(client: &Client) -> TestResult {
+        let uploads = client.list_multipart_uploads().bucket(BUCKET).prefix(KEY).send().await?;
+        assert!(uploads.uploads().is_empty(), "rejected operations must not leave an upload: {uploads:?}");
+        let head = client.head_object().bucket(BUCKET).key(KEY).send().await;
+        let status = head
+            .as_ref()
+            .err()
+            .and_then(|error| error.raw_response())
+            .map(|response| response.status().as_u16());
+        assert_eq!(
+            status,
+            Some(OBJECT_NOT_FOUND_STATUS),
+            "rejected operations must not publish {KEY}: {head:?}"
+        );
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn malformed_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
+        let (_env, client) = start_server_with_copy_source().await?;
+
+        assert_every_upload_operation_reports_no_such_upload(&client, MALFORMED_UPLOAD_ID).await;
+        assert_no_upload_or_object_published(&client).await
+    }
+
+    #[tokio::test]
+    async fn aborted_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
+        let (_env, client) = start_server_with_copy_source().await?;
+        let created = client.create_multipart_upload().bucket(BUCKET).key(KEY).send().await?;
+        let upload_id = created.upload_id().ok_or("CreateMultipartUpload returned no upload ID")?;
+        client
+            .abort_multipart_upload()
+            .bucket(BUCKET)
+            .key(KEY)
+            .upload_id(upload_id)
+            .send()
+            .await?;
+
+        assert_every_upload_operation_reports_no_such_upload(&client, upload_id).await;
+        assert_no_upload_or_object_published(&client).await
+    }
+}

crates/e2e_test/src/protocols/README.md

@@ -77,9 +77,7 @@ core protocol surface plus cross-protocol consistency:
   existing path returns an error, bad-password authentication is rejected
 - Cross-protocol via aws-sdk-s3: SFTP write then S3 read with SHA256 match,
   S3 write then SFTP read with SHA256 match
-- Cross-API directory visibility: SFTP-created sub-directory visible via S3
-  ListObjectsV2, S3-created `__XLDIR__` marker visible via SFTP readdir as a
-  directory entry
+- Cross-API directory visibility: SFTP-created sub-directory visible via S3 ListObjectsV2, and an S3 trailing-slash directory object visible via SFTP readdir as a directory entry
 
 #### sftp_compliance (`test_sftp_compliance_suite`)
 

crates/e2e_test/src/protocols/sftp_core.rs

@@ -394,9 +394,9 @@ pub async fn test_sftp_core_operations() -> Result<()> {
         info!("PASS: S3-written object matches via S3 GetObject and SFTP READ");
 
         // --- Cross-API directory visibility: SFTP mkdir, S3 ListObjectsV2 ---
-        // SFTP mkdir writes a __XLDIR__ marker. The rustfs S3 listing path
-        // decodes that marker back to a trailing-slash key, so the asserted
-        // pattern is "subdir_sftp/".
+        // SFTP mkdir writes a zero-byte object at the trailing-slash key.
+        // Storage encodes that key to the __XLDIR__ marker, and the S3
+        // listing path decodes the marker back to a trailing-slash key.
         info!("Testing SFTP: SFTP-created sub-directory visible via S3 ListObjectsV2");
         let sftp_subdir_name = "subdir_sftp";
         let sftp_subdir_path = format!("/{bucket}/{sftp_subdir_name}");
@@ -422,20 +422,21 @@ pub async fn test_sftp_core_operations() -> Result<()> {
         );
         info!("PASS: SFTP mkdir visible to S3 ListObjectsV2");
 
-        // --- Cross-API directory visibility: S3 marker, SFTP readdir ---
-        // aws-sdk-s3 PutObject writes a zero-byte marker keyed with the
-        // __XLDIR__ suffix. SFTP readdir must decode the marker back to a
-        // bare directory entry whose file_type reports as a directory.
-        info!("Testing SFTP: S3-created __XLDIR__ marker visible via SFTP readdir");
+        // --- Cross-API directory visibility: S3 directory object, SFTP readdir ---
+        // aws-sdk-s3 PutObject writes a zero-byte object at the trailing-slash
+        // key. The __XLDIR__ suffix is reserved on the client-key boundary, so
+        // a direct put of that suffix is rejected. Storage encodes the slash
+        // key, and SFTP readdir must show a directory entry.
+        info!("Testing SFTP: S3-created directory object visible via SFTP readdir");
         let s3_subdir_name = "subdir_s3";
-        let s3_subdir_marker_key = format!("{s3_subdir_name}{XLDIR_SUFFIX}");
+        let s3_subdir_key = format!("{s3_subdir_name}/");
         s3.put_object()
             .bucket(bucket)
-            .key(&s3_subdir_marker_key)
+            .key(&s3_subdir_key)
             .body(ByteStream::from_static(b""))
             .send()
             .await
-            .map_err(|e| anyhow!("S3 PutObject {}/{} failed: {:?}", bucket, s3_subdir_marker_key, e))?;
+            .map_err(|e| anyhow!("S3 PutObject {}/{} failed: {:?}", bucket, s3_subdir_key, e))?;
         let bucket_entries: Vec<(String, bool)> = sftp
             .read_dir(&bucket_path)
             .await?
@@ -444,9 +445,9 @@ pub async fn test_sftp_core_operations() -> Result<()> {
         let visible_via_sftp = bucket_entries.iter().any(|(name, is_dir)| name == s3_subdir_name && *is_dir);
         assert!(
             visible_via_sftp,
-            "S3-created marker must appear as a directory in SFTP readdir: entries were {bucket_entries:?}"
+            "S3-created directory object must appear as a directory in SFTP readdir: entries were {bucket_entries:?}"
         );
-        info!("PASS: S3 marker visible to SFTP readdir as a directory");
+        info!("PASS: S3 directory object visible to SFTP readdir as a directory");
 
         // --- Pre-cleanup of cross-protocol fixtures ---
         // Removes the new files and sub-directories so the existing rmdir

crates/e2e_test/src/quota_test.rs

@@ -412,6 +412,38 @@ mod integration_tests {
         );
     }
 
+    async fn upload_after_quota_reconciliation(
+        env: &QuotaTestEnv,
+        key: &str,
+        size_bytes: usize,
+    ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        let upload = async {
+            loop {
+                let result = env
+                    .client
+                    .put_object()
+                    .bucket(&env.bucket_name)
+                    .key(key)
+                    .body(aws_sdk_s3::primitives::ByteStream::from(vec![0u8; size_bytes]))
+                    .send()
+                    .await;
+                match result {
+                    Ok(_) => break,
+                    Err(error) => assert_quota_rejection(
+                        error.raw_response().map(|response| response.status().as_u16()),
+                        error.as_service_error(),
+                        &error,
+                    ),
+                }
+                sleep(Duration::from_secs(1)).await;
+            }
+        };
+        // The durable V1 ledger can retain the old charge for its 30-second reconciliation cooldown.
+        timeout(Duration::from_secs(45), upload)
+            .await
+            .map_err(|_| format!("PUT {key} did not succeed after quota reconciliation within 45 seconds").into())
+    }
+
     async fn assert_put_rejected_by_quota(env: &QuotaTestEnv, key: &str, size_bytes: usize) {
         let error = env
             .client
@@ -573,8 +605,7 @@ mod integration_tests {
 
         assert!(!env.object_exists("file1.txt").await?);
 
-        // Now we should be able to upload again (quota freed up)
-        env.upload_object("file3.txt", 256 * 1024).await?;
+        upload_after_quota_reconciliation(&env, "file3.txt", 256 * 1024).await?;
         assert!(env.object_exists("file3.txt").await?);
 
         env.cleanup_bucket().await?;
@@ -981,8 +1012,7 @@ mod integration_tests {
         );
         assert_eq!(delete_result.deleted().len(), 2);
 
-        // Now should be able to upload again (quota freed up)
-        env.upload_object("file3.txt", 256 * 1024).await?;
+        upload_after_quota_reconciliation(&env, "file3.txt", 256 * 1024).await?;
         assert!(env.object_exists("file3.txt").await?);
 
         env.cleanup_bucket().await?;

crates/e2e_test/src/reliant/grpc_lock_server.rs

@@ -42,6 +42,7 @@ fn lock_result_from_response(response: rustfs_lock::LockResponse) -> GenerallyLo
         success: response.success,
         error_info: response.error,
         lock_info: response.lock_info.and_then(|info| serde_json::to_string(&info).ok()),
+        ..Default::default()
     }
 }
 
@@ -50,6 +51,7 @@ fn lock_result_from_error(error: impl Into<String>) -> GenerallyLockResult {
         success: false,
         error_info: Some(error.into()),
         lock_info: None,
+        ..Default::default()
     }
 }
 
@@ -59,6 +61,7 @@ fn lock_result_from_release(lock_id: &rustfs_lock::LockId, success: bool) -> Gen
             success: true,
             error_info: None,
             lock_info: None,
+            ..Default::default()
         }
     } else {
         lock_result_from_error(format!("lock not found for release: {lock_id}"))
@@ -164,6 +167,7 @@ impl NodeService for MinimalLockNodeService {
                     success: false,
                     error_info: Some(format!("can not decode args, err: {err}")),
                     lock_info: None,
+                    ..Default::default()
                 }));
             }
         };
@@ -175,6 +179,7 @@ impl NodeService for MinimalLockNodeService {
                     success: result.success,
                     error_info: result.error,
                     lock_info: lock_info_json,
+                    ..Default::default()
                 }))
             }
             Err(err) => Ok(Response::new(GenerallyLockResponse {
@@ -184,6 +189,7 @@ impl NodeService for MinimalLockNodeService {
                     args.resource, args.owner, err
                 )),
                 lock_info: None,
+                ..Default::default()
             })),
         }
     }
@@ -197,6 +203,7 @@ impl NodeService for MinimalLockNodeService {
                     success: false,
                     error_info: Some(format!("can not decode args, err: {err}")),
                     lock_info: None,
+                    ..Default::default()
                 }));
             }
         };
@@ -208,6 +215,7 @@ impl NodeService for MinimalLockNodeService {
                     success: result.success,
                     error_info: result.error_info,
                     lock_info: None,
+                    ..Default::default()
                 }))
             }
             Err(err) => Ok(Response::new(GenerallyLockResponse {
@@ -217,6 +225,7 @@ impl NodeService for MinimalLockNodeService {
                     args.resource, args.owner, err
                 )),
                 lock_info: None,
+                ..Default::default()
             })),
         }
     }
@@ -230,6 +239,7 @@ impl NodeService for MinimalLockNodeService {
                     success: false,
                     error_info: Some(format!("can not decode args, err: {err}")),
                     lock_info: None,
+                    ..Default::default()
                 }));
             }
         };
@@ -241,6 +251,7 @@ impl NodeService for MinimalLockNodeService {
                     success: result.success,
                     error_info: result.error_info,
                     lock_info: None,
+                    ..Default::default()
                 }))
             }
             Err(err) => Ok(Response::new(GenerallyLockResponse {
@@ -250,6 +261,7 @@ impl NodeService for MinimalLockNodeService {
                     args.resource, args.owner, err
                 )),
                 lock_info: None,
+                ..Default::default()
             })),
         }
     }
@@ -263,6 +275,7 @@ impl NodeService for MinimalLockNodeService {
                     success: false,
                     error_info: Some(format!("can not decode args, err: {err}")),
                     lock_info: None,
+                    ..Default::default()
                 }));
             }
         };
@@ -272,11 +285,13 @@ impl NodeService for MinimalLockNodeService {
                 success,
                 error_info: None,
                 lock_info: None,
+                ..Default::default()
             })),
             Err(err) => Ok(Response::new(GenerallyLockResponse {
                 success: false,
                 error_info: Some(format!("can not refresh, err: {err}")),
                 lock_info: None,
+                ..Default::default()
             })),
         }
     }

crates/e2e_test/src/replication_extension_test.rs

@@ -31,8 +31,8 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{
-    BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DeleteMarkerEntry, ObjectVersion, ServerSideEncryption,
-    VersioningConfiguration,
+    BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Delete, DeleteMarkerEntry, ObjectIdentifier, ObjectVersion,
+    ServerSideEncryption, VersioningConfiguration,
 };
 use base64_simd::STANDARD as BASE64_STANDARD;
 use bytes::Bytes;
@@ -3729,6 +3729,8 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
     init_logging();
 
     let mut source_env = RustFSTestEnvironment::new().await?;
+    let source_log = format!("{}/source.log", source_env.temp_dir);
+    source_env.capture_log_path = Some(source_log.clone());
     let mut source_env_vars = replication_fast_env();
     source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
     source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
@@ -3791,19 +3793,66 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
         "target did not preserve the source delete-marker version ID"
     );
 
-    source_client
-        .delete_object()
+    let wait_for_source_writeback = || async {
+        let metrics_url = format!("{}/{source_bucket}?replication-metrics", source_env.url);
+        timeout(Duration::from_secs(30), async {
+            loop {
+                let response = signed_request(
+                    http::Method::GET,
+                    &metrics_url,
+                    &source_env.access_key,
+                    &source_env.secret_key,
+                    None,
+                    None,
+                )
+                .await?;
+                let metrics: serde_json::Value = response.error_for_status()?.json().await?;
+                let queued = metrics["queued"]["curr"]["count"]
+                    .as_f64()
+                    .ok_or("missing replication queue count")?;
+                if metrics["provider_available"] == true && queued == 0.0 {
+                    return Ok::<(), Box<dyn Error + Send + Sync>>(());
+                }
+                sleep(Duration::from_millis(50)).await;
+            }
+        })
+        .await?
+    };
+    wait_for_source_writeback().await?;
+
+    let purged = source_client
+        .delete_objects()
         .bucket(source_bucket)
-        .key(object_key)
-        .version_id(&purged_version_id)
+        .delete(
+            Delete::builder()
+                .objects(
+                    ObjectIdentifier::builder()
+                        .key(object_key)
+                        .version_id(&purged_version_id)
+                        .build()?,
+                )
+                .objects(
+                    ObjectIdentifier::builder()
+                        .key(object_key)
+                        .version_id(delete_marker_version_id)
+                        .build()?,
+                )
+                .build()?,
+        )
         .send()
         .await?;
+    assert!(purged.errors().is_empty(), "batch version purge failed: {:?}", purged.errors());
+    assert_eq!(purged.deleted().len(), 2, "both the data version and marker must be purged");
     assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
     let target_state = list_replication_state(&target_client, target_bucket).await?;
     assert!(
         target_state.iter().all(|entry| entry.version_id != purged_version_id),
         "target retained the explicitly purged object version"
     );
+    assert!(
+        target_state.iter().all(|entry| entry.version_id != delete_marker_version_id),
+        "target retained the explicitly purged delete marker"
+    );
     assert!(
         target_state
             .iter()
@@ -3819,6 +3868,19 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
         .await?;
     assert_eq!(retained.body.collect().await?.into_bytes().as_ref(), b"versioned replication payload v2");
 
+    // Wait for the source workers to finish their final writeback, which runs
+    // after the remote purge has already become visible (rustfs#9121).
+    wait_for_source_writeback().await?;
+    let logs = fs::read_to_string(source_log).await?;
+    let writeback_failures: Vec<_> = logs
+        .lines()
+        .filter(|line| line.contains("apply_replication_delete_state"))
+        .collect();
+    assert!(
+        writeback_failures.is_empty(),
+        "a completed batch purge must not report a source writeback failure: {writeback_failures:?}"
+    );
+
     Ok(())
 }
 

crates/e2e_test/src/storage_api.rs

@@ -94,3 +94,12 @@ pub(crate) mod scanner_peer_bucket {
     pub(crate) use ecstore_erasure::{ReedSolomonEncoder, calc_shard_size};
     pub(crate) use ecstore_rpc::PeerRestClient;
 }
+
+/// Storage handles used to seed a pre-validation bucket record while the fixture server is stopped.
+#[cfg(test)]
+pub(crate) mod bucket_tag_fixture {
+    pub(crate) use rustfs_ecstore::api::bucket::metadata::load_bucket_metadata;
+    pub(crate) use rustfs_ecstore::api::layout::{DisksLayout, EndpointServerPools};
+    pub(crate) use rustfs_ecstore::api::runtime::InstanceContext;
+    pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks_with_instance_ctx};
+}

crates/e2e_test/src/sts_query_compat_test.rs

@@ -360,12 +360,26 @@ impl OpaMock {
 
     pub(crate) async fn expect_bucket_tags(&mut self, action: &str, bucket: &str, department: Option<&str>) -> TestResult {
         let payload = self.next_request().await?;
+        Self::assert_bucket_tags(&payload, action, bucket, department);
+        Ok(())
+    }
+
+    async fn expect_versioned_tag_permission(&mut self, bucket: &str, object: &str, version: &str) -> TestResult {
+        let payload = self.next_request().await?;
+        Self::assert_bucket_tags(&payload, "s3:GetObjectVersionTagging", bucket, Some("finance"));
+        assert_eq!(payload["input"]["resource"]["object"], object);
+        assert_eq!(payload["input"]["context"]["conditions"]["versionid"], serde_json::json!([version]));
+        Ok(())
+    }
+
+    fn assert_bucket_tags(payload: &Value, action: &str, bucket: &str, department: Option<&str>) {
         let input = &payload["input"];
+        assert_eq!(input["identity"]["account"], "opabuckettags");
+        assert_eq!(input["identity"]["is_owner"], false);
         assert_eq!(input["action"], action);
         assert_eq!(input["resource"]["bucket"], bucket);
         let expected = department.map(|value| serde_json::json!([value]));
         assert_eq!(input["context"]["conditions"].get("ExistingBucketTag/department"), expected.as_ref());
-        Ok(())
     }
 
     async fn wait_for_validation(&mut self) -> TestResult {
@@ -678,22 +692,61 @@ async fn test_list_buckets_opa_contract() -> TestResult {
     Ok(())
 }
 
+/// Stop and reap the directory's owning server before opening the fixture store.
+async fn seed_legacy_duplicate_bucket_tags(env: &mut RustFSTestEnvironment, bucket: &str) -> TestResult {
+    use crate::storage_api::bucket_tag_fixture::{
+        DisksLayout, ECStore, EndpointServerPools, InstanceContext, init_local_disks_with_instance_ctx, load_bucket_metadata,
+    };
+
+    let server = env
+        .process
+        .as_mut()
+        .ok_or("missing server process before offline fixture write")?;
+    server.kill()?;
+    server.wait()?;
+    env.process = None;
+    let layout = DisksLayout::from_volumes(std::slice::from_ref(&env.temp_dir))?;
+    let (pools, _) = EndpointServerPools::create_server_endpoints(&env.address, &layout).await?;
+    let context = std::sync::Arc::new(InstanceContext::new());
+    init_local_disks_with_instance_ctx(&context, pools.clone()).await?;
+    let store = ECStore::new_with_instance_ctx(env.address.parse()?, pools, Default::default(), context.clone()).await?;
+    let _cancel = context
+        .background_cancel_token()
+        .expect("store binds its shutdown token")
+        .drop_guard();
+    let mut metadata = load_bucket_metadata(store.clone(), bucket).await?;
+    let incarnation = metadata.bucket_incarnation_id;
+    let policy = metadata.policy_config_json.clone();
+    assert!(!incarnation.is_nil());
+    assert!(!policy.is_empty(), "the bucket-policy Allow must survive legacy seeding");
+    // Encode the duplicate tag set with the same bucket record and erasure writer used by the store.
+    let xml = b"<Tagging><TagSet><Tag><Key>department</Key><Value>finance</Value></Tag><Tag><Key>department</Key><Value>engineering</Value></Tag></TagSet></Tagging>";
+    metadata.tagging_config_xml = xml.to_vec();
+    metadata.save_with_store(store.clone()).await?;
+    let persisted = load_bucket_metadata(store, bucket).await?;
+    assert_eq!(persisted.bucket_incarnation_id, incarnation);
+    assert_eq!(persisted.policy_config_json, policy);
+    assert_eq!(persisted.tagging_config_xml.as_slice(), xml);
+    Ok(())
+}
+
 #[tokio::test]
 async fn test_bucket_tags_opa_contract() -> TestResult {
     use aws_sdk_s3::primitives::ByteStream;
     use aws_sdk_s3::types::{
-        BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, VersioningConfiguration,
+        BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, TaggingDirective, VersioningConfiguration,
     };
 
     init_logging();
     let mut opa = OpaMock::start().await?;
     let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server_without_cleanup_with_env(&[
-        ("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str()),
+    let opa_url = opa.url.clone();
+    let server_env = [
+        ("RUSTFS_POLICY_PLUGIN_URL", opa_url.as_str()),
         ("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", OPA_AUTH_TOKEN),
         ("NO_PROXY", "127.0.0.1,localhost"),
-    ])
-    .await?;
+    ];
+    env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
     let secret = uuid::Uuid::new_v4().to_string();
     create_user(&env, "opabuckettags", &secret).await?;
     let client = aws_sdk_s3::Client::from_conf(build_test_s3_config(&env.url, "opabuckettags", &secret, None, "e2e-bucket-tags"));
@@ -733,7 +786,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
         .await?;
     assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
     opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
-    client
+    let head = client
         .head_object()
         .bucket(source)
         .key("report")
@@ -741,6 +794,8 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
         .send()
         .await?;
     opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
+    assert_eq!(head.tag_count(), Some(1));
+    opa.expect_versioned_tag_permission(source, "report", version).await?;
     client.list_objects_v2().bucket(source).send().await?;
     opa.expect_bucket_tags("s3:ListBucket", source, Some("finance")).await?;
     client.list_object_versions().bucket(source).send().await?;
@@ -750,28 +805,65 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
     admin.create_bucket().bucket(destination).send().await?;
     set_department(&admin, destination, "engineering").await?;
     let copy_source = format!("{source}/report?versionId={version}");
-    let error = client
-        .copy_object()
-        .bucket(destination)
-        .key("copied")
-        .copy_source(&copy_source)
-        .send()
-        .await
-        .expect_err("source tags must not authorize a different destination");
-    assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
-    opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
-    opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
-        .await?;
+    for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
+        let error = client
+            .copy_object()
+            .bucket(destination)
+            .key("copied")
+            .copy_source(&copy_source)
+            .set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
+            .set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
+            .send()
+            .await
+            .expect_err("source tags must not authorize a different destination");
+        assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
+        opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
+        if directive == TaggingDirective::Copy {
+            opa.expect_versioned_tag_permission(source, "report", version).await?;
+        }
+        opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
+            .await?;
+        let missing = admin
+            .head_object()
+            .bucket(destination)
+            .key("copied")
+            .send()
+            .await
+            .expect_err("denied COPY and REPLACE must leave no destination object");
+        assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
+    }
     set_department(&admin, destination, "finance").await?;
-    client
-        .copy_object()
-        .bucket(destination)
-        .key("copied")
-        .copy_source(&copy_source)
-        .send()
-        .await?;
-    opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
-    opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
+    for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
+        client
+            .copy_object()
+            .bucket(destination)
+            .key("copied")
+            .copy_source(&copy_source)
+            .set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
+            .set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
+            .send()
+            .await?;
+        opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
+        // The access hook checks preliminary tags, then the copy use case checks
+        // the exact source reader's tags before publishing the destination.
+        for _ in 0..2 {
+            if directive == TaggingDirective::Copy {
+                opa.expect_versioned_tag_permission(source, "report", version).await?;
+            }
+            opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
+        }
+        let tags = admin.get_object_tagging().bucket(destination).key("copied").send().await?;
+        assert_eq!(tags.tag_set().len(), 1);
+        assert_eq!(tags.tag_set()[0].key(), "department");
+        assert_eq!(
+            tags.tag_set()[0].value(),
+            if directive == TaggingDirective::Copy {
+                "engineering"
+            } else {
+                "legal"
+            }
+        );
+    }
 
     let upload = client
         .create_multipart_upload()
@@ -847,6 +939,14 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
     assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
     opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("engineering"))
         .await?;
+    let missing = admin
+        .head_object()
+        .bucket(destination)
+        .key("denied-source")
+        .send()
+        .await
+        .expect_err("a denied source must leave no destination object");
+    assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
     set_department(&client, source, "finance")
         .await
         .expect_err("requested tags must not authorize their own mutation");
@@ -885,7 +985,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
     assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
     opa.expect_bucket_tags("s3:GetObject", destination, Some("engineering"))
         .await?;
-    admin
+    let invalid_write = admin
         .put_bucket_tagging()
         .bucket(destination)
         .tagging(
@@ -895,7 +995,25 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
                 .build()?,
         )
         .send()
+        .await
+        .expect_err("new duplicate tags must be rejected before publication");
+    assert_eq!(invalid_write.as_service_error().and_then(ProvideErrorMetadata::code), Some("InvalidTag"));
+    assert_eq!(invalid_write.raw_response().map(|response| response.status().as_u16()), Some(400));
+
+    set_department(&admin, source, "finance").await?;
+    seed_legacy_duplicate_bucket_tags(&mut env, destination).await?;
+    env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
+    // Prove this process can authorize the same caller and read a healthy bucket's persisted tags.
+    let read = client
+        .get_object()
+        .bucket(source)
+        .key("report")
+        .version_id(version)
+        .send()
         .await?;
+    assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
+    opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
+
     let error = client
         .get_object()
         .bucket(destination)

crates/e2e_test/src/table_location_authorization_test.rs

@@ -114,10 +114,10 @@ fn staged_commit(metadata: &Value, location: &str) -> Value {
     })
 }
 
-async fn location_claim_preserves_other_users_objects_after_drop_and_restart(backing: &str) -> TestResult {
+async fn location_claim_preserves_other_users_objects_after_drop_and_restart() -> TestResult {
     init_logging();
     let mut env = RustFSTestEnvironment::new().await?;
-    let server_env = [("RUSTFS_TABLE_CATALOG_BACKING", backing)];
+    let server_env = [("RUSTFS_TABLE_CATALOG_BACKING", "object")];
     env.start_rustfs_server_without_cleanup_with_env(&[("RUSTFS_TABLE_CATALOG_BACKING", "object")])
         .await?;
     env.wait_for_server_ready().await?;
@@ -147,39 +147,6 @@ async fn location_claim_preserves_other_users_objects_after_drop_and_restart(bac
         StatusCode::OK,
     )
     .await?;
-    if backing == "durable-strong" {
-        catalog_request(
-            &env,
-            &root,
-            Method::POST,
-            &format!("/iceberg/v1/{bucket}/catalog/warehouse-index/backfill"),
-            None,
-            StatusCode::NO_CONTENT,
-        )
-        .await?;
-        let preflight = catalog_request(
-            &env,
-            &root,
-            Method::GET,
-            &format!("/iceberg/v1/{bucket}/catalog/migration"),
-            None,
-            StatusCode::OK,
-        )
-        .await?;
-        assert_eq!(preflight["blockers"], json!([]), "{preflight}");
-        let migration = catalog_request(
-            &env,
-            &root,
-            Method::POST,
-            &format!("/iceberg/v1/{bucket}/catalog/migration"),
-            None,
-            StatusCode::OK,
-        )
-        .await?;
-        assert_eq!(migration["ready_to_enable_durable_strong"], json!(true), "{migration}");
-        env.restart_server_preserving_data(vec![], &server_env).await?;
-        env.wait_for_server_ready().await?;
-    }
     admin_create_user(&env, &creator.0, &creator.1).await?;
     admin_create_user(&env, &reader.0, &reader.1).await?;
     let create_statement = json!({
@@ -341,7 +308,7 @@ async fn location_claim_preserves_other_users_objects_after_drop_and_restart(bac
         StatusCode::FORBIDDEN,
     )
     .await?;
-    if backing == "object" {
+    {
         catalog_request(
             &env,
             &creator,
@@ -408,10 +375,5 @@ async fn location_claim_preserves_other_users_objects_after_drop_and_restart(bac
 
 #[tokio::test]
 async fn object_catalog_location_claim_preserves_objects_after_drop_and_restart() -> TestResult {
-    location_claim_preserves_other_users_objects_after_drop_and_restart("object").await
-}
-
-#[tokio::test]
-async fn durable_strong_catalog_location_claim_preserves_objects_after_drop_and_restart() -> TestResult {
-    location_claim_preserves_other_users_objects_after_drop_and_restart("durable-strong").await
+    location_claim_preserves_other_users_objects_after_drop_and_restart().await
 }

crates/e2e_test/src/upgrade_compatibility_test.rs

@@ -48,6 +48,7 @@ const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
 const G09_EVIDENCE_DIR_ENV: &str = "RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR";
 const RC5_COMMIT: &str = "40a2470feb567201165a5b809b7598bb4b1f68f5";
 const STABLE_1_0_1_COMMIT: &str = "6de965ae3c965a78ff819fbcd7acd4aa44177d92";
+const STABLE_1_0_4_COMMIT: &str = "46337942d55f3a8e6a97b147ca98d9ea8115555d";
 const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
 const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI=";
 const PLAIN_BUCKET: &str = "upgrade-plain-data";
@@ -954,7 +955,8 @@ async fn exercise_mixed_cluster(
     phase: &str,
     current_node: usize,
     previous_node: usize,
-) -> TestResult {
+    previous_has_delete_recovery: bool,
+) -> Result<Option<String>, BoxError> {
     let clients = cluster.create_all_clients()?;
     wait_for_upgrade_write_readiness(&clients, MIXED_BUCKET, phase, LISTING_CONVERGENCE_TIMEOUT).await?;
     wait_for_drive_readmission(&clients, MIXED_BUCKET, phase, DRIVE_READMISSION_TIMEOUT).await?;
@@ -1009,6 +1011,30 @@ async fn exercise_mixed_cluster(
         vec![u8::try_from(MULTIPART_WORKERS - 1)?; 64 * 1024]
     );
 
+    // 1.0.4 already implements the delete decision protocol. Use a separate
+    // old-writer key so the original seed still spans the entire rollout.
+    if previous_has_delete_recovery {
+        let key = format!(".upgrade-delete/{phase}/previous-writer");
+        previous_client
+            .put_object()
+            .bucket(MIXED_BUCKET)
+            .key(&key)
+            .body(ByteStream::from_static(UPGRADE_READINESS_BODY))
+            .send()
+            .await?;
+        for client in &clients {
+            assert_eq!(read_object(client, MIXED_BUCKET, &key, None).await?.1, UPGRADE_READINESS_BODY);
+        }
+        upgrade_probe_client(current_client)
+            .delete_object()
+            .bucket(MIXED_BUCKET)
+            .key(&key)
+            .send()
+            .await?;
+        assert_mixed_deletes_visible(&clients, std::slice::from_ref(&key)).await?;
+        return Ok(Some(key));
+    }
+
     // Every online participant must understand the delete decision protocol.
     // An upgrade phase can serve reads/writes while intentionally refusing a
     // modern DELETE; rejecting it must leave the old writer's seed intact.
@@ -1029,6 +1055,31 @@ async fn exercise_mixed_cluster(
         );
     }
 
+    Ok(None)
+}
+
+async fn assert_mixed_deletes_visible(clients: &[Client], keys: &[String]) -> TestResult {
+    for (node, client) in clients.iter().enumerate() {
+        for key in keys {
+            let error = upgrade_probe_client(client)
+                .get_object()
+                .bucket(MIXED_BUCKET)
+                .key(key)
+                .send()
+                .await
+                .expect_err("a successful mixed-version DELETE must remain visible through every reader");
+            assert_eq!(
+                error.raw_response().map(|response| response.status().as_u16()),
+                Some(404),
+                "node {node}: {key}"
+            );
+            assert_eq!(
+                error.as_service_error().and_then(ProvideErrorMetadata::code),
+                Some("NoSuchKey"),
+                "node {node}: {key}"
+            );
+        }
+    }
     Ok(())
 }
 
@@ -2240,6 +2291,9 @@ async fn direct_upgrade_from_previous_release_preserves_object_contracts() -> Te
 async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts() -> TestResult {
     init_logging();
     let previous_binary = source_binary()?;
+    // Bind the expected protocol to the verified release source, never to the
+    // DELETE result itself. The older CI source pins retain their 503 contract.
+    let previous_has_delete_recovery = binary_source_revision(&previous_binary).await? == STABLE_1_0_4_COMMIT;
     let evidence_context = g09_evidence_context(&previous_binary).await?;
     let current_binary = rustfs_binary_path();
     let mut cluster = RustFSTestClusterEnvironment::new(MIXED_NODE_COUNT).await?;
@@ -2260,7 +2314,10 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
 
     cluster.stop_node(0)?;
     cluster.start_node_from_binary(0, &current_binary).await?;
-    exercise_mixed_cluster(&cluster, "one-current-node", 0, 1).await?;
+    let mut mixed_deleted_keys = Vec::new();
+    if let Some(key) = exercise_mixed_cluster(&cluster, "one-current-node", 0, 1, previous_has_delete_recovery).await? {
+        mixed_deleted_keys.push(key);
+    }
     // One controlled out-of-policy attempt proves fail-closed behavior below
     // native write quorum; positive MPU writers remain paused while mixed.
     assert_mixed_pending_refusal(&cluster, &pending, &sources).await?;
@@ -2268,7 +2325,9 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
     for (node_idx, phase) in [(1, "two-current-nodes"), (2, "one-previous-node")] {
         cluster.stop_node(node_idx)?;
         cluster.start_node_from_binary(node_idx, &current_binary).await?;
-        exercise_mixed_cluster(&cluster, phase, 0, 3).await?;
+        if let Some(key) = exercise_mixed_cluster(&cluster, phase, 0, 3, previous_has_delete_recovery).await? {
+            mixed_deleted_keys.push(key);
+        }
     }
 
     cluster.stop_node(3)?;
@@ -2278,6 +2337,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
     let completed_pending = finish_pending_multipart(&clients, &pending, pending_parts.concat()).await?;
     exercise_homogeneous_multipart(&cluster, "homogeneous-current").await?;
     verify_load_bodies(&clients, "homogeneous-previous", "multipart").await?;
+    assert_mixed_deletes_visible(&clients, &mixed_deleted_keys).await?;
 
     for (node_idx, client) in clients.iter().enumerate() {
         assert_eq!(
@@ -2326,6 +2386,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
     cluster.start_with_binary(&previous_binary).await?;
     prepare_release_baseline(&mut cluster, &previous_binary, "rollback").await?;
     let old_clients = cluster.create_all_clients()?;
+    assert_mixed_deletes_visible(&old_clients, &mixed_deleted_keys).await?;
     assert_resumed_multipart(&old_clients, &completed_pending).await?;
     for phase in ["homogeneous-previous", "homogeneous-current"] {
         verify_load_bodies(&old_clients, phase, "multipart").await?;
@@ -2390,6 +2451,8 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
                 "phases": phases,
                 "objects_per_phase": objects_per_phase,
                 "mixed_load": "plain-put",
+                "mixed_delete_protocol_supported": previous_has_delete_recovery,
+                "mixed_deleted_keys": mixed_deleted_keys,
                 "homogeneous_multipart_phases": ["homogeneous-previous", "homogeneous-current", "homogeneous-rollback"],
                 "multipart_objects_per_homogeneous_phase": MULTIPART_WORKERS * MULTIPART_UPLOADS_PER_WORKER,
                 "assertions": [
@@ -2422,7 +2485,11 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
                     "16x16 multipart load runs in homogeneous previous, current and rollback phases",
                     "one-current-node Complete returns typed 503 and preserves actual source authority and parts",
                     "the same legacy upload completes and replays once fully upgraded without adding a version",
-                    "modern DELETE fails closed and preserves every byte while old peers remain"
+                    if previous_has_delete_recovery {
+                        "mixed-version DELETE succeeds and stays absent through every reader after upgrade and rollback"
+                    } else {
+                        "modern DELETE fails closed and preserves every byte while old peers remain"
+                    }
                 ],
             }),
         )?;

crates/ecstore/Cargo.toml

@@ -31,6 +31,7 @@ workspace = true
 
 [features]
 default = []
+fips = ["rustfs-s3-client/fips", "rustfs-utils/fips", "rustfs-tls-runtime/fips"]
 diagnostic-read-version-caller = ["rustfs-io-metrics/diagnostic-read-version-caller"]
 # Sampled wall/CPU diagnostics only; does not change the read backend.
 diagnostic-local-batch = ["rustix/time"]
@@ -202,6 +203,9 @@ libc.workspace = true
 # (backlog#1178); "thread" provides cleanup affinity; "fs" is a workspace default.
 rustix = { workspace = true, features = ["process", "fs", "thread"] }
 rustfs-madmin.workspace = true
+rustfs-signer.workspace = true
+rustfs-tls-runtime.workspace = true
+quick-xml.workspace = true
 reqwest = { workspace = true }
 aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
 urlencoding = { workspace = true }

crates/ecstore/src/api/mod.rs

@@ -191,7 +191,11 @@ pub mod bucket {
             OBJECT_LOCK_CONFIG, UnreadableBucketConfig, is_unreadable_config_error, load_bucket_metadata,
             table_catalog_path_hash, unreadable_config_refusal,
         };
-        pub use crate::bucket::metadata::{BUCKET_DURABILITY_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
+        pub use crate::bucket::metadata::{
+            BUCKET_DURABILITY_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG, MAX_SWIFT_METADATA_COUNT, MAX_SWIFT_METADATA_VALUE_SIZE,
+            SWIFT_ACCOUNT_META_TAG_PREFIX, SWIFT_CONTAINER_META_TAG_PREFIX, SWIFT_TAGGING_MARKER,
+        };
+        pub use crate::bucket::metadata::{MAX_BUCKET_TAGS, is_swift_bucket_metadata_tag_key, validate_tag_set_entries};
     }
 
     pub mod config_parse_mode {
@@ -209,19 +213,20 @@ pub mod bucket {
     pub mod metadata_sys {
         pub use crate::bucket::metadata_sys::{
             BUCKET_CONFIG_PUBLISH_HOOK, BucketConfigPublishHook, BucketMetadataMutationGuard, BucketMetadataSys,
-            ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
+            ObjectLockConfigState, ScannerBucketIncarnationReadGuard, acquire_bucket_metadata_transaction_lock,
             acquire_bucket_metadata_transaction_lock_for_incarnation, acquire_scanner_bucket_incarnation_fence,
-            capture_bucket_metadata_incarnation, delete, delete_if_incarnation, delete_if_incarnation_at,
-            delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy, get_bucket_policy_raw,
-            get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
+            acquire_scanner_bucket_incarnation_read_fence, capture_bucket_metadata_incarnation, delete, delete_if_incarnation,
+            delete_if_incarnation_at, delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy,
+            get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
             get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
             get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config,
             get_on_demand_migration_config_in, get_public_access_block_config, get_quota_config, get_replication_config,
-            get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config,
-            init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata,
+            get_request_payment_config, get_s3_tagging_config, get_sse_config, get_tagging_config, get_versioning_config,
+            get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata,
             remove_bucket_metadata_if_absent, set_bucket_metadata, update, update_bucket_targets_under_transaction_lock,
             update_config_with, update_if_incarnation, update_if_incarnation_at, update_quota_if_incarnation,
-            update_quota_if_incarnation_at, update_under_transaction_lock, update_under_transaction_lock_at,
+            update_quota_if_incarnation_at, update_s3_tagging_config, update_under_transaction_lock,
+            update_under_transaction_lock_at,
         };
         #[cfg(feature = "test-util")]
         pub use crate::bucket::metadata_sys::{ConfigWriteLockProbe, test_support};
@@ -331,8 +336,8 @@ pub mod bucket {
     pub mod utils {
         pub use crate::bucket::utils::{
             check_bucket_and_object_names, check_list_objs_args, check_object_name_for_length_and_slash,
-            check_valid_bucket_name_strict, deserialize, has_bad_path_component, is_meta_bucketname, is_valid_object_prefix,
-            serialize,
+            check_valid_bucket_name_strict, deserialize, has_bad_path_component, has_xml_invalid_control_characters,
+            is_meta_bucketname, is_valid_object_prefix, serialize,
         };
     }
 
@@ -449,11 +454,18 @@ pub mod data_usage {
 pub mod disk {
     pub use crate::cluster::rpc::background_pipe::{BackgroundPipeReader, BackgroundPipeWriter, background_pipe};
     pub use crate::cluster::rpc::background_reader_stream::BackgroundReaderStream;
+    pub use crate::disk::BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION;
     pub use crate::disk::background_memory::BackgroundEncodeBuffer;
     pub use crate::disk::disk_store::get_object_disk_read_timeout;
+    #[cfg(feature = "test-util")]
+    pub use crate::disk::disk_store::wait_for_multipart_index_bootstrap;
     pub use crate::disk::local::{ReplacementExecutionLease, ScanGuard};
     #[cfg(feature = "test-util")]
     pub use crate::disk::local::{local_durability_modes, set_local_rename_data_fail_after_metadata_commit};
+    pub use crate::disk::multipart_index::{
+        MULTIPART_INDEX_RPC_MAX_BYTES, MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES, MultipartIndexPage, MultipartIndexRpcReply,
+        MultipartIndexRpcRequest,
+    };
     pub use crate::disk::multipart_source::{
         MULTIPART_SOURCE_PROTOCOL_VERSION, MultipartAbortCertificate, MultipartAbortProof, MultipartAbortProofKind,
         MultipartAbortRound, MultipartSourceOperation, MultipartSourcePhase, MultipartSourceRequest, MultipartSourceResponse,
@@ -466,13 +478,15 @@ pub mod disk {
     pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage};
     pub use crate::disk::{
         BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
-        CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration, CurrentDeleteTransition,
+        BoundedDeleteRecoveryDiscovery, BoundedDeleteRecoveryInspection, BoundedDeleteRecoverySettlement,
+        BoundedDeleteRecoverySettlementReply, CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration,
+        CurrentDeleteTransition, DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES, DELETE_RECOVERY_MAX_INSPECTION_ENTRIES,
         DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest,
         DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DeleteRecoveryState,
         Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, FileInfoVersions, FileReader, FileWriter,
-        GuardedDeleteVersionsPayload, GuardedFileInfoVersions, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize,
-        POOL_META_NATIVE_CAS_METADATA_SUFFIX, PartTransactionAction, PoolMetaNativeBackup, PoolMetaNativeInspection,
-        PoolMetaNativeObject, PoolMetaNativeRecoveryRequest, PoolMetaNativeRecoveryResponse,
+        GuardedDeleteVersionsPayload, GuardedFileInfoVersions, HEAL_REPLACEMENT_RECOVERY_DIR, HEALING_MARKER_PATH,
+        NsScannerOpenRequest, OldCurrentSize, POOL_META_NATIVE_CAS_METADATA_SUFFIX, PartTransactionAction, PoolMetaNativeBackup,
+        PoolMetaNativeInspection, PoolMetaNativeObject, PoolMetaNativeRecoveryRequest, PoolMetaNativeRecoveryResponse,
         QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET,
         ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, SnapshotLeaseToken,
         UpdateMetadataOpts, VolumeInfo, WalkDirOptions, is_quota_mutation_fence_path, new_disk,
@@ -617,7 +631,7 @@ pub mod rebalance {
 pub mod rio {
     pub use crate::io_support::rio::{
         DecryptReader, DynReader, EncryptReader, HardLimitReader, HashReader, ReadStream, Reader, WriteEncryption, WritePlan,
-        boxed_reader, compression_metadata_value, wrap_reader,
+        boxed_reader, compression_metadata_value, validate_compression_scheme, wrap_reader,
     };
 }
 
@@ -654,7 +668,9 @@ pub mod set_disk {
 
     #[cfg(feature = "test-util")]
     pub mod test_util {
-        pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test;
+        pub use crate::bucket::quota::reservation::{
+            fail_next_quota_ledger_save_for_test, initialize_quota_ledger_for_test, retry_quota_reconciliation_for_test,
+        };
         pub use crate::set_disk::ConditionalReadBenchmarkMetadataGuard;
         pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause};
 

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -16,7 +16,8 @@ use crate::bucket::metadata::BucketMetadata;
 use crate::bucket::metadata_sys::get_bucket_targets_config;
 use crate::bucket::metadata_sys::get_replication_config;
 use crate::bucket::remote_s3_client::{
-    PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, build_remote_s3_client,
+    PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, RemoteS3RetryPolicy,
+    UploadResponseDeadline, build_remote_s3_client,
 };
 use crate::bucket::replication::{ObjectLockIntegrity, object_lock_put_integrity, replication_etags_match};
 use crate::bucket::replication::{ReplicationStatusType, ReplicationTargetConfigBridge};
@@ -119,6 +120,7 @@ impl From<&BucketTarget> for RemoteS3EndpointSpec {
             ca_cert_pem: (!target.ca_cert_pem.trim().is_empty()).then(|| target.ca_cert_pem.clone()),
             connect_timeout: None,
             read_timeout: None,
+            operation_timeout: Some(TargetRequestTimeouts::from_env().request),
             // Replication has no retry budget of its own on the request path,
             // so it keeps the SDK's standard three attempts; stating it here
             // pins the behaviour to this line instead of an SDK default.
@@ -1191,6 +1193,7 @@ impl BucketTargetSys {
             health_check_duration: target.health_check_duration,
             replicate_sync: target.replication_sync,
             client: Arc::new(client),
+            timeouts: TargetRequestTimeouts::from_env(),
         })
     }
 
@@ -1922,6 +1925,83 @@ impl std::fmt::Display for S3ClientError {
     }
 }
 
+/// Replication request budgets, snapshotted when a target client is built.
+/// Streamed GET bodies are not subject to the upload-response budget.
+#[derive(Debug, Clone, Copy)]
+pub struct TargetRequestTimeouts {
+    request: Duration,
+    complete: Duration,
+    minimum_transfer_rate: u64,
+}
+
+impl Default for TargetRequestTimeouts {
+    fn default() -> Self {
+        Self {
+            request: Duration::from_secs(60),
+            complete: Duration::from_secs(15 * 60),
+            minimum_transfer_rate: 1024 * 1024,
+        }
+    }
+}
+
+impl TargetRequestTimeouts {
+    fn from_env() -> Self {
+        let defaults = Self::default();
+        let seconds = |name: &str, fallback: Duration| {
+            let value = rustfs_utils::get_env_u64(name, fallback.as_secs());
+            Duration::from_secs(if (1..=86400).contains(&value) {
+                value
+            } else {
+                fallback.as_secs()
+            })
+        };
+        let minimum_transfer_rate =
+            rustfs_utils::get_env_u64("RUSTFS_REPLICATION_MIN_TRANSFER_RATE_BYTES_PER_SEC", defaults.minimum_transfer_rate);
+        Self {
+            request: seconds("RUSTFS_REPLICATION_REQUEST_TIMEOUT_SECS", defaults.request),
+            complete: seconds("RUSTFS_REPLICATION_COMPLETE_TIMEOUT_SECS", defaults.complete),
+            minimum_transfer_rate: if minimum_transfer_rate == 0 {
+                defaults.minimum_transfer_rate
+            } else {
+                minimum_transfer_rate
+            },
+        }
+    }
+
+    fn upload_timeout(self, size: i64) -> Result<Duration, S3ClientError> {
+        let size = u64::try_from(size).map_err(|_| S3ClientError::new("replication upload size must be non-negative"))?;
+        // The SDK can make three attempts. Leave transmission headroom for
+        // all three; the separate response deadline still bounds a trickling
+        // target after each upload finishes.
+        let attempts = match REPLICATION_TARGET_RETRY_POLICY {
+            RemoteS3RetryPolicy::Disabled => 1,
+            RemoteS3RetryPolicy::Standard { max_attempts } => u64::from(max_attempts.max(1)),
+        };
+        let seconds = size
+            .div_ceil(self.minimum_transfer_rate)
+            .checked_mul(attempts)
+            .ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
+        let timeout = self
+            .request
+            .checked_add(Duration::from_secs(seconds))
+            .ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
+        if Instant::now().checked_add(timeout).is_none() {
+            return Err(S3ClientError::new("replication upload timeout exceeds supported duration"));
+        }
+        Ok(timeout)
+    }
+
+    fn upload_override(self, size: i64) -> Result<aws_sdk_s3::config::Builder, S3ClientError> {
+        Ok(aws_sdk_s3::config::Builder::new()
+            .timeout_config(
+                aws_smithy_types::timeout::TimeoutConfig::builder()
+                    .operation_timeout(self.upload_timeout(size)?)
+                    .build(),
+            )
+            .interceptor(UploadResponseDeadline(self.request)))
+    }
+}
+
 #[derive(Debug)]
 pub struct TargetClient {
     pub endpoint: String,
@@ -1935,6 +2015,7 @@ pub struct TargetClient {
     pub health_check_duration: Duration,
     pub replicate_sync: bool,
     pub client: Arc<S3Client>,
+    pub timeouts: TargetRequestTimeouts,
 }
 
 impl TargetClient {
@@ -2360,12 +2441,11 @@ impl TargetClient {
 
                 Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
             });
+        let mut config = self.timeouts.upload_override(size)?;
         if forwards_source_checksum {
-            operation = operation.config_override(
-                aws_sdk_s3::config::Builder::new()
-                    .request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired),
-            );
+            config = config.request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired);
         }
+        operation = operation.config_override(config);
         match operation.send().await {
             Ok(output) => {
                 // Under SSE-KMS/DSSE or SSE-C the target's ETag is not the MD5
@@ -2501,6 +2581,7 @@ impl TargetClient {
                 }
                 Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
             })
+            .config_override(self.timeouts.upload_override(size)?)
             .send()
             .await
         {
@@ -2538,6 +2619,13 @@ impl TargetClient {
                 }
                 Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
             })
+            .config_override(
+                aws_sdk_s3::config::Builder::new().timeout_config(
+                    aws_smithy_types::timeout::TimeoutConfig::builder()
+                        .operation_timeout(self.timeouts.complete)
+                        .build(),
+                ),
+            )
             .send()
             .await
         {
@@ -2966,11 +3054,241 @@ mod tests {
                 health_check_duration: Duration::from_secs(5),
                 replicate_sync: false,
                 client: Arc::new(client),
+                timeouts: TargetRequestTimeouts::default(),
             },
             request_headers,
         )
     }
 
+    #[derive(Debug, Clone)]
+    struct TimedResponseConnector {
+        upload_delay: Duration,
+        response_delay: Duration,
+        prefix_bytes: usize,
+        response: bytes::Bytes,
+        frames: Arc<std::sync::atomic::AtomicUsize>,
+        dropped: Arc<std::sync::atomic::AtomicUsize>,
+    }
+
+    struct ResponseDrop(Arc<std::sync::atomic::AtomicUsize>);
+    impl Drop for ResponseDrop {
+        fn drop(&mut self) {
+            self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+        }
+    }
+
+    impl SmithyHttpConnector for TimedResponseConnector {
+        fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
+            let settings = self.clone();
+            HttpConnectorFuture::new(async move {
+                http_body_util::BodyExt::collect(request.into_body())
+                    .await
+                    .map_err(aws_smithy_runtime_api::client::result::ConnectorError::io)?;
+                tokio::time::sleep(settings.upload_delay).await;
+                let guard = ResponseDrop(Arc::clone(&settings.dropped));
+                let stream = futures::stream::unfold(
+                    (settings.prefix_bytes, Some(settings.response), true, guard),
+                    move |(remaining, mut body, first, guard)| {
+                        let frames = Arc::clone(&settings.frames);
+                        async move {
+                            if remaining == 0 && body.is_none() {
+                                return None;
+                            }
+                            if !first {
+                                tokio::time::sleep(settings.response_delay).await;
+                            }
+                            let bytes = if remaining > 0 {
+                                bytes::Bytes::from_static(b" ")
+                            } else {
+                                body.take()?
+                            };
+                            frames.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+                            Some((
+                                Ok::<_, std::io::Error>(http_body::Frame::data(bytes)),
+                                (remaining.saturating_sub(1), body, false, guard),
+                            ))
+                        }
+                    },
+                )
+                .fuse();
+                let body = SdkBody::from_body_1_x(http_body_util::StreamBody::new(stream));
+                let mut response =
+                    HttpResponse::new(aws_smithy_runtime_api::http::StatusCode::try_from(200u16).expect("status"), body);
+                response.headers_mut().insert("etag", "\"test-etag\"");
+                Ok(response)
+            })
+        }
+    }
+
+    fn timed_response_target(
+        upload_delay: Duration,
+        response_delay: Duration,
+        prefix_bytes: usize,
+        response: &'static [u8],
+    ) -> (TargetClient, Arc<std::sync::atomic::AtomicUsize>, Arc<std::sync::atomic::AtomicUsize>) {
+        let frames = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let dropped = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let connector = SharedHttpConnector::new(TimedResponseConnector {
+            upload_delay,
+            response_delay,
+            prefix_bytes,
+            response: bytes::Bytes::from_static(response),
+            frames: Arc::clone(&frames),
+            dropped: Arc::clone(&dropped),
+        });
+        let http_client = http_client_fn(move |_settings, _components| connector.clone());
+        let (mut target, _) = header_recording_target_client(Vec::new());
+        target.timeouts = TargetRequestTimeouts {
+            request: Duration::from_secs(2),
+            complete: Duration::from_secs(10),
+            minimum_transfer_rate: 1024 * 1024,
+        };
+        target.client = Arc::new(S3Client::from_conf(
+            target
+                .client
+                .config()
+                .to_builder()
+                .http_client(http_client)
+                .retry_config(aws_sdk_s3::config::retry::RetryConfig::disabled())
+                .timeout_config(
+                    aws_smithy_types::timeout::TimeoutConfig::builder()
+                        .operation_timeout(target.timeouts.request)
+                        .build(),
+                )
+                .build(),
+        ));
+        (target, frames, dropped)
+    }
+
+    #[test]
+    fn replication_upload_budgets_preserve_large_transfers_and_reject_invalid_sizes() {
+        let policy = TargetRequestTimeouts::default();
+        assert_eq!(policy.upload_timeout(0).expect("empty upload"), Duration::from_secs(60));
+        assert_eq!(
+            policy.upload_timeout(5 * 1024 * 1024 * 1024).expect("maximum single PUT"),
+            Duration::from_secs(15420)
+        );
+        assert!(policy.upload_timeout(-1).is_err());
+        let slow = TargetRequestTimeouts {
+            minimum_transfer_rate: 1,
+            ..policy
+        };
+        assert!(slow.upload_timeout(i64::MAX).is_err());
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn replication_upload_response_deadline_is_independent_of_size_and_checksum_override() {
+        for operation in ["put", "forwarded-checksum", "part"] {
+            let (target, frames, dropped) = timed_response_target(Duration::from_secs(4), Duration::from_millis(500), 100, b"");
+            let body = vec![b'x'; 2 * 1024 * 1024];
+            let size = i64::try_from(body.len()).expect("size");
+            let started = tokio::time::Instant::now();
+            let error = if operation == "part" {
+                target
+                    .put_object_part(
+                        "bucket",
+                        "key",
+                        "upload",
+                        1,
+                        size,
+                        ByteStream::from(body),
+                        &PutObjectPartOptions::default(),
+                    )
+                    .await
+                    .expect_err("response deadline")
+            } else {
+                let mut opts = PutObjectOptions::default();
+                if operation == "forwarded-checksum" {
+                    opts.user_metadata.insert(
+                        "x-amz-checksum-sha256".to_owned(),
+                        "OoJ3yNhRwv3wwtZoGqEIPrPX9xwTnfLl+ka0wStN1g0=".to_owned(),
+                    );
+                }
+                target
+                    .put_object("bucket", "key", size, ByteStream::from(body), &opts)
+                    .await
+                    .expect_err("response deadline")
+            };
+            assert!(
+                started.elapsed() >= Duration::from_secs(6),
+                "upload gets its full independent window: {error}"
+            );
+            assert!(
+                started.elapsed() < target.timeouts.upload_timeout(size).expect("budget"),
+                "response must end before the larger total deadline"
+            );
+            assert!(
+                frames.load(std::sync::atomic::Ordering::SeqCst) >= 2,
+                "trickle delivered progress before failure"
+            );
+            assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1, "transport body released");
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn replication_slow_upload_preserves_transfer_window_and_streaming_get_body() {
+        let (target, _, _) = timed_response_target(Duration::from_secs(4), Duration::from_millis(100), 5, b"");
+        let body = vec![b'x'; 2 * 1024 * 1024];
+        let size = i64::try_from(body.len()).expect("size");
+        target
+            .put_object("bucket", "key", size, ByteStream::from(body), &PutObjectOptions::default())
+            .await
+            .expect("slow upload, prompt response");
+        let (target, _, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), 5, b"payload");
+        let response = target
+            .get_object("bucket", "key", None, None, None, HeaderMap::new())
+            .await
+            .expect("GET headers");
+        let started = tokio::time::Instant::now();
+        let body = response
+            .body
+            .collect()
+            .await
+            .expect("streamed GET remains outside upload deadline")
+            .into_bytes();
+        assert!(started.elapsed() > target.timeouts.request);
+        assert_eq!(&body[..], b"     payload");
+        assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn replication_complete_processing_whitespace_has_a_separate_bounded_window() {
+        const RESULT: &[u8] = b"<CompleteMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><ETag>\"etag\"</ETag></CompleteMultipartUploadResult>";
+        for prefix in [5, 100] {
+            let (target, frames, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), prefix, RESULT);
+            let started = tokio::time::Instant::now();
+            let result = target
+                .complete_multipart_upload(
+                    "bucket",
+                    "key",
+                    "upload",
+                    vec![CompletedPart::builder().part_number(1).e_tag("etag").build()],
+                    &PutObjectOptions::default(),
+                )
+                .await;
+            if prefix == 5 {
+                assert!(result.is_ok(), "legal processing whitespace: {result:?}");
+            } else {
+                assert!(result.is_err());
+                assert!(started.elapsed() <= Duration::from_secs(11));
+            }
+            assert!(started.elapsed() > target.timeouts.request);
+            assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
+            assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn replication_control_operation_deadline_covers_response_trickle() {
+        let (target, frames, dropped) =
+            timed_response_target(Duration::ZERO, Duration::from_millis(500), 100, b"<VersioningConfiguration/>");
+        let started = tokio::time::Instant::now();
+        assert!(target.get_bucket_versioning("bucket").await.is_err());
+        assert!(started.elapsed() <= Duration::from_secs(3));
+        assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
+        assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
+    }
+
     fn streaming_test_body(payload: &'static [u8]) -> ByteStream {
         let stream = tokio_util::io::ReaderStream::new(std::io::Cursor::new(payload));
         let body = http_body_util::StreamBody::new(futures::StreamExt::map(stream, |r| r.map(http_body::Frame::data)));
@@ -3279,6 +3597,7 @@ mod tests {
                 health_check_duration: Duration::from_secs(5),
                 replicate_sync: false,
                 client: Arc::new(client),
+                timeouts: TargetRequestTimeouts::default(),
             },
             request_uris,
         )
@@ -3591,6 +3910,7 @@ mod tests {
             health_check_duration: Duration::from_secs(5),
             replicate_sync: false,
             client: Arc::new(client),
+            timeouts: TargetRequestTimeouts::default(),
         })
     }
 

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -5487,6 +5487,14 @@ pub async fn put_restore_opts(
     if !oi.user_tags.is_empty() {
         meta.insert(AMZ_OBJECT_TAGGING.to_string(), (*oi.user_tags).clone());
     }
+    if let Some(expires) = oi.expires {
+        meta.insert(
+            "expires".to_string(),
+            expires
+                .format(&time::format_description::well_known::Rfc3339)
+                .map_err(std::io::Error::other)?,
+        );
+    }
     let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), rreq.days.unwrap_or(1));
     meta.insert(
         metadata_keys::RESTORE.to_string(),
@@ -5506,7 +5514,6 @@ pub async fn put_restore_opts(
         // computed MD5 is not the object's public plaintext ETag.
         preserve_etag: oi.etag.clone(),
         shard_integrity_write_mode: Some(oi.shard_integrity_write_mode()),
-        //expires:           oi.expires,
         ..Default::default()
     })
 }
@@ -7144,6 +7151,25 @@ mod tests {
             0,
             "tier generation lease should release after EOF"
         );
+        let unread = get_transitioned_object_reader_with_tier_manager(
+            &object_info.bucket,
+            &object_info.name,
+            &None,
+            &HeaderMap::new(),
+            &object_info,
+            &ObjectOptions::default(),
+            &manager,
+            None,
+        )
+        .await
+        .expect("open an unread transitioned response");
+        assert_eq!(TierConfigMgr::active_operation_lease_count(&manager, &tier).await, 1);
+        drop(unread);
+        assert_eq!(
+            TierConfigMgr::active_operation_lease_count(&manager, &tier).await,
+            0,
+            "canceling an unread response must release the tier lease"
+        );
     }
 
     #[cfg(feature = "test-util")]

crates/ecstore/src/bucket/metadata.rs

@@ -256,6 +256,128 @@ pub const BUCKET_POLICY_CONFIG: &str = "policy.json";
 pub const BUCKET_NOTIFICATION_CONFIG: &str = "notification.xml";
 pub const BUCKET_LIFECYCLE_CONFIG: &str = "lifecycle.xml";
 pub const BUCKET_SSECONFIG: &str = "bucket-encryption.xml";
+/// Validate tag entries supplied to the S3 API.
+pub fn validate_tag_set_entries(tag_set: &[s3s::dto::Tag]) -> std::result::Result<(), &'static str> {
+    validate_tag_entries(tag_set, validate_s3_tag_entry)
+}
+
+fn validate_tag_entries(
+    tag_set: &[s3s::dto::Tag],
+    mut validate_entry: impl FnMut(&str, &str) -> std::result::Result<(), &'static str>,
+) -> std::result::Result<(), &'static str> {
+    let mut keys = std::collections::HashSet::with_capacity(tag_set.len());
+    for tag in tag_set {
+        let key = tag
+            .key
+            .as_deref()
+            .filter(|key| !key.is_empty())
+            .ok_or("Tag key cannot be empty")?;
+        let value = tag.value.as_deref().ok_or("Tag value cannot be null")?;
+        validate_entry(key, value)?;
+        if !keys.insert(key) {
+            return Err("Cannot provide multiple Tags with the same key");
+        }
+    }
+    Ok(())
+}
+
+fn validate_s3_tag_entry(key: &str, value: &str) -> std::result::Result<(), &'static str> {
+    fn is_valid_tag_text(value: &str) -> bool {
+        static VALID_TAG_TEXT: std::sync::OnceLock<std::result::Result<regex::Regex, regex::Error>> = std::sync::OnceLock::new();
+        VALID_TAG_TEXT
+            .get_or_init(|| regex::Regex::new(r"^[\p{L}\p{N}\p{Z}+\-=._:/@]*$"))
+            .as_ref()
+            .is_ok_and(|pattern| pattern.is_match(value))
+    }
+    if key.encode_utf16().count() > 128 || !is_valid_tag_text(key) {
+        return Err("The TagKey you have provided is invalid");
+    }
+    if value.encode_utf16().count() > 256 || !is_valid_tag_text(value) {
+        return Err("The TagValue you have provided is invalid");
+    }
+    Ok(())
+}
+
+pub const MAX_BUCKET_TAGS: usize = 50;
+
+/// Maximum number of metadata headers on a Swift resource.
+pub const MAX_SWIFT_METADATA_COUNT: usize = 90;
+/// Maximum byte length of a Swift metadata value.
+pub const MAX_SWIFT_METADATA_VALUE_SIZE: usize = 256;
+
+/// Prefix for Swift account metadata persisted in tagging.xml.
+pub const SWIFT_ACCOUNT_META_TAG_PREFIX: &str = "swift-account-meta-";
+/// Prefix for Swift container metadata persisted in tagging.xml.
+pub const SWIFT_CONTAINER_META_TAG_PREFIX: &str = "swift-meta-";
+
+/// Marks a bucket whose shared tag config contains Swift-owned metadata.
+/// The key cannot be supplied through S3's tag alphabet.
+pub const SWIFT_TAGGING_MARKER: &str = "!rustfs-swift-metadata";
+
+/// Whether a persisted tag belongs to Swift's internal metadata namespace.
+pub fn is_swift_bucket_metadata_tag_key(key: &str) -> bool {
+    matches!(
+        key,
+        "swift-acl-read" | "swift-acl-write" | "swift-versions-location" | SWIFT_TAGGING_MARKER
+    ) || key.starts_with(SWIFT_ACCOUNT_META_TAG_PREFIX)
+        || key.starts_with(SWIFT_CONTAINER_META_TAG_PREFIX)
+}
+
+fn validate_persisted_bucket_tags(tags: &[s3s::dto::Tag]) -> std::result::Result<(), &'static str> {
+    const SWIFT_ACL_MAX_BYTES: usize = 8192;
+    const SWIFT_META_NAME_MAX_BYTES: usize = 128;
+    const MAX_PERSISTED_TAGS: usize = MAX_BUCKET_TAGS + 2 * MAX_SWIFT_METADATA_COUNT + 4;
+    if tags.len() > MAX_PERSISTED_TAGS {
+        return Err("Too many persisted bucket tags");
+    }
+
+    // Swift stores ACLs, versioning and metadata in the same XML as S3 tags.
+    // Only those known keys may carry values outside the S3 tag alphabet.
+    let mut account_metadata = 0;
+    let mut container_metadata = 0;
+    let mut s3_tags = 0;
+    validate_tag_entries(tags, |key, value| {
+        let swift_metadata = key
+            .strip_prefix(SWIFT_ACCOUNT_META_TAG_PREFIX)
+            .map(|name| (name, &mut account_metadata))
+            .or_else(|| {
+                key.strip_prefix(SWIFT_CONTAINER_META_TAG_PREFIX)
+                    .map(|name| (name, &mut container_metadata))
+            });
+        if let Some((name, count)) = swift_metadata
+            && name.len() <= SWIFT_META_NAME_MAX_BYTES
+            && http::HeaderName::from_bytes(name.as_bytes()).is_ok()
+            && value.len() <= MAX_SWIFT_METADATA_VALUE_SIZE
+            && http::HeaderValue::from_str(value).is_ok()
+        {
+            *count += 1;
+            if *count > MAX_SWIFT_METADATA_COUNT {
+                return Err("Too many Swift metadata tags");
+            }
+            return Ok(());
+        }
+
+        if key == SWIFT_TAGGING_MARKER && value == "1" {
+            return Ok(());
+        }
+
+        let valid_swift_header = match key {
+            "swift-acl-read" | "swift-acl-write" => value.len() <= SWIFT_ACL_MAX_BYTES,
+            "swift-versions-location" => value.len() <= 256 && !value.is_empty() && !value.contains('/'),
+            _ => false,
+        };
+        if valid_swift_header && http::HeaderValue::from_str(value).is_ok() {
+            return Ok(());
+        }
+
+        s3_tags += 1;
+        if s3_tags > MAX_BUCKET_TAGS {
+            return Err("Cannot have more than 50 tags per bucket");
+        }
+        validate_s3_tag_entry(key, value)
+    })
+}
+
 pub const BUCKET_TAGGING_CONFIG: &str = "tagging.xml";
 pub const BUCKET_QUOTA_CONFIG_FILE: &str = "quota.json";
 pub const OBJECT_LOCK_CONFIG: &str = "object-lock.xml";
@@ -990,6 +1112,13 @@ impl BucketMetadata {
                 self.encryption_config_updated_at = updated;
             }
             BUCKET_TAGGING_CONFIG => {
+                if !data.is_empty() {
+                    let tagging = deserialize::<Tagging>(&data).map_err(|_| {
+                        Error::InvalidArgument(self.name.clone(), String::new(), "Invalid bucket tagging XML".to_owned())
+                    })?;
+                    validate_persisted_bucket_tags(&tagging.tag_set)
+                        .map_err(|message| Error::InvalidArgument(self.name.clone(), String::new(), message.to_owned()))?;
+                }
                 self.tagging_config_xml = data;
                 // Drop the parsed form (like lifecycle above) so clearing the
                 // payload can't leave stale parsed tags to be cached.
@@ -1781,9 +1910,68 @@ mod test {
         assert_eq!(metadata.bucket_incarnation_id, incarnation);
     }
 
+    #[test]
+    fn persisted_bucket_tags_keep_separate_s3_and_swift_limits() {
+        use s3s::dto::Tag;
+
+        let tag = |key: String, value: &str| Tag {
+            key: Some(key),
+            value: Some(value.to_owned()),
+        };
+        let acl = tag("swift-acl-read".to_owned(), ".r:*,AUTH_other");
+        assert_eq!(
+            validate_tag_set_entries(std::slice::from_ref(&acl)),
+            Err("The TagValue you have provided is invalid")
+        );
+
+        let mut tags = vec![acl.clone()];
+        tags.extend((0..MAX_BUCKET_TAGS).map(|i| tag(format!("team-{i}"), "blue")));
+        tags.extend((0..MAX_SWIFT_METADATA_COUNT).map(|i| tag(format!("{SWIFT_CONTAINER_META_TAG_PREFIX}item!{i}"), "a,b")));
+        assert_eq!(validate_persisted_bucket_tags(&tags), Ok(()));
+
+        tags.push(tag(format!("{SWIFT_CONTAINER_META_TAG_PREFIX}overflow"), "a,b"));
+        assert_eq!(validate_persisted_bucket_tags(&tags), Err("Too many Swift metadata tags"));
+        tags.pop();
+
+        tags.push(acl);
+        assert_eq!(
+            validate_persisted_bucket_tags(&tags),
+            Err("Cannot provide multiple Tags with the same key")
+        );
+        tags.pop();
+
+        tags[0].value = Some(".r:*,".repeat(1700));
+        assert_eq!(validate_persisted_bucket_tags(&tags), Err("The TagValue you have provided is invalid"));
+    }
+
     /// backlog#2292: a replicated config is stamped with the source
     /// `updated_at` it was given, not the local clock, while the plain
     /// `update_config` entry keeps stamping the local clock.
+    #[test]
+    fn bucket_tag_updates_reject_unreadable_shapes_without_replacing_metadata() {
+        let mut metadata = BucketMetadata::new("validated-tags");
+        let valid = br#"<Tagging><TagSet><Tag><Key>team</Key><Value></Value></Tag></TagSet></Tagging>"#.to_vec();
+        metadata
+            .update_config(BUCKET_TAGGING_CONFIG, valid.clone())
+            .expect("empty value is valid");
+        let updated = metadata.tagging_config_updated_at;
+        for invalid in [
+            "<Tagging><TagSet><Tag><Key>team</Key></Tag></TagSet></Tagging>".to_owned(),
+            "<Tagging><TagSet><Tag><Key></Key><Value>x</Value></Tag></TagSet></Tagging>".to_owned(),
+            "<Tagging><TagSet><Tag><Key>team</Key><Value>x</Value></Tag><Tag><Key>team</Key><Value>y</Value></Tag></TagSet></Tagging>".to_owned(),
+            format!("<Tagging><TagSet>{}</TagSet></Tagging>", (0..51).map(|i| format!("<Tag><Key>k{i}</Key><Value>v</Value></Tag>")).collect::<String>()),
+        ] {
+            assert!(metadata.update_config_at(BUCKET_TAGGING_CONFIG, invalid.into_bytes(), updated + time::Duration::hours(1)).is_err());
+            assert_eq!(metadata.tagging_config_xml, valid);
+            assert_eq!(metadata.tagging_config_updated_at, updated);
+        }
+        metadata.tagging_config_xml = b"legacy corrupt tags".to_vec();
+        metadata
+            .update_config(BUCKET_TAGGING_CONFIG, Vec::new())
+            .expect("deletion repairs legacy corruption");
+        assert!(metadata.tagging_config_xml.is_empty());
+    }
+
     #[test]
     fn update_config_at_stamps_the_given_time_and_update_config_stamps_now() {
         let source_time = OffsetDateTime::now_utc() - time::Duration::hours(3);

crates/ecstore/src/bucket/metadata_sys.rs

@@ -22,6 +22,7 @@ use super::target::BucketTargets;
 use crate::bucket::bucket_target_sys::BucketTargetSys;
 use crate::bucket::metadata::{load_bucket_metadata_parse, load_bucket_metadata_parse_with_presence};
 use crate::bucket::utils::is_meta_bucketname;
+use crate::diagnostics::object_lock::ObjectLockAttempt;
 use crate::disk::RUSTFS_META_BUCKET;
 use crate::error::{Error, Result, is_err_bucket_not_found, is_err_strict_volume_not_found};
 use crate::object_api::ObjectOptions;
@@ -599,7 +600,9 @@ pub(crate) async fn get_bucket_incarnation_id_for_options_in(
     }
     let sys = bucket_metadata_sys_of(ctx)?;
     let sys = sys.read().await.clone();
-    let guard = acquire_bucket_metadata_transaction_read_lock_for_options_in(ctx, bucket, opts).await?;
+    let guard =
+        acquire_bucket_metadata_transaction_read_lock_for_options_in(ctx, bucket, opts, "bucket_incarnation_options_read")
+            .await?;
     sys.get_bucket_incarnation_id_under_transaction_lock(bucket, &guard).await
 }
 
@@ -866,6 +869,30 @@ impl BucketMetadataMutationGuard {
     }
 }
 
+/// Read-only scanner authority for one bucket incarnation.
+///
+/// Lock order is bucket lifecycle (read), then metadata transaction (read).
+/// This guard cannot authorize a configuration mutation or scanner ACK write.
+pub struct ScannerBucketIncarnationReadGuard {
+    bucket: String,
+    incarnation_id: Uuid,
+    lifecycle_guard: rustfs_lock::NamespaceLockGuard,
+    transaction_guard: rustfs_lock::NamespaceLockGuard,
+}
+
+impl ScannerBucketIncarnationReadGuard {
+    /// Returns the verified identity while both read fences remain valid.
+    ///
+    /// # Errors
+    /// Returns an error if either lock has lost its lease.
+    pub fn checked_bucket_incarnation(&self) -> Result<(&str, Uuid)> {
+        if self.lifecycle_guard.is_lock_lost() || self.transaction_guard.is_lock_lost() {
+            return Err(Error::other("scanner bucket incarnation read lock was lost"));
+        }
+        Ok((&self.bucket, self.incarnation_id))
+    }
+}
+
 async fn acquire_config_write_guard(sys: Arc<RwLock<BucketMetadataSys>>, bucket: &str) -> Result<BucketMetadataMutationGuard> {
     acquire_config_write_guard_for_incarnation(sys, bucket, None).await
 }
@@ -892,6 +919,68 @@ pub async fn acquire_scanner_bucket_incarnation_fence(
     acquire_config_write_guard_with_migration(sys, bucket, Some(expected_incarnation_id), false).await
 }
 
+/// Validate scanner read authority without serializing compatible metadata readers.
+///
+/// The returned guard retains both read locks. It never migrates metadata or
+/// creates an incarnation; mutation callers must use the exclusive scanner fence.
+///
+/// # Errors
+/// Returns an error for invalid bucket/owner identities, unavailable locks or
+/// storage, absent incarnation metadata, or a deleted/recreated bucket.
+pub async fn acquire_scanner_bucket_incarnation_read_fence(
+    bucket: &str,
+    expected_incarnation_id: Uuid,
+    expected_owner_id: Uuid,
+) -> Result<ScannerBucketIncarnationReadGuard> {
+    acquire_scanner_bucket_incarnation_read_fence_with_sys(
+        get_bucket_metadata_sys()?,
+        bucket,
+        expected_incarnation_id,
+        expected_owner_id,
+    )
+    .await
+}
+
+async fn acquire_scanner_bucket_incarnation_read_fence_with_sys(
+    sys: Arc<RwLock<BucketMetadataSys>>,
+    bucket: &str,
+    expected_incarnation_id: Uuid,
+    expected_owner_id: Uuid,
+) -> Result<ScannerBucketIncarnationReadGuard> {
+    super::utils::check_valid_bucket_name(bucket)?;
+    let metadata_sys = sys.read().await.clone();
+    if expected_owner_id.is_nil() || metadata_sys.object_store().id != expected_owner_id || expected_incarnation_id.is_nil() {
+        return Err(Error::other("scanner bucket incarnation owner does not match"));
+    }
+    let lifecycle_guard = metadata_sys.object_store().acquire_bucket_lifecycle_read_lock(bucket).await?;
+    let transaction_guard = await_bucket_namespace_operation(
+        Some(&lifecycle_guard),
+        bucket,
+        "scanner incarnation read transaction lock acquisition",
+        acquire_metadata_transaction_guard(
+            metadata_sys.object_store(),
+            bucket,
+            "scanner_incarnation_probe",
+            rustfs_lock::LockType::Shared,
+        ),
+    )
+    .await?;
+    let incarnation_id = validate_bucket_metadata_guard_incarnation(
+        &metadata_sys,
+        bucket,
+        Some(expected_incarnation_id),
+        &lifecycle_guard,
+        &transaction_guard,
+    )
+    .await?;
+    Ok(ScannerBucketIncarnationReadGuard {
+        bucket: bucket.to_owned(),
+        incarnation_id,
+        lifecycle_guard,
+        transaction_guard,
+    })
+}
+
 async fn acquire_config_write_guard_with_migration(
     sys: Arc<RwLock<BucketMetadataSys>>,
     bucket: &str,
@@ -917,15 +1006,47 @@ async fn acquire_config_write_guard_with_migration(
         Some(&lifecycle_guard),
         bucket,
         "bucket config transaction lock acquisition",
-        acquire_transaction_lock_with_sys(&sys, bucket),
+        acquire_transaction_lock_with_sys_for_purpose(
+            &sys,
+            bucket,
+            if migrate {
+                "bucket_config_update"
+            } else {
+                "scanner_incarnation_validation"
+            },
+        ),
     )
     .await?;
+    let current_incarnation_id = validate_bucket_metadata_guard_incarnation(
+        &metadata_sys,
+        bucket,
+        expected_incarnation_id,
+        &lifecycle_guard,
+        &transaction_guard,
+    )
+    .await?;
+
+    Ok(BucketMetadataMutationGuard {
+        bucket: bucket.to_string(),
+        incarnation_id: current_incarnation_id,
+        lifecycle_guard,
+        transaction_guard,
+    })
+}
+
+async fn validate_bucket_metadata_guard_incarnation(
+    metadata_sys: &BucketMetadataSys,
+    bucket: &str,
+    expected_incarnation_id: Option<Uuid>,
+    lifecycle_guard: &rustfs_lock::NamespaceLockGuard,
+    transaction_guard: &rustfs_lock::NamespaceLockGuard,
+) -> Result<Uuid> {
     await_bucket_namespace_operation(
-        Some(&lifecycle_guard),
+        Some(lifecycle_guard),
         bucket,
         "bucket config existence validation",
         await_bucket_namespace_operation(
-            Some(&transaction_guard),
+            Some(transaction_guard),
             bucket,
             "bucket config existence transaction validation",
             async {
@@ -943,11 +1064,11 @@ async fn acquire_config_write_guard_with_migration(
     )
     .await?;
     let current_incarnation_id = await_bucket_namespace_operation(
-        Some(&lifecycle_guard),
+        Some(lifecycle_guard),
         bucket,
         "bucket config incarnation validation",
         await_bucket_namespace_operation(
-            Some(&transaction_guard),
+            Some(transaction_guard),
             bucket,
             "bucket config incarnation transaction validation",
             load_bucket_incarnation(metadata_sys.object_store(), bucket),
@@ -960,12 +1081,7 @@ async fn acquire_config_write_guard_with_migration(
         return Err(Error::BucketNotFound(bucket.to_string()));
     }
 
-    Ok(BucketMetadataMutationGuard {
-        bucket: bucket.to_string(),
-        incarnation_id: current_incarnation_id,
-        lifecycle_guard,
-        transaction_guard,
-    })
+    Ok(current_incarnation_id)
 }
 
 /// Rewrite one config file while the caller already holds this bucket's
@@ -1174,11 +1290,23 @@ async fn delete_under_config_write_guard(
 /// targets one, so a writer on another node cannot land a whole-file save in
 /// the middle of this read-modify-write.
 pub async fn update_config_with<F>(bucket: &str, config_file: &str, mutate: F) -> Result<OffsetDateTime>
+where
+    F: FnOnce(&BucketMetadata) -> Result<Vec<u8>> + Send,
+{
+    update_config_with_expected(bucket, config_file, None, mutate).await
+}
+
+async fn update_config_with_expected<F>(
+    bucket: &str,
+    config_file: &str,
+    expected_incarnation_id: Option<Uuid>,
+    mutate: F,
+) -> Result<OffsetDateTime>
 where
     F: FnOnce(&BucketMetadata) -> Result<Vec<u8>> + Send,
 {
     let sys = get_bucket_metadata_sys()?;
-    let guard = Box::pin(acquire_config_write_guard(sys.clone(), bucket)).await?;
+    let guard = Box::pin(acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id)).await?;
     guard.ensure_valid(bucket)?;
     let metadata_sys = sys.read().await.clone();
     let updated = await_bucket_namespace_operation(
@@ -1197,6 +1325,70 @@ where
     Ok(updated)
 }
 
+fn swift_manages_bucket_tags(tagging: &Tagging) -> bool {
+    tagging
+        .tag_set
+        .iter()
+        .any(|tag| tag.key.as_deref() == Some(super::metadata::SWIFT_TAGGING_MARKER) && tag.value.as_deref() == Some("1"))
+}
+
+/// Read the S3 tag set while keeping Swift's internal tags private.
+pub async fn get_s3_tagging_config(bucket: &str) -> Result<(Tagging, OffsetDateTime)> {
+    let (mut tagging, updated) = get_tagging_config(bucket).await?;
+    if swift_manages_bucket_tags(&tagging) {
+        tagging.tag_set.retain(|tag| {
+            !tag.key
+                .as_deref()
+                .is_some_and(super::metadata::is_swift_bucket_metadata_tag_key)
+        });
+    }
+    Ok((tagging, updated))
+}
+
+/// Replace or clear S3 tags atomically while retaining Swift metadata.
+///
+/// The expected incarnation, when present, is checked under the bucket
+/// lifecycle and metadata guards. A failed write leaves the previous config
+/// intact. Repeated writes replace the same S3 set.
+pub async fn update_s3_tagging_config(
+    bucket: &str,
+    tagging: Option<Tagging>,
+    expected_incarnation_id: Option<Uuid>,
+) -> Result<OffsetDateTime> {
+    let tags = tagging.map(|tagging| tagging.tag_set).unwrap_or_default();
+    let invalid = |message: &str| Error::InvalidArgument(bucket.to_owned(), String::new(), message.to_owned());
+    if tags.len() > super::metadata::MAX_BUCKET_TAGS {
+        return Err(invalid("Cannot have more than 50 tags per bucket"));
+    }
+    super::metadata::validate_tag_set_entries(&tags).map_err(invalid)?;
+    update_config_with_expected(bucket, BUCKET_TAGGING_CONFIG, expected_incarnation_id, |metadata| {
+        let mut merged = metadata.tagging_config.clone().unwrap_or_default();
+        if swift_manages_bucket_tags(&merged) {
+            if tags.iter().any(|tag| {
+                tag.key
+                    .as_deref()
+                    .is_some_and(super::metadata::is_swift_bucket_metadata_tag_key)
+            }) {
+                return Err(invalid("Tag key is reserved for Swift metadata on this bucket"));
+            }
+            merged.tag_set.retain(|tag| {
+                tag.key
+                    .as_deref()
+                    .is_some_and(super::metadata::is_swift_bucket_metadata_tag_key)
+            });
+        } else {
+            merged.tag_set.clear();
+        }
+        merged.tag_set.extend(tags);
+        if merged.tag_set.is_empty() {
+            Ok(Vec::new())
+        } else {
+            super::utils::serialize(&merged).map_err(Error::other)
+        }
+    })
+    .await
+}
+
 /// Acquire a bucket's metadata transaction lock, held across a whole
 /// read-modify-write of its metadata file.
 ///
@@ -1235,13 +1427,18 @@ pub(crate) async fn acquire_bucket_metadata_transaction_lock_in(
 pub(crate) async fn acquire_bucket_metadata_transaction_read_lock_in(
     ctx: &crate::runtime::instance::InstanceContext,
     bucket: &str,
+) -> Result<rustfs_lock::NamespaceLockGuard> {
+    acquire_bucket_metadata_transaction_read_lock_in_for_purpose(ctx, bucket, "bucket_metadata_read").await
+}
+
+pub(crate) async fn acquire_bucket_metadata_transaction_read_lock_in_for_purpose(
+    ctx: &crate::runtime::instance::InstanceContext,
+    bucket: &str,
+    op: &'static str,
 ) -> Result<rustfs_lock::NamespaceLockGuard> {
     let sys = bucket_metadata_sys_of(ctx)?;
     let api = sys.read().await.object_store();
-    let lock = api
-        .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
-        .await?;
-    Ok(lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?)
+    acquire_metadata_transaction_guard(api, bucket, op, rustfs_lock::LockType::Shared).await
 }
 
 /// Readers already holding an Object Lock snapshot must share its guard:
@@ -1250,6 +1447,7 @@ pub(crate) async fn acquire_bucket_metadata_transaction_read_lock_for_options_in
     ctx: &crate::runtime::instance::InstanceContext,
     bucket: &str,
     opts: &ObjectOptions,
+    op: &'static str,
 ) -> Result<Arc<rustfs_lock::NamespaceLockGuard>> {
     if let Some(snapshot) = opts.object_lock_config_snapshot.as_ref() {
         let store = object_store_in(ctx).await?;
@@ -1259,12 +1457,22 @@ pub(crate) async fn acquire_bucket_metadata_transaction_read_lock_for_options_in
                 Error::other("Object Lock snapshot does not hold a valid metadata transaction fence for this bucket")
             });
     }
-    Ok(Arc::new(acquire_bucket_metadata_transaction_read_lock_in(ctx, bucket).await?))
+    Ok(Arc::new(
+        acquire_bucket_metadata_transaction_read_lock_in_for_purpose(ctx, bucket, op).await?,
+    ))
 }
 
 async fn acquire_transaction_lock_with_sys(
     sys: &Arc<RwLock<BucketMetadataSys>>,
     bucket: &str,
+) -> Result<rustfs_lock::NamespaceLockGuard> {
+    acquire_transaction_lock_with_sys_for_purpose(sys, bucket, "bucket_metadata_transaction_write").await
+}
+
+async fn acquire_transaction_lock_with_sys_for_purpose(
+    sys: &Arc<RwLock<BucketMetadataSys>>,
+    bucket: &str,
+    op: &'static str,
 ) -> Result<rustfs_lock::NamespaceLockGuard> {
     // Resolve the store under a short-lived read guard: this runs before the
     // write guard in `acquire_config_write_guards`, and must not still hold a
@@ -1273,12 +1481,22 @@ async fn acquire_transaction_lock_with_sys(
     let lock = api
         .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
         .await?;
-    let acquire = lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout());
+    let timeout = crate::set_disk::get_lock_acquire_timeout();
+    let mut attempt = ObjectLockAttempt::start(
+        op,
+        RUSTFS_META_BUCKET,
+        &bucket_metadata_transaction_lock_key(bucket),
+        None,
+        &lock,
+        "write",
+        timeout,
+    );
+    let acquire = lock.get_write_lock(timeout);
     #[cfg(any(test, feature = "test-util"))]
     {
         tokio::pin!(acquire);
         let mut notified = false;
-        let guard = futures::future::poll_fn(|cx| match std::future::Future::poll(acquire.as_mut(), cx) {
+        let result = futures::future::poll_fn(|cx| match std::future::Future::poll(acquire.as_mut(), cx) {
             std::task::Poll::Pending => {
                 if !notified {
                     notify_config_write_lock_attempt(bucket);
@@ -1288,11 +1506,38 @@ async fn acquire_transaction_lock_with_sys(
             }
             std::task::Poll::Ready(result) => std::task::Poll::Ready(result),
         })
-        .await?;
-        Ok(guard)
+        .await;
+        attempt.observe_acquisition(&result);
+        Ok(result?)
     }
     #[cfg(not(any(test, feature = "test-util")))]
-    Ok(acquire.await?)
+    {
+        let result = acquire.await;
+        attempt.observe_acquisition(&result);
+        Ok(result?)
+    }
+}
+
+pub(crate) async fn acquire_metadata_transaction_guard(
+    api: Arc<ECStore>,
+    bucket: &str,
+    op: &'static str,
+    lock_type: rustfs_lock::LockType,
+) -> Result<rustfs_lock::NamespaceLockGuard> {
+    let object = bucket_metadata_transaction_lock_key(bucket);
+    let lock = api.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
+    let timeout = crate::set_disk::get_lock_acquire_timeout();
+    let mode = match lock_type {
+        rustfs_lock::LockType::Shared => "read",
+        rustfs_lock::LockType::Exclusive => "write",
+    };
+    let mut attempt = ObjectLockAttempt::start(op, RUSTFS_META_BUCKET, &object, None, &lock, mode, timeout);
+    let result = match lock_type {
+        rustfs_lock::LockType::Shared => lock.get_read_lock(timeout).await,
+        rustfs_lock::LockType::Exclusive => lock.get_write_lock(timeout).await,
+    };
+    attempt.observe_acquisition(&result);
+    Ok(result?)
 }
 
 /// The lock resource name is deliberately still the `bucket-targets` one it
@@ -2794,13 +3039,13 @@ impl BucketMetadataSys {
     }
 
     async fn find_bucket_incarnation_id_from_disk(&self, bucket: &str) -> Result<Option<Uuid>> {
-        let transaction_lock = self
-            .object_store()
-            .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
-            .await?;
-        let _transaction_guard = transaction_lock
-            .get_read_lock(crate::set_disk::get_lock_acquire_timeout())
-            .await?;
+        let _transaction_guard = acquire_metadata_transaction_guard(
+            self.object_store(),
+            bucket,
+            "bucket_incarnation_disk_read",
+            rustfs_lock::LockType::Shared,
+        )
+        .await?;
         self.find_bucket_incarnation_id_under_transaction_lock(bucket, &_transaction_guard)
             .await
     }
@@ -2876,13 +3121,13 @@ impl BucketMetadataSys {
     /// revalidated at write quorum, so a stale snapshot can neither revert an
     /// acknowledged configuration update nor outlive a delete/recreate.
     async fn migrate_bucket_creation_commit(&self, bucket: &str) -> Result<BucketMetadataAuthority> {
-        let transaction_lock = self
-            .object_store()
-            .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
-            .await?;
-        let transaction_guard = transaction_lock
-            .get_write_lock(crate::set_disk::get_lock_acquire_timeout())
-            .await?;
+        let transaction_guard = acquire_metadata_transaction_guard(
+            self.object_store(),
+            bucket,
+            "bucket_creation_commit_migration",
+            rustfs_lock::LockType::Exclusive,
+        )
+        .await?;
         let namespace_lock = self.object_store().new_ns_lock(bucket, bucket).await?;
         let namespace_guard = namespace_lock
             .get_read_lock(crate::set_disk::get_lock_acquire_timeout())
@@ -2932,13 +3177,13 @@ impl BucketMetadataSys {
     }
 
     async fn migrate_legacy_metadata(&self, bucket: &str) -> Result<BucketMetadataAuthority> {
-        let transaction_lock = self
-            .object_store()
-            .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
-            .await?;
-        let _transaction_guard = transaction_lock
-            .get_write_lock(crate::set_disk::get_lock_acquire_timeout())
-            .await?;
+        let _transaction_guard = acquire_metadata_transaction_guard(
+            self.object_store(),
+            bucket,
+            "bucket_metadata_legacy_migration",
+            rustfs_lock::LockType::Exclusive,
+        )
+        .await?;
 
         let authority = self
             .load_authoritative_metadata_from_disk_under_transaction_lock(bucket)
@@ -3428,7 +3673,7 @@ mod tests {
         ),
         (
             BUCKET_TAGGING_CONFIG,
-            r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-🦀</Value></Tag></TagSet></Tagging>"#.as_bytes(),
+            r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-rust</Value></Tag></TagSet></Tagging>"#.as_bytes(),
         ),
         (BUCKET_REPLICATION_CONFIG, NEW_WRITER_REPLICATION_XML),
         (
@@ -3977,6 +4222,12 @@ mod tests {
                 .await
                 .is_err()
         );
+        assert!(
+            acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, Uuid::new_v4(), store.id)
+                .await
+                .is_err(),
+            "read-only scanner validation must not migrate an absent incarnation"
+        );
         assert!(load_bucket_incarnation(store, bucket).await.expect("read sidecar").is_none());
         assert!(
             sys.read()
@@ -4033,6 +4284,128 @@ mod tests {
         );
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    #[serial]
+    async fn scanner_incarnation_read_fence_shares_reads_and_blocks_mutations() {
+        let (_dirs, store) = isolated_store_over_temp_disks().await;
+        init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let sys = bucket_metadata_sys_of(&store.ctx).expect("metadata owner");
+        let bucket = "scanner-read-fence";
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create bucket");
+        let incarnation = store
+            .bucket_incarnation_id_from_disk(bucket)
+            .await
+            .expect("bucket incarnation");
+        let guard = acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, incarnation, store.id)
+            .await
+            .expect("read-only scanner fence");
+        assert_eq!(guard.checked_bucket_incarnation().expect("valid read fences"), (bucket, incarnation));
+
+        let transaction_lock = store
+            .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
+            .await
+            .expect("metadata transaction namespace");
+        assert!(
+            transaction_lock.get_write_lock(Duration::from_millis(20)).await.is_err(),
+            "scanner read authority must exclude metadata writers"
+        );
+        let lifecycle_lock = store
+            .new_ns_lock(bucket, crate::storage_api_contracts::bucket::BUCKET_LIFECYCLE_LOCK_OBJECT)
+            .await
+            .expect("bucket lifecycle namespace");
+        assert!(
+            lifecycle_lock.get_write_lock(Duration::from_millis(20)).await.is_err(),
+            "scanner read authority must exclude delete and recreation"
+        );
+        let another = timeout(
+            Duration::from_secs(1),
+            acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, incarnation, store.id),
+        )
+        .await
+        .expect("scanner readers must not serialize")
+        .expect("second read-only scanner fence");
+        let compatible = timeout(
+            Duration::from_secs(1),
+            acquire_bucket_metadata_transaction_read_lock_in(&store.ctx, bucket),
+        )
+        .await
+        .expect("metadata readers must remain compatible")
+        .expect("metadata transaction reader");
+        drop(compatible);
+        drop(another);
+
+        let mut mutation = Box::pin(acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(incarnation), false));
+        assert!(matches!(futures::poll!(&mut mutation), std::task::Poll::Pending));
+        let mut deletion = Box::pin(store.acquire_bucket_lifecycle_write_lock(bucket));
+        assert!(matches!(futures::poll!(&mut deletion), std::task::Poll::Pending));
+        drop(deletion);
+        drop(guard);
+        let mutation = timeout(Duration::from_secs(1), mutation)
+            .await
+            .expect("metadata mutation must progress after scanner read release")
+            .expect("exclusive mutation fence");
+        drop(mutation);
+        let lifecycle = timeout(Duration::from_secs(1), store.acquire_bucket_lifecycle_write_lock(bucket))
+            .await
+            .expect("bucket delete fence must progress after readers release")
+            .expect("exclusive lifecycle fence");
+        drop(lifecycle);
+
+        for (owner, expected) in [
+            (Uuid::new_v4(), incarnation),
+            (Uuid::nil(), incarnation),
+            (store.id, Uuid::nil()),
+        ] {
+            assert!(
+                acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, expected, owner)
+                    .await
+                    .is_err(),
+                "invalid owner or nil incarnation must not receive authority"
+            );
+        }
+        assert!(
+            acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, Uuid::new_v4(), store.id)
+                .await
+                .is_err(),
+            "a stale incarnation must not receive authority"
+        );
+        store
+            .delete_bucket(bucket, &DeleteBucketOptions::default())
+            .await
+            .expect("delete bucket");
+        assert!(
+            acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, incarnation, store.id)
+                .await
+                .is_err(),
+            "deleted bucket must not receive authority"
+        );
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("recreate bucket");
+        let replacement = store
+            .bucket_incarnation_id_from_disk(bucket)
+            .await
+            .expect("replacement incarnation");
+        assert_ne!(replacement, incarnation);
+        assert!(
+            acquire_scanner_bucket_incarnation_read_fence_with_sys(sys.clone(), bucket, incarnation, store.id)
+                .await
+                .is_err(),
+            "recreation must reject the previous scanner identity"
+        );
+        let guard = acquire_scanner_bucket_incarnation_read_fence_with_sys(sys, bucket, replacement, store.id)
+            .await
+            .expect("replacement read-only scanner fence");
+        assert_eq!(
+            guard.checked_bucket_incarnation().expect("replacement remains fenced"),
+            (bucket, replacement)
+        );
+    }
+
     #[tokio::test]
     async fn old_node_metadata_rewrite_cannot_replace_bucket_incarnation_sidecar() {
         let (dirs, ecstore) = isolated_store_over_temp_disks().await;
@@ -4556,8 +4929,13 @@ mod tests {
             std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created");
         }
         let mut bm = BucketMetadata::new(bucket);
-        bm.update_config(config_file, raw.to_vec())
-            .expect("raw config should be stored");
+        if config_file == crate::bucket::metadata::BUCKET_TAGGING_CONFIG {
+            // Model legacy persisted corruption without passing the write-time validator.
+            bm.tagging_config_xml = raw.to_vec();
+        } else {
+            bm.update_config(config_file, raw.to_vec())
+                .expect("raw config should be stored");
+        }
         sys.persist_new_and_set(bm).await.expect("initial metadata should persist");
     }
 
@@ -5738,7 +6116,8 @@ mod tests {
         // update only needs one interleaving to show up.
         const ROUNDS: usize = 8;
         for round in 0..ROUNDS {
-            let tagging = format!("<Tagging><Round>{round}</Round></Tagging>").into_bytes();
+            let tagging =
+                format!("<Tagging><TagSet><Tag><Key>round</Key><Value>{round}</Value></Tag></TagSet></Tagging>").into_bytes();
             let policy = format!(r#"{{"Version":"2012-10-17","Round":{round}}}"#).into_bytes();
 
             let start = Arc::new(tokio::sync::Barrier::new(2));
@@ -5811,7 +6190,7 @@ mod tests {
 
         let blocked = tokio::spawn({
             let node_b = node_b.clone();
-            async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging/>".to_vec()).await }
+            async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging><TagSet/></Tagging>".to_vec()).await }
         });
 
         // Long enough for the write to have finished had it not waited: the

crates/ecstore/src/bucket/quota/checker/tests/reporting_retirement.rs

@@ -119,6 +119,21 @@ async fn durable_quota_reporting_retires_degraded_floors_without_changing_ledger
             // Exercise the real storage entry point before testing the new
             // reporting semantics: the same ledger must admit and refuse
             // these writes even when a scanner floor claims a million bytes.
+            if protocol == 1 {
+                let mut bootstrap = PutObjReader::from_vec(vec![0x5a; 512]);
+                assert!(
+                    matches!(
+                        store
+                            .put_object(&bucket, "bootstrap", &mut bootstrap, &ObjectOptions::default())
+                            .await,
+                        Err(StorageError::SlowDown)
+                    ),
+                    "a missing authoritative ledger must not admit a write from scanner reporting"
+                );
+                crate::bucket::quota::reservation::retry_quota_reconciliation_for_test(store.clone(), &bucket)
+                    .await
+                    .expect("bootstrap authoritative usage through the production worker");
+            }
             let mut too_large = PutObjReader::from_vec(vec![0x5a; 8192]);
             assert!(
                 matches!(

crates/ecstore/src/bucket/remote_s3_client.rs

@@ -40,7 +40,9 @@ use aws_smithy_runtime_api::client::http::{
     HttpConnector as SmithyHttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn,
 };
 use aws_smithy_runtime_api::client::interceptors::Intercept;
-use aws_smithy_runtime_api::client::interceptors::context::BeforeTransmitInterceptorContextMut;
+use aws_smithy_runtime_api::client::interceptors::context::{
+    BeforeDeserializationInterceptorContextMut, BeforeTransmitInterceptorContextMut,
+};
 use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse};
 use aws_smithy_runtime_api::client::result::ConnectorError;
 use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents;
@@ -155,6 +157,8 @@ pub struct RemoteS3EndpointSpec {
     pub ca_cert_pem: Option<String>,
     pub connect_timeout: Option<Duration>,
     pub read_timeout: Option<Duration>,
+    /// Entire SDK call, including non-streaming response bodies and retries.
+    pub operation_timeout: Option<Duration>,
     /// How many wire requests one logical call may cost. Every consumer
     /// declares it; see [`RemoteS3RetryPolicy`].
     pub retry: RemoteS3RetryPolicy,
@@ -252,6 +256,78 @@ pub(crate) fn remote_sdk_credentials(credentials: &RemoteCredentials, now: Syste
     Ok(builder.build())
 }
 
+/// Only installed on non-streaming upload operations. The budget begins at
+/// response headers, independently of the potentially long request upload.
+#[derive(Debug)]
+pub(crate) struct UploadResponseDeadline(pub Duration);
+
+impl Intercept for UploadResponseDeadline {
+    fn name(&self) -> &'static str {
+        "RustfsUploadResponseDeadline"
+    }
+
+    fn modify_before_deserialization(
+        &self,
+        context: &mut BeforeDeserializationInterceptorContextMut<'_>,
+        _runtime_components: &RuntimeComponents,
+        _cfg: &mut ConfigBag,
+    ) -> Result<(), BoxError> {
+        let response = context.response_mut();
+        let inner = std::mem::replace(response.body_mut(), SdkBody::taken());
+        *response.body_mut() = SdkBody::from_body_1_x(UploadResponseBody {
+            inner: Some(inner),
+            deadline: Box::pin(tokio::time::sleep(self.0)),
+        });
+        Ok(())
+    }
+}
+
+struct UploadResponseBody {
+    inner: Option<SdkBody>,
+    deadline: std::pin::Pin<Box<tokio::time::Sleep>>,
+}
+
+impl http_body::Body for UploadResponseBody {
+    type Data = bytes::Bytes;
+    type Error = BoxError;
+
+    fn poll_frame(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Option<Result<http_body::Frame<Self::Data>, Self::Error>>> {
+        use std::future::Future as _;
+        use std::task::Poll;
+        if self.inner.is_none() {
+            return Poll::Ready(None);
+        }
+        // The absolute response budget never renews on data, empty frames,
+        // or trailers. A retry gets its own bounded response window.
+        if self.deadline.as_mut().poll(cx).is_ready() {
+            self.inner.take();
+            return Poll::Ready(Some(Err(std::io::Error::new(
+                std::io::ErrorKind::TimedOut,
+                "replication response body deadline exceeded",
+            )
+            .into())));
+        }
+        let Some(inner) = self.inner.as_mut() else {
+            return Poll::Ready(None);
+        };
+        let result = std::pin::Pin::new(inner).poll_frame(cx);
+        if matches!(result, Poll::Ready(None) | Poll::Ready(Some(Err(_)))) {
+            self.inner.take();
+        }
+        result
+    }
+
+    fn is_end_stream(&self) -> bool {
+        self.inner.as_ref().is_none_or(http_body::Body::is_end_stream)
+    }
+    fn size_hint(&self) -> http_body::SizeHint {
+        self.inner.as_ref().map(http_body::Body::size_hint).unwrap_or_default()
+    }
+}
+
 /// Appends a caller-identifying token to the SDK `User-Agent`. Runs after
 /// signing: SigV4 excludes `user-agent` from the canonical request, so the
 /// signature stays valid.
@@ -309,7 +385,7 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
         config_builder = config_builder.http_client(http_client);
     }
 
-    if spec.connect_timeout.is_some() || spec.read_timeout.is_some() {
+    if spec.connect_timeout.is_some() || spec.read_timeout.is_some() || spec.operation_timeout.is_some() {
         let mut timeouts = TimeoutConfig::builder();
         if let Some(connect_timeout) = spec.connect_timeout {
             timeouts = timeouts.connect_timeout(connect_timeout);
@@ -317,6 +393,9 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
         if let Some(read_timeout) = spec.read_timeout {
             timeouts = timeouts.read_timeout(read_timeout);
         }
+        if let Some(operation_timeout) = spec.operation_timeout {
+            timeouts = timeouts.operation_timeout(operation_timeout);
+        }
         config_builder = config_builder.timeout_config(timeouts.build());
     }
 
@@ -674,6 +753,7 @@ mod tests {
             ca_cert_pem: None,
             connect_timeout: None,
             read_timeout: None,
+            operation_timeout: None,
             retry: RemoteS3RetryPolicy::Disabled,
             user_agent_suffix: "",
         }
@@ -1027,11 +1107,16 @@ mod tests {
         spec.user_agent_suffix = "RustFS-Test/0.0";
         spec.connect_timeout = Some(Duration::from_secs(5));
         spec.read_timeout = Some(Duration::from_secs(5));
+        spec.operation_timeout = Some(Duration::from_secs(12));
         let config = build_remote_s3_config(&spec)
             .await
             .expect("spec should build")
             .http_client(http_client)
             .build();
+        assert_eq!(
+            config.timeout_config().and_then(TimeoutConfig::operation_timeout),
+            Some(Duration::from_secs(12))
+        );
         S3Client::from_conf(config)
             .head_bucket()
             .bucket("bucket")

crates/ecstore/src/bucket/replication/replication_metadata_boundary.rs

@@ -33,6 +33,7 @@ pub(crate) struct ReplicationMetadataStore;
 impl ReplicationMetadataStore {
     pub(crate) const MRF_REPLICATION_FILE: &'static str = "config/replication/mrf.bin";
     pub(crate) const MRF_REPLICATION_RECOVERY_LOCK: &'static str = "config/replication/mrf.bin.recovery";
+    pub(crate) const MRF_REPLICATION_REPLAY_STAMP: &'static str = "config/replication/mrf.bin.replay";
     pub(crate) const FORCE_DELETE_REPLICATION_FILE: &'static str = "config/replication/force-delete.bin";
     pub(crate) const FORCE_DELETE_REPLICATION_TRANSACTION_LOCK: &'static str = "config/replication/force-delete.bin.transaction";
 
@@ -116,6 +117,10 @@ mod tests {
             ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
             "config/replication/mrf.bin.recovery"
         );
+        assert_eq!(
+            ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
+            "config/replication/mrf.bin.replay"
+        );
         assert_eq!(
             ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE,
             "config/replication/force-delete.bin"

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -16,8 +16,8 @@ use super::replication_config_store::ReplicationConfigStore;
 use super::replication_error_boundary::{Error as EcstoreError, is_err_object_not_found, is_err_version_not_found};
 use super::replication_filemeta_boundary::{
     MrfOpKind, MrfReplicateEntry, REPLICATE_HEAL_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicatedTargetInfo,
-    ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision, replicate_decision_for_admitted_targets,
-    replication_statuses_map, version_purge_statuses_map,
+    ReplicationGenerationSnapshot, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision,
+    replicate_decision_for_admitted_targets, replication_statuses_map, version_purge_statuses_map,
 };
 use super::replication_lock_boundary::ReplicationLockTiming;
 use super::replication_logging::{EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REPLICATION};
@@ -86,6 +86,10 @@ const MRF_CORRUPT_FILE_PREFIX: &str = "config/replication/mrf.corrupt";
 const MRF_PENDING_CAP: usize = 200_000;
 const MRF_RETRY_INITIAL_DELAY: Duration = Duration::from_millis(100);
 const MRF_RETRY_MAX_DELAY: Duration = Duration::from_secs(5);
+/// Interval between runtime replays of the MRF journal, cluster-wide. A pass
+/// re-attempts every retained entry, so a target that stays down costs one
+/// attempt per entry per interval.
+const MRF_REPLAY_INTERVAL: Duration = Duration::from_secs(300);
 
 #[derive(Debug, Default)]
 pub struct DurableMrfBacklog {
@@ -615,7 +619,7 @@ async fn read_mrf_entries<S: ReplicationObjectIO>(storage: Arc<S>) -> Result<Vec
 }
 
 /// Acknowledge only the generation read by the recovery leader. Entries appended
-/// after that generation are retained as a suffix and replayed on the next startup.
+/// after that generation are retained as a suffix and replayed on the next pass.
 /// Lock order: recovery leader lock -> MRF journal object lock.
 async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
     storage: Arc<S>,
@@ -667,7 +671,7 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
     Err(EcstoreError::PreconditionFailed)
 }
 
-/// Acquires the MRF recovery leader lock for the startup replay.
+/// Acquires the MRF recovery leader lock for one journal replay pass.
 /// Returns `None` (after logging) when the lock cannot be created or another
 /// node is already processing the backlog.
 async fn acquire_mrf_recovery_guard<S: ReplicationStorage>(storage: &Arc<S>) -> Option<rustfs_lock::NamespaceLockGuard> {
@@ -1018,7 +1022,7 @@ fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_t
 }
 
 /// Acknowledges the replayed MRF prefix and returns the retained backlog.
-/// On acknowledgement failure the backlog is preserved for the next startup and
+/// On acknowledgement failure the backlog is preserved for the next replay pass and
 /// re-read (falling back to the replayed snapshot) so the published summary stays accurate.
 async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
     storage: &Arc<S>,
@@ -1033,7 +1037,7 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
                 component = LOG_COMPONENT_ECSTORE,
                 subsystem = LOG_SUBSYSTEM_REPLICATION,
                 error = %error,
-                "Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
+                "Failed to acknowledge the MRF recovery prefix; preserving it for the next replay pass"
             );
             match read_mrf_entries(storage.clone()).await {
                 Ok(current) => current,
@@ -1051,6 +1055,120 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
     }
 }
 
+fn unix_millis_now() -> i64 {
+    i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000).unwrap_or(i64::MAX)
+}
+
+/// Whether a periodic pass may run, given the start time of the last pass on
+/// any node. A stamp more than one interval ahead of this node's clock is skew
+/// or a bad value, so it does not suppress replay.
+fn mrf_replay_due(last_started_ms: Option<i64>, now_ms: i64, interval: Duration) -> bool {
+    let Some(last_started_ms) = last_started_ms else {
+        return true;
+    };
+    let interval_ms = i64::try_from(interval.as_millis()).unwrap_or(i64::MAX);
+    now_ms.saturating_sub(last_started_ms) >= interval_ms || last_started_ms.saturating_sub(now_ms) >= interval_ms
+}
+
+async fn read_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>) -> Option<i64> {
+    let data = ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP)
+        .await
+        .ok()?;
+    std::str::from_utf8(&data).ok()?.trim().parse().ok()
+}
+
+async fn write_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>, started_ms: i64) {
+    if let Err(error) = ReplicationConfigStore::save(
+        storage.clone(),
+        ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
+        started_ms.to_string().into_bytes(),
+    )
+    .await
+    {
+        warn!(
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REPLICATION,
+            error = %error,
+            "Failed to record the MRF replay start time"
+        );
+    }
+}
+
+/// One MRF journal replay pass. The recovery leader reads the journal,
+/// re-drives every entry, and acknowledges the replayed prefix; entries that
+/// could not be admitted, and entries appended meanwhile, stay for the next pass.
+///
+/// Every node runs the periodic loop, and the leader lock only serializes
+/// passes. With `min_interval`, a pass is skipped when a pass on any node
+/// started within that interval, read from the shared replay stamp, so a
+/// cluster replays once per interval rather than once per node. The startup
+/// pass passes `None` and always runs. A pass that finds entries records its
+/// start time. Returns whether the journal was read.
+/// Lock order: recovery leader lock -> replay stamp object lock.
+async fn replay_mrf_journal<S: ReplicationStorage>(storage: &Arc<S>, min_interval: Option<Duration>) -> bool {
+    let Some(recovery_guard) = acquire_mrf_recovery_guard(storage).await else {
+        return false;
+    };
+    let started_ms = unix_millis_now();
+    if let Some(interval) = min_interval
+        && !mrf_replay_due(read_mrf_replay_stamp(storage).await, started_ms, interval)
+    {
+        return false;
+    }
+
+    let Some(entries) = load_mrf_recovery_entries(storage).await else {
+        return true;
+    };
+    if !entries.is_empty() {
+        write_mrf_replay_stamp(storage, started_ms).await;
+    }
+    set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
+
+    let total = entries.len();
+    let mut queued_count = 0usize;
+    let mut retry_entries = Vec::new();
+
+    for entry in entries.iter() {
+        let Some(admission) = replay_mrf_entry(entry, storage, &mut retry_entries).await else {
+            continue;
+        };
+
+        if admission == ReplicationQueueAdmission::Missed {
+            retry_entries.push(entry.clone());
+        } else if admission == ReplicationQueueAdmission::Queued {
+            queued_count += 1;
+        }
+    }
+
+    let retained = resolve_retained_mrf_entries(storage, &recovery_guard, &entries, &retry_entries).await;
+    let retained_count = retained.len();
+    set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
+
+    if queued_count > 0 {
+        info!(
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REPLICATION,
+            recovered = queued_count,
+            total,
+            retained = retained_count,
+            "Replayed MRF entries admitted for retry"
+        );
+    }
+    true
+}
+
+/// Re-runs [`replay_mrf_journal`] every `period`. The recovery leader lock keeps
+/// one pass at a time across the cluster, including the startup pass, and the
+/// shared replay stamp spaces passes from different nodes by `period`.
+fn spawn_mrf_replay_loop<S: ReplicationStorage>(storage: Arc<S>, period: Duration) -> JoinHandle<()> {
+    tokio::spawn(async move {
+        loop {
+            tokio::time::sleep(period).await;
+            replay_mrf_journal(&storage, Some(period)).await;
+        }
+    })
+}
+
 #[derive(Debug, thiserror::Error)]
 #[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
 struct ResyncActiveConflictError {
@@ -1171,6 +1289,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
         // Start background tasks
         pool.start_mrf_persister().await;
         pool.start_mrf_processor().await;
+        pool.start_mrf_replayer().await;
         pool.start_force_delete_processor().await;
 
         pool
@@ -1462,7 +1581,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 
     /// Queues a replica task
     pub async fn queue_replica_task(&self, ri: ReplicateObjectInfo) -> ReplicationQueueAdmission {
-        // A version that is already queued or being uploaded is not driven a
+        // A source generation already queued or being uploaded is not driven a
         // second time: the scanner heal pass sees it as PENDING until the
         // first upload lands and would otherwise re-queue it every cycle
         // (backlog#2362). The key is released when the worker finishes, or
@@ -1639,55 +1758,26 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 
     /// Starts the MRF processor — one-shot at startup.
     ///
-    /// Reads the on-disk MRF file, re-injects admitted entries as Heal operations, and
-    /// rewrites any entries that could not be admitted for a later startup retry.
+    /// Replays the on-disk MRF journal once; [`Self::start_mrf_replayer`] repeats
+    /// the pass while the server runs.
     async fn start_mrf_processor(&self) {
         let storage = self.storage.clone();
-
         let handle = tokio::spawn(async move {
-            let Some(recovery_guard) = acquire_mrf_recovery_guard(&storage).await else {
-                return;
-            };
-
-            let Some(entries) = load_mrf_recovery_entries(&storage).await else {
-                return;
-            };
-            set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
-
-            let total = entries.len();
-            let mut queued_count = 0usize;
-            let mut retry_entries = Vec::new();
-
-            for entry in entries.iter() {
-                let Some(admission) = replay_mrf_entry(entry, &storage, &mut retry_entries).await else {
-                    continue;
-                };
-
-                if admission == ReplicationQueueAdmission::Missed {
-                    retry_entries.push(entry.clone());
-                } else if admission == ReplicationQueueAdmission::Queued {
-                    queued_count += 1;
-                }
-            }
-
-            let retained = resolve_retained_mrf_entries(&storage, &recovery_guard, &entries, &retry_entries).await;
-            let retained_count = retained.len();
-            set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
-
-            if queued_count > 0 {
-                info!(
-                    component = LOG_COMPONENT_ECSTORE,
-                    subsystem = LOG_SUBSYSTEM_REPLICATION,
-                    recovered = queued_count,
-                    total,
-                    retained = retained_count,
-                    "Replayed MRF entries admitted for retry"
-                );
-            }
+            replay_mrf_journal(&storage, None).await;
         });
         self.task_handles.lock().await.push(handle);
     }
 
+    /// Starts the runtime MRF replayer — ongoing background task.
+    ///
+    /// Entries reach the journal while the server runs (worker-queue overflow,
+    /// delete-marker purge intents whose watch window drained with targets still
+    /// dirty). Replaying only at startup left them waiting for a restart.
+    async fn start_mrf_replayer(&self) {
+        let handle = spawn_mrf_replay_loop(self.storage.clone(), MRF_REPLAY_INTERVAL);
+        self.task_handles.lock().await.push(handle);
+    }
+
     async fn start_force_delete_processor(&self) {
         let storage = self.storage.clone();
         let handle =
@@ -1755,7 +1845,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
     /// writes them to the on-disk MRF file every flush interval (default 10s,
     /// overridable via `RUSTFS_REPL_MRF_FLUSH_INTERVAL_MS`) or when 1 000 new
     /// entries accumulate. Runtime writers append under the journal lock; only the
-    /// startup recovery leader may remove an acknowledged prefix.
+    /// recovery leader of a replay pass may remove an acknowledged prefix.
     async fn start_mrf_persister(&self) {
         let Some(mut rx) = self.mrf_save_rx.lock().await.take() else {
             return;
@@ -2314,28 +2404,47 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 }
 
 /// Object versions currently queued or being uploaded, keyed by bucket,
-/// object name and version. `queue_replica_task` admits a version only once
+/// object name, version and source generation. `queue_replica_task` admits a generation only once
 /// while it is in flight; the scanner heal pass and MRF replays that arrive
 /// in the meantime are `Skipped` instead of driving a second complete upload
 /// (backlog#2362). Entries are removed when the worker finishes the task or
 /// when no worker accepted it.
 #[derive(Debug, Default)]
 pub(crate) struct ReplicationInFlight {
-    keys: std::sync::Mutex<std::collections::HashSet<(String, String, Option<uuid::Uuid>)>>,
+    keys: std::sync::Mutex<std::collections::HashSet<ReplicationInFlightKey>>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+struct ReplicationInFlightKey {
+    bucket: String,
+    object: String,
+    version_id: Option<uuid::Uuid>,
+    generation: ReplicationGenerationSnapshot,
+}
+
+impl From<&ReplicateObjectInfo> for ReplicationInFlightKey {
+    fn from(info: &ReplicateObjectInfo) -> Self {
+        Self {
+            bucket: info.bucket.clone(),
+            object: info.name.clone(),
+            version_id: info.version_id,
+            generation: info.replication_generation.clone(),
+        }
+    }
 }
 
 impl ReplicationInFlight {
-    fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<(String, String, Option<uuid::Uuid>)>> {
+    fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<ReplicationInFlightKey>> {
         self.keys.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
     }
 
-    /// Claim `ri`; `false` when the same version is already in flight.
+    /// Claim `ri`; a metadata mutation is independent of its earlier generation.
     fn try_begin(&self, ri: &ReplicateObjectInfo) -> bool {
-        self.lock().insert((ri.bucket.clone(), ri.name.clone(), ri.version_id))
+        self.lock().insert(ReplicationInFlightKey::from(ri))
     }
 
     fn finish(&self, ri: &ReplicateObjectInfo) {
-        self.lock().remove(&(ri.bucket.clone(), ri.name.clone(), ri.version_id));
+        self.lock().remove(&ReplicationInFlightKey::from(ri));
     }
 
     #[cfg(test)]
@@ -2348,26 +2457,21 @@ impl ReplicationInFlight {
 /// including when replication panics.
 struct ReplicationInFlightGuard {
     in_flight: Arc<ReplicationInFlight>,
-    key: ReplicateObjectInfo,
+    key: ReplicationInFlightKey,
 }
 
 impl ReplicationInFlightGuard {
     fn new(in_flight: Arc<ReplicationInFlight>, ri: &ReplicateObjectInfo) -> Self {
         Self {
             in_flight,
-            key: ReplicateObjectInfo {
-                bucket: ri.bucket.clone(),
-                name: ri.name.clone(),
-                version_id: ri.version_id,
-                ..Default::default()
-            },
+            key: ReplicationInFlightKey::from(ri),
         }
     }
 }
 
 impl Drop for ReplicationInFlightGuard {
     fn drop(&mut self) {
-        self.in_flight.finish(&self.key);
+        self.in_flight.lock().remove(&self.key);
     }
 }
 
@@ -3421,6 +3525,8 @@ mod tests {
         allow_first_read: Notify,
         read_count: AtomicUsize,
         delete_storage_calls: AtomicUsize,
+        delete_error: StdMutex<EcstoreError>,
+        replay_stamp: StdMutex<Option<Vec<u8>>>,
         reported_size: StdMutex<Option<i64>>,
         stream_read_bytes: Arc<AtomicUsize>,
         write_count: AtomicUsize,
@@ -3488,6 +3594,24 @@ mod tests {
             _h: Self::HeaderMap,
             _opts: &Self::ObjectOptions,
         ) -> Result<Self::GetObjectReader, Self::Error> {
+            if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
+                let stamp = self
+                    .shared
+                    .replay_stamp
+                    .lock()
+                    .expect("test replay stamp lock should not be poisoned")
+                    .clone()
+                    .ok_or(EcstoreError::FileNotFound)?;
+                return Ok(Self::GetObjectReader {
+                    object_info: ObjectInfo {
+                        size: i64::try_from(stamp.len()).expect("test stamp length should fit i64"),
+                        ..Default::default()
+                    },
+                    stream: Box::new(Cursor::new(stamp)),
+                    buffered_body: None,
+                    body_source: Default::default(),
+                });
+            }
             if !object.ends_with("/.replication/resync.bin")
                 && !object.ends_with("config/replication/mrf.bin")
                 && !object.ends_with("config/replication/force-delete.bin")
@@ -3551,6 +3675,16 @@ mod tests {
                 let lock = self.new_ns_lock(bucket, object).await?;
                 Some(lock.get_write_lock(Duration::from_secs(10)).await?)
             };
+            if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
+                let mut stamp = Vec::new();
+                data.stream.read_to_end(&mut stamp).await.map_err(EcstoreError::from)?;
+                *self
+                    .shared
+                    .replay_stamp
+                    .lock()
+                    .expect("test replay stamp lock should not be poisoned") = Some(stamp);
+                return Ok(ObjectInfo::default());
+            }
             if opts.http_preconditions.is_some()
                 && let Some(replacement) = self
                     .shared
@@ -3672,7 +3806,12 @@ mod tests {
             _opts: Self::ObjectOptions,
         ) -> Result<Self::ObjectInfo, Self::Error> {
             self.shared.delete_storage_calls.fetch_add(1, Ordering::SeqCst);
-            Err(EcstoreError::NotImplemented)
+            Err(self
+                .shared
+                .delete_error
+                .lock()
+                .expect("delete error lock should not be poisoned")
+                .clone())
         }
 
         async fn delete_objects(
@@ -3929,6 +4068,48 @@ mod tests {
         assert_eq!(current_queue(&pool, "in-flight-bucket").await, (3, 3 * 4096));
     }
 
+    #[tokio::test]
+    async fn queue_replica_task_admits_metadata_changes_during_an_initial_upload() {
+        let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
+        let (tx, mut rx) = mpsc::channel(4);
+        pool.workers.write().await.push(tx);
+        let initial = ReplicateObjectInfo {
+            bucket: "pending-tags-bucket".to_string(),
+            name: "object".to_string(),
+            version_id: Some(uuid::Uuid::new_v4()),
+            size: 4096,
+            replication_generation: ReplicationGenerationSnapshot {
+                mutation_id: Some(uuid::Uuid::new_v4().to_string()),
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        let metadata = ReplicateObjectInfo {
+            op_type: ReplicationType::Metadata,
+            user_tags: String::new(),
+            replication_generation: ReplicationGenerationSnapshot {
+                mutation_id: Some(uuid::Uuid::new_v4().to_string()),
+                ..Default::default()
+            },
+            ..initial.clone()
+        };
+        assert_eq!(pool.queue_replica_task(initial.clone()).await, ReplicationQueueAdmission::Queued);
+        assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Queued);
+        assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
+        assert_eq!(pool.in_flight.len(), 2);
+        drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &initial));
+        assert_eq!(pool.in_flight.len(), 1);
+        assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
+        for expected in [&initial, &metadata] {
+            let ReplicationOperation::Object(queued) = rx.recv().await.expect("queued operation") else {
+                panic!("expected object replication");
+            };
+            assert_eq!(queued.replication_generation, expected.replication_generation);
+        }
+        drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &metadata));
+        assert_eq!(pool.in_flight.len(), 0);
+    }
+
     #[tokio::test]
     async fn queue_replica_task_releases_the_version_when_no_worker_accepts_it() {
         let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
@@ -4171,6 +4352,8 @@ mod tests {
             allow_first_read: Notify::new(),
             read_count: AtomicUsize::new(0),
             delete_storage_calls: AtomicUsize::new(0),
+            delete_error: StdMutex::new(EcstoreError::NotImplemented),
+            replay_stamp: StdMutex::new(None),
             reported_size: StdMutex::new(None),
             stream_read_bytes: Arc::new(AtomicUsize::new(0)),
             write_count: AtomicUsize::new(0),
@@ -4221,6 +4404,103 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn delete_worker_already_purged_source_completes_without_retry() {
+        let arn = "arn:test";
+        for delete_marker_version in [false, true] {
+            for version_id in [Uuid::new_v4(), Uuid::nil()] {
+                for error in [
+                    EcstoreError::VersionNotFound("source".to_string(), "purged".to_string(), version_id.to_string()),
+                    EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
+                    EcstoreError::FileVersionNotFound,
+                    EcstoreError::FileNotFound,
+                ] {
+                    let shared = empty_resync_shared_state();
+                    *shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
+                    let storage = Arc::new(LoadResyncNodeStore::new("purged-source", shared.clone()));
+                    let delete = DeletedObjectReplicationInfo {
+                        bucket: "source".to_string(),
+                        delete_object: ReplicationDeletedObject {
+                            object_name: "purged".to_string(),
+                            version_id: (!delete_marker_version).then_some(version_id),
+                            delete_marker_version_id: delete_marker_version.then_some(version_id),
+                            replication_state: Some(ReplicationState {
+                                replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
+                                purge_targets: HashMap::from([(arn.to_string(), VersionPurgeStatusType::Complete)]),
+                                ..Default::default()
+                            }),
+                            ..Default::default()
+                        },
+                        op_type: ReplicationType::Delete,
+                        ..Default::default()
+                    };
+
+                    assert!(
+                        replicate_delete_with_outcome(delete, storage).await,
+                        "a confirmed target purge must complete when the source is already absent (marker={delete_marker_version}, version={version_id})"
+                    );
+                    assert_eq!(
+                        shared.delete_storage_calls.load(Ordering::SeqCst),
+                        2 + usize::from(!delete_marker_version && !version_id.is_nil()),
+                        "the worker must lock and attempt writeback, reading a non-null data version first"
+                    );
+                }
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn delete_worker_missing_source_does_not_hide_unfinished_replication() {
+        let arn = "arn:test";
+        for (version_purge, target_status, error) in [
+            (
+                true,
+                VersionPurgeStatusType::Pending,
+                EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
+            ),
+            (true, VersionPurgeStatusType::Complete, EcstoreError::ErasureWriteQuorum),
+            (
+                false,
+                VersionPurgeStatusType::Empty,
+                EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
+            ),
+        ] {
+            let shared = empty_resync_shared_state();
+            *shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
+            let storage = Arc::new(LoadResyncNodeStore::new("unfinished-delete", shared));
+            let delete = DeletedObjectReplicationInfo {
+                bucket: "source".to_string(),
+                delete_object: ReplicationDeletedObject {
+                    object_name: "purged".to_string(),
+                    version_id: version_purge.then(Uuid::new_v4),
+                    delete_marker: !version_purge,
+                    replication_state: Some(ReplicationState {
+                        replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
+                        targets: if version_purge {
+                            HashMap::new()
+                        } else {
+                            HashMap::from([(arn.to_string(), ReplicationStatusType::Completed)])
+                        },
+                        purge_targets: if version_purge {
+                            HashMap::from([(arn.to_string(), target_status)])
+                        } else {
+                            HashMap::new()
+                        },
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                },
+                op_type: ReplicationType::Delete,
+                ..Default::default()
+            };
+
+            assert!(
+                !replicate_delete_with_outcome(delete, storage).await,
+                "unfinished target work, source I/O failures, and marker creation must remain retryable"
+            );
+        }
+    }
+
     #[tokio::test]
     async fn delete_worker_preserves_persisted_admission_without_live_config() {
         let arn = "arn:test";
@@ -5304,6 +5584,8 @@ mod tests {
                 allow_first_read: Notify::new(),
                 read_count: AtomicUsize::new(0),
                 delete_storage_calls: AtomicUsize::new(0),
+                delete_error: StdMutex::new(EcstoreError::NotImplemented),
+                replay_stamp: StdMutex::new(None),
                 reported_size: StdMutex::new(None),
                 stream_read_bytes: Arc::new(AtomicUsize::new(0)),
                 write_count: AtomicUsize::new(0),
@@ -6268,6 +6550,108 @@ mod tests {
         assert_eq!(retained.target_arns, entry.target_arns);
     }
 
+    #[tokio::test]
+    async fn mrf_replay_loop_drains_entries_persisted_after_startup() {
+        let shared = empty_resync_shared_state();
+        let storage = Arc::new(LoadResyncNodeStore::new("mrf-runtime-replay", shared.clone()));
+        // A force-delete intent from a superseded generation replays as
+        // Skipped, so a pass that reaches it acknowledges it off the journal.
+        let stale_intent = |object: &str| MrfReplicateEntry {
+            bucket: "mrf-runtime-replay".to_string(),
+            object: object.to_string(),
+            op: MrfOpKind::Delete,
+            force_delete_id: Some(uuid::Uuid::new_v4()),
+            force_delete_local_commit: false,
+            target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
+            ..Default::default()
+        };
+        let journal_drained = || async {
+            tokio::time::timeout(Duration::from_secs(5), async {
+                while !shared.data.lock().expect("test data lock should not be poisoned").is_empty() {
+                    tokio::time::sleep(Duration::from_millis(10)).await;
+                }
+            })
+            .await
+            .is_ok()
+        };
+
+        let handle = spawn_mrf_replay_loop(storage, Duration::from_millis(20));
+        for object in ["first", "second"] {
+            *shared.data.lock().expect("test data lock should not be poisoned") =
+                encode_mrf_file(&[stale_intent(object)]).expect("MRF entry should encode");
+            assert!(
+                journal_drained().await,
+                "an MRF entry persisted while the server runs must be replayed without a restart ({object})"
+            );
+        }
+        handle.abort();
+    }
+
+    #[tokio::test]
+    async fn mrf_periodic_replay_runs_once_per_interval_across_nodes() {
+        let shared = empty_resync_shared_state();
+        let node_a = Arc::new(LoadResyncNodeStore::new("mrf-node-a", shared.clone()));
+        let node_b = Arc::new(LoadResyncNodeStore::new("mrf-node-b", shared.clone()));
+        let interval = Duration::from_secs(1);
+        let persist_backlog = |object: &str| {
+            let entry = MrfReplicateEntry {
+                bucket: "mrf-node-replay".to_string(),
+                object: object.to_string(),
+                op: MrfOpKind::Delete,
+                force_delete_id: Some(uuid::Uuid::new_v4()),
+                force_delete_local_commit: false,
+                target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
+                ..Default::default()
+            };
+            *shared.data.lock().expect("test data lock should not be poisoned") =
+                encode_mrf_file(&[entry]).expect("MRF entry should encode");
+        };
+        let journal_is_empty = || shared.data.lock().expect("test data lock should not be poisoned").is_empty();
+
+        persist_backlog("first");
+        assert!(replay_mrf_journal(&node_a, Some(interval)).await, "node A's periodic pass should replay");
+        assert!(journal_is_empty(), "node A's pass should acknowledge the replayed entry");
+
+        // Node B's timer fires within the same interval with a backlog present.
+        persist_backlog("second");
+        assert!(
+            !replay_mrf_journal(&node_b, Some(interval)).await,
+            "a node must not replay within one interval of another node's pass"
+        );
+        assert!(!journal_is_empty(), "the backlog should wait for the next interval");
+
+        assert!(
+            replay_mrf_journal(&node_b, None).await,
+            "the startup pass should replay regardless of the shared stamp"
+        );
+        assert!(journal_is_empty(), "the startup pass should acknowledge the replayed entry");
+
+        persist_backlog("third");
+        tokio::time::sleep(interval).await;
+        assert!(
+            replay_mrf_journal(&node_a, Some(interval)).await,
+            "a periodic pass should replay once the interval has elapsed"
+        );
+        assert!(journal_is_empty(), "the due pass should acknowledge the replayed entry");
+    }
+
+    #[test]
+    fn mrf_replay_due_tolerates_missing_and_skewed_stamps() {
+        let interval = Duration::from_secs(300);
+        let now = 1_000_000_000;
+        assert!(mrf_replay_due(None, now, interval));
+        assert!(!mrf_replay_due(Some(now - 299_999), now, interval));
+        assert!(mrf_replay_due(Some(now - 300_000), now, interval));
+        assert!(
+            !mrf_replay_due(Some(now + 1_000), now, interval),
+            "small clock skew should still space passes"
+        );
+        assert!(
+            mrf_replay_due(Some(now + 300_000), now, interval),
+            "a stamp an interval ahead must not suppress replay"
+        );
+    }
+
     #[test]
     fn mrf_entry_delete_marker_roundtrip() {
         let dm_vid = Uuid::new_v4();

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -2809,17 +2809,22 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
             true
         }
         Err(e) => {
-            error!(
-                event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
-                component = LOG_COMPONENT_ECSTORE,
-                subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
-                bucket = %bucket,
-                arn = %dobj.target_arn,
-                object = %dobj.delete_object.object_name,
-                operation = "apply_replication_delete_state",
-                error = %e,
-                "Replication target operation failed"
-            );
+            // A batch purge or another worker may have removed this version
+            // already. Remote failures still control the final outcome below.
+            let source_already_purged = is_version_purge && (is_err_object_not_found(&e) || is_err_version_not_found(&e));
+            if !source_already_purged {
+                error!(
+                    event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
+                    bucket = %bucket,
+                    arn = %dobj.target_arn,
+                    object = %dobj.delete_object.object_name,
+                    operation = "apply_replication_delete_state",
+                    error = %e,
+                    "Replication target operation failed"
+                );
+            }
             send_event(EventArgs {
                 event_name,
                 bucket_name: bucket.clone(),
@@ -2832,7 +2837,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
                 },
                 ..Default::default()
             });
-            false
+            source_already_purged
         }
     };
 
@@ -3010,7 +3015,7 @@ const DELETE_MARKER_PURGE_WATCH_INTERVAL: TokioDuration = TokioDuration::from_se
 /// If the marker disappears (deleted before or while the replica landed),
 /// purge the replicated marker from the targets, retrying failed targets on
 /// later rounds. When the window drains with targets still dirty, persist the
-/// purge intent as a durable MRF entry so the next startup replays it through
+/// purge intent as a durable MRF entry so the next MRF replay pass re-drives it through
 /// `purge_stale_delete_marker_targets`.
 async fn watch_and_purge_source_delete_marker<S: ReplicationStorage>(
     bucket: String,
@@ -3101,7 +3106,7 @@ async fn enqueue_delete_marker_purge_mrf(dobj: &DeletedObjectReplicationInfo, fa
 /// Heal callers retry for the full watch window because the startup MRF
 /// processor runs before bucket metadata (and thus target clients) finishes
 /// initializing — the first pass can see `target_client_missing` and a later
-/// round resolves the client; the replay loop is serial and startup-only, so
+/// round resolves the client; replay passes are serial background work, so
 /// blocking it for up to the window per dirty entry is acceptable. Live
 /// callers run on replication workers where a down target would pin a worker
 /// for the whole window, so they attempt once and lean on the durable intent
@@ -6604,6 +6609,7 @@ mod tests {
             health_check_duration: std::time::Duration::from_secs(5),
             replicate_sync: false,
             client: Arc::new(aws_sdk_s3::Client::from_conf(config)),
+            timeouts: Default::default(),
         })
     }
 

crates/ecstore/src/bucket/utils.rs

@@ -259,6 +259,10 @@ pub fn check_object_name_for_length_and_slash(bucket: &str, object: &str) -> Res
 pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) -> Result<()> {
     check_put_object_args(bucket, object)?;
 
+    if !data_movement && !is_meta_bucketname(bucket) && has_xml_invalid_control_characters(object) {
+        return Err(StorageError::ObjectNameInvalid(bucket.to_owned(), object.to_owned()));
+    }
+
     if cfg!(target_os = "windows") && !data_movement && !is_meta_bucketname(bucket) && object.contains('\\') {
         return Err(StorageError::InvalidArgument(
             bucket.to_owned(),
@@ -270,6 +274,14 @@ pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) ->
     Ok(())
 }
 
+/// New public keys must remain representable in an unencoded XML listing.
+/// Existing keys still use the read/delete validators so they can be removed.
+pub fn has_xml_invalid_control_characters(object: &str) -> bool {
+    object
+        .chars()
+        .any(|character| character < ' ' && !matches!(character, '\t' | '\n' | '\r'))
+}
+
 pub fn check_copy_obj_args(bucket: &str, object: &str) -> Result<()> {
     check_bucket_and_object_names(bucket, object)
 }

crates/ecstore/src/cache_value/metacache_set.rs

@@ -34,7 +34,7 @@ use tokio::sync::Mutex as TokioMutex;
 use tokio::task::JoinSet;
 use tokio::time::timeout;
 use tokio_util::sync::CancellationToken;
-use tracing::{debug, error, warn};
+use tracing::{Instrument, debug, error, warn};
 
 const LOG_COMPONENT_ECSTORE: &str = "ecstore";
 const LOG_SUBSYSTEM_METACACHE: &str = "metacache";
@@ -217,6 +217,8 @@ impl Drop for NotifyOnDrop {
 
 #[derive(Default)]
 pub struct ListPathRawOptions {
+    /// Retain native listing admission through producer cancellation/draining.
+    pub read_budget: Option<Arc<tokio::sync::OwnedSemaphorePermit>>,
     pub disks: Vec<Option<DiskStore>>,
     pub fallback_disks: Vec<Option<DiskStore>>,
     pub bucket: String,
@@ -224,6 +226,8 @@ pub struct ListPathRawOptions {
     pub recursive: bool,
     pub incl_deleted: bool,
     pub skip_hidden_prefix_check: bool,
+    pub hidden_entry_budget: usize,
+    pub pending_prefix: Option<String>,
     pub filter_prefix: Option<String>,
     pub forward_to: Option<String>,
     pub min_disks: usize,
@@ -253,6 +257,7 @@ pub struct ListPathRawOptions {
 impl Clone for ListPathRawOptions {
     fn clone(&self) -> Self {
         Self {
+            read_budget: self.read_budget.clone(),
             disks: self.disks.clone(),
             fallback_disks: self.fallback_disks.clone(),
             bucket: self.bucket.clone(),
@@ -260,6 +265,8 @@ impl Clone for ListPathRawOptions {
             recursive: self.recursive,
             incl_deleted: self.incl_deleted,
             skip_hidden_prefix_check: self.skip_hidden_prefix_check,
+            hidden_entry_budget: self.hidden_entry_budget,
+            pending_prefix: self.pending_prefix.clone(),
             filter_prefix: self.filter_prefix.clone(),
             forward_to: self.forward_to.clone(),
             min_disks: self.min_disks,
@@ -289,6 +296,8 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
         recursive: opts.recursive,
         incl_deleted: opts.incl_deleted,
         skip_hidden_prefix_check: opts.skip_hidden_prefix_check,
+        hidden_entry_budget: opts.hidden_entry_budget,
+        pending_prefix: opts.pending_prefix.clone(),
         report_notfound: opts.report_not_found,
         filter_prefix: opts.filter_prefix.clone(),
         forward_to: opts.forward_to.clone(),
@@ -301,13 +310,15 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
 }
 
 /// How a merged raw listing stream ended.
-#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
 pub struct ListPathRawOutcome {
     /// At least one producer stopped at its per-disk limit instead of at the
     /// end of its namespace, so the closed stream is not proof of exhaustion.
     /// Carried in-band by the metacache stream trailer, so remote producers
     /// report it as well.
     pub limit_reached: bool,
+    /// Inclusive sorted boundary proved by every reader before a bounded stop.
+    pub scan_cut: Option<String>,
 }
 
 pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> disk::error::Result<()> {
@@ -363,10 +374,15 @@ async fn list_path_raw_inner(
     let producer_errs: Arc<[OnceLock<DiskError>]> = (0..opts.disks.len()).map(|_| OnceLock::new()).collect::<Vec<_>>().into();
 
     let cancel_rx = CancellationToken::new();
+    let walk_options = Arc::new(walk_dir_options(&opts));
 
     for (disk_idx, disk) in opts.disks.iter().enumerate() {
         let opdisk = disk.clone();
-        let opts_clone = opts.clone();
+        let walk_options = walk_options.clone();
+        #[cfg(test)]
+        let test_reader_behavior = opts.test_reader_behaviors.get(disk_idx).cloned();
+        #[cfg(test)]
+        let test_disk_io_context_observed = opts.test_disk_io_context_observed.clone();
         let fallback_claim_tracker = fallback_claim_tracker.clone();
         let fds_clone = fds.clone();
         #[cfg(test)]
@@ -376,17 +392,19 @@ async fn list_path_raw_inner(
         let disk_io_context_clone = disk_io_context.clone();
         let (rd, wr) = tokio::io::duplex(64 * 1024);
         readers.push(MetacacheReader::new(rd));
-        jobs.spawn(with_optional_disk_io_context(disk_io_context_clone, async move {
+        let read_budget = opts.read_budget.clone();
+        let producer = with_optional_disk_io_context(disk_io_context_clone, async move {
+            let _read_budget = read_budget;
             let mut wr = PublishedBytesWriter::new(wr);
             #[cfg(test)]
-            if let Some(observer) = &opts_clone.test_disk_io_context_observed {
+            if let Some(observer) = &test_disk_io_context_observed {
                 observer.store(
                     crate::disk::current_disk_io_context().is_some_and(|context| context.class == DiskIoClass::Background),
                     std::sync::atomic::Ordering::Release,
                 );
             }
             #[cfg(test)]
-            let test_primary_error = if let Some(behavior) = opts_clone.test_reader_behaviors.get(disk_idx).cloned() {
+            let test_primary_error = if let Some(behavior) = test_reader_behavior {
                 match behavior {
                     TestReaderBehavior::Eof => return Ok(()),
                     TestReaderBehavior::Entries(entries) => {
@@ -443,8 +461,6 @@ async fn list_path_raw_inner(
                 None
             };
 
-            let wakl_opts = walk_dir_options(&opts_clone);
-
             let mut need_fallback = false;
             let mut last_err = None;
             #[cfg(test)]
@@ -459,7 +475,7 @@ async fn list_path_raw_inner(
 
             if !need_fallback && let Some(disk) = opdisk {
                 let primary_walk_started = std::time::Instant::now();
-                match disk.walk_dir(wakl_opts, &mut wr).await {
+                match disk.walk_dir((*walk_options).clone(), &mut wr).await {
                     Ok(_res) => {
                         rustfs_io_metrics::record_stage_duration(
                             "metacache_walk_dir_primary",
@@ -483,8 +499,8 @@ async fn list_path_raw_inner(
                                 event = EVENT_METACACHE_LISTING,
                                 component = LOG_COMPONENT_ECSTORE,
                                 subsystem = LOG_SUBSYSTEM_METACACHE,
-                                bucket = %opts_clone.bucket,
-                                path = %opts_clone.path,
+                                bucket = %walk_options.bucket,
+                                path = %walk_options.base_dir,
                                 disk_index = disk_idx,
                                 state = "walk_dir_missing_path",
                                 error = ?err,
@@ -495,8 +511,8 @@ async fn list_path_raw_inner(
                                 event = EVENT_METACACHE_LISTING,
                                 component = LOG_COMPONENT_ECSTORE,
                                 subsystem = LOG_SUBSYSTEM_METACACHE,
-                                bucket = %opts_clone.bucket,
-                                path = %opts_clone.path,
+                                bucket = %walk_options.bucket,
+                                path = %walk_options.base_dir,
                                 disk_index = disk_idx,
                                 state = "walk_dir_failed",
                                 error = ?err,
@@ -598,8 +614,8 @@ async fn list_path_raw_inner(
                             event = EVENT_METACACHE_LISTING,
                             component = LOG_COMPONENT_ECSTORE,
                             subsystem = LOG_SUBSYSTEM_METACACHE,
-                            bucket = %opts_clone.bucket,
-                            path = %opts_clone.path,
+                            bucket = %walk_options.bucket,
+                            path = %walk_options.base_dir,
                             disk_index = disk_idx,
                             state = "fallback_disk_missing_for_path",
                             error = ?err,
@@ -610,8 +626,8 @@ async fn list_path_raw_inner(
                             event = EVENT_METACACHE_LISTING,
                             component = LOG_COMPONENT_ECSTORE,
                             subsystem = LOG_SUBSYSTEM_METACACHE,
-                            bucket = %opts_clone.bucket,
-                            path = %opts_clone.path,
+                            bucket = %walk_options.bucket,
+                            path = %walk_options.base_dir,
                             disk_index = disk_idx,
                             state = "fallback_disk_missing",
                             error = ?err,
@@ -626,7 +642,7 @@ async fn list_path_raw_inner(
                 }
 
                 let fallback_walk_started = std::time::Instant::now();
-                match disk.as_ref().walk_dir(walk_dir_options(&opts_clone), &mut wr).await {
+                match disk.as_ref().walk_dir((*walk_options).clone(), &mut wr).await {
                     Ok(_r) => {
                         rustfs_io_metrics::record_stage_duration(
                             "metacache_walk_dir_fallback",
@@ -652,8 +668,8 @@ async fn list_path_raw_inner(
                                 event = EVENT_METACACHE_LISTING,
                                 component = LOG_COMPONENT_ECSTORE,
                                 subsystem = LOG_SUBSYSTEM_METACACHE,
-                                bucket = %opts_clone.bucket,
-                                path = %opts_clone.path,
+                                bucket = %walk_options.bucket,
+                                path = %walk_options.base_dir,
                                 disk_index = disk_idx,
                                 state = "fallback_walk_dir_missing_path",
                                 error = ?err,
@@ -664,8 +680,8 @@ async fn list_path_raw_inner(
                                 event = EVENT_METACACHE_LISTING,
                                 component = LOG_COMPONENT_ECSTORE,
                                 subsystem = LOG_SUBSYSTEM_METACACHE,
-                                bucket = %opts_clone.bucket,
-                                path = %opts_clone.path,
+                                bucket = %walk_options.bucket,
+                                path = %walk_options.base_dir,
                                 disk_index = disk_idx,
                                 state = "fallback_walk_dir_failed",
                                 error = ?err,
@@ -687,12 +703,13 @@ async fn list_path_raw_inner(
 
             // warn!("list_path_raw: while need_fallback done");
             Ok(())
-        }));
+        });
+        jobs.spawn(producer.in_current_span());
     }
 
     let revjob_rx = rx.clone();
     let mut revjobs = JoinSet::new();
-    revjobs.spawn(async move {
+    let merge = async move {
         // Consumer-side peek timeout, and a caveat worth understanding
         // (rustfs/backlog#1217).
         //
@@ -722,6 +739,8 @@ async fn list_path_raw_inner(
         // Sticky across rounds: once any reader's trailer says its producer
         // stopped at the per-disk limit, the merged stream may hide more keys.
         let mut limit_reached = false;
+        let mut scan_cut = None;
+        let mut last_scanned_key: Option<String> = None;
 
         loop {
             let mut current = MetaCacheEntry::default();
@@ -733,7 +752,7 @@ async fn list_path_raw_inner(
             // );
 
             if revjob_rx.is_cancelled() {
-                return Ok(ListPathRawOutcome { limit_reached });
+                return Ok(ListPathRawOutcome { limit_reached, scan_cut });
             }
 
             let mut top_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
@@ -793,7 +812,12 @@ async fn list_path_raw_inner(
                                     continue;
                                 }
                                 // eof
-                                limit_reached |= r.limit_reached() == Some(true);
+                                if r.limit_reached() == Some(true) {
+                                    limit_reached = true;
+                                    scan_cut = Some(last_scanned_key.clone().ok_or_else(|| {
+                                        DiskError::other("bounded listing reader ended without a scan boundary")
+                                    })?);
+                                }
                                 at_eof += 1;
                                 // warn!("list_path_raw: peek eof, disk: {}", i);
                                 continue;
@@ -867,7 +891,7 @@ async fn list_path_raw_inner(
                             *r = MetacacheReader::new(detached_rd);
                             continue;
                         }
-                        PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached }),
+                        PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
                     }
                 };
 
@@ -926,7 +950,7 @@ async fn list_path_raw_inner(
                 if let Some(finished_fn) = opts.finished.as_ref() {
                     tokio::select! {
                         biased;
-                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
+                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
                         _ = finished_fn(&errs) => {}
                     }
                 }
@@ -976,6 +1000,14 @@ async fn list_path_raw_inner(
                 return Err(classify_listing_quorum_failure(&failures));
             }
 
+            // The next head from a bounded reader is unknown. The previous
+            // globally consumed group is a safe cut; consuming another reader's
+            // later head would turn this absence of evidence into a skipped key.
+            // Check quorum failures above before accepting the bounded result.
+            if limit_reached {
+                break;
+            }
+
             // Break if all at EOF or error.
             if at_eof + has_err == readers.len() {
                 if has_err > 0
@@ -983,7 +1015,7 @@ async fn list_path_raw_inner(
                 {
                     tokio::select! {
                         biased;
-                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
+                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
                         _ = finished_fn(&errs) => {}
                     }
                 }
@@ -994,6 +1026,7 @@ async fn list_path_raw_inner(
                 break;
             }
 
+            last_scanned_key = Some(current.name.clone());
             if agree == readers.len() && !opts.preserve_replica_metadata {
                 for r in readers.iter_mut() {
                     if !r.skip_current() {
@@ -1005,7 +1038,7 @@ async fn list_path_raw_inner(
                     // warn!("list_path_raw: agreed_fn start, current: {:?}", &current.name);
                     tokio::select! {
                         biased;
-                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
+                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
                         result = agreed_fn(current) => result?
                     }
                     // warn!("list_path_raw: agreed_fn done");
@@ -1029,13 +1062,14 @@ async fn list_path_raw_inner(
             if let Some(partial_fn) = opts.partial.as_ref() {
                 tokio::select! {
                     biased;
-                    _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
+                    _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
                     _ = partial_fn(MetaCacheEntries(top_entries), &errs) => {}
                 }
             }
         }
-        Ok(ListPathRawOutcome { limit_reached })
-    });
+        Ok(ListPathRawOutcome { limit_reached, scan_cut })
+    };
+    revjobs.spawn(merge.in_current_span());
 
     let merge_started = std::time::Instant::now();
     let Some(revjob) = revjobs.join_next().await else {
@@ -1199,6 +1233,24 @@ mod tests {
         assert!(!options.skip_total_timeout);
     }
 
+    #[test]
+    fn cloned_reader_options_preserve_hidden_scan_budget_and_pending_prefix() {
+        let request = ListPathRawOptions {
+            bucket: "versioned".to_owned(),
+            hidden_entry_budget: 100,
+            pending_prefix: Some("pending/".to_owned()),
+            forward_to: Some("pending/0099".to_owned()),
+            ..Default::default()
+        };
+        // Both walks are built once from the shared options; the per-job clone
+        // must carry every field that the walk derives from the request.
+        let job = request.clone();
+        let walk = walk_dir_options(&job);
+        assert_eq!(walk.hidden_entry_budget, 100);
+        assert_eq!(walk.pending_prefix.as_deref(), request.pending_prefix.as_deref());
+        assert_eq!(walk.forward_to.as_deref(), Some("pending/0099"));
+    }
+
     #[tokio::test]
     async fn list_path_raw_empty_disks_returns_read_quorum() {
         let err = list_path_raw(CancellationToken::new(), ListPathRawOptions::default())
@@ -1249,6 +1301,163 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn list_path_raw_fanout_retains_only_linear_disk_references() {
+        let root = tempfile::tempdir().expect("create disk root");
+        let endpoint = crate::disk::endpoint::Endpoint::try_from(root.path().to_str().expect("UTF-8 disk path"))
+            .expect("parse disk endpoint");
+        let local = Arc::new(
+            crate::disk::local::LocalDisk::new(&endpoint, false)
+                .await
+                .expect("open local disk"),
+        );
+        let wrapper = crate::disk::disk_store::LocalDiskWrapper::new(local, false);
+        let disk = Arc::new(crate::disk::Disk::Local(Box::new(wrapper)));
+        let baseline = Arc::strong_count(&disk);
+
+        for primary_count in [1, 4, 16] {
+            for fallback_count in [0, 2] {
+                let entered: Vec<_> = (0..primary_count).map(|_| Arc::new(tokio::sync::Notify::new())).collect();
+                let stopped: Vec<_> = (0..primary_count).map(|_| Arc::new(tokio::sync::Notify::new())).collect();
+                let behaviors = entered
+                    .iter()
+                    .zip(&stopped)
+                    .map(|(entered, stopped)| TestReaderBehavior::StallWithSignals {
+                        entered: entered.clone(),
+                        stopped: stopped.clone(),
+                    })
+                    .collect();
+                let cancel = CancellationToken::new();
+                let listing = tokio::spawn(list_path_raw(
+                    cancel.clone(),
+                    ListPathRawOptions {
+                        disks: vec![Some(disk.clone()); primary_count],
+                        fallback_disks: vec![Some(disk.clone()); fallback_count],
+                        min_disks: primary_count,
+                        test_reader_behaviors: behaviors,
+                        peek_timeout: Some(Duration::from_secs(60)),
+                        ..Default::default()
+                    },
+                ));
+                for signal in &entered {
+                    timeout(Duration::from_secs(2), signal.notified())
+                        .await
+                        .expect("every producer must enter the real fanout before counting references");
+                }
+
+                // The merge owns the configured primary/fallback slots, each producer
+                // owns its primary, and the shared fallback queue owns each fallback.
+                let expected = baseline + 2 * (primary_count + fallback_count);
+                let retained = Arc::strong_count(&disk);
+                cancel.cancel();
+                timeout(Duration::from_secs(2), listing)
+                    .await
+                    .expect("cancellation must stop the listing")
+                    .expect("listing task must join")
+                    .expect("external cancellation must finish cleanly");
+                for signal in &stopped {
+                    timeout(Duration::from_secs(2), signal.notified())
+                        .await
+                        .expect("cancellation must drop every producer");
+                }
+                assert_eq!(Arc::strong_count(&disk), baseline, "cancellation must release every fanout reference");
+                assert_eq!(
+                    retained, expected,
+                    "fanout must retain linear references for {primary_count} primary and {fallback_count} fallback slots"
+                );
+            }
+        }
+    }
+
+    #[tokio::test(flavor = "current_thread")]
+    async fn list_path_raw_producer_and_merge_preserve_request_span() {
+        use tracing_subscriber::{Layer, layer::SubscriberExt};
+
+        struct ProducerSpan(Arc<AtomicBool>);
+        #[derive(Default)]
+        struct StateField(bool);
+        impl tracing::field::Visit for StateField {
+            fn record_debug(&mut self, _: &tracing::field::Field, _: &dyn std::fmt::Debug) {}
+
+            fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
+                self.0 |= field.name() == "state" && value == "walk_dir_missing_path";
+            }
+        }
+        impl<S> Layer<S> for ProducerSpan
+        where
+            S: tracing::Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a>,
+        {
+            fn on_event(&self, event: &tracing::Event<'_>, context: tracing_subscriber::layer::Context<'_, S>) {
+                let mut state = StateField::default();
+                event.record(&mut state);
+                if state.0
+                    && context
+                        .lookup_current()
+                        .is_some_and(|span| span.name() == "flink_list_request")
+                {
+                    self.0.store(true, std::sync::atomic::Ordering::Release);
+                }
+            }
+        }
+
+        // The subscriber is thread-local; keep the spawned tasks on this runtime thread.
+        let producer_observed = Arc::new(AtomicBool::new(false));
+        let subscriber = tracing_subscriber::registry().with(ProducerSpan(producer_observed.clone()));
+        let _subscriber = tracing::subscriber::set_default(subscriber);
+        let request = tracing::info_span!("flink_list_request", request_id = "list-span-regression");
+        let root = tempfile::tempdir().expect("create disk root");
+        let endpoint = crate::disk::endpoint::Endpoint::try_from(root.path().to_str().expect("UTF-8 disk path"))
+            .expect("parse disk endpoint");
+        let local = Arc::new(
+            crate::disk::local::LocalDisk::new(&endpoint, false)
+                .await
+                .expect("open local disk"),
+        );
+        let wrapper = crate::disk::disk_store::LocalDiskWrapper::new(local, false);
+        let disk = Arc::new(crate::disk::Disk::Local(Box::new(wrapper)));
+        let error = list_path_raw(
+            CancellationToken::new(),
+            ListPathRawOptions {
+                disks: vec![Some(disk)],
+                bucket: "missing-bucket".to_owned(),
+                min_disks: 1,
+                ..Default::default()
+            },
+        )
+        .instrument(request.clone())
+        .await
+        .expect_err("missing bucket must fail the real disk producer");
+        assert_eq!(error, DiskError::VolumeNotFound);
+        assert!(
+            producer_observed.load(std::sync::atomic::Ordering::Acquire),
+            "disk error must retain the request span"
+        );
+
+        let merge_observed = Arc::new(AtomicBool::new(false));
+        let merge_sink = merge_observed.clone();
+        let request_id = request.id();
+        list_path_raw(
+            CancellationToken::new(),
+            ListPathRawOptions {
+                disks: vec![None],
+                min_disks: 1,
+                test_reader_behaviors: vec![TestReaderBehavior::Entries(vec![fallback_test_entry()])],
+                agreed: Some(Box::new(move |_| {
+                    merge_sink.store(tracing::Span::current().id() == request_id, std::sync::atomic::Ordering::Release);
+                    Box::pin(async { Ok(()) })
+                })),
+                ..Default::default()
+            },
+        )
+        .instrument(request)
+        .await
+        .expect("merge should deliver the test entry");
+        assert!(
+            merge_observed.load(std::sync::atomic::Ordering::Acquire),
+            "merge callback must retain the request span"
+        );
+    }
+
     #[tokio::test]
     async fn scanner_joinset_walk_uses_the_local_disk_background_gate() {
         let root = tempfile::tempdir().expect("local disk root should be created");
@@ -1316,6 +1525,7 @@ mod tests {
             outcome.limit_reached,
             "a reader whose trailer reports a limit stop must mark the merged stream as possibly incomplete"
         );
+        assert_eq!(outcome.scan_cut.as_deref(), Some("bucket/object"));
     }
 
     #[tokio::test]
@@ -1692,6 +1902,42 @@ mod tests {
         assert_eq!(seen.lock().expect("seen mutex poisoned").as_slice(), &["bucket/object".to_string()]);
     }
 
+    #[tokio::test]
+    async fn list_read_budget_is_retained_by_raw_producer_until_cancellation() {
+        let semaphore = Arc::new(tokio::sync::Semaphore::new(4));
+        let grant = Arc::new(semaphore.clone().acquire_many_owned(4).await.expect("listing grant"));
+        let weak_grant = Arc::downgrade(&grant);
+        let entered = Arc::new(tokio::sync::Notify::new());
+        let stopped = Arc::new(tokio::sync::Notify::new());
+        let worker = tokio::spawn(list_path_raw(
+            CancellationToken::new(),
+            ListPathRawOptions {
+                read_budget: Some(grant),
+                disks: vec![None],
+                min_disks: 1,
+                test_reader_behaviors: vec![TestReaderBehavior::StallWithSignals {
+                    entered: entered.clone(),
+                    stopped: stopped.clone(),
+                }],
+                ..Default::default()
+            },
+        ));
+        timeout(Duration::from_secs(5), entered.notified())
+            .await
+            .expect("producer entered");
+        assert!(weak_grant.strong_count() >= 2, "producer must retain a grant independently of the parent");
+        assert_eq!(semaphore.available_permits(), 0);
+        worker.abort();
+        assert!(worker.await.expect_err("parent cancelled").is_cancelled());
+        timeout(Duration::from_secs(5), stopped.notified())
+            .await
+            .expect("producer stopped");
+        let _next = timeout(Duration::from_secs(5), semaphore.acquire_many(4))
+            .await
+            .expect("cancelled listing releases all permits")
+            .expect("budget stays open");
+    }
+
     #[tokio::test]
     async fn list_path_raw_aborts_unresponsive_producer_after_quorum_eof() {
         let result = timeout(

crates/ecstore/src/cluster/rpc/http_auth.rs

@@ -32,7 +32,6 @@ use crate::storage_api_contracts::internode::{
     PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_CAPABILITY_VERSION,
 };
 
-use hmac::{Hmac, KeyInit, Mac};
 use http::uri::Authority;
 use http::{HeaderMap, HeaderValue, Method, Uri};
 #[cfg(test)]
@@ -47,6 +46,7 @@ use rustfs_io_metrics::internode_metrics::{
     global_internode_metrics,
 };
 use rustfs_object_data_cache::{MemoryBasis, resolve_effective_memory};
+use rustfs_utils::crypto::{HmacSha256, KeyInit, Mac};
 use rustfs_utils::get_env_bool;
 use sha2::Digest as _;
 use sha2::Sha256;
@@ -58,8 +58,6 @@ use time::OffsetDateTime;
 use tracing::{error, info, warn};
 use uuid::Uuid;
 
-type HmacSha256 = Hmac<Sha256>;
-
 const SIGNATURE_HEADER: &str = "x-rustfs-signature";
 pub const RPC_BACKGROUND_ADMISSION_HEADER: &str = "x-rustfs-rpc-background-admission";
 const RPC_BACKGROUND_ADMISSION_V1: &str = "v1";
@@ -368,10 +366,84 @@ fn resolve_replay_cache_capacity() -> usize {
     decision.capacity
 }
 
+const RPC_NONCE_EXPIRATIONS_PER_PAGE: usize = 1_024;
+
+struct RpcNonceExpiration {
+    nonce: Uuid,
+    expiry_offset_ns: u32,
+    valid_until_offset_secs: i32,
+}
+
+struct RpcNonceExpirationPage {
+    expires_at: Instant,
+    valid_until: i64,
+    entries: VecDeque<RpcNonceExpiration>,
+}
+
+#[derive(Default)]
+struct RpcNonceExpirations {
+    pages: VecDeque<RpcNonceExpirationPage>,
+}
+
+impl RpcNonceExpirations {
+    fn push_back(&mut self, expires_at: Instant, valid_until: i64, nonce: Uuid) {
+        // Use exact offsets and start another page if either offset is too large.
+        // Fully drained pages can be freed while newer nonces are still valid.
+        if let Some(page) = self.pages.back_mut()
+            && page.entries.len() < RPC_NONCE_EXPIRATIONS_PER_PAGE
+            && let Some(offset) = expires_at.checked_duration_since(page.expires_at)
+            && let Ok(expiry_offset_ns) = u32::try_from(offset.as_nanos())
+            && let Some(offset) = valid_until.checked_sub(page.valid_until)
+            && let Ok(valid_until_offset_secs) = i32::try_from(offset)
+        {
+            page.entries.push_back(RpcNonceExpiration {
+                nonce,
+                expiry_offset_ns,
+                valid_until_offset_secs,
+            });
+        } else {
+            self.pages.push_back(RpcNonceExpirationPage {
+                expires_at,
+                valid_until,
+                entries: VecDeque::from([RpcNonceExpiration {
+                    nonce,
+                    expiry_offset_ns: 0,
+                    valid_until_offset_secs: 0,
+                }]),
+            });
+        }
+    }
+
+    fn pop_expired(&mut self, now: Instant, wall_time: i64) -> Option<Uuid> {
+        let page = self.pages.front_mut()?;
+        let entry = page.entries.front()?;
+        let elapsed = now.checked_duration_since(page.expires_at)?;
+        let valid_until = page.valid_until + i64::from(entry.valid_until_offset_secs);
+        if elapsed <= Duration::from_nanos(u64::from(entry.expiry_offset_ns)) || wall_time <= valid_until {
+            return None;
+        }
+
+        let nonce = entry.nonce;
+        page.entries.pop_front();
+        if page.entries.is_empty() {
+            self.pages.pop_front();
+            if self.pages.is_empty() {
+                self.pages.shrink_to_fit();
+            }
+        }
+        Some(nonce)
+    }
+
+    #[cfg(test)]
+    fn capacity(&self) -> usize {
+        self.pages.iter().map(|page| page.entries.capacity()).sum()
+    }
+}
+
 #[derive(Default)]
 struct RpcNonceCache {
     nonces: HashSet<Uuid>,
-    expirations: VecDeque<(Instant, i64, Uuid)>,
+    expirations: RpcNonceExpirations,
     max_wall_time: i64,
 }
 
@@ -427,16 +499,13 @@ fn publish_nonce_cache_metrics(metrics: Option<RpcNonceCacheMetrics<'_>>) {
 impl RpcNonceCache {
     fn remove_expired(&mut self, now: Instant, wall_time: i64) -> usize {
         let mut removed = 0;
-        while matches!(
-            self.expirations.front(),
-            Some((expires_at, valid_until, _)) if *expires_at < now && *valid_until < wall_time
-        ) {
-            let Some((_, _, nonce)) = self.expirations.pop_front() else {
-                break;
-            };
+        while let Some(nonce) = self.expirations.pop_expired(now, wall_time) {
             self.nonces.remove(&nonce);
             removed += 1;
         }
+        if removed > 0 && self.nonces.is_empty() {
+            self.nonces.shrink_to_fit();
+        }
         removed
     }
 
@@ -470,7 +539,7 @@ impl RpcNonceCache {
         }
         self.nonces.insert(record.nonce);
         self.expirations
-            .push_back((record.expires_at, record.signed_at.saturating_add(SIGNATURE_VALID_DURATION), record.nonce));
+            .push_back(record.expires_at, record.signed_at.saturating_add(SIGNATURE_VALID_DURATION), record.nonce);
         (
             Ok(()),
             Some(RpcNonceCacheMetrics {
@@ -1590,7 +1659,13 @@ fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &Hea
     if boot_epoch != tonic_rpc_boot_epoch() {
         return Err(std::io::Error::other("RPC boot epoch is stale"));
     }
-    check_and_record_tonic_nonce(nonce, signed_at, path)
+    check_and_record_tonic_nonce(nonce, signed_at, path)?;
+    // A body-bound request also carries a usable v2 signature. Consume both
+    // nonces so stripping the replay-scope headers cannot authorize it again.
+    if content_sha256 != UNSIGNED_PAYLOAD {
+        verify_tonic_rpc_signature_with_strictness(audience, path, headers, true)?;
+    }
+    Ok(())
 }
 
 /// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
@@ -2581,6 +2656,43 @@ mod tests {
         assert_eq!(path_error.to_string(), "Invalid RPC replay scope signature");
     }
 
+    #[test]
+    fn replay_scope_consumes_the_body_bound_v2_nonce() {
+        ensure_test_rpc_secret();
+        let audience = "node-a:9000";
+        let path = "/node_service.NodeService/RecoverDeleteTransaction";
+        let digest = "a".repeat(64);
+        for v3_first in [true, false] {
+            let mut headers =
+                gen_tonic_signature_headers(audience, "node_service.NodeService", "RecoverDeleteTransaction", Some(&digest))
+                    .expect("body-bound v2 headers should build");
+            let timestamp = headers[TIMESTAMP_HEADER].to_str().expect("timestamp").to_string();
+            headers.extend(
+                gen_tonic_replay_scope_headers(audience, path, &timestamp, &digest, tonic_rpc_boot_epoch())
+                    .expect("replay-scope headers should build"),
+            );
+            let mut downgraded = headers.clone();
+            for header in [
+                RPC_REPLAY_SCOPE_VERSION_HEADER,
+                RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
+                RPC_REPLAY_SCOPE_NONCE_HEADER,
+                RPC_BOOT_EPOCH_HEADER,
+            ] {
+                downgraded.remove(header);
+            }
+            let (first, replay) = if v3_first {
+                (&headers, &downgraded)
+            } else {
+                (&downgraded, &headers)
+            };
+            verify_tonic_rpc_signature_with_policy(audience, path, first, true, false, false)
+                .expect("the first delivery must be accepted");
+            let error = verify_tonic_rpc_signature_with_policy(audience, path, replay, true, false, false)
+                .expect_err("a mutation cannot replay through the other signature generation");
+            assert_eq!(error.to_string(), "RPC request replay detected");
+        }
+    }
+
     #[test]
     fn replay_scope_rejects_partial_metadata_and_stale_epoch_without_fallback() {
         ensure_test_rpc_secret();
@@ -3137,6 +3249,51 @@ mod tests {
         assert!(cache.nonces.contains(&nonce_b));
     }
 
+    #[test]
+    fn nonce_cache_releases_expiration_storage_after_a_burst() {
+        let start = Instant::now();
+        let expiry = start + REPLAY_CACHE_RETENTION;
+        let mut cache = RpcNonceCache::default();
+
+        for nonce in 1..=4_096 {
+            let record = test_nonce_record(Uuid::from_u128(nonce), 100, start, 100, expiry, 4_096);
+            cache.check_and_record(record).0.expect("first use of nonce");
+        }
+        assert!(cache.expirations.capacity() >= 4_096);
+        assert_eq!(cache.remove_expired(expiry, 701), 0);
+        assert_eq!(cache.remove_expired(expiry + Duration::from_nanos(1), 400), 0);
+        assert_eq!(cache.remove_expired(expiry + Duration::from_nanos(1), 701), 4_096);
+        assert!(cache.nonces.is_empty());
+        assert_eq!(cache.nonces.capacity(), 0);
+        assert_eq!(cache.expirations.capacity(), 0, "expired entries must release their expiration storage");
+    }
+
+    #[test]
+    fn nonce_expirations_preserve_exact_deadlines_across_pages() {
+        let expiry = Instant::now();
+        let first = Uuid::from_u128(1);
+        let second = Uuid::from_u128(2);
+        let third = Uuid::from_u128(3);
+        let fourth = Uuid::from_u128(4);
+        let mut queue = super::RpcNonceExpirations::default();
+        let next = expiry + Duration::from_nanos(1);
+        let later = expiry + Duration::from_secs(5);
+
+        queue.push_back(expiry, 501, first);
+        queue.push_back(next, 500, second);
+        queue.push_back(later, 505, third);
+        queue.push_back(later, i64::MAX, fourth);
+        assert_eq!(queue.pages.len(), 3);
+        assert_eq!(queue.pop_expired(next, 501), None);
+        assert_eq!(queue.pop_expired(next, 502), Some(first));
+        assert_eq!(queue.pop_expired(next, 502), None);
+        assert_eq!(queue.pop_expired(later, 502), Some(second));
+        assert_eq!(queue.pages.len(), 2);
+        assert_eq!(queue.pop_expired(later, 506), None);
+        assert_eq!(queue.pop_expired(later + Duration::from_nanos(1), 506), Some(third));
+        assert_eq!(queue.pop_expired(later + Duration::from_nanos(1), i64::MAX), None);
+    }
+
     #[test]
     fn nonce_cache_metrics_mark_successful_records_only() {
         let now = Instant::now();

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -2014,9 +2014,21 @@ impl PeerRestClient {
             }
             result
         })
-        .await
-        .unwrap_or_else(|_| Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))));
-        self.finalize_result(result).await
+        .await;
+        self.settle_bucket_metadata_reload(bucket, result).await
+    }
+
+    async fn settle_bucket_metadata_reload(
+        &self,
+        bucket: &str,
+        result: std::result::Result<Result<()>, tokio::time::error::Elapsed>,
+    ) -> Result<()> {
+        match result {
+            Ok(result) => self.finalize_result(result).await,
+            // A local operation deadline does not establish that the peer is
+            // unreachable. In particular, it must not gate IAM revocations.
+            Err(_) => Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))),
+        }
     }
 
     async fn load_bucket_metadata_once(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
@@ -4753,6 +4765,31 @@ mod tests {
         assert!(client.offline.load(Ordering::Acquire));
     }
 
+    #[tokio::test]
+    async fn bucket_metadata_deadline_does_not_gate_healthy_peer_notifications() {
+        let client = test_peer_client();
+        let elapsed = tokio::time::timeout(std::time::Duration::ZERO, std::future::pending::<Result<()>>()).await;
+        let error = client
+            .settle_bucket_metadata_reload("slow-policy", elapsed)
+            .await
+            .expect_err("local budget expires");
+        assert!(error.to_string().contains("timed out"));
+        assert!(
+            !client.offline.load(Ordering::Acquire),
+            "local work must not suppress subsequent IAM RPCs"
+        );
+        client
+            .settle_bucket_metadata_reload("healthy", Ok(Ok(())))
+            .await
+            .expect("peer reply");
+        let error = client
+            .settle_bucket_metadata_reload("unreachable", Ok(Err(Error::from(tonic::Status::unavailable("connection refused")))))
+            .await
+            .expect_err("real transport failure");
+        assert!(error.to_string().contains("connection refused"));
+        assert!(client.offline.load(Ordering::Acquire), "transport failures must still enter recovery");
+    }
+
     #[tokio::test]
     async fn peer_rest_client_finalize_result_keeps_online_for_business_errors() {
         let client = test_peer_client();

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -27,13 +27,13 @@ use crate::disk::multipart_source::{
     MULTIPART_SOURCE_PROTOCOL_VERSION, MultipartSourceOperation, MultipartSourceRequest, MultipartSourceResponse,
 };
 use crate::disk::{
-    BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, ConditionalFileUpdate, DELETE_RECOVERY_PROTOCOL_VERSION,
-    DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest, DeleteRecoveryInspection,
-    DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation,
-    DiskOption, FileInfoVersions, FileReader, FileWriter, GuardedDeleteVersionsPayload, GuardedFileInfoVersions,
-    PartTransactionAction, PoolMetaNativeRecoveryRequest, PoolMetaNativeRecoveryResponse, ReadMultipleReq, ReadMultipleResp,
-    ReadOptions, RenameDataResp, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
-    batch_read_version_one_by_one,
+    BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION, BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, ConditionalFileUpdate,
+    DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest,
+    DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DiskAPI, DiskInfo,
+    DiskInfoOptions, DiskLocation, DiskOption, FileInfoVersions, FileReader, FileWriter, GuardedDeleteVersionsPayload,
+    GuardedFileInfoVersions, PartTransactionAction, PoolMetaNativeRecoveryRequest, PoolMetaNativeRecoveryResponse,
+    ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo,
+    WalkDirOptions, batch_read_version_one_by_one,
     disk_store::{
         DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, SKIP_IF_SUCCESS_BEFORE,
         get_drive_active_check_interval, get_drive_active_check_timeout, get_drive_disk_info_timeout, get_drive_list_dir_timeout,
@@ -996,21 +996,32 @@ fn internode_idempotent_read_retries() -> usize {
 static PREWARMED_PEERS: std::sync::LazyLock<std::sync::Mutex<std::collections::HashSet<String>>> =
     std::sync::LazyLock::new(|| std::sync::Mutex::new(std::collections::HashSet::new()));
 
+fn claim_control_channel_prewarm(addr: &str) -> bool {
+    let Ok(mut prewarmed) = PREWARMED_PEERS.lock() else {
+        return false;
+    };
+    prewarmed.insert(addr.to_owned())
+}
+
+fn release_control_channel_prewarm(addr: &str) {
+    if let Ok(mut prewarmed) = PREWARMED_PEERS.lock() {
+        prewarmed.remove(addr);
+    }
+}
+
 /// Best-effort background prewarm of a peer's control channel (grpc-optimization P3-1). Deduped per
-/// peer; a dial failure just falls through to the existing lazy connect and recovery monitor.
+/// peer; a dial failure releases the marker so a later disk construction can retry the prewarm.
 fn spawn_control_channel_prewarm(addr: String) {
-    {
-        let Ok(mut prewarmed) = PREWARMED_PEERS.lock() else {
-            return;
-        };
-        if !prewarmed.insert(addr.clone()) {
-            return;
-        }
+    if !claim_control_channel_prewarm(&addr) {
+        return;
     }
     tokio::spawn(async move {
         match node_service_time_out_client_no_auth(&addr).await {
             Ok(_) => debug!(addr = %addr, "internode control channel prewarmed"),
-            Err(err) => debug!(addr = %addr, error = %err, "internode control channel prewarm failed (best-effort)"),
+            Err(err) => {
+                release_control_channel_prewarm(&addr);
+                debug!(addr = %addr, error = %err, "internode control channel prewarm failed (best-effort)");
+            }
         }
     });
 }
@@ -1021,6 +1032,7 @@ impl RemoteDisk {
         volume: &str,
         payload: Vec<u8>,
         probe_only: bool,
+        bounded_prepare: bool,
     ) -> Result<DeleteVersionsResponse> {
         if payload.len() > rustfs_protos::DELETE_RECOVERY_MAX_PAYLOAD_BYTES {
             return Err(DiskError::FileCorrupt);
@@ -1045,6 +1057,8 @@ impl RemoteDisk {
                     .map_err(|status| {
                         if status.code() == Code::Unimplemented {
                             DiskError::GuardedDeleteUnsupported
+                        } else if bounded_prepare && status.code() == Code::InvalidArgument {
+                            DiskError::UnsupportedDisk
                         } else {
                             status.into()
                         }
@@ -1078,11 +1092,22 @@ impl RemoteDisk {
         volume: &str,
         operation: DeleteRecoveryRpcOperation,
     ) -> Result<DeleteRecoveryRpcReply> {
+        let protocol_version = match &operation {
+            DeleteRecoveryRpcOperation::ApplyVersion { opts, .. } | DeleteRecoveryRpcOperation::ApplyVersions { opts, .. }
+                if opts.requires_bounded_delete_recovery =>
+            {
+                BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION
+            }
+            _ => DELETE_RECOVERY_PROTOCOL_VERSION,
+        };
         let payload = rmp_serde::to_vec_named(&operation)?;
         if payload.len() > operation.max_wire_payload_bytes() {
             return Err(Error::other("delete recovery operation exceeds payload limit"));
         }
-        let settle_timeout = if matches!(&operation, DeleteRecoveryRpcOperation::Settle { .. }) {
+        let settle_timeout = if matches!(
+            &operation,
+            DeleteRecoveryRpcOperation::Settle { .. } | DeleteRecoveryRpcOperation::SettleBounded(_)
+        ) {
             Some(crate::disk::get_delete_recovery_timeout()?)
         } else {
             None
@@ -1105,7 +1130,7 @@ impl RemoteDisk {
                 let mut request = Request::new(RecoverDeleteTransactionRequest {
                     disk: self.endpoint.to_string(),
                     volume: volume.to_owned(),
-                    protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
+                    protocol_version,
                     operation: payload.into(),
                 });
                 let canonical_body = rustfs_protos::canonical_recover_delete_transaction_request_body(request.get_ref())
@@ -1123,7 +1148,7 @@ impl RemoteDisk {
                         }
                     })?
                     .into_inner();
-                if response.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION {
+                if response.protocol_version != protocol_version {
                     return Err(DiskError::UnsupportedDisk);
                 }
                 if !response.success {
@@ -1637,6 +1662,7 @@ impl RemoteDisk {
         let initial_probe_ok = Self::perform_connectivity_check(&addr).await.is_ok();
         if initial_probe_ok {
             health.record_operation_success(&endpoint, "connectivity_probe_success");
+            rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
         } else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
             warn!(
                 event = EVENT_REMOTE_DISK_HEALTH,
@@ -1708,6 +1734,7 @@ impl RemoteDisk {
                     // Perform basic connectivity check
                     if Self::perform_connectivity_check(&addr).await.is_ok() {
                         health.record_operation_success(&endpoint, "connectivity_probe_success");
+                        rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
                     } else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
                         warn!(
                             event = EVENT_REMOTE_DISK_HEALTH,
@@ -1778,6 +1805,7 @@ impl RemoteDisk {
                 }
                 _ = interval.tick() => {
                     if Self::perform_recovery_probe(&addr, &endpoint).await.is_ok() {
+                        rustfs_io_metrics::internode_metrics::record_peer_reachable(&addr);
                         let became_online = health.mark_recovery_success(&endpoint, "disk_info_probe_success");
                         debug!(
                             event = EVENT_REMOTE_DISK_HEALTH,
@@ -2009,6 +2037,7 @@ impl RemoteDisk {
             let operation_result = operation().await;
             if operation_result.is_ok() {
                 self.health.record_operation_success(&self.endpoint, "operation_success");
+                rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
             }
             self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
                 .await;
@@ -2023,6 +2052,7 @@ impl RemoteDisk {
                 // Log success; the waiting guard balances every exit path.
                 if operation_result.is_ok() {
                     self.health.record_operation_success(&self.endpoint, "operation_success");
+                    rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
                 }
                 self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
                     .await;
@@ -2711,8 +2741,56 @@ impl RemoteDisk {
 
 #[async_trait::async_trait]
 impl DiskAPI for RemoteDisk {
+    async fn list_multipart_index_roots(
+        &self,
+        bucket: &str,
+        marker: Option<&str>,
+    ) -> Result<crate::disk::multipart_index::MultipartIndexPage> {
+        use crate::disk::multipart_index::{MULTIPART_INDEX_RPC_MAX_BYTES, MultipartIndexRpcReply, MultipartIndexRpcRequest};
+        let expected_disk = self
+            .id
+            .lock()
+            .await
+            .filter(|id| !id.is_nil())
+            .ok_or(DiskError::DiskNotFound)?;
+        let query = MultipartIndexRpcRequest::new(bucket, marker);
+        let endpoint = self.endpoint.to_string();
+        let canonical = query.canonical_request(&endpoint)?;
+        let opts = serde_json::to_string(&query)?;
+        self.execute_with_timeout(
+            || async {
+                let mut client = self.get_client().await?;
+                let mut request = Request::new(DiskInfoRequest {
+                    disk: endpoint.clone(),
+                    opts: opts.clone(),
+                });
+                crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
+                let response = client.disk_info(request).await?.into_inner();
+                if !response.success {
+                    return Err(response.error.unwrap_or_default().into());
+                }
+                if response.disk_info.len() > MULTIPART_INDEX_RPC_MAX_BYTES {
+                    return Err(DiskError::FileCorrupt);
+                }
+                // An old server ignores the additive query and returns ordinary DiskInfo.
+                // It cannot satisfy the dedicated reply schema or the fresh challenge.
+                let reply: MultipartIndexRpcReply =
+                    serde_json::from_str(&response.disk_info).map_err(|_| DiskError::MethodNotAllowed)?;
+                reply.validate(&query, expected_disk)?;
+                crate::cluster::rpc::verify_tonic_rpc_response_proof(
+                    &query.canonical_response(&endpoint, &reply.page)?,
+                    &reply.proof,
+                )
+                .map_err(Error::other)?;
+                Ok(reply.page)
+            },
+            get_drive_disk_info_timeout(),
+        )
+        .await
+    }
+
     async fn supports_guarded_delete_versions(&self) -> Result<()> {
-        let response = self.guarded_delete_versions_rpc("", Vec::new(), true).await?;
+        let response = self.guarded_delete_versions_rpc("", Vec::new(), true, false).await?;
         if !response.errors.is_empty() || !response.item_errors.is_empty() {
             return Err(DiskError::FileCorrupt);
         }
@@ -2725,7 +2803,12 @@ impl DiskAPI for RemoteDisk {
         opts: DeleteOptions,
     ) -> Vec<Option<DiskError>> {
         let count = versions.len();
-        let payload = GuardedDeleteVersionsPayload { versions, opts };
+        let bounded_prepare = opts.requires_bounded_delete_recovery;
+        let payload = GuardedDeleteVersionsPayload {
+            versions,
+            opts,
+            bounded_recovery_protocol: bounded_prepare.then_some(BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION),
+        };
         if let Err(error) = payload.validate() {
             return vec![Some(error); count];
         }
@@ -2733,7 +2816,10 @@ impl DiskAPI for RemoteDisk {
             Ok(encoded) => encoded,
             Err(error) => return vec![Some(error.into()); count],
         };
-        match self.guarded_delete_versions_rpc(volume, encoded, false).await {
+        match self
+            .guarded_delete_versions_rpc(volume, encoded, false, bounded_prepare)
+            .await
+        {
             Ok(response) => {
                 // New guarded peers must return typed errors, never lossy strings.
                 if response.item_errors.len() != count {
@@ -3624,7 +3710,13 @@ impl DiskAPI for RemoteDisk {
                         );
                     }
                     let rpc_started = read_version_stage_timer(batch_read_version_attribution_enabled);
-                    let response = match client.batch_read_version(request).await {
+                    let response_result = client.batch_read_version(request).await;
+                    if let Some(rpc_started) = rpc_started {
+                        crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_duration(
+                            rpc_started.elapsed(),
+                        );
+                    }
+                    let response = match response_result {
                         Ok(response) => {
                             record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
                             response.into_inner()
@@ -4294,6 +4386,38 @@ impl DiskAPI for RemoteDisk {
         }
     }
 
+    async fn discover_delete_transactions_bounded(
+        &self,
+        volume: &str,
+        object: &str,
+        max_entries: u32,
+    ) -> Result<crate::disk::BoundedDeleteRecoveryDiscovery> {
+        // Older peers reject this variant. Do not retry through the unbounded protocol;
+        // they are reported as unsupported participants (rolling-upgrade contract).
+        match self
+            .recover_delete_transaction(
+                volume,
+                DeleteRecoveryRpcOperation::DiscoverBounded {
+                    object: object.to_owned(),
+                    max_entries,
+                },
+            )
+            .await
+        {
+            Ok(DeleteRecoveryRpcReply::DiscoveredBounded(page)) => Ok(page),
+            Ok(_) => Err(Error::FileCorrupt),
+            Err(error)
+                if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
+                    || matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
+                    .is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
+            {
+                // No paginated fallback through the unbounded protocol; the peer
+                // is an unsupported participant until it is upgraded.
+                Err(Error::UnsupportedDisk)
+            }
+            Err(error) => Err(error),
+        }
+    }
     async fn discover_delete_transactions(
         &self,
         volume: &str,
@@ -4329,6 +4453,62 @@ impl DiskAPI for RemoteDisk {
             _ => Err(Error::other("unexpected delete discovery reply")),
         }
     }
+    async fn settle_delete_transaction_bounded(
+        &self,
+        volume: &str,
+        request: crate::disk::BoundedDeleteRecoverySettlement,
+        _namespace_owner: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<crate::disk::BoundedDeleteRecoverySettlementReply> {
+        match self
+            .recover_delete_transaction(volume, DeleteRecoveryRpcOperation::SettleBounded(request))
+            .await
+        {
+            Ok(DeleteRecoveryRpcReply::SettledBounded(reply)) => Ok(reply),
+            Ok(_) => Err(Error::FileCorrupt),
+            Err(error)
+                if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
+                    || matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
+                    .is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
+            {
+                // No paginated fallback through the unbounded protocol; the peer
+                // is an unsupported participant until it is upgraded.
+                Err(Error::UnsupportedDisk)
+            }
+            Err(error) => Err(error),
+        }
+    }
+    async fn inspect_delete_transaction_bounded(
+        &self,
+        volume: &str,
+        object: &str,
+        transaction: Uuid,
+        max_entries: u32,
+    ) -> Result<crate::disk::BoundedDeleteRecoveryInspection> {
+        match self
+            .recover_delete_transaction(
+                volume,
+                DeleteRecoveryRpcOperation::InspectBounded {
+                    object: object.to_owned(),
+                    transaction,
+                    max_entries,
+                },
+            )
+            .await
+        {
+            Ok(DeleteRecoveryRpcReply::InspectedBounded(page)) => Ok(page),
+            Ok(_) => Err(Error::FileCorrupt),
+            Err(error)
+                if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
+                    || matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
+                    .is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
+            {
+                // No paginated fallback through the unbounded protocol; the peer
+                // is an unsupported participant until it is upgraded.
+                Err(Error::UnsupportedDisk)
+            }
+            Err(error) => Err(error),
+        }
+    }
     async fn inspect_delete_transaction(
         &self,
         volume: &str,
@@ -4551,7 +4731,10 @@ impl DiskAPI for RemoteDisk {
                     read_multiple_req_bin: read_multiple_req_bin.into(),
                 });
 
-                let response = client.read_multiple(request).await?.into_inner();
+                let rpc_started = Instant::now();
+                let response = client.read_multiple(request).await;
+                crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_multiple_duration(rpc_started.elapsed());
+                let response = response?.into_inner();
 
                 if !response.success {
                     return Err(response.error.unwrap_or_default().into());
@@ -4602,6 +4785,7 @@ impl DiskAPI for RemoteDisk {
                 attach_mutation_body_digest(&mut request, canonical_body, "write_all")?;
 
                 crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_request();
+                let rpc_started = Instant::now();
                 let response = match client.write_all(request).await {
                     Ok(response) => response.into_inner(),
                     Err(err) => {
@@ -4610,6 +4794,7 @@ impl DiskAPI for RemoteDisk {
                     }
                 };
 
+                crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_duration(rpc_started.elapsed());
                 crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_sent_bytes(data_len);
 
                 if !response.success {
@@ -4707,6 +4892,7 @@ impl DiskAPI for RemoteDisk {
                 });
 
                 crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_request();
+                let rpc_started = Instant::now();
                 let response = match client.read_all(request).await {
                     Ok(response) => response.into_inner(),
                     Err(err) => {
@@ -4715,6 +4901,8 @@ impl DiskAPI for RemoteDisk {
                     }
                 };
 
+                crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_duration(rpc_started.elapsed());
+
                 if !response.success {
                     crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_error();
                     return Err(response.error.unwrap_or_default().into());
@@ -4849,6 +5037,113 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn bounded_delete_discovery_lost_remote_reply_leaves_no_cursor() {
+        runtime_sources::ensure_test_rpc_secret();
+        let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
+            .await
+            .expect("test peer must bind");
+        let directory = tempfile::tempdir().expect("temporary disk");
+        let endpoint = Endpoint::try_from(directory.path().to_str().expect("disk path")).expect("endpoint");
+        let disk = Arc::new(
+            crate::disk::local::LocalDisk::new(&endpoint, false)
+                .await
+                .expect("local disk"),
+        );
+        disk.make_volume("bucket").await.expect("bucket");
+        let prefix = directory.path().join("bucket/prefix");
+        std::fs::create_dir(&prefix).expect("prefix");
+        for index in 0..64 {
+            std::fs::create_dir(prefix.join(format!("child-{index}"))).expect("legitimate child");
+        }
+        let fixture = Arc::new(TestBoundedDiscovery {
+            disk: Arc::clone(&disk),
+            response_gate: TestFormatReadGate::default(),
+        });
+        *peer.peer.bounded_discovery.lock().expect("bounded fixture") = Some(Arc::clone(&fixture));
+        let remote = remote_disk_for_addr(&peer.addr).await;
+        let task = tokio::spawn(async move { remote.discover_delete_transactions_bounded("bucket", "prefix", 8).await });
+        tokio::time::timeout(Duration::from_secs(5), fixture.response_gate.entered.notified())
+            .await
+            .expect("disk completed bounded scan");
+        task.abort();
+        assert!(task.await.expect_err("caller cancelled").is_cancelled());
+        assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
+        fixture.response_gate.release.cancel();
+        peer.stop().await;
+        assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
+        assert!(prefix.join("child-63").is_dir());
+    }
+
+    #[tokio::test]
+    async fn multipart_index_old_disk_info_handler_is_not_a_ready_empty_index() {
+        runtime_sources::ensure_test_rpc_secret();
+        let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
+            .await
+            .expect("old peer must bind");
+        let remote = remote_disk_for_addr(&peer.addr).await;
+        remote
+            .set_disk_id(Some(Uuid::new_v4()))
+            .await
+            .expect("expected disk identity");
+        assert!(matches!(
+            remote.list_multipart_index_roots("bucket", None).await,
+            Err(DiskError::MethodNotAllowed)
+        ));
+        assert_eq!(
+            peer.peer.disk_info_calls.load(Ordering::Acquire),
+            1,
+            "exercise the actual legacy JSON handler"
+        );
+        peer.stop().await;
+    }
+
+    #[tokio::test]
+    async fn bounded_delete_discovery_old_peer_rejects_without_paginated_fallback() {
+        runtime_sources::ensure_test_rpc_secret();
+        let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
+            .await
+            .expect("old peer must bind");
+        let remote = remote_disk_for_addr(&peer.addr).await;
+        assert_eq!(
+            remote
+                .discover_delete_transactions_bounded("bucket", "prefix", 4096)
+                .await
+                .expect_err("old peer cannot prove a bounded inventory"),
+            DiskError::UnsupportedDisk
+        );
+        let transaction = Uuid::new_v4();
+        assert_eq!(
+            remote
+                .inspect_delete_transaction_bounded("bucket", "prefix", transaction, 65536)
+                .await
+                .expect_err("old peer cannot inspect with a budget"),
+            DiskError::UnsupportedDisk
+        );
+        assert_eq!(
+            remote
+                .settle_delete_transaction_bounded(
+                    "bucket",
+                    crate::disk::BoundedDeleteRecoverySettlement {
+                        object: "prefix".to_owned(),
+                        transaction,
+                        expected_fingerprint: [0; 32],
+                        action: DeleteRecoveryAction::PublishCommit,
+                        max_entries: 65536,
+                    },
+                    None
+                )
+                .await
+                .expect_err("old peer cannot settle with a budget"),
+            DiskError::UnsupportedDisk
+        );
+        assert_eq!(
+            *peer.peer.unsupported_methods.lock().expect("observed methods"),
+            vec!["/node_service.NodeService/RecoverDeleteTransaction"; 3]
+        );
+        peer.stop().await;
+    }
+
     #[tokio::test]
     async fn multipart_source_recovery_old_peer_rejects_without_generic_fallback() {
         runtime_sources::ensure_test_rpc_secret();
@@ -5740,10 +6035,18 @@ mod tests {
         object_read_all_disks: Arc<StdMutex<Vec<String>>>,
         format_data: Bytes,
         read_all_data: Bytes,
+        fail_format_reads: Arc<AtomicBool>,
         next_format_read_gate: Arc<StdMutex<Option<Arc<TestFormatReadGate>>>>,
+        bounded_discovery: Arc<StdMutex<Option<Arc<TestBoundedDiscovery>>>>,
         unsupported_methods: Arc<StdMutex<Vec<String>>>,
     }
 
+    #[derive(Debug)]
+    struct TestBoundedDiscovery {
+        disk: Arc<crate::disk::local::LocalDisk>,
+        response_gate: TestFormatReadGate,
+    }
+
     #[derive(Debug, Default)]
     struct TestFormatReadGate {
         entered: tokio::sync::Notify,
@@ -5759,7 +6062,9 @@ mod tests {
                 object_read_all_disks: Arc::default(),
                 format_data,
                 read_all_data,
+                fail_format_reads: Arc::default(),
                 next_format_read_gate: Arc::default(),
+                bounded_discovery: Arc::default(),
                 unsupported_methods: Arc::default(),
             }
         }
@@ -5859,6 +6164,56 @@ mod tests {
                         Ok(grpc.unary(method, request).await)
                     })
                 }
+                "/node_service.NodeService/RecoverDeleteTransaction"
+                    if self.peer.bounded_discovery.lock().expect("bounded fixture").is_some() =>
+                {
+                    #[derive(Clone)]
+                    struct BoundedSvc(Arc<TestBoundedDiscovery>);
+                    impl tonic::server::UnaryService<RecoverDeleteTransactionRequest> for BoundedSvc {
+                        type Response = rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse;
+                        type Future = Pin<Box<dyn Future<Output = std::result::Result<Response<Self::Response>, Status>> + Send>>;
+                        fn call(&mut self, request: Request<RecoverDeleteTransactionRequest>) -> Self::Future {
+                            let fixture = Arc::clone(&self.0);
+                            Box::pin(async move {
+                                let request = request.into_inner();
+                                let operation: DeleteRecoveryRpcOperation =
+                                    rustfs_protos::decode_delete_recovery_payload(&request.operation)
+                                        .map_err(|_| Status::invalid_argument("invalid bounded request"))?;
+                                let DeleteRecoveryRpcOperation::DiscoverBounded { object, max_entries } = operation else {
+                                    return Err(Status::invalid_argument("unexpected fallback"));
+                                };
+                                let page = fixture
+                                    .disk
+                                    .discover_delete_transactions_bounded(&request.volume, &object, max_entries)
+                                    .await
+                                    .map_err(|_| Status::internal("bounded discovery failed"))?;
+                                // Lose the response only after the real disk operation has completed.
+                                fixture.response_gate.entered.notify_one();
+                                fixture.response_gate.release.cancelled().await;
+                                let result = rmp_serde::to_vec_named(&DeleteRecoveryRpcReply::DiscoveredBounded(page))
+                                    .map_err(|_| Status::internal("encode bounded reply"))?;
+                                Ok(Response::new(rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse {
+                                    success: true,
+                                    protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
+                                    result: result.into(),
+                                    error: None,
+                                }))
+                            })
+                        }
+                    }
+                    let fixture = self
+                        .peer
+                        .bounded_discovery
+                        .lock()
+                        .expect("bounded fixture")
+                        .clone()
+                        .expect("configured fixture");
+                    Box::pin(async move {
+                        let codec = tonic_prost::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec);
+                        Ok(grpc.unary(BoundedSvc(fixture), request).await)
+                    })
+                }
                 "/node_service.NodeService/ReadAll" => {
                     #[derive(Clone)]
                     struct ReadAllSvc(AuthenticatedReadPeer);
@@ -5882,6 +6237,9 @@ mod tests {
                                         gate.entered.notify_one();
                                         gate.release.cancelled().await;
                                     }
+                                    if peer.fail_format_reads.load(Ordering::Acquire) {
+                                        return Err(Status::unavailable("format read temporarily unavailable"));
+                                    }
                                     peer.format_data.clone()
                                 } else {
                                     peer.object_read_all_disks
@@ -8847,6 +9205,124 @@ mod tests {
         peer.stop().await;
     }
 
+    #[tokio::test]
+    #[serial(remote_disk_recovery_probe)]
+    async fn endpoint_reconnect_failure_preserves_registered_remote_recovery() {
+        temp_env::async_with_vars(
+            [
+                (crate::disk::disk_store::ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, Some("true")),
+                (rustfs_config::ENV_DRIVE_RETURNING_SUCCESS_THRESHOLD, Some("1")),
+            ],
+            async {
+                runtime_sources::ensure_test_rpc_secret();
+                let mut reference = crate::layout::format::FormatV3::new(1, 1);
+                reference.erasure.this = reference.erasure.sets[0][0];
+                let peer = TestGrpcPeer::spawn(
+                    Bytes::from(reference.to_json().expect("format should serialize")),
+                    Bytes::from_static(b"recovered-peer-data"),
+                )
+                .await
+                .expect("authenticated regression peer is required");
+                let set = missing_read_set(&peer, reference).await;
+                let disk = crate::disk::new_disk(
+                    &set.set_endpoints[0],
+                    &DiskOption {
+                        cleanup: false,
+                        health_check: true,
+                    },
+                )
+                .await
+                .expect("registered remote disk should construct");
+                let crate::disk::Disk::Remote(remote) = disk.as_ref() else {
+                    panic!("fixture must create a remote disk");
+                };
+                assert!(remote.health_check);
+                remote.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
+                set.disks.write().await[0] = Some(Arc::clone(&disk));
+
+                peer.peer.fail_format_reads.store(true, Ordering::Release);
+                set.connect_disks().await;
+                peer.peer.fail_format_reads.store(false, Ordering::Release);
+                let cancelled_after_failure = remote.cancel_token.is_cancelled();
+                assert!(peer.peer.read_all_calls() > 0, "reconnect must attempt the failing format RPC");
+                assert!(
+                    Arc::ptr_eq(&disk, set.disks.read().await[0].as_ref().expect("existing slot must remain")),
+                    "a failed reconnect must not publish its unvalidated candidate"
+                );
+                remote.spawn_recovery_monitor_if_needed();
+                let recovery = time::timeout(Duration::from_secs(5), async {
+                    while !disk.is_online().await {
+                        tokio::task::yield_now().await;
+                    }
+                    disk.read_all("bucket", "object").await
+                })
+                .await;
+                disk.close().await.expect("stop the registered disk's monitoring");
+                peer.stop().await;
+                assert!(!cancelled_after_failure, "failed replacement must retain the registered monitor");
+                assert_eq!(
+                    recovery
+                        .expect("failed reconnect must leave the registered disk able to recover without another sweep")
+                        .expect("the recovered registered disk must serve real I/O"),
+                    Bytes::from_static(b"recovered-peer-data")
+                );
+            },
+        )
+        .await;
+    }
+
+    #[tokio::test]
+    #[serial(remote_disk_recovery_probe)]
+    async fn endpoint_reconnect_stops_old_remote_monitor_only_after_publication() {
+        runtime_sources::ensure_test_rpc_secret();
+        let mut reference = crate::layout::format::FormatV3::new(1, 1);
+        reference.erasure.this = reference.erasure.sets[0][0];
+        let peer = TestGrpcPeer::spawn(
+            Bytes::from(reference.to_json().expect("format should serialize")),
+            Bytes::from_static(b"replacement-peer-data"),
+        )
+        .await
+        .expect("authenticated regression peer is required");
+        let set = missing_read_set(&peer, reference).await;
+        let disk = crate::disk::new_disk(
+            &set.set_endpoints[0],
+            &DiskOption {
+                cleanup: false,
+                health_check: true,
+            },
+        )
+        .await
+        .expect("registered remote disk should construct");
+        let crate::disk::Disk::Remote(remote) = disk.as_ref() else {
+            panic!("fixture must create a remote disk");
+        };
+        remote.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
+        set.disks.write().await[0] = Some(Arc::clone(&disk));
+        let gate = peer.peer.pause_next_format_read();
+        let pending_set = Arc::clone(&set);
+        let pending = tokio::spawn(async move { pending_set.connect_disks().await });
+        time::timeout(Duration::from_secs(5), gate.entered.notified())
+            .await
+            .expect("replacement must reach the format validation barrier");
+        let cancelled_before_publication = remote.cancel_token.is_cancelled();
+        gate.release.cancel();
+        pending.await.expect("validated replacement must finish publication");
+        let replacement = set.disks.read().await[0].clone().expect("replacement must occupy the slot");
+        let readback = replacement.read_all("bucket", "object").await;
+        replacement.close().await.expect("stop replacement monitoring");
+        peer.stop().await;
+        assert!(
+            !cancelled_before_publication,
+            "the registered monitor must survive pending format validation"
+        );
+        assert!(!Arc::ptr_eq(&disk, &replacement), "validated replacement must be published");
+        assert!(remote.cancel_token.is_cancelled(), "the displaced monitor must stop after publication");
+        assert_eq!(
+            readback.expect("replacement must serve real I/O"),
+            Bytes::from_static(b"replacement-peer-data")
+        );
+    }
+
     #[tokio::test]
     #[serial(remote_disk_recovery_probe)]
     async fn endpoint_reconnect_admits_ready_drive_while_another_format_probe_is_blocked() {
@@ -11543,4 +12019,15 @@ mod tests {
 
         accept_task.abort();
     }
+
+    #[test]
+    fn prewarm_claim_is_deduplicated_and_released_after_failure() {
+        let addr = "http://prewarm-retry-test.invalid:9000";
+        release_control_channel_prewarm(addr);
+        assert!(claim_control_channel_prewarm(addr));
+        assert!(!claim_control_channel_prewarm(addr));
+        release_control_channel_prewarm(addr);
+        assert!(claim_control_channel_prewarm(addr));
+        release_control_channel_prewarm(addr);
+    }
 }

crates/ecstore/src/cluster/rpc/runtime_sources.rs

@@ -481,6 +481,14 @@ pub(crate) fn record_remote_disk_grpc_write_all_request() {
         .record_outgoing_request_for_operation_and_backend(INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC);
 }
 
+pub(crate) fn record_remote_disk_grpc_write_all_duration(duration: Duration) {
+    global_internode_metrics().record_duration_for_operation_and_backend(
+        INTERNODE_OPERATION_GRPC_WRITE_ALL,
+        INTERNODE_TRANSPORT_BACKEND_GRPC,
+        duration,
+    );
+}
+
 pub(crate) fn record_remote_disk_grpc_write_all_sent_bytes(bytes: usize) {
     global_internode_metrics().record_sent_bytes_for_operation_and_backend(
         INTERNODE_OPERATION_GRPC_WRITE_ALL,
@@ -521,6 +529,14 @@ pub(crate) fn record_remote_disk_grpc_read_all_request() {
         .record_outgoing_request_for_operation_and_backend(INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC);
 }
 
+pub(crate) fn record_remote_disk_grpc_read_all_duration(duration: Duration) {
+    global_internode_metrics().record_duration_for_operation_and_backend(
+        INTERNODE_OPERATION_GRPC_READ_ALL,
+        INTERNODE_TRANSPORT_BACKEND_GRPC,
+        duration,
+    );
+}
+
 #[cfg(feature = "diagnostic-read-version-caller")]
 fn read_version_request_mode(opts: &ReadOptions) -> ReadVersionRequestMode {
     match (opts.read_data, opts.healing, opts.incl_free_versions) {
@@ -598,6 +614,17 @@ pub(crate) fn record_remote_disk_grpc_batch_read_version_stage(stage: &'static s
     );
 }
 
+pub(crate) fn record_remote_disk_grpc_batch_read_version_duration(duration: Duration) {
+    if !rustfs_io_metrics::get_stage_metrics_enabled() {
+        return;
+    }
+    global_internode_metrics().record_duration_for_operation_and_backend(
+        INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
+        INTERNODE_TRANSPORT_BACKEND_GRPC,
+        duration,
+    );
+}
+
 pub(crate) fn record_remote_disk_grpc_batch_read_version_error() {
     if !rustfs_io_metrics::get_stage_metrics_enabled() {
         return;
@@ -690,6 +717,14 @@ pub(crate) fn record_remote_disk_grpc_read_multiple_recv_bytes(bytes: usize) {
     record_grpc_payload_size(INTERNODE_OPERATION_GRPC_READ_MULTIPLE, bytes);
 }
 
+pub(crate) fn record_remote_disk_grpc_read_multiple_duration(duration: Duration) {
+    global_internode_metrics().record_duration_for_operation_and_backend(
+        INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
+        INTERNODE_TRANSPORT_BACKEND_GRPC,
+        duration,
+    );
+}
+
 /// Payload-size threshold (bytes) above which a unary internode gRPC response is counted as a
 /// "large payload" for alerting. Env-overridable via `RUSTFS_INTERNODE_RPC_LARGE_PAYLOAD_WARN_BYTES`.
 fn internode_rpc_large_payload_warn_bytes() -> usize {

crates/ecstore/src/core/pool_meta_native_integration_tests.rs

@@ -18,6 +18,7 @@ use crate::storage_api_contracts::namespace::NamespaceLocking as _;
 use codec::{fixture_inline_pool_meta_replicas as replicas, fixture_split_inline_pool_meta_replicas as split};
 use std::path::PathBuf;
 use tokio::fs;
+use uuid::Uuid;
 
 struct PhysicalFixture {
     _directories: Vec<tempfile::TempDir>,
@@ -216,6 +217,264 @@ async fn pool_meta_native_physical_split_without_ordinary_commit_performs_no_mut
         "all current heads, identities and complete backups must remain unchanged"
     );
     assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
+    state
+        .ensure_write_safe("ordinary authority temporarily unavailable")
+        .expect("missing ordinary authority alone must not install a permanent corruption latch");
+}
+
+async fn interrupted_first_commit_fixture() -> (PhysicalFixture, PoolMetaCommittedCandidate) {
+    let fixture = fixture(3).await;
+    let PoolMetaReplica::Valid {
+        previous: Some(previous),
+        committed: false,
+        ..
+    } = decode_pool_meta_replica(fixture.pending.clone())
+    else {
+        panic!("fixture must retain its acknowledged predecessor");
+    };
+    let physical = disks(&fixture.store).await;
+    let prepared = replicas(&fixture.pending, OffsetDateTime::now_utc()).await;
+    for (index, disk) in physical[0].iter().enumerate() {
+        fs::write(object_path(disk).join("xl.meta"), &prepared[index])
+            .await
+            .expect("one pool must retain an ordinary-readable PREPARE");
+    }
+    (fixture, *previous)
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_physical_interrupted_first_commit_restores_proven_predecessor() {
+    let (mut fixture, previous) = interrupted_first_commit_fixture().await;
+    let before = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+    assert!(
+        before[1..]
+            .iter()
+            .all(|read| matches!(read.replica, PoolMetaReplica::Unreadable(_)))
+    );
+    assert!(matches!(before[0].replica, PoolMetaReplica::Valid { committed: false, .. }));
+    let mut state = clean_state(&fixture);
+    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
+    assert!(
+        repair(&fixture.store, &mut state, &fence)
+            .await
+            .expect("ordinary PREPARE must authorize restoring only its acknowledged predecessor")
+    );
+    state
+        .ensure_write_safe("first commit rollback")
+        .expect("rollback must finish its transaction");
+    assert_ne!(
+        previous.canonical, fixture.canonical,
+        "the interrupted COMMIT must not become authoritative"
+    );
+    fixture.canonical = previous.canonical;
+    fixture.revision = previous.revision;
+    assert_committed(&fixture).await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_physical_interrupted_rollback_retains_authority_and_resumes() {
+    let (mut fixture, previous) = interrupted_first_commit_fixture().await;
+    let pause = Arc::new(NativeApplyPause::default());
+    let reached = pause.reached.notified();
+    tokio::pin!(reached);
+    reached.as_mut().enable();
+    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
+    let mut state = clean_state(&fixture);
+    let recovery = tokio::spawn({
+        let store = Arc::clone(&fixture.store);
+        let pause = Arc::clone(&pause);
+        async move {
+            AFTER_NATIVE_POOL_APPLY
+                .scope(pause, async move { repair(&store, &mut state, &fence).await })
+                .await
+        }
+    });
+    tokio::time::timeout(std::time::Duration::from_secs(10), reached)
+        .await
+        .expect("real rollback should pause after one pool is durably restored");
+    let partial = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+    assert!(matches!(partial[0].replica, PoolMetaReplica::Valid { committed: false, .. }));
+    assert!(matches!(&partial[1].replica, PoolMetaReplica::Valid { canonical, committed: true, .. }
+        if canonical == &previous.canonical));
+    assert!(matches!(partial[2].replica, PoolMetaReplica::Unreadable(_)));
+    recovery.abort();
+    assert!(
+        recovery
+            .await
+            .expect_err("the paused repair must be cancelled")
+            .is_cancelled()
+    );
+    let mut state = clean_state(&fixture);
+    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
+    assert!(
+        repair(&fixture.store, &mut state, &fence)
+            .await
+            .expect("restart must resume from the retained ordinary PREPARE")
+    );
+    fixture.canonical = previous.canonical;
+    fixture.revision = previous.revision;
+    assert_committed(&fixture).await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_physical_rollback_resumes_after_last_prepare_loses_read_quorum() {
+    let (mut fixture, previous) = interrupted_first_commit_fixture().await;
+    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
+    let inspected = inspect(&fixture.store.pools, &fence)
+        .await
+        .expect("physical rollback inventory");
+    let replacements = plan(
+        &inspected,
+        &fixture.canonical,
+        fixture.revision,
+        &fixture.identity,
+        Some(&previous.canonical),
+    )
+    .await
+    .expect("the ordinary PREPARE must authorize its predecessor");
+    for (pool, count) in [(1, 4), (2, 4), (0, 2)] {
+        for (index, replacement) in replacements[pool].iter().enumerate().take(count) {
+            let snapshot = &inspected[pool].snapshots[index];
+            assert_eq!(
+                inspected[pool].disks[index]
+                    .pool_meta_native_recovery(Request::Apply {
+                        object: PoolMetaNativeObject::PoolBin,
+                        expected_disk_id: snapshot.disk_id,
+                        expected_current_sha256: snapshot.current.as_ref().map(|raw| Sha256::digest(raw).into()),
+                        expected_backup_inventory_sha256: snapshot.inventory_sha256().expect("original inventory hash"),
+                        replacement: replacement.clone(),
+                    })
+                    .await
+                    .expect("partial rollback must use the real conditional publication and sync path"),
+                Response::Applied
+            );
+        }
+    }
+    let partial = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+    assert!(matches!(partial[0].replica, PoolMetaReplica::Unreadable(_)));
+    assert!(partial[1..].iter().all(|read| matches!(&read.replica,
+        PoolMetaReplica::Valid { canonical, committed: true, .. } if canonical == &previous.canonical)));
+    let mut state = clean_state(&fixture);
+    assert!(
+        repair(&fixture.store, &mut state, &fence)
+            .await
+            .expect("restored ordinary COMMIT must finish rollback after the last PREPARE becomes split")
+    );
+    fixture.canonical = previous.canonical;
+    fixture.revision = previous.revision;
+    assert_committed(&fixture).await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_physical_rollback_backup_does_not_poison_reused_generation() {
+    let (mut fixture, previous) = interrupted_first_commit_fixture().await;
+    let mut state = clean_state(&fixture);
+    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
+    assert!(repair(&fixture.store, &mut state, &fence).await.expect("initial rollback"));
+    let revision = PoolMetaRevision {
+        transaction_id: Some(Uuid::new_v4()),
+        ..fixture.revision
+    };
+    let mut meta = previous.meta.clone();
+    meta.pools[0].last_update += Duration::seconds(1);
+    let canonical = encode_pool_meta_v3_envelope(&meta, revision, true, None).expect("later COMMIT");
+    let pending = encode_pool_meta_v3_envelope(&meta, revision, false, Some(&previous)).expect("later PREPARE");
+    let modified = OffsetDateTime::now_utc() + Duration::seconds(1);
+    let split = split(&pending, &canonical, modified).await;
+    let commit = replicas(&canonical, modified).await;
+    let prepared = replicas(&pending, modified).await;
+    let directory = Uuid::new_v4();
+    let retained = snapshots(&fixture.store).await;
+    for (pool_index, pool) in disks(&fixture.store).await.into_iter().enumerate() {
+        for (disk_index, disk) in pool.iter().enumerate() {
+            assert!(
+                !retained[pool_index][disk_index].backups.is_empty(),
+                "the abandoned PREPARE must remain as real on-disk evidence"
+            );
+            let object = object_path(disk);
+            fs::write(object.join("xl.meta"), &if pool_index < 2 { &split } else { &commit }[disk_index])
+                .await
+                .expect("install later partial COMMIT without clearing old backups");
+            fs::create_dir(object.join(directory.to_string()))
+                .await
+                .expect("later backup directory");
+            fs::write(object.join(directory.to_string()).join("xl.meta.bkp"), &prepared[disk_index])
+                .await
+                .expect("later PREPARE backup");
+        }
+    }
+    assert!(
+        repair(&fixture.store, &mut state, &fence)
+            .await
+            .expect("an ordinary later COMMIT must survive the abandoned sibling PREPARE")
+    );
+    fixture.canonical = canonical;
+    fixture.revision = revision;
+    assert_committed(&fixture).await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_physical_rollback_rejects_retained_commit_without_mutation() {
+    let (fixture, _) = interrupted_first_commit_fixture().await;
+    let physical = disks(&fixture.store).await;
+    let commit = replicas(&fixture.canonical, OffsetDateTime::now_utc()).await;
+    let directory = Uuid::new_v4();
+    let object = object_path(&physical[0][0]);
+    fs::create_dir(object.join(directory.to_string()))
+        .await
+        .expect("retained COMMIT directory");
+    fs::write(object.join(directory.to_string()).join("xl.meta.bkp"), &commit[0])
+        .await
+        .expect("retain evidence that the interrupted transaction may already have committed");
+    let before = snapshots(&fixture.store).await;
+    let mut state = clean_state(&fixture);
+    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
+    assert!(repair(&fixture.store, &mut state, &fence).await.is_err());
+    assert_eq!(snapshots(&fixture.store).await, before, "ambiguous COMMIT evidence must remain untouched");
+    assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_physical_rollback_rejects_competing_prepares_without_mutation() {
+    let (fixture, previous) = interrupted_first_commit_fixture().await;
+    let pending = encode_pool_meta_v3_envelope(
+        &previous.meta,
+        PoolMetaRevision {
+            transaction_id: Some(Uuid::new_v4()),
+            ..fixture.revision
+        },
+        false,
+        Some(&previous),
+    )
+    .expect("competing PREPARE must remain structurally valid");
+    let physical = disks(&fixture.store).await;
+    let prepared = replicas(&pending, OffsetDateTime::now_utc()).await;
+    for (index, disk) in physical[1].iter().enumerate() {
+        fs::write(object_path(disk).join("xl.meta"), &prepared[index])
+            .await
+            .expect("second pool must expose the competing transaction through ordinary reads");
+    }
+    let before = snapshots(&fixture.store).await;
+    let mut state = clean_state(&fixture);
+    let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
+    let error = repair(&fixture.store, &mut state, &fence)
+        .await
+        .expect_err("two distinct prepared transactions must not authorize a guessed rollback");
+    assert_eq!(
+        error
+            .pool_metadata_failure()
+            .expect("transaction conflict must be typed")
+            .kind,
+        crate::error::PoolMetadataFailure::RecoveryRequired
+    );
+    assert_eq!(snapshots(&fixture.store).await, before, "competing transactions must remain untouched");
+    assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
 }
 
 struct IdentityEpochFork {
@@ -467,7 +726,7 @@ async fn pool_meta_native_physical_partial_apply_resumes_after_conditional_rejec
     let inspected = inspect(&fixture.store.pools, &fence)
         .await
         .expect("real native snapshots should be obtained");
-    let replacements = plan(&inspected, &fixture.canonical, fixture.revision, &fixture.identity)
+    let replacements = plan(&inspected, &fixture.canonical, fixture.revision, &fixture.identity, None)
         .await
         .expect("physical recovery plan should validate");
     let snapshot = &inspected[0].snapshots[0];
@@ -628,3 +887,559 @@ async fn pool_meta_native_physical_three_plus_one_checkpoint_converges_all_shard
         );
     }
 }
+
+async fn owner_repair_no_backups_fixture() -> PhysicalFixture {
+    let fixture = fixture(0).await;
+    for pool in disks(&fixture.store).await {
+        for disk in pool {
+            let object = object_path(&disk);
+            let mut entries = fs::read_dir(&object).await.expect("fixture metadata directory");
+            while let Some(entry) = entries.next_entry().await.expect("fixture directory entry") {
+                if entry.file_type().await.expect("fixture entry type").is_dir() {
+                    fs::remove_dir_all(entry.path())
+                        .await
+                        .expect("remove task-owned fixture rollback directory");
+                }
+            }
+        }
+    }
+    fixture
+}
+
+async fn owner_repair_identity_heads(store: &ECStore) -> Vec<Vec<(Uuid, Bytes)>> {
+    let mut all = Vec::new();
+    for pool in &store.pools {
+        let set = pool.get_disks_by_key(POOL_META_IDENTITY_NAME);
+        let members = set
+            .disks
+            .read()
+            .await
+            .iter()
+            .cloned()
+            .map(|disk| disk.expect("identity fixture disk must be online"))
+            .collect::<Vec<_>>();
+        let mut heads = Vec::new();
+        for disk in members {
+            match disk
+                .pool_meta_native_recovery(Request::InspectIdentity {})
+                .await
+                .expect("physical identity inspection")
+            {
+                Response::IdentityInspected { disk_id, raw } => heads.push((disk_id, raw)),
+                _ => panic!("identity inspection must return its physical head"),
+            }
+        }
+        all.push(heads);
+    }
+    all
+}
+
+async fn legacy_native_owner_fixture(version: u16, stale: bool) -> (PhysicalFixture, Vec<u8>) {
+    let fixture = owner_repair_no_backups_fixture().await;
+    let mut meta = match decode_pool_meta_replica(fixture.canonical.clone()) {
+        PoolMetaReplica::Valid {
+            meta, committed: true, ..
+        } => meta,
+        _ => panic!("fixture must provide valid committed pool metadata"),
+    };
+    meta.version = version;
+    let raw = meta
+        .encode_config_data_for_v2_gate(version == POOL_META_VERSION)
+        .expect("legacy fixture should encode in its original version");
+    assert_eq!(u16::from_le_bytes([raw[2], raw[3]]), version);
+    let all = disks(&fixture.store).await;
+    let now = OffsetDateTime::now_utc();
+    for pool in &all {
+        let heads = replicas(&raw, now).await;
+        for (index, disk) in pool.iter().enumerate() {
+            fs::write(object_path(disk).join("xl.meta"), &heads[index])
+                .await
+                .expect("install exact legacy head");
+        }
+    }
+    if stale {
+        let old = replicas(&raw, now - Duration::seconds(10)).await;
+        fs::write(object_path(&all[0][0]).join("xl.meta"), &old[0])
+            .await
+            .expect("install stale same-raw legacy head");
+    }
+    (fixture, raw)
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_v1_with_all_upgrade_gates_disabled_keeps_exact_raw() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            let (fixture, raw) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+            let guard = owner(&fixture.store).await;
+            let mut state = fixture.store.pool_meta_save_gate.lock().await;
+            let result = tokio::time::timeout(
+                std::time::Duration::from_secs(30),
+                fixture.store.heal_pool_meta_replica_under_fence(0, guard, &mut state),
+            )
+            .await
+            .expect("legacy native heal must terminate");
+            assert!(result.is_ok(), "fully online legacy native heal should succeed: {result:?}");
+            drop(state);
+            let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            for read in &reads {
+                assert!(
+                    matches!(&read.replica, PoolMetaReplica::Valid { raw: bytes, revision, .. }
+                    if bytes == &raw && revision.version == POOL_META_V1_VERSION),
+                    "native heal must not upgrade a V1 raw payload when both upgrade gates are disabled"
+                );
+            }
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_legacy_source_with_trailing_bytes_without_writes() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            for version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
+                let (fixture, raw) = legacy_native_owner_fixture(version, false).await;
+                let mut source_raw = raw.clone();
+                // A second MessagePack value is readable by legacy decoding but
+                // must not be copied as part of an unverified repair payload.
+                source_raw.push(0xc0);
+                assert!(
+                    matches!(decode_pool_meta_replica(source_raw.clone()), PoolMetaReplica::Valid { raw, .. } if raw == source_raw),
+                    "ordinary legacy reads must remain compatible with trailing bytes"
+                );
+                let all_disks = disks(&fixture.store).await;
+                let source_heads = replicas(&source_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+                for (index, disk) in all_disks[0].iter().enumerate() {
+                    fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
+                        .await
+                        .expect("install readable legacy source with trailing bytes");
+                }
+                let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
+                    .await
+                    .expect("legacy semantic selection must remain readable");
+                assert_eq!(selection.legacy_raw.as_deref(), Some(source_raw.as_slice()));
+                let before = snapshots(&fixture.store).await;
+                let identities = owner_repair_identity_heads(&fixture.store).await;
+                let guard = owner(&fixture.store).await;
+                let mut state = fixture.store.pool_meta_save_gate.lock().await;
+                let result = tokio::time::timeout(
+                    std::time::Duration::from_secs(30),
+                    fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
+                )
+                .await
+                .expect("legacy native heal must terminate");
+                drop(state);
+                assert!(result.is_err(), "version {version}: unparsed trailing source bytes must not be republished");
+                assert_eq!(snapshots(&fixture.store).await, before, "version {version}: no pool.bin head or backup may change");
+                assert_eq!(
+                    owner_repair_identity_heads(&fixture.store).await,
+                    identities,
+                    "version {version}: no identity shard may change"
+                );
+                let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+                assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &source_raw));
+                assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw: target, .. } if target == &raw));
+            }
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_mixed_v1_v2_preserves_observed_format_floor() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            for first_version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
+                for target_pool in [0, 1] {
+                    let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+                    let mut meta = match decode_pool_meta_replica(raw_v1.clone()) {
+                        PoolMetaReplica::Valid { meta, committed: true, .. } => meta,
+                        _ => panic!("initial V1 metadata must decode"),
+                    };
+                    meta.version = POOL_META_VERSION;
+                    let raw_v2 = meta.encode_config_data_for_v2_gate(true).expect("equivalent V2 payload must encode");
+                    assert_eq!(u16::from_le_bytes([raw_v1[2], raw_v1[3]]), POOL_META_V1_VERSION);
+                    assert_eq!(u16::from_le_bytes([raw_v2[2], raw_v2[3]]), POOL_META_VERSION);
+                    let semantic_v1 = match decode_pool_meta_replica(raw_v1.clone()) {
+                        PoolMetaReplica::Valid { canonical, .. } => canonical,
+                        _ => panic!("V1 payload must remain valid"),
+                    };
+                    let semantic_v2 = match decode_pool_meta_replica(raw_v2.clone()) {
+                        PoolMetaReplica::Valid { canonical, .. } => canonical,
+                        _ => panic!("V2 payload must remain valid"),
+                    };
+                    assert_eq!(semantic_v1, semantic_v2, "the two formats must carry equivalent metadata");
+                    let other_version = if first_version == POOL_META_V1_VERSION { POOL_META_VERSION } else { POOL_META_V1_VERSION };
+                    // Pool two follows pool zero; pools zero and one cover both
+                    // orderings and both target formats in separate runs.
+                    let versions = [first_version, other_version, first_version];
+                    let physical = disks(&fixture.store).await;
+                    let timestamp = OffsetDateTime::now_utc();
+                    for (pool_index, pool) in physical.iter().enumerate() {
+                        let raw = if versions[pool_index] == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
+                        let heads = replicas(raw, timestamp).await;
+                        for (index, disk) in pool.iter().enumerate() {
+                            fs::write(object_path(disk).join("xl.meta"), &heads[index]).await.expect("install mixed legacy pool head");
+                        }
+                    }
+                    let before = snapshots(&fixture.store).await;
+                    let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
+                        .await.expect("equivalent legacy pools should be selectable");
+                    assert_eq!(selection.revision.version, POOL_META_VERSION, "the observed V2 format floor must be retained");
+                    assert_eq!(selection.canonical.as_deref(), Some(semantic_v2.as_slice()));
+                    let guard = owner(&fixture.store).await;
+                    let mut state = fixture.store.pool_meta_save_gate.lock().await;
+                    tokio::time::timeout(std::time::Duration::from_secs(30),
+                        fixture.store.heal_pool_meta_replica_under_fence(target_pool, guard, &mut state))
+                        .await.expect("mixed legacy heal must terminate")
+                        .expect("equivalent legacy metadata should heal without downgrading the format floor");
+                    drop(state);
+                    let after = snapshots(&fixture.store).await;
+                    let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+                    for (pool_index, read) in reads.iter().enumerate() {
+                        let expected_version = if pool_index == target_pool { POOL_META_VERSION } else { versions[pool_index] };
+                        let expected_raw = if expected_version == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
+                        assert!(matches!(&read.replica,
+                            PoolMetaReplica::Valid { raw, canonical, revision, committed: true, .. }
+                                if raw == expected_raw && canonical == &semantic_v2 && revision.version == expected_version
+                        ), "pool {pool_index}, first={first_version}, target={target_pool}: preserve equivalent payload and expected legacy version");
+                        for (index, snapshot) in after[pool_index].iter().enumerate() {
+                            codec::validate_inline_pool_meta_shard(snapshot.current.as_ref().expect("physical head"), index, expected_raw)
+                                .await.expect("every physical shard must match its selected legacy payload");
+                        }
+                        if pool_index != target_pool {
+                            assert_eq!(after[pool_index], before[pool_index], "healing one pool must not modify other pools");
+                        }
+                    }
+                }
+            }
+        },
+    ).await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_v1_and_v3_prepare_in_either_pool_order() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            for prepare_pool in [0, 1] {
+                let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+                let (legacy_meta, legacy_canonical) = match decode_pool_meta_replica(raw_v1.clone()) {
+                    PoolMetaReplica::Valid {
+                        meta,
+                        canonical,
+                        revision,
+                        committed: true,
+                        ..
+                    } if revision.version == POOL_META_V1_VERSION => (meta, canonical),
+                    _ => panic!("baseline V1 metadata must be committed and readable"),
+                };
+                let previous = PoolMetaCommittedCandidate {
+                    canonical: legacy_canonical,
+                    meta: legacy_meta.clone(),
+                    revision: PoolMetaRevision::legacy(POOL_META_V1_VERSION),
+                };
+                let mut next = legacy_meta;
+                next.version = POOL_META_GENERATION_VERSION;
+                next.pools[0].last_update += Duration::seconds(1);
+                let revision = PoolMetaRevision {
+                    version: POOL_META_GENERATION_VERSION,
+                    cluster_id: fixture.revision.cluster_id,
+                    epoch: fixture.revision.epoch,
+                    generation: 1,
+                    transaction_id: Some(Uuid::new_v4()),
+                };
+                let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&previous))
+                    .expect("V3 PREPARE with V1 predecessor must encode");
+                assert!(matches!(decode_pool_meta_replica(pending.clone()),
+                    PoolMetaReplica::Valid { committed: false, revision: actual, .. } if actual == revision));
+                let heads = replicas(&pending, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+                let physical = disks(&fixture.store).await;
+                for (index, disk) in physical[prepare_pool].iter().enumerate() {
+                    fs::write(object_path(disk).join("xl.meta"), &heads[index])
+                        .await
+                        .expect("install task-owned V3 PREPARE evidence");
+                }
+                let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+                assert!(matches!(&reads[prepare_pool].replica,
+                    PoolMetaReplica::Valid { raw, committed: false, revision: actual, .. }
+                        if raw == &pending && *actual == revision));
+                let other_pool = 1 - prepare_pool;
+                assert!(matches!(&reads[other_pool].replica,
+                    PoolMetaReplica::Valid { raw, committed: true, revision: actual, .. }
+                        if raw == &raw_v1 && actual.version == POOL_META_V1_VERSION));
+                let before = snapshots(&fixture.store).await;
+                let identities = owner_repair_identity_heads(&fixture.store).await;
+                let guard = owner(&fixture.store).await;
+                let mut state = fixture.store.pool_meta_save_gate.lock().await;
+                let result = tokio::time::timeout(
+                    std::time::Duration::from_secs(30),
+                    fixture
+                        .store
+                        .heal_pool_meta_replica_under_fence(prepare_pool, guard, &mut state),
+                )
+                .await
+                .expect("mixed V1/V3 PREPARE heal must terminate");
+                drop(state);
+                assert!(
+                    result.is_err(),
+                    "pool {prepare_pool}: native heal must not overwrite a V3 PREPARE with legacy data"
+                );
+                assert_eq!(
+                    snapshots(&fixture.store).await,
+                    before,
+                    "pool {prepare_pool}: rejection must retain all physical head and backup evidence"
+                );
+                assert_eq!(
+                    owner_repair_identity_heads(&fixture.store).await,
+                    identities,
+                    "pool {prepare_pool}: rejection must not mutate identity evidence"
+                );
+            }
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_v1_legacy_dont_save_true_without_writes() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+            let meta = match decode_pool_meta_replica(safe_v1.clone()) {
+                PoolMetaReplica::Valid {
+                    meta, committed: true, ..
+                } => meta,
+                _ => panic!("baseline V1 metadata must decode"),
+            };
+            let statuses = meta
+                .pools
+                .iter()
+                .map(|pool| {
+                    assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
+                    LegacyPoolStatus {
+                        id: pool.id,
+                        cmd_line: pool.cmd_line.clone(),
+                        last_update: pool.last_update,
+                        decommission: None,
+                    }
+                })
+                .collect();
+            let legacy = LegacyPoolMeta {
+                version: POOL_META_V1_VERSION,
+                pools: statuses,
+                dont_save: true,
+            };
+            let mut dangerous_raw = Vec::new();
+            dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
+            dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
+            dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
+            let old_reader: LegacyPoolMeta =
+                rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode source");
+            assert!(old_reader.dont_save, "the old reader would treat this source as permanently do-not-save");
+            assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
+            let source = match decode_pool_meta_replica(dangerous_raw.clone()) {
+                PoolMetaReplica::Valid {
+                    canonical,
+                    revision,
+                    committed: true,
+                    ..
+                } => {
+                    assert_eq!(revision.version, POOL_META_V1_VERSION);
+                    canonical
+                }
+                _ => panic!("current reader must recognize the legacy source"),
+            };
+            assert!(matches!(decode_pool_meta_replica(safe_v1.clone()),
+                PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
+                    if canonical == source && revision.version == POOL_META_V1_VERSION));
+            let physical = disks(&fixture.store).await;
+            let source_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+            for (index, disk) in physical[0].iter().enumerate() {
+                fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
+                    .await
+                    .expect("install legacy source on pool zero");
+            }
+            let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
+            assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
+            let before = snapshots(&fixture.store).await;
+            let identities = owner_repair_identity_heads(&fixture.store).await;
+            let guard = owner(&fixture.store).await;
+            let mut state = fixture.store.pool_meta_save_gate.lock().await;
+            let result = tokio::time::timeout(
+                std::time::Duration::from_secs(30),
+                fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
+            )
+            .await
+            .expect("unsafe legacy heal must terminate");
+            drop(state);
+            let error = result.expect_err("native heal must reject a source whose old-reader dont_save is true");
+            assert!(
+                error.to_string().contains("dont_save"),
+                "rejection must identify the unsafe legacy field: {error}"
+            );
+            assert_eq!(
+                snapshots(&fixture.store).await,
+                before,
+                "rejection must preserve every original head and backup"
+            );
+            assert_eq!(
+                owner_repair_identity_heads(&fixture.store).await,
+                identities,
+                "rejection must not write identity shards"
+            );
+            let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw));
+            assert!(
+                matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1),
+                "the dangerous source bytes must not be copied into the target pool"
+            );
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_target_v1_dont_save_true_without_writes() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+            let meta = match decode_pool_meta_replica(safe_v1.clone()) {
+                PoolMetaReplica::Valid {
+                    meta, committed: true, ..
+                } => meta,
+                _ => panic!("baseline V1 metadata must decode"),
+            };
+            let statuses = meta
+                .pools
+                .iter()
+                .map(|pool| {
+                    assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
+                    LegacyPoolStatus {
+                        id: pool.id,
+                        cmd_line: pool.cmd_line.clone(),
+                        last_update: pool.last_update,
+                        decommission: None,
+                    }
+                })
+                .collect();
+            let legacy = LegacyPoolMeta {
+                version: POOL_META_V1_VERSION,
+                pools: statuses,
+                dont_save: true,
+            };
+            let mut dangerous_raw = Vec::new();
+            dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
+            dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
+            dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
+            let old_reader: LegacyPoolMeta =
+                rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode target");
+            assert!(old_reader.dont_save, "the old reader interprets the target as do-not-save");
+            assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
+            let source_canonical = match decode_pool_meta_replica(safe_v1.clone()) {
+                PoolMetaReplica::Valid {
+                    canonical,
+                    revision,
+                    committed: true,
+                    ..
+                } => {
+                    assert_eq!(revision.version, POOL_META_V1_VERSION);
+                    canonical
+                }
+                _ => panic!("safe pool-zero source must decode"),
+            };
+            assert!(
+                matches!(decode_pool_meta_replica(dangerous_raw.clone()),
+                PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
+                    if canonical == source_canonical && revision.version == POOL_META_V1_VERSION),
+                "the dangerous target must remain semantically equivalent to the safe source"
+            );
+            let physical = disks(&fixture.store).await;
+            let target_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+            for (index, disk) in physical[1].iter().enumerate() {
+                fs::write(object_path(disk).join("xl.meta"), &target_heads[index])
+                    .await
+                    .expect("install legacy dangerous target head");
+            }
+            let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
+            assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
+            let before = snapshots(&fixture.store).await;
+            let identities = owner_repair_identity_heads(&fixture.store).await;
+            let guard = owner(&fixture.store).await;
+            let mut state = fixture.store.pool_meta_save_gate.lock().await;
+            let result = tokio::time::timeout(
+                std::time::Duration::from_secs(30),
+                fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
+            )
+            .await
+            .expect("unsafe legacy target heal must terminate");
+            drop(state);
+            let error = result.expect_err("native heal must reject a target whose old-reader dont_save is true");
+            assert!(
+                error.to_string().contains("dont_save"),
+                "rejection must identify the unsafe legacy field: {error}"
+            );
+            assert_eq!(
+                snapshots(&fixture.store).await,
+                before,
+                "rejection must preserve all physical heads and backups"
+            );
+            assert_eq!(
+                owner_repair_identity_heads(&fixture.store).await,
+                identities,
+                "rejection must not write identity shards"
+            );
+            let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1));
+            assert!(
+                matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw),
+                "unsafe target evidence must not be overwritten by the safe source"
+            );
+        },
+    )
+    .await;
+}

crates/ecstore/src/core/pool_meta_native_recovery.rs

@@ -30,6 +30,7 @@ pub(super) struct NativeApplyPause {
 #[cfg(test)]
 tokio::task_local! {
     pub(super) static AFTER_NATIVE_APPLY: Arc<NativeApplyPause>;
+    pub(super) static AFTER_NATIVE_POOL_APPLY: Arc<NativeApplyPause>;
 }
 
 struct InspectedPool {
@@ -168,6 +169,7 @@ async fn plan(
     canonical: &[u8],
     revision: PoolMetaRevision,
     identity: &[u8],
+    rollback: Option<&[u8]>,
 ) -> Result<Vec<Vec<Bytes>>> {
     let mut pending = Vec::<Vec<u8>>::new();
     let mut predecessors = Vec::<Vec<u8>>::new();
@@ -223,23 +225,83 @@ async fn plan(
             }
         }
     }
-    let mut obsolete = Vec::<Vec<u8>>::new();
-    for payload in &decoded {
-        match decode_pool_meta_replica(payload.clone()) {
-            PoolMetaReplica::Valid {
+    // The last ordinary PREPARE can itself become split during rollback. In
+    // that case another pool already exposes the restored ordinary COMMIT.
+    // Its authority permits continuing the rollback, never promoting a child
+    // transaction reconstructed from disk evidence.
+    let mut continuation: Option<(Vec<u8>, PoolMetaRevision)> = None;
+    if rollback.is_none() {
+        let mut outstanding = Vec::new();
+        for pool in inspected {
+            for (index, snapshot) in pool.snapshots.iter().enumerate() {
+                let raw = snapshot.current.as_ref().ok_or_else(invalid)?;
+                if codec::validate_inline_pool_meta_shard(raw, index, canonical).await.is_err() {
+                    outstanding.push((index, raw));
+                }
+            }
+        }
+        for payload in &decoded {
+            if let PoolMetaReplica::Valid {
                 canonical: candidate,
                 revision: candidate_revision,
                 committed: false,
                 previous: Some(previous),
                 ..
-            } if candidate_revision == revision && candidate == canonical => {
-                if !pending.contains(payload) {
-                    pending.push(payload.clone());
+            } = decode_pool_meta_replica(payload.clone())
+                && previous.revision == revision
+                && previous.canonical == canonical
+            {
+                let mut matches = true;
+                for (index, raw) in &outstanding {
+                    matches &= codec::validate_inline_pool_meta_shard(raw, *index, payload).await.is_ok()
+                        || codec::validate_inline_pool_meta_shard(raw, *index, &candidate).await.is_ok();
                 }
-                if !predecessors.contains(&previous.canonical) {
-                    predecessors.push(previous.canonical);
+                if matches {
+                    if !outstanding.is_empty()
+                        && continuation.as_ref().is_some_and(|(selected, selected_revision)| {
+                            selected != &candidate || *selected_revision != candidate_revision
+                        })
+                    {
+                        return Err(known_validation(invalid(), "native_transaction_validation"));
+                    }
+                    continuation.get_or_insert((candidate, candidate_revision));
                 }
             }
+        }
+    }
+    let (canonical, revision, rollback) = match &continuation {
+        Some((transaction, next_revision)) => (transaction.as_slice(), *next_revision, Some(canonical)),
+        None => (canonical, revision, rollback),
+    };
+    for payload in &decoded {
+        if let PoolMetaReplica::Valid {
+            canonical: candidate,
+            revision: candidate_revision,
+            committed: false,
+            previous: Some(previous),
+            ..
+        } = decode_pool_meta_replica(payload.clone())
+            && candidate_revision == revision
+            && candidate == canonical
+        {
+            pending.push(payload.clone());
+            if !predecessors.contains(&previous.canonical) {
+                predecessors.push(previous.canonical);
+            }
+        }
+    }
+    if pending.len() > 1 {
+        return Err(known_validation(
+            Error::other("native V3 prepare records disagree on the predecessor"),
+            "native_transaction_validation",
+        ));
+    }
+    let mut obsolete = Vec::<Vec<u8>>::new();
+    for payload in &decoded {
+        if pending.contains(payload) {
+            continue;
+        }
+        match decode_pool_meta_replica(payload.clone()) {
             PoolMetaReplica::Valid { revision: older, .. }
                 if older.is_generation_protocol()
                     && older.cluster_id == revision.cluster_id
@@ -251,6 +313,23 @@ async fn plan(
                 // fully reconstructed and bound into the native CAS.
                 obsolete.push(payload.clone());
             }
+            PoolMetaReplica::Valid {
+                revision: abandoned,
+                committed: false,
+                previous: Some(previous),
+                ..
+            } if abandoned.is_generation_protocol()
+                && abandoned.cluster_id == revision.cluster_id
+                && abandoned.epoch == revision.epoch
+                && abandoned.generation == revision.generation
+                && abandoned.transaction_id != revision.transaction_id
+                && predecessors.contains(&previous.canonical) =>
+            {
+                // A rollback retains the failed PREPARE as evidence. A later
+                // transaction may reuse that generation, but only a backup of
+                // an uncommitted sibling with the same predecessor is obsolete.
+                obsolete.push(payload.clone());
+            }
             PoolMetaReplica::Valid {
                 canonical: candidate,
                 revision: candidate_revision,
@@ -272,13 +351,7 @@ async fn plan(
             _ => {}
         }
     }
-    if pending.len() > 1 {
-        return Err(known_validation(
-            Error::other("native V3 prepare records disagree on the predecessor"),
-            "native_transaction_validation",
-        ));
-    }
-    if pending.is_empty() {
+    if pending.is_empty() || rollback.is_some_and(|previous| !predecessors.iter().any(|candidate| candidate == previous)) {
         return Err(invalid());
     }
     for payload in &decoded {
@@ -294,13 +367,25 @@ async fn plan(
         for (index, snapshot) in pool.snapshots.iter().enumerate() {
             let raw = snapshot.current.as_ref().ok_or_else(invalid)?;
             let mut matched = false;
-            for payload in std::iter::once(canonical).chain(pending.iter().map(Vec::as_slice)) {
+            for payload in std::iter::once(canonical)
+                .chain(pending.iter().map(Vec::as_slice))
+                .chain(rollback)
+            {
                 matched |= codec::validate_inline_pool_meta_shard(raw, index, payload).await.is_ok();
             }
             if !matched {
                 return Err(invalid());
             }
             for backup in &snapshot.backups {
+                if rollback.is_some()
+                    && codec::validate_inline_pool_meta_shard(&backup.data, index, canonical)
+                        .await
+                        .is_ok()
+                {
+                    // A retained COMMIT might have been authoritative before
+                    // this interruption. Never roll it back from PREPARE alone.
+                    return Err(invalid());
+                }
                 let mut matched = false;
                 for payload in std::iter::once(canonical).chain(
                     pending
@@ -323,7 +408,7 @@ async fn plan(
     let mut replacements = Vec::with_capacity(inspected.len());
     for pool in inspected {
         replacements.push(
-            codec::encode_inline_pool_meta_replicas(&current(pool)?, canonical, modified)
+            codec::encode_inline_pool_meta_replicas(&current(pool)?, rollback.unwrap_or(canonical), modified)
                 .await
                 .map_err(|error| codec_validation(error, "native_metadata_validation"))?,
         );
@@ -338,10 +423,12 @@ pub(super) async fn repair(
 ) -> Result<bool> {
     fence.ensure_held()?;
     let reads = read_pool_meta_replicas(store.pools.clone(), true).await;
-    if !reads
+    let unreadable = reads
         .iter()
-        .any(|read| matches!(read.replica, PoolMetaReplica::Unreadable(_)))
-    {
+        .enumerate()
+        .filter_map(|(index, read)| matches!(read.replica, PoolMetaReplica::Unreadable(_)).then_some(index))
+        .collect::<Vec<_>>();
+    if unreadable.is_empty() {
         return Ok(false);
     }
     state.ensure_write_safe("native pool metadata recovery")?;
@@ -360,8 +447,9 @@ pub(super) async fn repair(
     {
         return Err(invalid());
     }
-    // Only an ordinary, successfully read COMMIT grants authority. Rebuilding
-    // two erasure shards below proves integrity, never a commit decision.
+    // Ordinary reads grant authority: a COMMIT selects its own state, while a
+    // PREPARE can select only its acknowledged predecessor. Reconstruction
+    // below proves integrity and never promotes an interrupted transaction.
     let ordinary_commits = reads
         .iter()
         .filter_map(|read| match &read.replica {
@@ -375,7 +463,22 @@ pub(super) async fn repair(
             _ => None,
         })
         .collect::<Vec<_>>();
-    if ordinary_commits.is_empty() {
+    let ordinary_prepares = reads
+        .iter()
+        .filter_map(|read| match &read.replica {
+            PoolMetaReplica::Valid {
+                canonical,
+                revision,
+                committed: false,
+                previous: Some(previous),
+                ..
+            } if revision.is_generation_protocol() && previous.revision.is_generation_protocol() => {
+                Some((canonical.clone(), *revision, previous.canonical.clone()))
+            }
+            _ => None,
+        })
+        .collect::<Vec<_>>();
+    if ordinary_commits.is_empty() && ordinary_prepares.is_empty() {
         return Err(invalid());
     }
     let selection = select_pool_meta_replica_reads(reads)
@@ -395,10 +498,20 @@ pub(super) async fn repair(
         return Err(invalid());
     }
     let canonical = selection.canonical.as_ref().ok_or_else(invalid)?;
+    let mut rollback = None;
+    for (pending, revision, previous) in &ordinary_prepares {
+        if previous == canonical {
+            if rollback.is_some_and(|candidate| candidate != (pending, *revision)) {
+                return Err(latch_validation(state, known_validation(invalid(), "native_transaction_validation")));
+            }
+            rollback = Some((pending, *revision));
+        }
+    }
     if !selection.revision.is_generation_protocol()
-        || !ordinary_commits
-            .iter()
-            .any(|(raw, revision)| raw == canonical && *revision == selection.revision)
+        || (rollback.is_none()
+            && !ordinary_commits
+                .iter()
+                .any(|(raw, revision)| raw == canonical && *revision == selection.revision))
     {
         return Err(invalid());
     }
@@ -445,13 +558,18 @@ pub(super) async fn repair(
         .await
         .map_err(|error| latch_validation(state, error))?;
     let inspected = inspect(&store.pools, fence).await?;
-    let replacements = plan(&inspected, canonical, selection.revision, &identity_bytes)
+    let (transaction, revision) = rollback.unwrap_or((canonical, selection.revision));
+    let replacements = plan(&inspected, transaction, revision, &identity_bytes, rollback.map(|_| canonical.as_slice()))
         .await
         .map_err(|error| latch_validation(state, error))?;
     fence.ensure_held()?;
     let mut arm = state.arm_transaction();
     arm.phase = Some("native_repair_apply");
-    for (pool, replacements) in inspected.iter().zip(replacements) {
+    let mut apply_order = inspected.iter().zip(replacements).enumerate().collect::<Vec<_>>();
+    // Preserve ordinary authority until all unreadable pools are repaired.
+    // Cancellation can then resume from a PREPARE or a restored COMMIT.
+    apply_order.sort_by_key(|(index, _)| !unreadable.contains(index));
+    for (_, (pool, replacements)) in apply_order {
         for ((disk, snapshot), replacement) in pool.disks.iter().zip(&pool.snapshots).zip(replacements) {
             fence.ensure_held()?;
             let request = Request::Apply {
@@ -466,6 +584,11 @@ pub(super) async fn repair(
                 _ => return Err(invalid()),
             }
         }
+        #[cfg(test)]
+        if let Ok(pause) = AFTER_NATIVE_POOL_APPLY.try_with(Arc::clone) {
+            pause.reached.notify_one();
+            pause.resume.notified().await;
+        }
     }
     #[cfg(test)]
     if let Ok(pause) = AFTER_NATIVE_APPLY.try_with(Arc::clone) {
@@ -627,7 +750,7 @@ mod tests {
     #[tokio::test]
     async fn pool_meta_native_plan_restores_physical_split_with_fresh_common_identity() {
         let f = fixture().await;
-        let repaired = plan(&f.pools, &f.commit, f.revision, &f.identity).await.unwrap();
+        let repaired = plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.unwrap();
         assert_eq!(repaired.len(), 4);
         for pool in repaired {
             assert_eq!(
@@ -648,7 +771,7 @@ mod tests {
         let mut f = fixture().await;
         let wrong = replicas(b"different identity", f.modified).await;
         f.pools[0].snapshots[0].identity = Some(wrong[0].clone());
-        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
+        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
     }
 
     #[tokio::test]
@@ -671,10 +794,10 @@ mod tests {
         .unwrap();
         let wrong = replicas(&higher, f.modified + Duration::seconds(2)).await;
         f.pools[0].snapshots[0].current = Some(wrong[0].clone());
-        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
+        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
         f.pools[0].snapshots[0].current = Some(replicas(&f.pending, f.modified).await[0].clone());
         f.pools[0].snapshots[0].backups[0].data = wrong[0].clone();
-        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
+        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
     }
 
     #[tokio::test]
@@ -717,17 +840,17 @@ mod tests {
             }
         }
         assert!(older_pending != f.pending);
-        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_ok());
+        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_ok());
     }
 
     #[tokio::test]
     async fn pool_meta_native_plan_rejects_missing_or_corrupt_evidence() {
         let mut f = fixture().await;
         let before = f.pools[1].snapshots[2].current.take();
-        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
+        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
         f.pools[1].snapshots[2].current = before;
         f.pools[1].snapshots[2].backups[0].data = Bytes::from_static(b"invalid metadata");
-        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
+        assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
     }
 }
 

crates/ecstore/src/core/pool_meta_recovery_codec.rs

@@ -315,7 +315,7 @@ pub(super) async fn encode_inline_pool_meta_replicas(
         .encode_with_shard_integrity(
             Cursor::new(Bytes::copy_from_slice(canonical)),
             &mut writers,
-            raw.len(),
+            crate::erasure::coding::encode::ShardWritePolicy::object(raw.len()),
             1,
             IntegrityEncodeMode::Inline(canonical.len()),
             protected,

crates/ecstore/src/core/pools_test.rs

@@ -1616,7 +1616,7 @@ mod decommission_lock_order_tests {
             run_routine_to_a_deferral_pause(
                 &store,
                 1,
-                "target pool 1 no longer fits the selected mutation: source pool 0 reservation estimate is exhausted",
+                "target pool 1 no longer fits the selected mutation: target pool 1 reservation requires",
             )
             .await;
         });
@@ -1647,7 +1647,7 @@ mod decommission_lock_order_tests {
             run_routine_to_a_deferral_pause(
                 &store,
                 1,
-                "target pool 1 no longer fits the selected mutation: source pool 0 reservation estimate is exhausted",
+                "target pool 1 no longer fits the selected mutation: target pool 1 reservation requires",
             )
             .await;
         });
@@ -6062,6 +6062,12 @@ mod decommission_lock_order_tests {
             )
             .await
             .expect("read the next source object for the multipart capacity oracle");
+        // Reconciled bytes remain charged. With no physical room for a new
+        // reservation, a distinct mutation must fail instead of reusing them.
+        let mut exhausted_capacity = capacity_snapshot();
+        exhausted_capacity[2] = DecommissionPoolCapacityInfo::for_test(2, layout, 0, target_total, target_total);
+        set_decommission_capacity_info_overrides_for_test(other_store.id, vec![exhausted_capacity]);
+
         let next_result = tokio::time::timeout(
             Duration::from_secs(30),
             data_movement::migrate_decommission_object(
@@ -7827,6 +7833,12 @@ mod decommission_lock_order_tests {
             .expect("same-mutation retry should publish a readable target");
         assert_eq!(retry_info.size, i64::try_from(body.len()).expect("test body size fits i64"));
 
+        // Reconciled bytes remain charged. With no physical room for a new
+        // reservation, a distinct mutation must fail instead of reusing them.
+        let mut exhausted_capacity = capacity_snapshot();
+        exhausted_capacity[2] = DecommissionPoolCapacityInfo::for_test(2, layout, 0, target_total, target_total);
+        set_decommission_capacity_info_overrides_for_test(other_store.id, vec![exhausted_capacity]);
+
         let next_object = "data-movement-put-after-retry.bin";
         let next_version_id = uuid::Uuid::new_v4().to_string();
         let mut next_opts = ObjectOptions {
@@ -7841,7 +7853,11 @@ mod decommission_lock_order_tests {
         let next_result = other_store
             .put_object_for_data_movement(RUSTFS_META_BUCKET, next_object, &mut next_data, &next_opts, None)
             .await;
-        assert!(next_result.is_err(), "a different mutation must not reuse A's recovered capacity");
+        let next_error = match next_result {
+            Err(error) | Ok((_, Err(error))) => error,
+            Ok((_, Ok(_))) => panic!("a different mutation must not reuse A's recovered capacity"),
+        };
+        assert!(next_error.to_string().contains("decommission_capacity_blocked"), "{next_error}");
         let next_target_err = other_store.pools[2]
             .get_object_info(
                 RUSTFS_META_BUCKET,
@@ -9929,6 +9945,12 @@ mod decommission_lock_order_tests {
             )
             .await
             .expect("read the next source object for the capacity oracle");
+        // Reconciled bytes remain charged. With no physical room for a new
+        // reservation, a distinct mutation must fail instead of reusing them.
+        let mut exhausted_capacity = capacity_snapshot();
+        exhausted_capacity[2] = DecommissionPoolCapacityInfo::for_test(2, layout, 0, target_total, target_total);
+        set_decommission_capacity_info_overrides_for_test(other_store.id, vec![exhausted_capacity]);
+
         let next_result = tokio::time::timeout(
             Duration::from_secs(30),
             data_movement::migrate_decommission_object(
@@ -10190,11 +10212,20 @@ mod decommission_lock_order_tests {
         assert_eq!(reconciled_reservation.pending_target_physical_bytes, 0);
         assert_eq!(reconciled_reservation.consumed_target_physical_bytes, 1);
 
+        // Reconciled bytes remain charged. With no physical room for a new
+        // reservation, a distinct mutation must fail instead of reusing them.
+        let mut exhausted_capacity = capacity_snapshot();
+        exhausted_capacity[2] = DecommissionPoolCapacityInfo::for_test(2, layout, 1, 2, 1);
+        set_decommission_capacity_info_overrides_for_test(other_store.id, vec![exhausted_capacity]);
+
         let next_after_reconcile = other_store
             .delete_object(&bucket, next_object, next_opts)
             .await
             .expect_err("the consumed exact-fit capacity must reject the next marker");
-        assert!(next_after_reconcile.to_string().contains("decommission_capacity_blocked"));
+        assert!(
+            next_after_reconcile.to_string().contains("decommission_capacity_blocked"),
+            "{next_after_reconcile}"
+        );
         let next_target_err = other_store.pools[2]
             .get_object_info(
                 &bucket,

crates/ecstore/src/core/sets.rs

@@ -317,8 +317,9 @@ impl Sets {
         &self.ctx
     }
 
-    #[cfg(test)]
-    pub(crate) async fn stop_endpoint_monitor_for_test(&self) {
+    /// Stop endpoint reconnection and wait for any in-flight pass before a test owns disk probes.
+    #[cfg(any(test, feature = "test-util"))]
+    pub async fn stop_endpoint_monitor_for_test(&self) {
         if let Some(exit_signal) = &self.exit_signal {
             let _ = exit_signal.send(());
             // Wait for an in-flight reconnect pass as well as its receiver to exit.

crates/ecstore/src/data_movement/e2e_barrier.rs

@@ -31,6 +31,8 @@ use tokio::fs;
 use tokio::time::Instant;
 
 pub(crate) const BARRIER_DIR_ENV: &str = "RUSTFS_E2E_DATA_MOVEMENT_BARRIER_DIR";
+/// Rebalance holds its admission fence; no version of this entry has moved.
+pub(crate) const STAGE_AFTER_ENTRY_ADMISSION: &str = "after_entry_admission";
 /// Every target part is staged and the target version is not yet published.
 pub(crate) const STAGE_BEFORE_TARGET_COMMIT: &str = "before_target_commit";
 /// The target commit is durable and the source preflight matched; the source

crates/ecstore/src/data_usage/mod.rs

@@ -34,7 +34,7 @@ pub use local_snapshot::{LocalUsageSnapshot, read_snapshot as read_local_snapsho
 use rustfs_data_usage::{
     BucketTargetUsageInfo, BucketUsageInfo, CompressionTotalInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME,
     DataUsageCache, DataUsageInfo, DiskUsageStatus, LEGACY_DATA_USAGE_OBJECT_NAME, SizeHistogram, VersionsHistogram,
-    observed_data_usage_is_newer,
+    observed_data_usage_is_newer, observed_data_usage_is_newer_than_identity,
 };
 use rustfs_io_metrics::record_system_path_failure;
 use rustfs_utils::path::SLASH_SEPARATOR;
@@ -227,7 +227,10 @@ pub async fn lookup_degraded_bucket_usage_baseline(store: Arc<ECStore>, bucket:
 /// endpoints for up to `DATA_USAGE_CACHE_TTL_SECS` between backend reads.
 #[derive(Debug, Clone)]
 struct CachedDataUsageSnapshot {
+    /// Last successful load, retained through failures for lineage comparison.
     info: Option<DataUsageInfo>,
+    /// Controls serving independently from the retained ordering evidence.
+    last_load_failed: bool,
     loaded_at: tokio::time::Instant,
     /// Pre-discard per-bucket sizes from the same load, retained even when the
     /// snapshot is incomplete and its bucket data is discarded. Consumed only
@@ -240,6 +243,7 @@ impl CachedDataUsageSnapshot {
     fn result<T>(&self, project: impl FnOnce(&DataUsageInfo) -> T) -> Result<T, Error> {
         self.info
             .as_ref()
+            .filter(|_| !self.last_load_failed)
             .map(project)
             .ok_or_else(|| Error::other("data usage snapshot load recently failed"))
     }
@@ -275,6 +279,17 @@ fn cache_data_usage_snapshot_result<T>(
             mut degraded_baseline,
             observed_unavailable,
         }) => {
+            // A primary read can fall back to an older backup. Refresh the
+            // retry age without replacing a newer snapshot this process has
+            // already read; snapshot size itself is not an ordering signal.
+            if let Some(previous) = cache.as_mut()
+                && let Some(cached_info) = previous.info.as_ref()
+                && snapshot_predates(&info, cached_info)
+            {
+                previous.loaded_at = loaded_at;
+                previous.last_load_failed = false;
+                return Some(Ok(project(cached_info)));
+            }
             // A flaky observed read must not shrink quota coverage for a TTL
             // window: carry the previous refresh's baseline entries forward,
             // letting the fresh (authoritative) values win where they exist.
@@ -286,21 +301,23 @@ fn cache_data_usage_snapshot_result<T>(
             let projected = project(&info);
             *cache = Some(CachedDataUsageSnapshot {
                 info: Some(info),
+                last_load_failed: false,
                 loaded_at,
                 degraded_baseline,
             });
             Ok(projected)
         }
         Err(e) => {
-            // Keep the previous baseline through a failed refresh: quota
-            // admission must not lose its last grounded values because one
-            // backend read errored.
-            let degraded_baseline = cache.take().map(|cached| cached.degraded_baseline).unwrap_or_default();
-            *cache = Some(CachedDataUsageSnapshot {
+            // A failed load is negatively cached, but cannot erase the last
+            // good snapshot's ordering proof or grounded quota baseline.
+            let cached = cache.get_or_insert_with(|| CachedDataUsageSnapshot {
                 info: None,
+                last_load_failed: true,
                 loaded_at,
-                degraded_baseline,
+                degraded_baseline: HashMap::new(),
             });
+            cached.loaded_at = loaded_at;
+            cached.last_load_failed = true;
             Err(e)
         }
     })
@@ -1245,7 +1262,7 @@ async fn load_data_usage_from_backend_with_baseline(store: Arc<ECStore>) -> Resu
     let (loaded_snapshot, source) = load_data_usage_snapshot(store.clone()).await?;
     // The observed-newness gate below compares against the snapshot as
     // persisted, before normalization demotes or discards anything.
-    let authoritative_as_persisted = loaded_snapshot.clone();
+    let authoritative_as_persisted = loaded_snapshot.snapshot_identity();
     let (info, mut degraded_baseline) = normalize_loaded_data_usage(loaded_snapshot, source.is_authoritative()).await;
 
     // A bucket without a converged scanner cycle behind it — a freshly joined
@@ -1263,7 +1280,7 @@ async fn load_data_usage_from_backend_with_baseline(store: Arc<ECStore>) -> Resu
     // read per snapshot refresh.
     let mut observed_unavailable = false;
     match load_observed_data_usage_snapshot(store).await {
-        Ok(Some(observed)) if observed_data_usage_is_newer(&observed, &authoritative_as_persisted) => {
+        Ok(Some(observed)) if observed_data_usage_is_newer_than_identity(&observed, authoritative_as_persisted) => {
             backfill_degraded_baseline_from_observed(&mut degraded_baseline, &observed);
         }
         Ok(_) => {}
@@ -2367,6 +2384,9 @@ fn unknown_bucket_reload_due(cache: &Option<CachedDataUsageSnapshot>, bucket: &s
     let Some(cached) = cache.as_ref() else {
         return false;
     };
+    if cached.last_load_failed {
+        return false;
+    }
     let Some(info) = cached.info.as_ref() else {
         return false;
     };
@@ -2382,11 +2402,35 @@ fn unknown_bucket_reload_due(cache: &Option<CachedDataUsageSnapshot>, bucket: &s
         .is_none_or(|attempted_at| now.duration_since(attempted_at) >= refresh_age)
 }
 
-/// Whether a freshly loaded snapshot is older than the cached one, as when a
-/// primary read fails and the loader falls back to the previous `.bkp` copy.
-fn snapshot_predates(loaded: Option<SystemTime>, cached: Option<SystemTime>) -> bool {
+/// Compare authoritative lineage by the writer's epoch, cycle, then timestamp
+/// order. Within one lineage, compare the observation's coordinated position
+/// before its timestamp; a later observation cannot revive an older lineage.
+fn snapshot_predates(loaded: &DataUsageInfo, cached: &DataUsageInfo) -> bool {
+    let loaded_baseline = loaded
+        .usage_snapshot_authoritative_baseline
+        .unwrap_or_else(|| loaded.snapshot_identity());
+    let cached_baseline = cached
+        .usage_snapshot_authoritative_baseline
+        .unwrap_or_else(|| cached.snapshot_identity());
+    let loaded_position = (loaded_baseline.scanner_epoch, loaded_baseline.scanner_cycle);
+    let cached_position = (cached_baseline.scanner_epoch, cached_baseline.scanner_cycle);
+    if loaded_position != cached_position {
+        return loaded_position < cached_position;
+    }
+    let (loaded, cached) = if loaded_baseline == cached_baseline {
+        let loaded_observation_position = loaded.scanner_epoch.zip(loaded.scanner_cycle);
+        let cached_observation_position = cached.scanner_epoch.zip(cached.scanner_cycle);
+        if loaded_observation_position != cached_observation_position {
+            return loaded_observation_position < cached_observation_position;
+        }
+        (loaded.last_update, cached.last_update)
+    } else {
+        (loaded_baseline.last_update, cached_baseline.last_update)
+    };
     match (loaded, cached) {
-        (Some(loaded), Some(cached)) => loaded < cached,
+        (Some(loaded), Some(cached)) => {
+            loaded < cached && !rustfs_data_usage::usage_last_update_is_untrusted_future(cached, SystemTime::now())
+        }
         (None, Some(_)) => true,
         _ => false,
     }
@@ -2427,22 +2471,15 @@ where
         return false;
     };
     let mut cache = data_usage_snapshot_cache().write().await;
-    let cached_last_update = cache
-        .as_ref()
-        .and_then(|cached| cached.info.as_ref())
-        .and_then(|info| info.last_update);
-    if snapshot_predates(loaded.info.last_update, cached_last_update) {
-        return false;
-    }
     cache_data_usage_snapshot_result(
         &mut cache,
         Ok(loaded),
         tokio::time::Instant::now(),
         refresh_generation,
         data_usage_snapshot_generation(),
-        |_| (),
+        |info| info.buckets_usage.contains_key(bucket),
     )
-    .is_some()
+    .is_some_and(|result| matches!(result, Ok(true)))
 }
 
 // Calls of `reload_unknown_bucket_usage` per bucket. Unit tests do not run in
@@ -4224,6 +4261,7 @@ mod tests {
         let degraded_baseline = info.bucket_sizes.clone();
         *data_usage_snapshot_cache().write().await = Some(CachedDataUsageSnapshot {
             info: Some(info),
+            last_load_failed: false,
             loaded_at: tokio::time::Instant::now()
                 .checked_sub(age)
                 .expect("the monotonic clock must reach back by the requested age"),
@@ -4700,6 +4738,355 @@ mod tests {
         assert_snapshot_bucket(&cached, "bucket");
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn expired_usage_snapshot_refresh_keeps_newer_counts_over_an_older_backup() {
+        use crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
+        use crate::config::com::save_config;
+
+        let (_dirs, store) = isolated_store_over_temp_disks().await;
+        let mut current = usage_snapshot_with_new_bucket(SystemTime::UNIX_EPOCH + Duration::from_secs(2), 100);
+        current.scanner_epoch = Some(4);
+        current.scanner_cycle = Some(11);
+        current.usage_snapshot_converged = Some(true);
+        let mut backup = data_usage_info_for_test("existing", 1, 5, SystemTime::UNIX_EPOCH + Duration::from_secs(1));
+        backup.scanner_epoch = Some(4);
+        backup.scanner_cycle = Some(10);
+        backup.usage_snapshot_converged = Some(true);
+        for (path, info) in [
+            (DATA_USAGE_OBJ_NAME_PATH.as_str(), &current),
+            (DATA_USAGE_OBJ_BACKUP_PATH.as_str(), &backup),
+        ] {
+            save_config(store.clone(), path, serde_json::to_vec(info).expect("encode usage snapshot"))
+                .await
+                .expect("persist initial usage snapshot");
+        }
+        invalidate_data_usage_snapshot_cache().await;
+        for loaded in [
+            load_data_usage_from_backend_cached(store.clone()).await,
+            load_admin_data_usage_from_backend_cached(store.clone()).await,
+        ] {
+            assert_eq!(loaded.expect("prime usage cache"), current);
+        }
+
+        // Bucket cleanup can reduce usage without advancing the scanner
+        // position. Only its publication timestamp advances.
+        let mut after_delete = data_usage_info_for_test("existing", 1, 3, SystemTime::UNIX_EPOCH + Duration::from_secs(3));
+        after_delete.scanner_epoch = current.scanner_epoch;
+        after_delete.scanner_cycle = current.scanner_cycle;
+        after_delete.usage_snapshot_converged = Some(true);
+        let mut stale_observation =
+            data_usage_info_for_test("retired-incarnation", 99, 999, SystemTime::UNIX_EPOCH + Duration::from_secs(4));
+        stale_observation.scanner_epoch = Some(4);
+        stale_observation.scanner_cycle = Some(12);
+        stale_observation.usage_snapshot_converged = Some(false);
+        stale_observation.usage_snapshot_authoritative_baseline = Some(backup.snapshot_identity());
+        assert!(observed_data_usage_is_newer(&stale_observation, &backup));
+        assert!(!observed_data_usage_is_newer(&stale_observation, &current));
+        save_config(
+            store.clone(),
+            DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
+            serde_json::to_vec(&stale_observation).expect("encode observation of an older namespace"),
+        )
+        .await
+        .expect("persist an observation anchored to the old backup");
+        for (payload, expected) in [
+            (b"not a usage snapshot".to_vec(), &current),
+            (serde_json::to_vec(&after_delete).expect("encode post-delete usage"), &after_delete),
+        ] {
+            save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), payload)
+                .await
+                .expect("replace primary usage snapshot");
+            for cache in [data_usage_snapshot_cache(), admin_data_usage_snapshot_cache()] {
+                cache
+                    .write()
+                    .await
+                    .as_mut()
+                    .expect("usage cache should be populated")
+                    .loaded_at = tokio::time::Instant::now() - Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
+            }
+            for loaded in [
+                load_data_usage_from_backend_cached(store.clone()).await,
+                load_admin_data_usage_from_backend_cached(store.clone()).await,
+            ] {
+                assert_eq!(loaded.expect("refresh an expired usage cache"), *expected);
+            }
+            assert!(
+                !data_usage_snapshot_cache()
+                    .read()
+                    .await
+                    .as_ref()
+                    .expect("quota cache should remain populated")
+                    .degraded_baseline
+                    .contains_key("retired-incarnation"),
+                "a newer observation anchored to an older namespace must not backfill ghost usage"
+            );
+        }
+        invalidate_data_usage_snapshot_cache().await;
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn expired_admin_usage_refresh_orders_observations_by_generation_before_time() {
+        use crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
+        use crate::config::com::save_config;
+
+        let (_dirs, store) = isolated_store_over_temp_disks().await;
+        let mut authoritative = data_usage_info_for_test("bucket", 1, 10, SystemTime::UNIX_EPOCH + Duration::from_secs(100));
+        authoritative.scanner_epoch = Some(7);
+        authoritative.scanner_cycle = Some(10);
+        authoritative.usage_snapshot_converged = Some(true);
+        save_config(
+            store.clone(),
+            DATA_USAGE_OBJ_NAME_PATH.as_str(),
+            serde_json::to_vec(&authoritative).expect("encode authoritative usage"),
+        )
+        .await
+        .expect("persist the common authoritative baseline");
+        let observation = |epoch, cycle, seconds, objects, size| {
+            let mut info =
+                data_usage_info_for_test("bucket", objects, size, SystemTime::UNIX_EPOCH + Duration::from_secs(seconds));
+            info.scanner_epoch = Some(epoch);
+            info.scanner_cycle = Some(cycle);
+            info.usage_snapshot_converged = Some(false);
+            info.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity());
+            info
+        };
+        let first = observation(7, 11, 200, 11, 1_100);
+        let newer_cycle = observation(7, 12, 150, 12, 1_200);
+        let older_cycle = observation(7, 11, 300, 99, 9_900);
+        let newer_epoch = observation(8, 1, 50, 1, 100);
+        invalidate_data_usage_snapshot_cache().await;
+
+        for (observed, expected) in [
+            (&first, &first),
+            (&newer_cycle, &newer_cycle),
+            (&older_cycle, &newer_cycle),
+            (&newer_epoch, &newer_epoch),
+        ] {
+            assert!(
+                observed_data_usage_is_newer(observed, &authoritative),
+                "the persisted observation must pass the real loader's selection gate"
+            );
+            save_config(
+                store.clone(),
+                DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
+                serde_json::to_vec(observed).expect("encode scanner observation"),
+            )
+            .await
+            .expect("persist scanner observation");
+            if let Some(cached) = admin_data_usage_snapshot_cache().write().await.as_mut() {
+                cached.loaded_at = tokio::time::Instant::now() - Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
+            }
+            let loaded = load_admin_data_usage_from_backend_cached(store.clone())
+                .await
+                .expect("refresh admin usage through the real backend loader");
+            assert_eq!(
+                &loaded,
+                expected,
+                "same-baseline observation ordering must follow epoch/cycle before wall clock: incoming={:?}",
+                observed.snapshot_identity()
+            );
+        }
+        invalidate_data_usage_snapshot_cache().await;
+    }
+
+    #[test]
+    fn expired_usage_snapshot_refresh_accepts_new_authority_over_a_later_old_observation() {
+        let loaded_at = tokio::time::Instant::now();
+        let baseline = data_usage_info_for_test("bucket", 3, 42, SystemTime::UNIX_EPOCH + Duration::from_secs(1));
+        let mut observed = data_usage_info_for_test("bucket", 5, 60, SystemTime::UNIX_EPOCH + Duration::from_secs(4));
+        observed.usage_snapshot_converged = Some(false);
+        observed.usage_snapshot_authoritative_baseline = Some(baseline.snapshot_identity());
+        let mut cache = Some(CachedDataUsageSnapshot {
+            info: Some(observed),
+            last_load_failed: false,
+            loaded_at,
+            degraded_baseline: HashMap::new(),
+        });
+        let after_delete = data_usage_info_for_test("bucket", 1, 7, SystemTime::UNIX_EPOCH + Duration::from_secs(3));
+        let refreshed = cache_data_usage_snapshot_result(
+            &mut cache,
+            Ok(loaded_usage_for_test(after_delete.clone())),
+            loaded_at,
+            1,
+            1,
+            DataUsageInfo::clone,
+        )
+        .expect("matching refresh generation")
+        .expect("a newer authority must retire observations of its predecessor");
+        assert_eq!(refreshed, after_delete);
+    }
+
+    #[test]
+    fn expired_usage_snapshot_refresh_orders_lineage_by_epoch_cycle_then_time() {
+        let loaded_at = tokio::time::Instant::now();
+        for (cached_position, loaded_position, loaded_time, accept) in [
+            ((Some(2), Some(10)), (Some(1), Some(99)), 110, false),
+            ((Some(2), Some(10)), (Some(2), Some(9)), 110, false),
+            ((Some(2), Some(10)), (Some(3), Some(1)), 90, true),
+            ((Some(2), Some(10)), (Some(2), Some(11)), 90, true),
+            ((Some(2), Some(10)), (Some(2), Some(10)), 110, true),
+            ((Some(3), None), (Some(2), Some(10)), 110, false),
+            ((Some(2), Some(10)), (Some(3), None), 90, true),
+            ((None, None), (None, None), 110, true),
+            ((None, None), (None, None), 90, false),
+        ] {
+            let mut previous = data_usage_info_for_test("bucket", 3, 42, SystemTime::UNIX_EPOCH + Duration::from_secs(100));
+            (previous.scanner_epoch, previous.scanner_cycle) = cached_position;
+            let mut loaded = data_usage_info_for_test("bucket", 1, 7, SystemTime::UNIX_EPOCH + Duration::from_secs(loaded_time));
+            (loaded.scanner_epoch, loaded.scanner_cycle) = loaded_position;
+            let expected = if accept { &loaded } else { &previous };
+            let mut cache = Some(CachedDataUsageSnapshot {
+                info: Some(previous.clone()),
+                last_load_failed: false,
+                loaded_at,
+                degraded_baseline: HashMap::new(),
+            });
+            let refreshed = cache_data_usage_snapshot_result(
+                &mut cache,
+                Ok(loaded_usage_for_test(loaded.clone())),
+                loaded_at,
+                1,
+                1,
+                DataUsageInfo::clone,
+            )
+            .expect("matching refresh generation")
+            .expect("a valid usage snapshot remains available");
+            assert_eq!(
+                &refreshed, expected,
+                "lineage ordering must match scanner publication: cached={cached_position:?}, loaded={loaded_position:?}, loaded_time={loaded_time}"
+            );
+        }
+    }
+
+    #[test]
+    fn expired_usage_snapshot_refresh_preserves_rejection_backoff_and_baseline() {
+        let loaded_at = tokio::time::Instant::now();
+        let expected = data_usage_info_for_test("bucket", 3, 42, SystemTime::UNIX_EPOCH + Duration::from_secs(2));
+        let mut cache = Some(CachedDataUsageSnapshot {
+            info: Some(expected.clone()),
+            last_load_failed: false,
+            loaded_at,
+            degraded_baseline: HashMap::from([("bucket".to_string(), 42)]),
+        });
+        let refreshed_at = loaded_at + Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
+        let refreshed = cache_data_usage_snapshot_result(
+            &mut cache,
+            Ok(loaded_usage_for_test(data_usage_info_for_test("bucket", 1, 7, SystemTime::UNIX_EPOCH))),
+            refreshed_at,
+            1,
+            1,
+            DataUsageInfo::clone,
+        )
+        .expect("matching refresh generation")
+        .expect("the newer cached snapshot remains available");
+        assert_eq!(refreshed, expected);
+        let retained = cache.as_ref().expect("cache should remain populated");
+        assert_eq!(retained.degraded_baseline.get("bucket"), Some(&42));
+        assert_eq!(retained.loaded_at, refreshed_at, "an older fallback must retain the TTL backoff");
+
+        let result =
+            cache_data_usage_snapshot_result(&mut cache, Err(Error::ErasureReadQuorum), refreshed_at, 1, 1, DataUsageInfo::clone)
+                .expect("matching refresh generation");
+        assert!(matches!(result, Err(Error::ErasureReadQuorum)));
+        let failed = cache.as_ref().expect("negative cache should be populated");
+        assert!(failed.last_load_failed);
+        assert_eq!(failed.info.as_ref(), Some(&expected), "failure must retain the last good ordering proof");
+    }
+
+    #[test]
+    fn expired_usage_snapshot_refresh_accepts_clock_recovery() {
+        let loaded_at = tokio::time::Instant::now();
+        let now = SystemTime::now();
+        let future = now + rustfs_data_usage::USAGE_LAST_UPDATE_FUTURE_TOLERANCE + Duration::from_secs(60);
+        let mut cache = Some(CachedDataUsageSnapshot {
+            info: Some(data_usage_info_for_test("bucket", 3, 42, future)),
+            last_load_failed: false,
+            loaded_at,
+            degraded_baseline: HashMap::new(),
+        });
+        let recovered = data_usage_info_for_test("bucket", 1, 7, now);
+        let refreshed = cache_data_usage_snapshot_result(
+            &mut cache,
+            Ok(loaded_usage_for_test(recovered.clone())),
+            loaded_at,
+            1,
+            1,
+            DataUsageInfo::clone,
+        )
+        .expect("matching refresh generation")
+        .expect("a clock recovery snapshot must not be frozen behind an untrusted timestamp");
+        assert_eq!(refreshed, recovered);
+    }
+
+    #[test]
+    fn expired_usage_snapshot_refresh_keeps_last_good_lineage_across_a_failed_load() {
+        let loaded_at = tokio::time::Instant::now();
+        let current = data_usage_info_for_test("bucket", 3, 42, SystemTime::UNIX_EPOCH + Duration::from_secs(2));
+        let mut cache = None;
+        let first = cache_data_usage_snapshot_result(
+            &mut cache,
+            Ok(loaded_usage_for_test(current.clone())),
+            loaded_at,
+            1,
+            1,
+            DataUsageInfo::clone,
+        )
+        .expect("matching refresh generation")
+        .expect("prime the last successful snapshot");
+        assert_eq!(first, current);
+
+        let failed_at = loaded_at + Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
+        let failure =
+            cache_data_usage_snapshot_result(&mut cache, Err(Error::ErasureReadQuorum), failed_at, 1, 1, DataUsageInfo::clone)
+                .expect("matching refresh generation");
+        assert!(matches!(failure, Err(Error::ErasureReadQuorum)));
+        assert!(
+            fresh_cached_data_usage_snapshot(
+                &cache,
+                failed_at + Duration::from_secs(DATA_USAGE_UNKNOWN_BUCKET_REFRESH_SECS),
+                Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS),
+                |_| panic!("a failed load must not serve its retained snapshot as a success"),
+            )
+            .expect("the failed load remains inside its negative TTL")
+            .is_err()
+        );
+        assert!(
+            !unknown_bucket_reload_due(
+                &cache,
+                "unknown",
+                failed_at + Duration::from_secs(DATA_USAGE_UNKNOWN_BUCKET_REFRESH_SECS),
+            ),
+            "an unknown bucket must not bypass the failed load's TTL"
+        );
+
+        let recovered_at = failed_at + Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
+        let recovered = cache_data_usage_snapshot_result(
+            &mut cache,
+            Ok(loaded_usage_for_test(data_usage_info_for_test("bucket", 1, 7, SystemTime::UNIX_EPOCH))),
+            recovered_at,
+            1,
+            1,
+            DataUsageInfo::clone,
+        )
+        .expect("matching refresh generation")
+        .expect("a readable older backup must preserve the last good lineage");
+        assert_eq!(recovered, current);
+        assert_eq!(
+            cache.as_ref().expect("recovered cache").degraded_baseline.get("bucket"),
+            Some(&42),
+            "a failed load must not erase the baseline's ordering proof"
+        );
+        assert_eq!(
+            fresh_cached_data_usage_snapshot(&cache, recovered_at, Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS), |info| info
+                .objects_total_size,)
+            .expect("the recovered load starts a fresh TTL")
+            .expect("the last successful snapshot is available again"),
+            42
+        );
+    }
+
     #[test]
     #[serial]
     fn unavailable_observed_read_keeps_previous_baseline_coverage() {
@@ -4707,6 +5094,7 @@ mod tests {
         let refresh_generation = data_usage_snapshot_generation();
         let mut cache = Some(CachedDataUsageSnapshot {
             info: Some(data_usage_info_for_test("bucket", 1, 42, SystemTime::UNIX_EPOCH)),
+            last_load_failed: false,
             loaded_at,
             degraded_baseline: HashMap::from([("observed-only".to_string(), 7_u64), ("covered".to_string(), 1)]),
         });
@@ -4740,6 +5128,7 @@ mod tests {
         let refresh_generation = data_usage_snapshot_generation();
         let mut cache = Some(CachedDataUsageSnapshot {
             info: Some(data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH)),
+            last_load_failed: false,
             loaded_at,
             degraded_baseline: HashMap::new(),
         });
@@ -5928,6 +6317,7 @@ mod tests {
         let mut snapshot_cache = data_usage_snapshot_cache().write().await;
         *snapshot_cache = Some(CachedDataUsageSnapshot {
             info: Some(successor),
+            last_load_failed: false,
             loaded_at: tokio::time::Instant::now(),
             degraded_baseline: HashMap::new(),
         });
@@ -5999,6 +6389,7 @@ mod tests {
         let mut snapshot_cache = data_usage_snapshot_cache().write().await;
         *snapshot_cache = Some(CachedDataUsageSnapshot {
             info: Some(successor),
+            last_load_failed: false,
             loaded_at: tokio::time::Instant::now(),
             degraded_baseline: HashMap::new(),
         });
@@ -6051,6 +6442,7 @@ mod tests {
         let stale = data_usage_info_for_test(BUCKET, 7, 294, SystemTime::now());
         *data_usage_snapshot_cache().write().await = Some(CachedDataUsageSnapshot {
             info: Some(stale),
+            last_load_failed: false,
             loaded_at: tokio::time::Instant::now(),
             degraded_baseline: HashMap::new(),
         });

crates/ecstore/src/data_usage/tests/snapshot_projection.rs

@@ -21,6 +21,7 @@ fn scalar_projection_borrows_snapshot_and_preserves_unknown_and_zero() {
     info.buckets_usage.insert("empty".to_string(), BucketUsageInfo::default());
     let cache = Some(CachedDataUsageSnapshot {
         info: Some(info),
+        last_load_failed: false,
         loaded_at,
         degraded_baseline: HashMap::new(),
     });
@@ -66,6 +67,7 @@ fn projection_is_skipped_for_missing_failed_expired_or_invalidated_snapshots() {
 
     cache = Some(CachedDataUsageSnapshot {
         info: Some(data_usage_info_for_test("known", 1, 42, SystemTime::UNIX_EPOCH)),
+        last_load_failed: false,
         loaded_at,
         degraded_baseline: HashMap::new(),
     });
@@ -127,6 +129,33 @@ fn projected_publication_moves_bucket_maps_into_cache() {
         cached.buckets_usage.get_key_value("known").expect("cached bucket").0.as_ptr(),
         original_key
     );
+
+    let failure = cache_data_usage_snapshot_result(
+        &mut cache,
+        Err(Error::ErasureReadQuorum),
+        tokio::time::Instant::now(),
+        1,
+        1,
+        |_: &DataUsageInfo| -> u64 { panic!("a failed load must not project its retained snapshot") },
+    )
+    .expect("matching refresh generation");
+    assert!(matches!(failure, Err(Error::ErasureReadQuorum)));
+    let retained = cache
+        .as_ref()
+        .expect("negative cache")
+        .info
+        .as_ref()
+        .expect("last good ordering proof");
+    assert_eq!(
+        retained
+            .buckets_usage
+            .get_key_value("known")
+            .expect("retained bucket")
+            .0
+            .as_ptr(),
+        original_key,
+        "a failed load must retain the snapshot without copying its bucket maps"
+    );
 }
 
 #[test]

crates/ecstore/src/diagnostics/object_lock.rs

@@ -167,6 +167,16 @@ impl ObjectLockAttempt {
             }
         }
     }
+
+    /// Correlate a raw guard's acquisition with its peer identities without
+    /// claiming ownership of the guard's lifetime. Peer release events retain
+    /// the identities needed to reconstruct when the actual owner releases it.
+    pub(crate) fn observe_acquisition(&mut self, result: &Result<NamespaceLockGuard, LockError>) {
+        self.observe(result);
+        if let Some(context) = self.0.as_mut() {
+            context.finished = true;
+        }
+    }
 }
 
 fn duration_ms(duration: Duration) -> u64 {
@@ -500,6 +510,45 @@ pub(crate) mod tests {
         )
     }
 
+    #[test]
+    #[serial_test::serial]
+    fn raw_guard_acquisition_diagnostic_does_not_claim_guard_lifetime() {
+        temp_env::with_vars([(rustfs_config::ENV_OBJECT_LOCK_DIAG_ENABLE, Some("true"))], || {
+            let capture = CapturedLockEvents::default();
+            let runtime = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .expect("raw guard diagnostic runtime");
+            tracing::subscriber::with_default(capture.subscriber(), || {
+                runtime.block_on(async {
+                    let lock = namespace();
+                    let mut attempt = ObjectLockAttempt::start(
+                        "bucket_incarnation_disk_read",
+                        "bucket",
+                        "transaction.lock",
+                        None,
+                        &lock,
+                        "read",
+                        Duration::from_secs(1),
+                    );
+                    let result = lock.get_read_lock(Duration::from_secs(1)).await;
+                    attempt.observe_acquisition(&result);
+                    drop(attempt);
+                    let guard = result.expect("retain the raw guard after diagnostic completion");
+                    assert!(!guard.is_lock_lost());
+                    let states = capture
+                        .rows()
+                        .iter()
+                        .map(|row| row["fields"]["state"].as_str().expect("diagnostic state string").to_owned())
+                        .collect::<Vec<_>>();
+                    assert_eq!(states, ["acquiring", "acquired"]);
+                    drop(guard);
+                    assert_eq!(capture.rows().len(), 2, "the acquisition observer must not invent a guard release");
+                });
+            });
+        });
+    }
+
     #[test]
     #[serial_test::serial]
     fn object_lock_diagnostics_disabled_does_not_allocate_context_or_emit() {

crates/ecstore/src/disk/disk_store.rs

@@ -55,6 +55,20 @@ use tokio_util::sync::CancellationToken;
 use tracing::{info, warn};
 use uuid::Uuid;
 
+/// Join the local disk's actual bootstrap worker for deterministic coverage tests.
+/// Completion may mean legacy fallback after a failed scan; callers must assert
+/// the expected index result separately. Remote disks have no local worker.
+#[cfg(any(test, feature = "test-util"))]
+pub async fn wait_for_multipart_index_bootstrap(disk: &crate::disk::DiskStore) -> Result<()> {
+    match disk.as_ref() {
+        crate::disk::Disk::Local(local) => {
+            local.disk.wait_for_multipart_index_bootstrap().await;
+            Ok(())
+        }
+        crate::disk::Disk::Remote(_) => Err(DiskError::MethodNotAllowed),
+    }
+}
+
 /// Disk health status constants
 const DISK_HEALTH_OK: u32 = 0;
 const DISK_HEALTH_FAULTY: u32 = 1;
@@ -2443,6 +2457,7 @@ impl LocalDiskWrapper {
         object: &str,
         transaction: uuid::Uuid,
         namespace_owner: Option<Arc<dyn Send + Sync>>,
+        work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
     ) -> Result<Option<(crate::disk::DeleteRecoveryJournal, bool, bool)>> {
         self.track_disk_health_mutation(
             "settle_delete_transaction",
@@ -2450,7 +2465,7 @@ impl LocalDiskWrapper {
             || {
                 Box::pin(async {
                     self.disk
-                        .try_publish_prepared_delete_local(volume, object, transaction, namespace_owner)
+                        .try_publish_prepared_delete_local_with_budget(volume, object, transaction, namespace_owner, work_budget)
                         .await
                 })
             },
@@ -2466,6 +2481,7 @@ impl LocalDiskWrapper {
         transaction: uuid::Uuid,
         journal: &crate::disk::DeleteRecoveryJournal,
         namespace_owner: Option<Arc<dyn Send + Sync>>,
+        work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
     ) -> Result<bool> {
         self.track_disk_health_mutation(
             "settle_delete_transaction",
@@ -2473,7 +2489,14 @@ impl LocalDiskWrapper {
             || {
                 Box::pin(async {
                     self.disk
-                        .try_finish_and_finalize_prepared_delete_local(volume, object, transaction, journal, namespace_owner)
+                        .try_finish_and_finalize_prepared_delete_local_with_budget(
+                            volume,
+                            object,
+                            transaction,
+                            journal,
+                            namespace_owner,
+                            work_budget,
+                        )
                         .await
                 })
             },
@@ -2728,6 +2751,18 @@ impl LocalDiskWrapper {
 
 #[async_trait::async_trait]
 impl DiskAPI for LocalDiskWrapper {
+    async fn list_multipart_index_roots(
+        &self,
+        bucket: &str,
+        marker: Option<&str>,
+    ) -> Result<crate::disk::multipart_index::MultipartIndexPage> {
+        if self.health.is_faulty() {
+            return Err(DiskError::FaultyDisk);
+        }
+        self.check_disk_stale().await?;
+        self.disk.list_multipart_index_roots(bucket, marker).await
+    }
+
     async fn supports_guarded_delete_versions(&self) -> Result<()> {
         // The local implementation shares this binary's capability. Availability
         // and disk identity remain mutation checks and write-quorum inputs.
@@ -2781,6 +2816,62 @@ impl DiskAPI for LocalDiskWrapper {
         )
         .await
     }
+    async fn discover_delete_transactions_bounded(
+        &self,
+        volume: &str,
+        object: &str,
+        max_entries: u32,
+    ) -> Result<crate::disk::BoundedDeleteRecoveryDiscovery> {
+        self.track_disk_health_with_op(
+            "discover_delete_transactions",
+            || async {
+                self.disk
+                    .discover_delete_transactions_bounded(volume, object, max_entries)
+                    .await
+            },
+            get_max_timeout_duration(),
+        )
+        .await
+    }
+
+    async fn inspect_delete_transaction_bounded(
+        &self,
+        volume: &str,
+        object: &str,
+        transaction: uuid::Uuid,
+        max_entries: u32,
+    ) -> Result<crate::disk::BoundedDeleteRecoveryInspection> {
+        self.track_disk_health_with_op(
+            "inspect_delete_transaction",
+            || async {
+                self.disk
+                    .inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
+                    .await
+            },
+            get_max_timeout_duration(),
+        )
+        .await
+    }
+
+    async fn settle_delete_transaction_bounded(
+        &self,
+        volume: &str,
+        request: crate::disk::BoundedDeleteRecoverySettlement,
+        namespace_owner: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<crate::disk::BoundedDeleteRecoverySettlementReply> {
+        self.track_disk_health_mutation(
+            "settle_delete_transaction",
+            DiskMetricMutation::Delete,
+            || async {
+                self.disk
+                    .settle_delete_transaction_bounded(volume, request, namespace_owner)
+                    .await
+            },
+            crate::disk::get_delete_recovery_timeout()?,
+        )
+        .await
+    }
+
     async fn discover_delete_transactions(
         &self,
         volume: &str,
@@ -3749,7 +3840,11 @@ mod tests {
         assert!(futures::poll!(&mut operation).is_pending());
         time::advance(SLOW_DISK_OPERATION_THRESHOLD - Duration::from_millis(1)).await;
         assert!(futures::poll!(&mut operation).is_pending());
-        assert!(logs.contents().is_empty(), "normal disk latency should not warn");
+        assert!(
+            !logs.contents().contains("disk_operation_slow") && !logs.contents().contains("WARN"),
+            "normal disk latency should not warn: {}",
+            logs.contents()
+        );
         time::advance(Duration::from_millis(1)).await;
         assert!(futures::poll!(&mut operation).is_pending());
         let warning = logs.contents();

crates/ecstore/src/disk/error.rs

@@ -172,6 +172,8 @@ pub enum DiskError {
 
     #[error("guarded batch deletion is not supported by this disk")]
     GuardedDeleteUnsupported,
+    #[error("foreground delete recovery work limit reached")]
+    DeleteRecoveryWorkLimit,
 
     #[error("erasure write quorum")]
     ErasureWriteQuorum,
@@ -770,6 +772,7 @@ impl Clone for DiskError {
             DiskError::MethodNotAllowed => DiskError::MethodNotAllowed,
             DiskError::ConditionalDeleteConflict => DiskError::ConditionalDeleteConflict,
             DiskError::GuardedDeleteUnsupported => DiskError::GuardedDeleteUnsupported,
+            DiskError::DeleteRecoveryWorkLimit => DiskError::DeleteRecoveryWorkLimit,
             DiskError::ErasureWriteQuorum => DiskError::ErasureWriteQuorum,
             DiskError::ErasureReadQuorum => DiskError::ErasureReadQuorum,
             DiskError::ShortWrite => DiskError::ShortWrite,
@@ -829,6 +832,7 @@ impl DiskError {
             DiskError::RemoteClientUnavailable(_) => 0x2B,
             DiskError::ConditionalDeleteConflict => 0x2C,
             DiskError::GuardedDeleteUnsupported => 0x2D,
+            DiskError::DeleteRecoveryWorkLimit => 0x2E,
         }
     }
 
@@ -879,6 +883,7 @@ impl DiskError {
             0x2B => Some(DiskError::RemoteClientUnavailable(String::new())),
             0x2C => Some(DiskError::ConditionalDeleteConflict),
             0x2D => Some(DiskError::GuardedDeleteUnsupported),
+            0x2E => Some(DiskError::DeleteRecoveryWorkLimit),
             _ => None,
         }
     }
@@ -1662,6 +1667,7 @@ mod tests {
         for (error, code) in [
             (DiskError::ConditionalDeleteConflict, 0x2C),
             (DiskError::GuardedDeleteUnsupported, 0x2D),
+            (DiskError::DeleteRecoveryWorkLimit, 0x2E),
         ] {
             assert_eq!(error.to_u32(), code);
             assert_eq!(DiskError::from_u32(code), Some(error.clone()));

crates/ecstore/src/disk/fd_cache_tests.rs

@@ -183,3 +183,459 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
         assert_eq!(fresh.len, 11);
     }
 }
+
+#[tokio::test]
+async fn fd_cache_tracker_skips_unmatched_invalidations_and_handles_deep_prefixes() {
+    let (_directory, file) = fixture();
+    let mut cache = FdCache::new();
+
+    // Unmatched invalidations on an empty cache must not spawn the maintenance task.
+    cache.invalidate_under("v", "missing/prefix");
+    cache.invalidate_volume("v");
+    cache.invalidate_exact("v", "missing/prefix/part.1").await;
+    cache.clear();
+    assert!(
+        cache.maintenance_task.get().is_none(),
+        "empty-cache invalidations must not spawn maintenance"
+    );
+
+    // Populate a normal entry and verify matched vs unmatched filter gates directly,
+    // and ensure unmatched invalidations do not enqueue pending removals.
+    let cached_key = key("v", "obj-1/0198a3f0/part.1", false);
+    cache.insert(cached_key.clone(), Arc::clone(&file)).await;
+    assert_eq!(cache.tracker.live_entries(), 1);
+    assert!(cache.tracker.may_match_volume("v"));
+    assert!(!cache.tracker.may_match_volume("other-volume"));
+    assert!(cache.tracker.may_match_prefix("v", "obj-1"));
+    assert!(cache.tracker.may_match_prefix("v", "obj-1/0198a3f0"));
+    assert!(!cache.tracker.may_match_prefix("v", "obj-2/0198a3f0"));
+    assert!(cache.tracker.may_match_exact("v", "obj-1/0198a3f0/part.1"));
+    assert!(!cache.tracker.may_match_exact("v", "obj-1/0198a3f0/xl.meta"));
+
+    cache.invalidate_volume("other-volume");
+    cache.invalidate_under("v", "obj-2/0198a3f0");
+    cache.invalidate_exact("v", "obj-1/0198a3f0/xl.meta").await;
+    assert!(
+        !cache
+            .tracker
+            .has_pending_invalidations
+            .load(std::sync::atomic::Ordering::SeqCst),
+        "unmatched invalidations must not enqueue maintenance removals"
+    );
+    assert!(cache.get(&cached_key).await.is_some());
+
+    // Deeply nested path (> FD_CACHE_MAX_INDEXED_PREFIXES slashes) must fall back
+    // to per-volume unindexed_counts without degrading other volumes.
+    let other_vol_key = key("v2", "shallow/part.1", false);
+    cache.insert(other_vol_key.clone(), Arc::clone(&file)).await;
+    let deep_path = "c1/c2/c3/c4/c5/c6/c7/c8/c9/c10/c11/c12/c13/c14/part.1";
+    let deep_key = key("v", deep_path, false);
+    cache.insert(deep_key.clone(), Arc::clone(&file)).await;
+    assert_eq!(
+        cache
+            .tracker
+            .unindexed_counts
+            .iter()
+            .map(|c| c.load(std::sync::atomic::Ordering::Acquire))
+            .sum::<u32>(),
+        1
+    );
+    assert!(
+        !cache.tracker.may_match_prefix("v2", "unmatched/prefix"),
+        "deep key in volume v must not disable prefix filtering in volume v2"
+    );
+    cache.invalidate_under("v", "c1/c2/c3/c4/c5/c6/c7/c8/c9/c10/c11/c12/c13");
+    assert!(cache.get(&deep_key).await.is_none());
+    cache.invalidate_volume("v2");
+    assert!(cache.get(&other_vol_key).await.is_none());
+
+    // Replacing an invalidated key before maintenance drains pending_invalidations
+    // must release the old descriptor immediately and keep the replacement live
+    // even when a strong reference to the old CachedFdEntry shell survives until
+    // maintenance runs its Arc::ptr_eq check.
+    let old_file = Arc::new(tempfile::tempfile().expect("create replaced descriptor"));
+    let old_weak = Arc::downgrade(&old_file);
+    cache.insert(cached_key.clone(), old_file).await;
+    let held_old_shell = cache.cache.get(&cached_key).expect("cached shell must exist");
+    cache.invalidate_under("v", "obj-1/0198a3f0");
+    assert!(cache.get(&cached_key).await.is_none());
+    cache.insert(cached_key.clone(), Arc::clone(&file)).await;
+    assert!(
+        old_weak.upgrade().is_none(),
+        "replacing an entry must retire and drop the previous descriptor immediately"
+    );
+    tokio::time::timeout(std::time::Duration::from_secs(5), async {
+        while cache
+            .tracker
+            .has_pending_invalidations
+            .load(std::sync::atomic::Ordering::SeqCst)
+        {
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("maintenance should drain pending invalidations");
+    drop(held_old_shell);
+    assert!(
+        cache.get(&cached_key).await.is_some(),
+        "stale pending invalidation must not evict a newer replacement"
+    );
+    assert_eq!(
+        cache
+            .tracker
+            .unindexed_counts
+            .iter()
+            .map(|c| c.load(std::sync::atomic::Ordering::Acquire))
+            .sum::<u32>(),
+        0
+    );
+
+    // Dropping a cache while a live non-invalidated entry is present (even if a
+    // cloned Moka handle outlives FdCache and the maintenance task is aborted)
+    // must retire the descriptor and release FdCacheTracker without cycles.
+    let live_drop_file = Arc::new(tempfile::tempfile().expect("create live drop descriptor"));
+    let live_drop_weak = Arc::downgrade(&live_drop_file);
+    let shutdown_file = Arc::new(tempfile::tempfile().expect("create shutdown descriptor"));
+    let shutdown_weak = Arc::downgrade(&shutdown_file);
+    let tracker_weak = Arc::downgrade(&cache.tracker);
+    let cloned_moka = cache.cache.clone();
+    cache.insert(key("v", "live-on-drop/part.1", false), live_drop_file).await;
+    cache.insert(key("v", "shutdown/part.1", false), shutdown_file).await;
+    cache.invalidate_under("v", "shutdown");
+    if let Some(handle) = cache.maintenance_task.take() {
+        handle.abort();
+        let _ = handle.await;
+    }
+    drop(cache);
+    assert!(
+        live_drop_weak.upgrade().is_none(),
+        "Drop for FdCache must retire live descriptors even if a cloned Moka handle survives"
+    );
+    assert!(
+        shutdown_weak.upgrade().is_none(),
+        "dropping cache when maintenance is aborted must still release descriptors"
+    );
+    assert!(
+        tracker_weak.upgrade().is_none(),
+        "dropping cache must release FdCacheTracker without reference cycles"
+    );
+    drop(cloned_moka);
+
+    // Short-TTL expired entries (including mixed expired and fresh entries) must
+    // be retired by invalidate_under and clear() even though Moka's iterator skips
+    // expired entries.
+    let ttl_cache = FdCache::with_ttl(std::time::Duration::from_millis(15));
+    let expired_file = Arc::new(tempfile::tempfile().expect("create short-ttl descriptor"));
+    let expired_weak = Arc::downgrade(&expired_file);
+    ttl_cache.insert(key("v", "ttl-obj/part.1", false), expired_file).await;
+    tokio::time::sleep(std::time::Duration::from_millis(30)).await;
+    let fresh_file = Arc::new(tempfile::tempfile().expect("create fresh descriptor"));
+    let fresh_weak = Arc::downgrade(&fresh_file);
+    ttl_cache.insert(key("v", "ttl-obj/part.2", false), fresh_file).await;
+    ttl_cache.invalidate_under("v", "ttl-obj");
+    tokio::time::timeout(std::time::Duration::from_secs(5), async {
+        while expired_weak.upgrade().is_some() || fresh_weak.upgrade().is_some() {
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("invalidate_under must retire both expired and fresh matching descriptors");
+    assert_eq!(ttl_cache.tracker.live_entries(), 0);
+
+    let clear_expired_file = Arc::new(tempfile::tempfile().expect("create clear short-ttl descriptor"));
+    let clear_expired_weak = Arc::downgrade(&clear_expired_file);
+    ttl_cache
+        .insert(key("v", "ttl-clear/part.1", false), clear_expired_file)
+        .await;
+    tokio::time::sleep(std::time::Duration::from_millis(30)).await;
+    let clear_live_file = Arc::new(tempfile::tempfile().expect("create clear live descriptor"));
+    let clear_live_weak = Arc::downgrade(&clear_live_file);
+    ttl_cache.insert(key("v", "ttl-clear/part.2", false), clear_live_file).await;
+    ttl_cache.clear();
+    assert!(
+        clear_expired_weak.upgrade().is_none() && clear_live_weak.upgrade().is_none(),
+        "clear() must synchronously retire both TTL-expired and live descriptors"
+    );
+    assert_eq!(ttl_cache.tracker.live_entries(), 0);
+}
+
+#[tokio::test]
+async fn fd_cache_lockstep_oracle_equivalence_and_counter_conservation() {
+    use std::collections::HashMap;
+    use std::sync::atomic::Ordering;
+
+    let (_directory, file) = fixture();
+    let cache = FdCache::with_ttl(std::time::Duration::from_secs(3600));
+    let mut oracle: HashMap<FdKey, Arc<FdCacheEntry>> = HashMap::new();
+
+    let volumes = ["vol-a", "vol-b", "vol-c", ".rustfs.sys/multipart"];
+    let paths = [
+        "a/b",
+        "a/b/part.1",
+        "a/b/dir/part.2",
+        "a/bc",
+        "a/bc/part.1",
+        "prefix/obj-0001.bin/0198a3f0/part.1",
+        "prefix/obj-0002.bin/0198a3f0/part.1",
+        "d1/d2/d3/d4/d5/d6/d7/d8/d9/d10/d11/d12/d13/part.1",
+    ];
+    let prefixes = [
+        "a",
+        "a/",
+        "a/b",
+        "a/b/",
+        "a/b/dir",
+        "a/bc",
+        "prefix",
+        "prefix/obj-0001.bin",
+        "prefix/obj-0001.bin/0198a3f0",
+        "d1/d2/d3/d4/d5/c_missing",
+        "d1/d2/d3/d4/d5/d6/d7/d8/d9/d10/d11/d12/d13",
+        "nonexistent/prefix",
+    ];
+
+    let mut candidate_keys = Vec::new();
+    for v in volumes {
+        for p in paths {
+            for direct in [false, true] {
+                candidate_keys.push(key(v, p, direct));
+            }
+        }
+    }
+
+    let mut rng = 0x9e37_79b9_7f4a_7c15u64;
+    let mut next_u64 = || {
+        rng ^= rng << 13;
+        rng ^= rng >> 7;
+        rng ^= rng << 17;
+        rng
+    };
+
+    for step in 0..1000 {
+        let op = next_u64() % 100;
+        if op < 45 {
+            // Insert or replace (either fresh or stale generation)
+            let k = candidate_keys[(next_u64() as usize) % candidate_keys.len()].clone();
+            let entry = Arc::new(FdCacheEntry {
+                file: Arc::clone(&file),
+                len: step,
+                invalidated: AtomicBool::new(false),
+            });
+            let cur_gen = cache.generation();
+            let stale = (next_u64() % 5) == 0 && cur_gen > 0;
+            let gen_to_use = if stale { cur_gen - 1 } else { cur_gen };
+            cache.insert_if_fresh(k.clone(), Arc::clone(&entry), gen_to_use).await;
+            if !stale {
+                oracle.insert(k, entry);
+            }
+        } else if op < 65 {
+            // Exact invalidation
+            let v = volumes[(next_u64() as usize) % volumes.len()];
+            let p = paths[(next_u64() as usize) % paths.len()];
+            cache.invalidate_exact(v, p).await;
+            oracle.remove(&key(v, p, false));
+            oracle.remove(&key(v, p, true));
+        } else if op < 88 {
+            // Prefix invalidation
+            let v = volumes[(next_u64() as usize) % volumes.len()];
+            let raw_prefix = prefixes[(next_u64() as usize) % prefixes.len()];
+            cache.invalidate_under(v, raw_prefix);
+            let trimmed = raw_prefix.trim_end_matches('/');
+            oracle.retain(|k, _| {
+                !(k.volume == v && (k.path == trimmed || k.path.strip_prefix(trimmed).is_some_and(|r| r.starts_with('/'))))
+            });
+        } else if op < 97 {
+            // Volume invalidation
+            let v = volumes[(next_u64() as usize) % volumes.len()];
+            cache.invalidate_volume(v);
+            oracle.retain(|k, _| k.volume != v);
+        } else {
+            // Full clear
+            cache.clear();
+            oracle.clear();
+        }
+
+        // Verify lockstep equivalence against the reference oracle after every operation.
+        for k in &candidate_keys {
+            let got = cache.get(k).await;
+            let expected = oracle.get(k);
+            match (got, expected) {
+                (Some(g), Some(e)) => assert!(
+                    Arc::ptr_eq(&g, e),
+                    "step {step}: pointer mismatch for key ({}, {}, {})",
+                    k.volume,
+                    k.path,
+                    k.direct
+                ),
+                (None, None) => {}
+                (g, e) => panic!(
+                    "step {step}: oracle mismatch for key ({}, {}, {}): got={}, expected={}",
+                    k.volume,
+                    k.path,
+                    k.direct,
+                    g.is_some(),
+                    e.is_some()
+                ),
+            }
+        }
+    }
+
+    // Clear and wait for maintenance to drain all entries, then verify exact counter conservation.
+    cache.clear();
+    tokio::time::timeout(std::time::Duration::from_secs(5), async {
+        while cache.tracker.live_entries() != 0 || cache.tracker.has_pending_invalidations.load(Ordering::SeqCst) {
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("clear must drain all live entries and pending invalidations");
+
+    assert_eq!(cache.tracker.live_entries(), 0);
+    assert!(cache.tracker.unindexed_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.volume_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.exact_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.prefix_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.prefix_shards.iter().all(|s| s.lock().is_empty()));
+    assert!(cache.tracker.entry_shards.iter().all(|s| s.lock().is_empty()));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn fd_cache_concurrent_insert_and_invalidation_stress() {
+    use std::sync::atomic::{AtomicU64, Ordering};
+
+    let (_directory, file) = fixture();
+    let cache = Arc::new(FdCache::new());
+
+    // Deterministic Dekker-partner interleaving check: pause `CachedFdEntry::new`
+    // at `entry_shards[shard].lock()` after it has passed the initial generation
+    // check and incremented `volume_counts`, bump generation via an unmatched
+    // invalidation, and verify the post-`new()` recheck refuses publication and
+    // preserves the existing entry.
+    let seam_key = key("v-seam", "obj-1/part.1", false);
+    cache.insert(seam_key.clone(), Arc::clone(&file)).await;
+    let initial_entry = cache.get(&seam_key).await.expect("initial entry must exist");
+    let (vol_hash, vol_slot) = super::fd_cache_volume_hash_and_slot("v-seam");
+    let entry_shard = (vol_hash as usize) & (super::FD_CACHE_PREFIX_SHARDS - 1);
+    let stale_gen = cache.generation();
+    let stale_entry = Arc::new(FdCacheEntry {
+        file: Arc::clone(&file),
+        len: 999_999,
+        invalidated: AtomicBool::new(false),
+    });
+    let blocked_insert = {
+        let hold = cache.tracker.entry_shards[entry_shard].lock();
+        let cache_clone = Arc::clone(&cache);
+        let seam_key_clone = seam_key.clone();
+        let task = tokio::spawn(async move {
+            cache_clone.insert_if_fresh(seam_key_clone, stale_entry, stale_gen).await;
+        });
+        let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
+        while cache.tracker.volume_counts[usize::from(vol_slot)].load(Ordering::SeqCst) < 2 {
+            assert!(
+                std::time::Instant::now() < deadline,
+                "insert never reached CachedFdEntry::new with the previous entry still live"
+            );
+            std::thread::yield_now();
+        }
+        cache.invalidate_under("v-seam", "unmatched-prefix");
+        drop(hold);
+        task
+    };
+    blocked_insert.await.expect("blocked insert task should complete");
+    let preserved = cache
+        .get(&seam_key)
+        .await
+        .expect("refused stale insert must not retire a valid previous entry");
+    assert!(Arc::ptr_eq(&preserved, &initial_entry));
+    cache.invalidate_volume("v-seam");
+
+    let floor_v1 = Arc::new(AtomicU64::new(0));
+    let floor_v2 = Arc::new(AtomicU64::new(0));
+    let mut handles = Vec::new();
+
+    for t in 0..8usize {
+        let cache = Arc::clone(&cache);
+        let file = Arc::clone(&file);
+        let floor_v1 = Arc::clone(&floor_v1);
+        let floor_v2 = Arc::clone(&floor_v2);
+        handles.push(tokio::spawn(async move {
+            for i in 0..250u64 {
+                let is_v1 = (t + i as usize).is_multiple_of(2);
+                let vol = if is_v1 { "v1" } else { "v2" };
+                let path = format!("dir-{}/obj-{}/part.1", t % 4, i % 16);
+                let k = key(vol, &path, i.is_multiple_of(2));
+                let g = cache.generation();
+                let entry = Arc::new(FdCacheEntry {
+                    file: Arc::clone(&file),
+                    len: g,
+                    invalidated: AtomicBool::new(false),
+                });
+                cache.insert_if_fresh(k.clone(), entry, g).await;
+                let min_gen = if is_v1 {
+                    floor_v1.load(Ordering::SeqCst)
+                } else {
+                    floor_v2.load(Ordering::SeqCst)
+                };
+                if let Some(hit) = cache.get(&k).await {
+                    assert!(
+                        hit.len >= min_gen,
+                        "stale descriptor served after volume invalidation: opened_gen={}, floor_gen={min_gen}",
+                        hit.len
+                    );
+                }
+            }
+        }));
+    }
+
+    for t in 0..4usize {
+        let cache = Arc::clone(&cache);
+        let floor_v1 = Arc::clone(&floor_v1);
+        let floor_v2 = Arc::clone(&floor_v2);
+        handles.push(tokio::spawn(async move {
+            for i in 0..250u64 {
+                let is_v1 = (t + i as usize).is_multiple_of(2);
+                let vol = if is_v1 { "v1" } else { "v2" };
+                match i % 4 {
+                    0 => cache.invalidate_under(vol, &format!("dir-{}/obj-{}", t % 4, i % 16)),
+                    1 => {
+                        let p = format!("dir-{}/obj-{}/part.1", t % 4, i % 16);
+                        cache.invalidate_exact(vol, &p).await;
+                    }
+                    2 => {
+                        let before = cache.generation();
+                        cache.invalidate_volume(vol);
+                        let floor = before + 1;
+                        if is_v1 {
+                            floor_v1.fetch_max(floor, Ordering::SeqCst);
+                        } else {
+                            floor_v2.fetch_max(floor, Ordering::SeqCst);
+                        }
+                    }
+                    _ => cache.invalidate_under("uncached-vol", "uncached/prefix"),
+                }
+                tokio::task::yield_now().await;
+            }
+        }));
+    }
+
+    for h in handles {
+        h.await.expect("worker task should succeed");
+    }
+
+    cache.clear();
+    tokio::time::timeout(std::time::Duration::from_secs(5), async {
+        while cache.tracker.live_entries() != 0 || cache.tracker.has_pending_invalidations.load(Ordering::SeqCst) {
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("clear after concurrent stress must drain all live entries");
+
+    assert_eq!(cache.tracker.live_entries(), 0);
+    assert!(cache.tracker.unindexed_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.volume_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.exact_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.prefix_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
+    assert!(cache.tracker.prefix_shards.iter().all(|s| s.lock().is_empty()));
+    assert!(cache.tracker.entry_shards.iter().all(|s| s.lock().is_empty()));
+}

crates/ecstore/src/disk/fs.rs

@@ -148,47 +148,58 @@ pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
     native_io(move || create_dir_all_std(&path)).await
 }
 
-fn is_dir_error(e: &io::Error) -> bool {
+fn is_dir_error(path: &Path, e: &io::Error) -> bool {
     e.raw_os_error() == Some(libc::EISDIR)
         || e.kind() == io::ErrorKind::IsADirectory
         // macOS: remove_file on a directory returns EPERM
         || (cfg!(target_os = "macos") && e.raw_os_error() == Some(libc::EPERM))
+        || is_windows_dir_access_denied(path, e)
+}
+
+/// Windows reports `remove_file` on a directory (or directory symlink) as
+/// ERROR_ACCESS_DENIED, which is indistinguishable from a real permission
+/// failure without inspecting the entry itself.
+#[cfg(windows)]
+fn is_windows_dir_access_denied(path: &Path, e: &io::Error) -> bool {
+    use std::os::windows::fs::FileTypeExt;
+
+    e.kind() == io::ErrorKind::PermissionDenied
+        && std::fs::symlink_metadata(path).is_ok_and(|metadata| {
+            let file_type = metadata.file_type();
+            file_type.is_dir() || file_type.is_symlink_dir()
+        })
+}
+
+#[cfg(not(windows))]
+fn is_windows_dir_access_denied(_path: &Path, _e: &io::Error) -> bool {
+    false
 }
 
 #[tracing::instrument(level = "debug", skip_all)]
 pub async fn remove(path: impl AsRef<Path>) -> io::Result<()> {
-    // Try remove_file first; fall back to remove_dir if it's a directory
-    let file_path = path.as_ref().to_path_buf();
-    match native_io(move || std::fs::remove_file(file_path)).await {
-        Ok(()) => Ok(()),
-        Err(e) if is_dir_error(&e) => {
-            let path = path.as_ref().to_path_buf();
-            native_io(move || std::fs::remove_dir(path)).await
-        }
-        Err(e) => Err(e),
-    }
+    let path = path.as_ref().to_path_buf();
+    native_io(move || remove_std(path)).await
 }
 
 #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
 pub async fn remove_all(path: impl AsRef<Path>) -> io::Result<()> {
     // Try remove_file first; fall back to remove_dir_all if it's a directory
-    let file_path = path.as_ref().to_path_buf();
-    match native_io(move || std::fs::remove_file(file_path)).await {
+    let path = path.as_ref().to_path_buf();
+    native_io(move || match std::fs::remove_file(&path) {
         Ok(()) => Ok(()),
-        Err(e) if is_dir_error(&e) => {
-            let path = path.as_ref().to_path_buf();
-            native_io(move || std::fs::remove_dir_all(path)).await
-        }
+        Err(e) if is_dir_error(&path, &e) => std::fs::remove_dir_all(&path),
         Err(e) => Err(e),
-    }
+    })
+    .await
 }
 
 #[tracing::instrument(level = "debug", skip_all)]
 pub fn remove_std(path: impl AsRef<Path>) -> io::Result<()> {
     // Try remove_file first; fall back to remove_dir if it's a directory
-    match std::fs::remove_file(path.as_ref()) {
+    let path = path.as_ref();
+    match std::fs::remove_file(path) {
         Ok(()) => Ok(()),
-        Err(e) if is_dir_error(&e) => std::fs::remove_dir(path.as_ref()),
+        Err(e) if is_dir_error(path, &e) => std::fs::remove_dir(path),
         Err(e) => Err(e),
     }
 }

crates/ecstore/src/disk/local/commit.rs

@@ -15,6 +15,8 @@
 //! Single-disk object rename publication and rollback. The shared execution core
 //! retains instrumentation, mutation leases, and commit guards through the syscall.
 
+#[cfg(target_os = "linux")]
+use super::has_rename_data_test_failpoints;
 #[cfg(all(test, windows))]
 use super::run_destination_commit_directory_preparation;
 #[cfg(any(not(windows), test))]
@@ -96,7 +98,7 @@ struct RenameCommitContext<'a> {
     data_paths: Option<&'a (PathBuf, PathBuf)>,
 }
 
-struct StagedRollbackBackup(PathBuf);
+pub(super) struct StagedRollbackBackup(PathBuf);
 
 impl StagedRollbackBackup {
     fn path(&self) -> &Path {
@@ -198,7 +200,7 @@ pub(super) fn create_local_inline_rollback_backup(
             return Err(err);
         }
     } else if sync && !old_metadata.is_empty() {
-        std::fs::File::open(&backup_path)?.sync_data()?;
+        os::open_file_for_sync_std(&backup_path)?.sync_data()?;
     }
     Ok(backup_path)
 }
@@ -233,11 +235,13 @@ fn rollback_inline_metadata_std(
     }
 }
 
-fn fsync_inline_commit_std(
+pub(super) fn fsync_inline_commit_std(
     dst: &Path,
     dst_volume_dir: &Path,
     destination_was_absent: bool,
+    rollback_data_dir: Option<Uuid>,
     local_backup: Option<&StagedRollbackBackup>,
+    new_data_dir: Option<&Path>,
     commit_guard: &os::RenameCommitGuard,
 ) -> std::io::Result<()> {
     if let Some(dst_parent) = dst.parent() {
@@ -247,7 +251,10 @@ fn fsync_inline_commit_std(
                 rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC,
                 fsync_started,
             );
-            rollback_inline_metadata_std(dst, None, local_backup)?;
+            rollback_inline_metadata_std(dst, rollback_data_dir, local_backup)?;
+            if let Some(new_data_path) = new_data_dir {
+                remove_dir_all_if_exists(new_data_path)?;
+            }
             return Err(err);
         }
         rustfs_io_metrics::record_put_object_stage_duration_from(
@@ -268,7 +275,10 @@ fn fsync_inline_commit_std(
                     rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC,
                     fsync_started,
                 );
-                rollback_inline_metadata_std(dst, None, local_backup)?;
+                rollback_inline_metadata_std(dst, rollback_data_dir, local_backup)?;
+                if let Some(new_data_path) = new_data_dir {
+                    remove_dir_all_if_exists(new_data_path)?;
+                }
                 return Err(err);
             }
             rustfs_io_metrics::record_put_object_stage_duration_from(
@@ -486,6 +496,9 @@ impl LocalDisk {
             state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
         })?;
         crate::hp_guard!("LocalDisk::rename_data");
+        if dst_volume == super::super::RUSTFS_META_MULTIPART_BUCKET {
+            self.multipart_index.record_info(dst_path, &fi).await?;
+        }
         #[cfg(feature = "e2e-test-hooks")]
         if fi.is_healing() {
             wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?;
@@ -824,7 +837,7 @@ impl LocalDisk {
                         None
                     }
                 });
-                if rollback_data_dir.is_some() {
+                if rollback_data_dir.is_some() && has_rename_data_test_failpoints(&dst_path_str) {
                     return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
                 }
                 #[cfg(test)]
@@ -847,6 +860,9 @@ impl LocalDisk {
                     &dst_path_str,
                     version_id,
                 );
+                if let Some(old_data_dir) = has_old_data_dir.as_ref() {
+                    let _ = xlmeta.data.remove_two(version_id, *old_data_dir);
+                }
                 xlmeta.add_version(fi_moved)?;
                 let version_signature = rename_data_versions_signature(&xlmeta);
                 let new_dst_buf = xlmeta.marshal_msg().map_err(std::io::Error::other)?;
@@ -866,8 +882,9 @@ impl LocalDisk {
                 }
 
                 let mut staged_rollback_path = None;
-                if let Some(ref old_metadata) = has_dst_buf
-                    && (sync_commit_metadata || cfg!(test))
+                if (!no_inline || rollback_data_dir.is_none())
+                    && let Some(ref old_metadata) = has_dst_buf
+                    && (rollback_data_dir.is_some() || sync_commit_metadata || cfg!(test))
                 {
                     let backup_path = create_local_inline_rollback_backup(
                         &dst_file_path_owned,
@@ -968,9 +985,52 @@ impl LocalDisk {
                         }
                         return Ok(FusedCommitAttempt::RestoreDataSource(guard, DiskError::Unexpected));
                     }
+                    if let (Some(old_data_dir), Some(dst_buf)) = (rollback_data_dir, has_dst_buf.as_ref()) {
+                        let backup_parent = dst_parent_owned.join(old_data_dir.to_string());
+                        if let Err(err) =
+                            os::mkdir_all_below_existing_base_std(&backup_parent, &skip_parent, &publication_root_owned)
+                        {
+                            return Ok(FusedCommitAttempt::RestoreDataSource(guard, to_file_error(err).into()));
+                        }
+                        let backup_path_guard = match guard.create_destination_directory_for_path_access(&backup_parent) {
+                            Ok(g) => g,
+                            Err(err) => return Ok(FusedCommitAttempt::RestoreDataSource(guard, to_file_error(err).into())),
+                        };
+                        let backup_path = backup_parent.join(STORAGE_FORMAT_FILE_BACKUP);
+                        if let Err(err) = check_path_length(backup_path.to_string_lossy().as_ref()) {
+                            return Ok(FusedCommitAttempt::RestoreDataSource(guard, err));
+                        }
+                        #[cfg(test)]
+                        run_owned_file_before_open(&backup_path);
+                        if let Err(err) = backup_path_guard.write_file_for_path_access(
+                            &backup_path,
+                            dst_buf.as_ref(),
+                            sync_commit_metadata,
+                            sync_commit_metadata,
+                        ) {
+                            return Ok(FusedCommitAttempt::RestoreDataSource(guard, to_file_error(err).into()));
+                        }
+                    }
                     if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, &dst_path_str) {
                         return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
                     }
+                } else if let (Some(old_data_dir), Some(staged_backup)) =
+                    (rollback_data_dir, staged_rollback_path.as_ref().map(StagedRollbackBackup::path))
+                {
+                    let backup_parent = dst_parent_owned.join(old_data_dir.to_string());
+                    let backup_path = backup_parent.join(STORAGE_FORMAT_FILE_BACKUP);
+                    if let Err(err) = os::rename_all_with_precondition_and_parent_sync_std(
+                        staged_backup,
+                        &backup_path,
+                        &dst_volume_dir_owned,
+                        &publication_root_owned,
+                        None,
+                        sync_commit_metadata,
+                    ) {
+                        let _ = remove_file_if_exists(staged_backup);
+                        return Ok(FusedCommitAttempt::CommitErr(to_file_error(err).into()));
+                    }
+                    staged_rollback_path = None;
                 }
 
                 let rename_res = if should_fail_commit_rename(&dst_path_str) {
@@ -998,15 +1058,16 @@ impl LocalDisk {
 
                 if should_fail_after_metadata_commit(&dst_path_str) {
                     if no_inline {
-                        rollback_inline_metadata_std(&dst_file_path_owned, None, staged_rollback_path.as_ref())?;
+                        rollback_inline_metadata_std(&dst_file_path_owned, rollback_data_dir, staged_rollback_path.as_ref())?;
                         if let Some((_, new_data_path)) = has_data_dir_path_owned.as_ref() {
                             remove_dir_all_if_exists(new_data_path)?;
                         }
                         return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
                     }
-                    let err = rollback_inline_metadata_std(&dst_file_path_owned, None, staged_rollback_path.as_ref())
-                        .err()
-                        .unwrap_or_else(|| std::io::Error::other("test fail after metadata commit"));
+                    let err =
+                        rollback_inline_metadata_std(&dst_file_path_owned, rollback_data_dir, staged_rollback_path.as_ref())
+                            .err()
+                            .unwrap_or_else(|| std::io::Error::other("test fail after metadata commit"));
                     return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
                 }
 
@@ -1019,14 +1080,16 @@ impl LocalDisk {
                         &dst_file_path_owned,
                         &dst_volume_dir_owned,
                         destination_was_absent,
+                        rollback_data_dir,
                         staged_rollback_path.as_ref(),
+                        has_data_dir_path_owned
+                            .as_ref()
+                            .filter(|_| no_inline)
+                            .map(|(_, new_data_path)| new_data_path.as_path()),
                         &guard,
                     )
                 {
                     if no_inline {
-                        if let Some((_, new_data_path)) = has_data_dir_path_owned.as_ref() {
-                            remove_dir_all_if_exists(new_data_path)?;
-                        }
                         return Ok(FusedCommitAttempt::CommitErr(to_file_error(err).into()));
                     }
                     return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
@@ -1042,9 +1105,9 @@ impl LocalDisk {
 
                 Ok(FusedCommitAttempt::Committed(Box::new(RenameDataResp {
                     multipart_source: None,
-                    old_data_dir: None,
-                    rollback_data_dir: None,
-                    cleanup_data_dir: None,
+                    old_data_dir: has_old_data_dir,
+                    rollback_data_dir,
+                    cleanup_data_dir: has_old_data_dir,
                     sign: version_signature,
                     old_current_size,
                     old_current_source_checked,
@@ -1533,8 +1596,9 @@ impl LocalDisk {
                     let dst_file_path = dst_file_path.clone();
                     let owned_directory_guard = commit.directory_guard.clone();
                     let owned_mutation_lease = commit.mutation_lease.clone();
+                    let sync = tmp_meta_sync != SyncMode::None;
                     async move {
-                        os::run_blocking_namespace_operation(owned_mutation_lease, move || {
+                        os::run_blocking_namespace_operation_with_sync(owned_mutation_lease, sync, move || {
                             #[cfg(test)]
                             run_owned_file_before_open(&src_file_path);
                             let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard(
@@ -1542,7 +1606,7 @@ impl LocalDisk {
                                 &dst_file_path,
                                 &owned_directory_guard,
                             )?;
-                            prepared_metadata_source.write_all(&new_dst_buf, tmp_meta_sync != SyncMode::None)?;
+                            prepared_metadata_source.write_all(&new_dst_buf, sync)?;
                             Ok(prepared_metadata_source)
                         })
                         .await
@@ -1594,12 +1658,13 @@ impl LocalDisk {
                 );
             }
             if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref() {
-                #[cfg(target_os = "linux")]
+                #[cfg(any(target_os = "linux", windows))]
                 let payload_guard = if multipart_publication.is_some() {
                     Some(
                         os::multipart_payload_commit_guard(
                             &commit.directory_guard,
                             src_data_path.parent().ok_or(DiskError::InvalidPath)?,
+                            commit.publication_root,
                             commit.mutation_lease.clone(),
                         )
                         .await
@@ -1608,9 +1673,9 @@ impl LocalDisk {
                 } else {
                     None
                 };
-                #[cfg(target_os = "linux")]
+                #[cfg(any(target_os = "linux", windows))]
                 let directory_guard = payload_guard.as_ref().unwrap_or(&commit.directory_guard);
-                #[cfg(not(target_os = "linux"))]
+                #[cfg(not(any(target_os = "linux", windows)))]
                 let directory_guard = &commit.directory_guard;
                 if let Err(err) = os::rename_all_with_commit_guard(
                     src_data_path,
@@ -1705,13 +1770,15 @@ impl LocalDisk {
                 // sync finish. A detached spawn_blocking writer could survive
                 // cancellation and later truncate a newer transaction's
                 // deterministic rollback backup.
-                let write_result = os::run_blocking_namespace_operation(commit.mutation_lease.clone(), move || {
+                let lease = commit.mutation_lease.clone();
+                let sync = backup_sync != SyncMode::None;
+                let write_result = os::run_blocking_namespace_operation_with_sync(lease, sync, move || {
                     #[cfg(test)]
                     run_owned_file_before_open(&backup_path);
                     backup_path_guard.write_file_for_path_access(
                         &backup_path,
                         backup_bytes.as_ref(),
-                        backup_sync != SyncMode::None,
+                        sync,
                         backup_sync == SyncMode::FileAndDir,
                     )
                 })
@@ -2151,7 +2218,9 @@ impl LocalDisk {
                             &dst,
                             &dst_volume_dir_for_commit,
                             destination_was_absent,
+                            rollback_data_dir,
                             staged_rollback_path.as_ref(),
+                            None,
                             &rename_commit_guard_for_preparation,
                         )
                     {

crates/ecstore/src/disk/local/delete_recovery.rs

@@ -5,6 +5,28 @@ use crate::disk::{
 };
 use sha2::{Digest, Sha256};
 
+pub(super) struct RecoveryOperationScope {
+    pub(super) budget: Arc<crate::object_api::DeleteRecoveryWorkBudget>,
+    may_publish_receipt: bool,
+    receipt_published: std::sync::atomic::AtomicBool,
+}
+
+tokio::task_local! {
+    static RECOVERY_OPERATION: Arc<RecoveryOperationScope>;
+}
+
+pub(super) fn recovery_operation_scope() -> Option<Arc<RecoveryOperationScope>> {
+    RECOVERY_OPERATION.try_with(Arc::clone).ok()
+}
+
+impl RecoveryOperationScope {
+    pub(super) fn receipt_published(&self) {
+        if self.may_publish_receipt {
+            self.receipt_published.store(true, std::sync::atomic::Ordering::Release);
+        }
+    }
+}
+
 pub(super) const PREPARED: &str = "delete-prepared";
 pub(super) const TARGET: &str = "delete-target";
 pub(super) const FINISHED: &str = "delete-commit-finished";
@@ -19,24 +41,35 @@ struct DiscoverySession {
     touched: Instant,
 }
 
+/// Plain (non-transaction) children of an object-scoped discovery root cost one
+/// budget entry per this many names: enough to bound a hostile prefix without
+/// making a real folder prefix undeletable.
+const PLAIN_CHILDREN_PER_BUDGET_ENTRY: u32 = 64;
+
 impl DiscoverySession {
     /// Consume one bounded step, returning ownership only when traversal remains.
-    fn advance(mut self, limit: u32) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>)> {
+    fn advance(self, limit: u32) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>)> {
+        self.advance_bounded(limit, super::super::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES)
+            .map(|(transactions, next, _)| (transactions, next))
+    }
+
+    fn advance_bounded(mut self, limit: u32, max_entries: u32) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>, u32)> {
         if self.frames.is_empty() {
             match std::fs::read_dir(&self.root) {
                 Ok(entries) => self.frames.push((String::new(), entries)),
                 Err(err) if err.kind() == ErrorKind::NotFound => {
-                    return Ok((Vec::new(), None));
+                    return Ok((Vec::new(), None, 0));
                 }
                 Err(err) => return Err(err),
             }
         }
         let mut transactions = BTreeSet::new();
         let mut seen_candidates = HashSet::new();
-        let mut visited = 0usize;
+        let mut visited = 0u32;
+        let mut plain_children = 0u32;
         while !self.frames.is_empty()
             && transactions.len() < usize::try_from(limit).map_err(|_| std::io::Error::from(ErrorKind::InvalidInput))?
-            && visited < 4096
+            && visited < max_entries
         {
             let (parent, entries) = self
                 .frames
@@ -49,7 +82,6 @@ impl DiscoverySession {
                     continue;
                 }
             };
-            visited += 1;
             let name = entry
                 .file_name()
                 .into_string()
@@ -57,6 +89,19 @@ impl DiscoverySession {
             if !safe_path(&name, false) || name.contains('/') {
                 return Err(std::io::Error::from(ErrorKind::InvalidData));
             }
+            // An object-scoped scan only ever descends into transaction (UUID)
+            // directories at the root. Plain child keys of a prefix directory are
+            // skipped by name without a stat; they are charged at one entry per
+            // readdir batch so the work stays bounded while deleting a missing key
+            // that names a prefix with thousands of children can still complete.
+            if self.object.is_some() && parent.is_empty() && !canonical_uuid(&name).is_some_and(|id| !id.is_nil()) {
+                plain_children += 1;
+                if plain_children.is_multiple_of(PLAIN_CHILDREN_PER_BUDGET_ENTRY) {
+                    visited += 1;
+                }
+                continue;
+            }
+            visited += 1;
             let relative = if parent.is_empty() {
                 name.clone()
             } else {
@@ -156,7 +201,7 @@ impl DiscoverySession {
             .map(|(object, transaction)| DeleteTransactionId { object, transaction })
             .collect();
         let next = (!self.frames.is_empty()).then_some(self);
-        Ok((transactions, next))
+        Ok((transactions, next, visited))
     }
 
     async fn has_evidence(mut self, disk: &LocalDisk, path: &Path) -> Result<bool> {
@@ -226,6 +271,9 @@ pub(in crate::disk) fn generation(mut canonical: FileInfo) -> Result<crate::disk
         compat::SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
         compat::SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
     ] {
+        if canonical.metadata.is_empty() {
+            break;
+        }
         if !compat::contains_key_str(&canonical.metadata, suffix) {
             continue;
         }
@@ -256,22 +304,24 @@ pub(in crate::disk) fn generation(mut canonical: FileInfo) -> Result<crate::disk
         }
         compat::insert_str(&mut canonical.metadata, suffix, value);
     }
-    fn ordered(value: serde_json::Value) -> serde_json::Value {
+    fn sort_json_in_place(value: &mut serde_json::Value) {
         match value {
             serde_json::Value::Object(map) => {
-                let sorted: std::collections::BTreeMap<_, _> = map.into_iter().collect();
-                serde_json::Value::Object(sorted.into_iter().map(|(key, value)| (key, ordered(value))).collect())
+                map.sort_keys();
+                map.values_mut().for_each(sort_json_in_place);
             }
-            serde_json::Value::Array(values) => serde_json::Value::Array(values.into_iter().map(ordered).collect()),
-            other => other,
+            serde_json::Value::Array(values) => values.iter_mut().for_each(sort_json_in_place),
+            _ => {}
         }
     }
+    let mut json_value = serde_json::to_value(&canonical)?;
+    sort_json_in_place(&mut json_value);
     Ok(crate::disk::DeleteRecoveryVersionIdentity {
         version_id: canonical.version_id,
         mod_time: canonical.mod_time,
         data_dir: canonical.data_dir,
         deleted: canonical.deleted,
-        metadata_digest: digest(&serde_json::to_vec(&ordered(serde_json::to_value(canonical)?))?),
+        metadata_digest: digest(&serde_json::to_vec(&json_value)?),
     })
 }
 
@@ -382,6 +432,13 @@ fn journal_bytes(journal: &DeleteRecoveryJournal) -> Result<Vec<u8>> {
     if bytes.len() > usize::try_from(MAX_METADATA).map_err(|_| DiskError::FileCorrupt)? - 128 {
         return Err(DiskError::FileCorrupt);
     }
+    // MessagePack encoding of `DeleteRecoveryRpcReply::Inspected` is more compact
+    // than JSON; only construct and serialize `projected` when the JSON size plus
+    // `current_versions` and fixed envelope could approach the RPC payload limit.
+    let rpc_budget = rustfs_protos::DELETE_RECOVERY_MAX_PAYLOAD_BYTES.saturating_sub(4096);
+    if bytes.len().saturating_mul(2).saturating_add(4096) <= rpc_budget {
+        return Ok(bytes);
+    }
     let projected = crate::disk::DeleteRecoveryRpcReply::Inspected(DeleteRecoveryInspection {
         protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
         id: DeleteTransactionId {
@@ -398,7 +455,7 @@ fn journal_bytes(journal: &DeleteRecoveryJournal) -> Result<Vec<u8>> {
         explicit_delete_safe: false,
         fingerprint: [0; 32],
     });
-    if rmp_serde::to_vec_named(&projected)?.len() > rustfs_protos::DELETE_RECOVERY_MAX_PAYLOAD_BYTES - 4096 {
+    if rmp_serde::to_vec_named(&projected)?.len() > rpc_budget {
         return Err(DiskError::FileCorrupt);
     }
     Ok(bytes)
@@ -580,12 +637,41 @@ fn explicit_delete_referenced_data_dirs(
     Ok(Some(referenced))
 }
 
-fn explicit_delete_plan(
+pub(super) async fn next_recovery_entry(entries: &mut tokio::fs::ReadDir) -> std::io::Result<Option<tokio::fs::DirEntry>> {
+    if let Some(scope) = recovery_operation_scope() {
+        scope.budget.reserve(1).map_err(std::io::Error::from)?;
+    }
+    entries.next_entry().await
+}
+
+pub(super) fn recovery_entries(
+    mut entries: std::fs::ReadDir,
+    budget: Option<&crate::object_api::DeleteRecoveryWorkBudget>,
+) -> impl Iterator<Item = std::io::Result<std::fs::DirEntry>> + '_ {
+    let mut finished = false;
+    std::iter::from_fn(move || {
+        if finished {
+            return None;
+        }
+        if let Some(budget) = budget
+            && let Err(error) = budget.reserve(1)
+        {
+            finished = true;
+            return Some(Err(error.into()));
+        }
+        let entry = entries.next();
+        finished = entry.is_none();
+        entry
+    })
+}
+
+fn explicit_delete_plan_bounded(
     root: &Path,
     transaction: Uuid,
     original: Option<&FileMeta>,
     journal: Option<&DeleteRecoveryJournal>,
     current_missing: bool,
+    budget: Option<&crate::object_api::DeleteRecoveryWorkBudget>,
 ) -> Result<Option<ExplicitDeletePlan>> {
     let Some(referenced) = explicit_delete_referenced_data_dirs(original, journal, current_missing)? else {
         return Ok(None);
@@ -602,7 +688,7 @@ fn explicit_delete_plan(
     let mut candidates = Vec::new();
     match std::fs::read_dir(&rollback) {
         Ok(entries) => {
-            for entry in entries {
+            for entry in recovery_entries(entries, budget) {
                 let entry = entry.map_err(to_file_error)?;
                 let kind = entry.file_type().map_err(to_file_error)?;
                 let name = entry.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?;
@@ -634,7 +720,7 @@ fn explicit_delete_plan(
     }
     match std::fs::read_dir(root) {
         Ok(entries) => {
-            for entry in entries {
+            for entry in recovery_entries(entries, budget) {
                 let entry = entry.map_err(to_file_error)?;
                 let name = entry.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?;
                 let Some(id) = canonical_uuid(&name).filter(|id| !id.is_nil()) else {
@@ -658,7 +744,7 @@ fn explicit_delete_plan(
     candidates.dedup();
     for dir in candidates {
         let mut has_parts = false;
-        for entry in std::fs::read_dir(&dir).map_err(to_file_error)? {
+        for entry in recovery_entries(std::fs::read_dir(&dir).map_err(to_file_error)?, budget) {
             let entry = entry.map_err(to_file_error)?;
             if !entry.file_type().map_err(to_file_error)?.is_file() {
                 return Ok(None);
@@ -697,15 +783,27 @@ fn canonical_uuid(name: &str) -> Option<Uuid> {
     Uuid::parse_str(name).ok().filter(|id| name == id.hyphenated().to_string())
 }
 
-pub(super) fn inspection_with_markers(
+pub(super) fn missing_marker_preserves_child_prefix(journal: &DeleteRecoveryJournal) -> bool {
+    journal.original_digest.is_none()
+        && journal.original_versions.is_empty()
+        && journal.data_dirs.is_empty()
+        && journal.target_digest.is_some()
+        && !journal.requests.is_empty()
+        && journal
+            .requests
+            .iter()
+            .all(|request| request.deleted && request.version_id.is_some_and(|id| !id.is_nil()))
+}
+
+pub(super) fn inspection_with_markers_bounded(
     root: &Path,
     volume: &str,
     object: &str,
     transaction: Uuid,
+    budget: Option<&crate::object_api::DeleteRecoveryWorkBudget>,
 ) -> Result<(DeleteRecoveryInspection, bool)> {
     let rollback = root.join(transaction.to_string());
     let before = regular_metadata_bytes(&rollback.join(STORAGE_FORMAT_FILE_BACKUP))?;
-    let current = regular_metadata_bytes(&root.join(STORAGE_FORMAT_FILE))?;
     let prepared = regular_bytes(&rollback.join(PREPARED))?;
     let target = regular_metadata_bytes(&rollback.join(TARGET))?;
     let receipt_bytes = regular_bytes(&rollback.join(DELETE_QUORUM_COMMITTED_FILE))?;
@@ -739,6 +837,20 @@ pub(super) fn inspection_with_markers(
     {
         return Err(DiskError::FileCorrupt);
     }
+    // A failed creation of a marker over a child named `xl.meta` did not
+    // replace that directory. Older releases nevertheless left a journal.
+    // Only this bound, originally absent generation may treat it as absence.
+    let current_path = root.join(STORAGE_FORMAT_FILE);
+    let current = if !committed
+        && before.is_none()
+        && (journal.as_ref().is_some_and(missing_marker_preserves_child_prefix)
+            || (journal.is_none() && target.is_none() && receipt_bytes.is_none()))
+        && std::fs::symlink_metadata(&current_path).is_ok_and(|metadata| metadata.is_dir())
+    {
+        None
+    } else {
+        regular_metadata_bytes(&current_path)?
+    };
     let backup_digest = before.as_deref().map(digest);
     let current_digest = current.as_deref().map(digest);
     let prepared_digest = prepared.as_deref().map(digest);
@@ -781,7 +893,7 @@ pub(super) fn inspection_with_markers(
         Err(err) => return Err(to_file_error(err).into()),
     };
     if let Some(entries) = entries {
-        for entry in entries {
+        for entry in recovery_entries(entries, budget) {
             let entry = entry.map_err(to_file_error)?;
             let kind = entry.file_type().map_err(to_file_error)?;
             if !kind.is_file() && !kind.is_dir() {
@@ -798,7 +910,7 @@ pub(super) fn inspection_with_markers(
             descriptions.push((name.clone(), kind.is_dir(), entry.metadata().map_err(to_file_error)?.len()));
             if kind.is_dir() && id.is_some_and(|id| !id.is_nil()) {
                 has_other_uuid_dir |= id != Some(transaction);
-                for child in std::fs::read_dir(entry.path()).map_err(to_file_error)? {
+                for child in recovery_entries(std::fs::read_dir(entry.path()).map_err(to_file_error)?, budget) {
                     let child = child.map_err(to_file_error)?;
                     let kind = child.file_type().map_err(to_file_error)?;
                     if !kind.is_dir() && !kind.is_file() {
@@ -844,7 +956,7 @@ pub(super) fn inspection_with_markers(
                     }
                     if name == transaction.to_string() && kind.is_dir() {
                         has_staged_dir = true;
-                        for file in std::fs::read_dir(child.path()).map_err(to_file_error)? {
+                        for file in recovery_entries(std::fs::read_dir(child.path()).map_err(to_file_error)?, budget) {
                             let file = file.map_err(to_file_error)?;
                             let kind = file.file_type().map_err(to_file_error)?;
                             let file_name = file.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?;
@@ -968,7 +1080,8 @@ pub(super) fn inspection_with_markers(
                 .is_some_and(|referenced| !referenced.contains(&transaction))
         }
         DeleteRecoveryState::LegacyAmbiguous | DeleteRecoveryState::NewPrepared => {
-            explicit_delete_plan(root, transaction, before_meta.as_ref(), journal.as_ref(), current.is_none())?.is_some()
+            explicit_delete_plan_bounded(root, transaction, before_meta.as_ref(), journal.as_ref(), current.is_none(), budget)?
+                .is_some()
         }
         _ => false,
     };
@@ -1002,21 +1115,17 @@ pub(super) fn inspection_with_markers(
 
 type PreparedDeleteReceiptEntry = (DeleteRecoveryJournal, bool);
 
-static PREPARED_DELETE_RECEIPTS: std::sync::LazyLock<parking_lot::Mutex<HashMap<PathBuf, PreparedDeleteReceiptEntry>>> =
-    std::sync::LazyLock::new(|| parking_lot::Mutex::new(HashMap::new()));
+const PREPARED_DELETE_RECEIPTS_PER_SHARD: usize = 64;
 
-fn put_prepared_delete_receipt(rollback: PathBuf, entry: PreparedDeleteReceiptEntry) {
-    let mut map = PREPARED_DELETE_RECEIPTS.lock();
-    if map.len() >= 4096
-        && let Some(evict_key) = map.keys().next().cloned()
-    {
-        map.remove(&evict_key);
-    }
-    map.insert(rollback, entry);
+static PREPARED_DELETE_RECEIPTS: std::sync::LazyLock<os::ShardedPathCache<PreparedDeleteReceiptEntry>> =
+    std::sync::LazyLock::new(|| os::ShardedPathCache::new(PREPARED_DELETE_RECEIPTS_PER_SHARD));
+
+fn put_prepared_delete_receipt(rollback: impl AsRef<Path>, entry: PreparedDeleteReceiptEntry) {
+    PREPARED_DELETE_RECEIPTS.insert(rollback.as_ref(), entry);
 }
 
 fn take_prepared_delete_receipt(rollback: &Path) -> Option<PreparedDeleteReceiptEntry> {
-    PREPARED_DELETE_RECEIPTS.lock().remove(rollback)
+    PREPARED_DELETE_RECEIPTS.remove(rollback)
 }
 
 impl LocalDisk {
@@ -1043,6 +1152,7 @@ impl LocalDisk {
         let volume = volume.to_owned();
         let object = object.to_owned();
         let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner).await;
+        let recovery_scope = recovery_operation_scope();
         os::run_blocking_namespace_operation(lease, move || {
             Ok((|| {
                 let entries = match std::fs::read_dir(&root) {
@@ -1052,7 +1162,7 @@ impl LocalDisk {
                 };
                 let mut predecessors = BTreeSet::new();
                 let mut visited = 0usize;
-                for entry in entries {
+                for entry in recovery_entries(entries, recovery_scope.as_ref().map(|scope| scope.budget.as_ref())) {
                     visited += 1;
                     if visited > 65536 {
                         return Err(DiskError::DiskOngoingReq);
@@ -1205,6 +1315,42 @@ impl LocalDisk {
             .map_err(|error| to_file_error(error).into())
     }
 
+    #[cfg(test)]
+    pub(crate) fn public_delete_discovery_sessions_for_test(&self) -> usize {
+        DISCOVERY_SESSIONS
+            .lock()
+            .values()
+            .filter(|session| session.disk_root == self.io_root())
+            .count()
+    }
+
+    pub(super) async fn delete_recovery_discover_bounded(
+        &self,
+        volume: &str,
+        object: &str,
+        max_entries: u32,
+    ) -> Result<super::super::BoundedDeleteRecoveryDiscovery> {
+        if !(1..=super::super::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES).contains(&max_entries) {
+            return Err(DiskError::InvalidPath);
+        }
+        let path = self.delete_discovery_path(volume, Some(object))?;
+        let session = self.new_delete_discovery_session(volume, Some(object), &path)?;
+        let lease = os::acquire_namespace_mutation_lease_with_owner(&path, None).await;
+        self.validate_delete_discovery_owner(&session)?;
+        // The blocking owner drops every directory handle even if the RPC waiter disappears.
+        // A partial inventory is never evidence of absence and never enters DISCOVERY_SESSIONS.
+        os::run_blocking_namespace_operation(lease, move || {
+            let (transactions, next, visited) = session.advance_bounded(256, max_entries)?;
+            Ok(super::super::BoundedDeleteRecoveryDiscovery {
+                transactions: if next.is_none() { transactions } else { Vec::new() },
+                visited,
+                complete: next.is_none(),
+            })
+        })
+        .await
+        .map_err(|error| to_file_error(error).into())
+    }
+
     pub(super) async fn delete_recovery_discover(
         &self,
         volume: &str,
@@ -1252,6 +1398,42 @@ impl LocalDisk {
             .await
     }
 
+    pub(super) async fn delete_recovery_inspect_bounded(
+        &self,
+        volume: &str,
+        object: &str,
+        transaction: Uuid,
+        max_entries: u32,
+    ) -> Result<super::super::BoundedDeleteRecoveryInspection> {
+        if !safe_path(object, false)
+            || transaction.is_nil()
+            || !(1..=super::super::DELETE_RECOVERY_MAX_INSPECTION_ENTRIES).contains(&max_entries)
+        {
+            return Err(DiskError::InvalidPath);
+        }
+        let root = self.get_object_path(volume, object)?;
+        let owner = os::acquire_metadata_mutation_lease(&root, None).await;
+        let lease = os::acquire_namespace_mutation_lease_with_owner(&root, Some(owner)).await;
+        let volume = volume.to_owned();
+        let object = object.to_owned();
+        // Same lane as the unbounded inspection: a bulk delete inspects one
+        // transaction per key on every drive, and the one-worker foreground
+        // namespace lane would serialize those behind every other mutation.
+        os::run_blocking_namespace_operation_with_sync(lease, true, move || {
+            let budget = crate::object_api::DeleteRecoveryWorkBudget::new(max_entries);
+            Ok(
+                inspection_with_markers_bounded(&root, &volume, &object, transaction, Some(&budget)).map(|(inspection, _)| {
+                    super::super::BoundedDeleteRecoveryInspection {
+                        inspection,
+                        visited: max_entries - budget.remaining(),
+                    }
+                }),
+            )
+        })
+        .await
+        .map_err(to_file_error)?
+    }
+
     pub(super) async fn delete_recovery_inspect(
         &self,
         volume: &str,
@@ -1291,9 +1473,22 @@ impl LocalDisk {
         let volume = volume.to_owned();
         let object = object.to_owned();
         let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner).await;
-        os::run_blocking_namespace_operation(lease, move || Ok(inspection_with_markers(&root, &volume, &object, transaction)))
-            .await
-            .map_err(to_file_error)?
+        let scope = recovery_operation_scope();
+        // Journal inspections ride the durable-mutation lane (the fused delete
+        // prepare already does): a bulk delete publishes one transaction per
+        // key, and the one-worker foreground namespace lane would serialize
+        // those per-drive publishes behind every other mutation.
+        os::run_blocking_namespace_operation_with_sync(lease, true, move || {
+            Ok(inspection_with_markers_bounded(
+                &root,
+                &volume,
+                &object,
+                transaction,
+                scope.as_ref().map(|scope| scope.budget.as_ref()),
+            ))
+        })
+        .await
+        .map_err(to_file_error)?
     }
 
     async fn activate_delete_preparation(
@@ -1491,6 +1686,49 @@ impl LocalDisk {
         self.activate_delete_preparation(&journal, original, target, owner).await
     }
 
+    pub(super) async fn delete_recovery_settle_bounded(
+        &self,
+        volume: &str,
+        request: super::super::BoundedDeleteRecoverySettlement,
+        owner: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<super::super::BoundedDeleteRecoverySettlementReply> {
+        if !(1..=super::super::DELETE_RECOVERY_MAX_INSPECTION_ENTRIES).contains(&request.max_entries) {
+            return Err(DiskError::InvalidPath);
+        }
+        let scope = Arc::new(RecoveryOperationScope {
+            budget: Arc::new(crate::object_api::DeleteRecoveryWorkBudget::new(request.max_entries)),
+            may_publish_receipt: matches!(
+                &request.action,
+                DeleteRecoveryAction::PublishCommit | DeleteRecoveryAction::AdoptCommit { .. }
+            ),
+            receipt_published: std::sync::atomic::AtomicBool::new(false),
+        });
+        let result = RECOVERY_OPERATION
+            .scope(
+                Arc::clone(&scope),
+                self.delete_recovery_settle(
+                    volume,
+                    &request.object,
+                    request.transaction,
+                    request.expected_fingerprint,
+                    request.action,
+                    owner,
+                ),
+            )
+            .await;
+        let outcome = match result {
+            Ok(inspection) => super::super::DeleteRecoverySettlementOutcome::Inspected(Box::new(inspection)),
+            Err(DiskError::DeleteRecoveryWorkLimit) if scope.receipt_published.load(std::sync::atomic::Ordering::Acquire) => {
+                super::super::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred
+            }
+            Err(error) => return Err(error),
+        };
+        Ok(super::super::BoundedDeleteRecoverySettlementReply {
+            outcome,
+            visited: request.max_entries - scope.budget.remaining(),
+        })
+    }
+
     pub(crate) async fn delete_recovery_settle(
         &self,
         volume: &str,
@@ -1689,14 +1927,26 @@ impl LocalDisk {
             let plan_root = root.clone();
             let missing = observed.current_digest.is_none();
             let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner.clone()).await;
-            let plan = os::run_blocking_namespace_operation(lease, move || {
+            let scope = recovery_operation_scope();
+            // Same durable-mutation lane as the journal inspection above: this
+            // plan closure runs inside every settle on the bulk-delete publish
+            // path.
+            let plan = os::run_blocking_namespace_operation_with_sync(lease, true, move || {
                 Ok((|| {
                     let before =
                         regular_metadata_bytes(&plan_root.join(transaction.to_string()).join(STORAGE_FORMAT_FILE_BACKUP))?
                             .as_deref()
                             .map(FileMeta::load)
                             .transpose()?;
-                    explicit_delete_plan(&plan_root, transaction, before.as_ref(), None, missing)?.ok_or(DiskError::FileCorrupt)
+                    explicit_delete_plan_bounded(
+                        &plan_root,
+                        transaction,
+                        before.as_ref(),
+                        None,
+                        missing,
+                        scope.as_ref().map(|scope| scope.budget.as_ref()),
+                    )?
+                    .ok_or(DiskError::FileCorrupt)
                 })())
             })
             .await
@@ -1961,6 +2211,7 @@ impl LocalDisk {
                 let (rollback_buf, root_buf, volume_str, object_str) =
                     (rollback.clone(), root.clone(), volume.to_owned(), object.to_owned());
                 let publication_root = self.publication_root.clone();
+                let scope = recovery_operation_scope();
                 return os::run_blocking_namespace_operation_with_sync(receipt_lease, sync, move || {
                     let _publication_root = publication_root;
                     for (path, _) in &file_leases {
@@ -1993,8 +2244,14 @@ impl LocalDisk {
                             }
                         }
                     }
-                    let inspected =
-                        inspection_with_markers(&root_buf, &volume_str, &object_str, transaction).map(|(state, _)| state);
+                    let inspected = inspection_with_markers_bounded(
+                        &root_buf,
+                        &volume_str,
+                        &object_str,
+                        transaction,
+                        scope.as_ref().map(|scope| scope.budget.as_ref()),
+                    )
+                    .map(|(state, _)| state);
                     drop((file_leases, prune_dirs));
                     Ok(inspected)
                 })
@@ -2151,6 +2408,7 @@ impl LocalDisk {
         let volume_str = volume.to_owned();
         let object_str = object.to_owned();
 
+        let recovery_scope = recovery_operation_scope();
         let res = os::run_blocking_namespace_operation_with_sync(
             lease,
             sync,
@@ -2197,7 +2455,9 @@ impl LocalDisk {
                         let mut predecessors = BTreeSet::new();
                         let mut sole_transaction = true;
                         if let Ok(entries) = std::fs::read_dir(&file_path) {
-                            for (visited, entry) in entries.enumerate() {
+                            for (visited, entry) in
+                                recovery_entries(entries, recovery_scope.as_ref().map(|scope| scope.budget.as_ref())).enumerate()
+                            {
                                 if visited >= 65536 {
                                     return Err(DiskError::DiskOngoingReq);
                                 }
@@ -2410,12 +2670,25 @@ impl LocalDisk {
         Ok(Some(()))
     }
 
+    #[cfg(test)]
     pub(crate) async fn try_publish_prepared_delete_local(
         &self,
         volume: &str,
         object: &str,
         transaction: Uuid,
         namespace_owner: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<Option<(DeleteRecoveryJournal, bool, bool)>> {
+        self.try_publish_prepared_delete_local_with_budget(volume, object, transaction, namespace_owner, None)
+            .await
+    }
+
+    pub(crate) async fn try_publish_prepared_delete_local_with_budget(
+        &self,
+        volume: &str,
+        object: &str,
+        transaction: Uuid,
+        namespace_owner: Option<Arc<dyn Send + Sync>>,
+        work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
     ) -> Result<Option<(DeleteRecoveryJournal, bool, bool)>> {
         self.reject_multipart_delete_recovery_mutation(volume, object)?;
         if !safe_path(object, false) || transaction.is_nil() {
@@ -2494,7 +2767,7 @@ impl LocalDisk {
                 if sole_transaction {
                     match std::fs::read_dir(&root) {
                         Ok(entries) => {
-                            for entry in entries {
+                            for entry in recovery_entries(entries, work_budget.as_deref()) {
                                 let entry = entry?;
                                 let Ok(name) = entry.file_name().into_string() else {
                                     sole_transaction = false;
@@ -2542,6 +2815,7 @@ impl LocalDisk {
         .map_err(to_file_error)?
     }
 
+    #[cfg(test)]
     pub(crate) async fn try_finish_and_finalize_prepared_delete_local(
         &self,
         volume: &str,
@@ -2549,6 +2823,26 @@ impl LocalDisk {
         transaction: Uuid,
         journal: &DeleteRecoveryJournal,
         namespace_owner: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<bool> {
+        self.try_finish_and_finalize_prepared_delete_local_with_budget(
+            volume,
+            object,
+            transaction,
+            journal,
+            namespace_owner,
+            None,
+        )
+        .await
+    }
+
+    pub(crate) async fn try_finish_and_finalize_prepared_delete_local_with_budget(
+        &self,
+        volume: &str,
+        object: &str,
+        transaction: Uuid,
+        journal: &DeleteRecoveryJournal,
+        namespace_owner: Option<Arc<dyn Send + Sync>>,
+        work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
     ) -> Result<bool> {
         fn ignore_not_found(res: std::io::Result<()>) -> std::io::Result<()> {
             match res {
@@ -2615,7 +2909,7 @@ impl LocalDisk {
                             had_physical_data_dirs = true;
                             let mut removable = Vec::new();
                             let mut preserve_dir = false;
-                            for entry in entries {
+                            for entry in recovery_entries(entries, work_budget.as_deref()) {
                                 let entry = entry?;
                                 let name = entry.file_name();
                                 let Some(name) = name.to_str() else {
@@ -4119,6 +4413,100 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn delete_marker_collision_rejects_new_journals_and_recovers_legacy_proof() {
+        let (temp, disk, _info, original) = fixture().await;
+        let root = disk.io_get_object_path("recovery", "object").expect("object path");
+        let staged_child = temp.path().join("child-fixture");
+        fs::rename(&root, &staged_child).await.expect("move intact object");
+        fs::create_dir(&root).await.expect("create parent prefix");
+        fs::rename(staged_child, root.join(STORAGE_FORMAT_FILE))
+            .await
+            .expect("seed an independent object named object/xl.meta");
+        let transaction = Uuid::new_v4();
+        let marker = FileInfo {
+            volume: "recovery".to_owned(),
+            name: "object".to_owned(),
+            version_id: Some(Uuid::new_v4()),
+            deleted: true,
+            mod_time: Some(OffsetDateTime::now_utc()),
+            ..Default::default()
+        };
+        disk.delete_version_recoverable(
+            "recovery",
+            "object",
+            marker.clone(),
+            true,
+            DeleteOptions {
+                old_data_dir: Some(transaction),
+                ..Default::default()
+            },
+        )
+        .await
+        .expect_err("a child xl.meta directory cannot be replaced by a marker");
+        assert!(!root.join(transaction.to_string()).exists(), "rejection must precede journal publication");
+
+        // Reproduce the durable evidence written by releases that prepared a
+        // missing-object marker before attempting to replace the directory.
+        disk.prepare_delete_recovery(
+            "recovery",
+            "object",
+            None,
+            &[marker],
+            None,
+            DeletePreparationContext {
+                transaction,
+                bucket_incarnation: None,
+                namespace_owner: None,
+            },
+        )
+        .await
+        .expect("seed legacy prepared marker");
+        let journal_path = root.join(transaction.to_string()).join(PREPARED);
+        let journal_bytes = fs::read(&journal_path).await.expect("prepared journal");
+        let mut incompatible: DeleteRecoveryJournal = serde_json::from_slice(&journal_bytes).expect("journal");
+        incompatible.original_digest = Some([7; 32]);
+        fs::write(&journal_path, serde_json::to_vec(&incompatible).expect("serialize journal"))
+            .await
+            .expect("write incompatible evidence");
+        assert!(
+            disk.inspect_delete_transaction("recovery", "object", transaction)
+                .await
+                .is_err(),
+            "a journal that claims existing metadata cannot ignore a directory collision"
+        );
+        fs::write(&journal_path, journal_bytes)
+            .await
+            .expect("restore exact legacy proof");
+        let observed = disk
+            .inspect_delete_transaction("recovery", "object", transaction)
+            .await
+            .expect("inspect legacy proof");
+        assert_eq!(observed.state, DeleteRecoveryState::NewPrepared);
+        let recovered = disk
+            .settle_delete_transaction("recovery", "object", transaction, observed.fingerprint, DeleteRecoveryAction::Rollback)
+            .await
+            .expect("rollback legacy marker without unlinking the child directory");
+        assert_eq!(recovered.state, DeleteRecoveryState::Absent);
+        assert!(!root.join(transaction.to_string()).exists());
+        assert_eq!(
+            disk.read_all("recovery", "object/xl.meta/xl.meta")
+                .await
+                .expect("child metadata")
+                .as_ref(),
+            original
+        );
+        assert!(
+            disk.read_version("", "recovery", "object/xl.meta", "", &Default::default())
+                .await
+                .is_ok()
+        );
+        assert!(
+            discover_all_keys(&disk, Some("object")).await.is_empty(),
+            "no proof may keep replacement heal pending"
+        );
+    }
+
     // Data movement copies a delete marker onto a target that holds no
     // metadata yet. Rolling back that sub-quorum write removes the marker it
     // published, and must not report the backup it never wrote as missing:
@@ -5762,6 +6150,152 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn delete_recovery_bounded_inspection_charges_root_and_uuid_children() {
+        let (_temp, disk, _, _) = fixture().await;
+        let transaction = Uuid::new_v4();
+        apply(&disk, transaction).await;
+        let full = disk
+            .inspect_delete_transaction_bounded("recovery", "object", transaction, 65536)
+            .await
+            .expect("full bounded inspection");
+        assert!(full.visited > 1, "both root and transaction directory entries are charged");
+        let error = disk
+            .inspect_delete_transaction_bounded("recovery", "object", transaction, full.visited - 1)
+            .await
+            .expect_err("nested entry exceeds allowance");
+        assert_eq!(error, DiskError::DeleteRecoveryWorkLimit);
+        let unchanged = disk
+            .inspect_delete_transaction("recovery", "object", transaction)
+            .await
+            .expect("unchanged proof");
+        assert_eq!(full.inspection.fingerprint, unchanged.fingerprint);
+    }
+
+    #[tokio::test]
+    async fn delete_recovery_receipt_publication_distinguishes_deferred_inspection_from_write_failure() {
+        for write_fails in [false, true] {
+            let (_temp, disk, _, _) = fixture().await;
+            let transaction = Uuid::new_v4();
+            apply(&disk, transaction).await;
+            let full = disk
+                .inspect_delete_transaction_bounded("recovery", "object", transaction, 65536)
+                .await
+                .expect("measure preparation work");
+            let receipt = disk
+                .io_get_object_path("recovery", &format!("object/{transaction}/{DELETE_QUORUM_COMMITTED_FILE}"))
+                .expect("receipt path");
+            if write_fails {
+                let destination = receipt.clone();
+                set_owned_file_before_open(&receipt, move || std::fs::create_dir(&destination).expect("reject receipt rename"));
+            }
+            let result = disk
+                .settle_delete_transaction_bounded(
+                    "recovery",
+                    super::super::super::BoundedDeleteRecoverySettlement {
+                        object: "object".to_owned(),
+                        transaction,
+                        expected_fingerprint: full.inspection.fingerprint,
+                        action: DeleteRecoveryAction::PublishCommit,
+                        max_entries: full.visited,
+                    },
+                    None,
+                )
+                .await;
+            if write_fails {
+                assert!(result.is_err(), "a failed receipt write never counts as publication");
+                assert!(receipt.is_dir());
+            } else {
+                let result = result.expect("persisted receipt is acknowledged independently of post-write inspection");
+                assert!(matches!(
+                    result.outcome,
+                    super::super::super::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred
+                ));
+                assert_eq!(result.visited, full.visited);
+                let state = disk
+                    .inspect_delete_transaction("recovery", "object", transaction)
+                    .await
+                    .expect("durable receipt");
+                assert_eq!(state.state, DeleteRecoveryState::QuorumCommitted);
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn delete_recovery_bounded_discovery_refuses_partial_inventory_without_public_cursor() {
+        let (_temp, disk, _, _) = fixture().await;
+        let path = disk.delete_discovery_path("recovery", Some("prefix")).expect("prefix path");
+        fs::create_dir_all(&path).await.expect("physical prefix");
+        // Transaction-shaped children are the entries a bounded probe pays for.
+        for _ in 0..64 {
+            fs::create_dir(path.join(Uuid::new_v4().to_string()))
+                .await
+                .expect("legitimate child");
+        }
+        for _ in 0..140 {
+            let page = disk
+                .discover_delete_transactions_bounded("recovery", "prefix", 8)
+                .await
+                .expect("bounded probe");
+            assert_eq!(page.visited, 8);
+            assert!(!page.complete, "an empty partial page cannot establish absence");
+            assert!(page.transactions.is_empty());
+        }
+        let page = disk
+            .discover_delete_transactions_bounded("recovery", "prefix", 128)
+            .await
+            .expect("complete probe");
+        assert!(page.complete);
+        assert_eq!(page.visited, 64);
+        assert!(page.transactions.is_empty());
+        let transaction = Uuid::new_v4();
+        apply(&disk, transaction).await;
+        let before = disk
+            .inspect_delete_transaction("recovery", "object", transaction)
+            .await
+            .expect("before probe");
+        let page = disk
+            .discover_delete_transactions_bounded("recovery", "object", 4096)
+            .await
+            .expect("complete evidence");
+        assert!(page.complete);
+        assert!(page.transactions.iter().any(|id| id.transaction == transaction));
+        let after = disk
+            .inspect_delete_transaction("recovery", "object", transaction)
+            .await
+            .expect("after probe");
+        assert_eq!(before.fingerprint, after.fingerprint);
+        assert!(
+            DISCOVERY_SESSIONS
+                .lock()
+                .values()
+                .all(|session| session.disk_root != disk.io_root())
+        );
+    }
+
+    #[tokio::test]
+    async fn delete_recovery_cancelled_bounded_discovery_drops_private_session() {
+        let (_temp, disk, _, _) = fixture().await;
+        let path = disk.delete_discovery_path("recovery", Some("object")).expect("probe scope");
+        let lease = os::acquire_namespace_mutation_lease_with_owner(&path, None).await;
+        let mut probe = Box::pin(disk.discover_delete_transactions_bounded("recovery", "object", 4096));
+        assert!(futures::poll!(probe.as_mut()).is_pending());
+        drop(probe);
+        assert!(
+            DISCOVERY_SESSIONS
+                .lock()
+                .values()
+                .all(|session| session.disk_root != disk.io_root())
+        );
+        drop(lease);
+        assert!(
+            disk.discover_delete_transactions_bounded("recovery", "object", 4096)
+                .await
+                .expect("subsequent probe")
+                .complete
+        );
+    }
+
     #[tokio::test]
     async fn delete_recovery_evidence_probes_do_not_register_public_cursors() {
         let (_temp, disk, _, _) = fixture().await;
@@ -6646,6 +7180,86 @@ mod tests {
         assert!(!root.join(data_dir.to_string()).exists(), "physical data_dir must be removed");
         assert!(!root.exists(), "empty object directory must be pruned");
     }
+
+    #[cfg(all(any(test, feature = "test-util"), not(windows)))]
+    #[tokio::test]
+    async fn delete_recovery_receipt_formatting_and_post_finished_unlink_interruption() {
+        let (_temp, disk, info, _) = fixture().await;
+        let root = disk.io_get_object_path("recovery", "object").expect("object path");
+
+        let mut v1 = info.clone();
+        v1.version_id = Some(Uuid::new_v4());
+        let mut v2 = info.clone();
+        v2.version_id = Some(Uuid::new_v4());
+        let mut meta = FileMeta::new();
+        meta.add_version(v1.clone()).expect("add v1");
+        meta.add_version(v2.clone()).expect("add v2");
+        fs::write(root.join(STORAGE_FORMAT_FILE), meta.marshal_msg().expect("marshal meta"))
+            .await
+            .expect("write xl.meta");
+
+        let mut rev_info = v1.clone();
+        rev_info
+            .metadata
+            .extend([("x-amz-meta-z".into(), "1".into()), ("x-amz-meta-a".into(), "2".into())]);
+        let mut fwd_info = v1.clone();
+        fwd_info
+            .metadata
+            .extend([("x-amz-meta-a".into(), "2".into()), ("x-amz-meta-z".into(), "1".into())]);
+        assert_eq!(
+            generation(rev_info).expect("rev").metadata_digest,
+            generation(fwd_info).expect("fwd").metadata_digest
+        );
+
+        let transaction = Uuid::new_v4();
+        let opts = DeleteOptions {
+            recoverable_delete: true,
+            ..Default::default()
+        };
+        assert_eq!(
+            disk.try_fused_recoverable_delete_version("recovery", "object", &v2, &opts, transaction, None)
+                .await
+                .expect("fused versioned delete preparation"),
+            Some(())
+        );
+        let (journal, sole_transaction, target_satisfied) = disk
+            .try_publish_prepared_delete_local("recovery", "object", transaction, None)
+            .await
+            .expect("local publish")
+            .expect("versioned target satisfied");
+        assert!(sole_transaction && target_satisfied && journal.target_digest.is_some());
+
+        let rollback = root.join(transaction.to_string());
+        // Real finalizer failure at the second fsync_dir_std(&rollback) (after [TARGET, PREPARED, FINISHED] unlink)
+        // must leave a durable finished=true receipt that inspects as CommitFinished.
+        let rollback_for_hook = rollback.clone();
+        let _hook = os::prepared_publication_test_hooks::install_at(
+            os::prepared_publication_test_hooks::Stage::Remove,
+            &rollback.join(FINISHED),
+            move || {
+                os::fsync_dir_recorder::set_failure(&rollback_for_hook, std::io::ErrorKind::Other);
+            },
+        );
+        assert!(
+            disk.try_finish_and_finalize_prepared_delete_local("recovery", "object", transaction, &journal, None)
+                .await
+                .is_err()
+        );
+        assert!(!rollback.join(FINISHED).exists());
+        assert_eq!(
+            disk.inspect_delete_transaction("recovery", "object", transaction)
+                .await
+                .expect("post-FINISHED-unlink failure")
+                .state,
+            DeleteRecoveryState::CommitFinished
+        );
+        assert!(
+            disk.try_finish_and_finalize_prepared_delete_local("recovery", "object", transaction, &journal, None)
+                .await
+                .expect("finalize")
+        );
+        assert!(!rollback.exists());
+    }
 }
 
 #[cfg(test)]

crates/ecstore/src/disk/local/multipart_source_state.rs

@@ -117,6 +117,7 @@ pub(super) struct LoadedSource {
 }
 
 pub(super) struct SourceStore {
+    multipart_index: Arc<super::multipart_index::IndexOwner>,
     pub disk_id: Uuid,
     pub physical_slot: usize,
     pub disk_ids: Vec<Uuid>,
@@ -186,6 +187,7 @@ impl LocalDisk {
         let logical = self.get_object_path(RUSTFS_META_MULTIPART_BUCKET, path)?;
         let lease = os::acquire_namespace_mutation_lease_with_owner(&logical, owner.clone()).await;
         Ok(SourceStore {
+            multipart_index: self.multipart_index.clone(),
             disk_id,
             physical_slot,
             disk_ids,
@@ -270,21 +272,17 @@ struct ActiveMultipartSourceEntry {
     parts: std::collections::HashMap<usize, (FileStamp, rustfs_filemeta::ObjectPartInfo)>,
 }
 
-static ACTIVE_MULTIPART_SOURCE_HEADS: std::sync::LazyLock<
-    parking_lot::Mutex<std::collections::HashMap<PathBuf, ActiveMultipartSourceEntry>>,
-> = std::sync::LazyLock::new(|| parking_lot::Mutex::new(std::collections::HashMap::new()));
+const ACTIVE_MULTIPART_SOURCE_HEADS_PER_SHARD: usize = 64;
 
-pub(super) fn put_active_multipart_source_head(source: PathBuf, raw: Bytes, fi: FileInfo) {
+static ACTIVE_MULTIPART_SOURCE_HEADS: std::sync::LazyLock<os::ShardedPathCache<ActiveMultipartSourceEntry>> =
+    std::sync::LazyLock::new(|| os::ShardedPathCache::new(ACTIVE_MULTIPART_SOURCE_HEADS_PER_SHARD));
+
+pub(super) fn put_active_multipart_source_head(source: impl AsRef<Path>, raw: Bytes, fi: FileInfo) {
+    let source = source.as_ref();
     let Some(head_stamp) = FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) else {
         return;
     };
-    let mut map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
-    if map.len() >= 4096
-        && let Some(evict_key) = map.keys().next().cloned()
-    {
-        map.remove(&evict_key);
-    }
-    map.insert(
+    ACTIVE_MULTIPART_SOURCE_HEADS.insert(
         source,
         ActiveMultipartSourceEntry {
             raw,
@@ -304,56 +302,50 @@ fn stamp_matches_active_disk(source: &Path, expected_head_stamp: FileStamp) -> b
 }
 
 pub(super) fn get_active_multipart_source_head(source: &Path) -> Option<(Bytes, FileInfo)> {
-    let (raw, fi, head_stamp) = {
-        let map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
-        let entry = map.get(source)?;
-        (entry.raw.clone(), entry.fi.clone(), entry.head_stamp)
-    };
+    let (raw, fi, head_stamp) =
+        ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| (entry.raw.clone(), entry.fi.clone(), entry.head_stamp))?;
     if !stamp_matches_active_disk(source, head_stamp) {
-        ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
+        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
         return None;
     }
     Some((raw, fi))
 }
 
 pub(super) fn has_active_multipart_source_head(source: &Path) -> bool {
-    let Some(head_stamp) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get(source).map(|entry| entry.head_stamp) else {
+    let Some(head_stamp) = ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| entry.head_stamp) else {
         return false;
     };
     if !stamp_matches_active_disk(source, head_stamp) {
-        ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
+        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
         return false;
     }
     true
 }
 
 pub(super) fn has_clean_active_multipart_source(source: &Path) -> bool {
-    let Some((head_stamp, clean)) = ACTIVE_MULTIPART_SOURCE_HEADS
-        .lock()
-        .get(source)
-        .map(|entry| (entry.head_stamp, entry.unsettled_parts.is_empty()))
+    let Some((head_stamp, clean)) =
+        ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| (entry.head_stamp, entry.unsettled_parts.is_empty()))
     else {
         return false;
     };
     if !stamp_matches_active_disk(source, head_stamp) {
-        ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
+        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
         return false;
     }
     clean
 }
 
 pub(super) fn begin_active_multipart_part_transaction(source: &Path, part_number: usize) {
-    if let Some(entry) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get_mut(source) {
+    ACTIVE_MULTIPART_SOURCE_HEADS.with_get_mut(source, |entry| {
         entry.unsettled_parts.insert(part_number);
         entry.parts.remove(&part_number);
-    }
+    });
 }
 
 pub(super) fn record_active_multipart_part_committed(source: &Path, part: rustfs_filemeta::ObjectPartInfo) {
     let Some(data_dir) = ACTIVE_MULTIPART_SOURCE_HEADS
-        .lock()
-        .get(source)
-        .and_then(|entry| entry.fi.data_dir)
+        .with_get(source, |entry| entry.fi.data_dir)
+        .flatten()
         .filter(|dir| !dir.is_nil())
     else {
         return;
@@ -362,9 +354,9 @@ pub(super) fn record_active_multipart_part_committed(source: &Path, part: rustfs
     let Some(part_stamp) = FileStamp::from_path(&part_meta_path) else {
         return;
     };
-    if let Some(entry) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get_mut(source) {
+    ACTIVE_MULTIPART_SOURCE_HEADS.with_get_mut(source, |entry| {
         entry.parts.insert(part.number, (part_stamp, part));
-    }
+    });
 }
 
 pub(super) fn finish_active_multipart_part_transaction(
@@ -373,34 +365,29 @@ pub(super) fn finish_active_multipart_part_transaction(
     committed: bool,
     retired_existing_txn: bool,
 ) {
-    let mut map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
     let Some(part_number) = part_number.filter(|_| committed) else {
-        map.remove(source);
+        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
         return;
     };
     if !retired_existing_txn {
         return;
     }
-    if let Some(entry) = map.get_mut(source)
-        && !entry.unsettled_parts.remove(&part_number)
-    {
-        map.remove(source);
-    }
+    ACTIVE_MULTIPART_SOURCE_HEADS.remove_if_mut(source, |entry| !entry.unsettled_parts.remove(&part_number));
 }
 
 pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize) -> Option<rustfs_filemeta::ObjectPartInfo> {
-    let (head_stamp, data_dir, expected_stamp, part) = {
-        let map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
-        let entry = map.get(source)?;
-        if !entry.unsettled_parts.is_empty() {
-            return None;
-        }
-        let data_dir = entry.fi.data_dir.filter(|dir| !dir.is_nil())?;
-        let (expected_stamp, part) = entry.parts.get(&part_number)?;
-        (entry.head_stamp, data_dir, *expected_stamp, part.clone())
-    };
+    let (head_stamp, data_dir, expected_stamp, part) = ACTIVE_MULTIPART_SOURCE_HEADS
+        .with_get(source, |entry| {
+            if !entry.unsettled_parts.is_empty() {
+                return None;
+            }
+            let data_dir = entry.fi.data_dir.filter(|dir| !dir.is_nil())?;
+            let (expected_stamp, part) = entry.parts.get(&part_number)?;
+            Some((entry.head_stamp, data_dir, *expected_stamp, part.clone()))
+        })
+        .flatten()?;
     if !stamp_matches_active_disk(source, head_stamp) {
-        ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
+        ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
         return None;
     }
     let part_meta_path = source.join(data_dir.to_string()).join(format!("part.{part_number}.meta"));
@@ -411,11 +398,11 @@ pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize
 }
 
 pub(super) fn invalidate_active_multipart_source_head(source: &Path) {
-    ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
+    ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
 }
 
 pub(super) fn invalidate_active_multipart_sources_under(prefix: &Path) {
-    ACTIVE_MULTIPART_SOURCE_HEADS.lock().retain(|key, _| !key.starts_with(prefix));
+    ACTIVE_MULTIPART_SOURCE_HEADS.retain_outside_prefix(prefix);
 }
 
 /// Canonical source mutations are admitted under SourceRW at the syscall,
@@ -770,6 +757,9 @@ impl SourceStore {
         os::sync_multipart_owned_record_parents_options(source, parent, existed, sync_source_root, base, root)
     }
     pub async fn replace_head(&self, bytes: &[u8]) -> Result<()> {
+        self.multipart_index
+            .record_head(&format!("{}/xl.meta", self.path), bytes)
+            .await?;
         invalidate_active_multipart_source_head(&self.source);
         let path = self.source.join(STORAGE_FORMAT_FILE);
         let base = self.base.clone();
@@ -1527,7 +1517,7 @@ impl LocalDisk {
                     }
                     Err(error) => return Err(error),
                 };
-                let _directory = os::pin_multipart_source_directory(&directory, base, root)?;
+                let directory_pin = os::pin_multipart_source_directory(&directory, base, root)?;
                 if !parts.is_empty() {
                     let anchor_path = directory.join(RETIREMENT_ANCHOR);
                     let bytes = os::read_multipart_source_metadata(&anchor_path, base, root)?;
@@ -1560,7 +1550,13 @@ impl LocalDisk {
                 }
                 // An anchor-less empty DD is only rmdir'ed. A new child makes
                 // this fail closed; no recursive removal can consume its bytes.
+                // A Windows pin refuses DELETE sharing, so the leaf pin is
+                // released first; the object pin still holds the parent.
+                #[cfg(windows)]
+                drop(directory_pin);
                 std::fs::remove_dir(&directory)?;
+                #[cfg(not(windows))]
+                drop(directory_pin);
                 if sync {
                     os::fsync_dir_std(&object_path)?;
                 }
@@ -1936,6 +1932,7 @@ impl NativeMultipartPublication {
         *self.validated_prepared.lock() = Some(validated);
         Ok(())
     }
+    #[cfg(target_os = "linux")]
     pub(super) fn prepare_and_install_head_on_thread(
         &self,
         new_metadata: &[u8],

crates/ecstore/src/disk/local/multipart_staging.rs

@@ -398,7 +398,9 @@ impl LocalDisk {
         let source_path = source_path.to_owned();
         let incoming = incoming.clone();
         let sync_directory = effective_durability(destination_volume).syncs_data_shards();
-        let fresh_head = os::run_cancellable_blocking_namespace_operation(Arc::clone(&staging.lease), move |cancelled| {
+        let sync_lane = sync_directory && incoming.data.is_none() && !incoming.is_remote();
+        let lease = Arc::clone(&staging.lease);
+        let fresh_head = os::run_cancellable_blocking_namespace_operation_with_sync(lease, sync_lane, move |cancelled| {
             if cancelled() {
                 return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "multipart staging cancelled"));
             }

crates/ecstore/src/disk/mod.rs

@@ -27,6 +27,7 @@ mod io_admission;
 #[cfg(test)]
 pub(crate) mod lifecycle_test_io;
 pub mod local;
+pub mod multipart_index;
 pub mod multipart_source;
 pub mod os;
 
@@ -52,6 +53,10 @@ pub const RUSTFS_META_MULTIPART_BUCKET: &str = ".rustfs.sys/multipart";
 pub const RUSTFS_META_TMP_BUCKET: &str = ".rustfs.sys/tmp";
 pub const RUSTFS_META_TMP_DELETED_BUCKET: &str = ".rustfs.sys/tmp/.trash";
 pub const BUCKET_META_PREFIX: &str = "buckets";
+/// Directory under `BUCKET_META_PREFIX` that holds drive-replacement heal
+/// intents, seals and recovery proofs. It shares the bucket-metadata root with
+/// a user bucket of the same name, so bucket deletion must leave it in place.
+pub const HEAL_REPLACEMENT_RECOVERY_DIR: &str = "ahm-replacement";
 pub const FORMAT_CONFIG_FILE: &str = "format.json";
 /// Per-disk marker present while an erasure-set heal is rebuilding this disk.
 /// `LocalDisk::disk_info` reports `healing = true` while the file exists.
@@ -414,10 +419,11 @@ impl Disk {
         object: &str,
         transaction: Uuid,
         namespace_owner: Option<Arc<dyn Send + Sync>>,
+        work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
     ) -> Result<Option<(DeleteRecoveryJournal, bool, bool)>> {
         match self {
             Self::Local(disk) => {
-                disk.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner)
+                disk.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner, work_budget)
                     .await
             }
             Self::Remote(_) => Ok(None),
@@ -431,11 +437,19 @@ impl Disk {
         transaction: Uuid,
         journal: &DeleteRecoveryJournal,
         namespace_owner: Option<Arc<dyn Send + Sync>>,
+        work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
     ) -> Result<bool> {
         match self {
             Self::Local(disk) => {
-                disk.try_finish_and_finalize_prepared_delete_local(volume, object, transaction, journal, namespace_owner)
-                    .await
+                disk.try_finish_and_finalize_prepared_delete_local(
+                    volume,
+                    object,
+                    transaction,
+                    journal,
+                    namespace_owner,
+                    work_budget,
+                )
+                .await
             }
             Self::Remote(_) => Ok(false),
         }
@@ -512,6 +526,17 @@ impl Disk {
 
 #[async_trait::async_trait]
 impl DiskAPI for Disk {
+    async fn list_multipart_index_roots(
+        &self,
+        bucket: &str,
+        marker: Option<&str>,
+    ) -> Result<multipart_index::MultipartIndexPage> {
+        match self {
+            Self::Local(disk) => disk.list_multipart_index_roots(bucket, marker).await,
+            Self::Remote(disk) => disk.list_multipart_index_roots(bucket, marker).await,
+        }
+    }
+
     async fn supports_guarded_delete_versions(&self) -> Result<()> {
         match self {
             Self::Local(disk) => disk.supports_guarded_delete_versions().await,
@@ -555,6 +580,17 @@ impl DiskAPI for Disk {
             Self::Remote(disk) => disk.has_delete_transaction_evidence(volume, object).await,
         }
     }
+    async fn discover_delete_transactions_bounded(
+        &self,
+        volume: &str,
+        object: &str,
+        max_entries: u32,
+    ) -> Result<BoundedDeleteRecoveryDiscovery> {
+        match self {
+            Self::Local(disk) => disk.discover_delete_transactions_bounded(volume, object, max_entries).await,
+            Self::Remote(disk) => disk.discover_delete_transactions_bounded(volume, object, max_entries).await,
+        }
+    }
     async fn discover_delete_transactions(
         &self,
         volume: &str,
@@ -567,6 +603,35 @@ impl DiskAPI for Disk {
             Self::Remote(disk) => disk.discover_delete_transactions(volume, object, cursor, limit).await,
         }
     }
+    async fn settle_delete_transaction_bounded(
+        &self,
+        volume: &str,
+        request: BoundedDeleteRecoverySettlement,
+        namespace_owner: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<BoundedDeleteRecoverySettlementReply> {
+        match self {
+            Self::Local(disk) => disk.settle_delete_transaction_bounded(volume, request, namespace_owner).await,
+            Self::Remote(disk) => disk.settle_delete_transaction_bounded(volume, request, namespace_owner).await,
+        }
+    }
+    async fn inspect_delete_transaction_bounded(
+        &self,
+        volume: &str,
+        object: &str,
+        transaction: Uuid,
+        max_entries: u32,
+    ) -> Result<BoundedDeleteRecoveryInspection> {
+        match self {
+            Self::Local(disk) => {
+                disk.inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
+                    .await
+            }
+            Self::Remote(disk) => {
+                disk.inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
+                    .await
+            }
+        }
+    }
     async fn inspect_delete_transaction(
         &self,
         volume: &str,
@@ -1513,6 +1578,15 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
     fn host_name(&self) -> String;
     fn endpoint(&self) -> Endpoint;
     async fn close(&self) -> Result<()>;
+    /// Return candidates only while this disk maintains complete boot-local coverage.
+    /// Unsupported or unready disks must never return an authoritative empty page.
+    async fn list_multipart_index_roots(
+        &self,
+        _bucket: &str,
+        _marker: Option<&str>,
+    ) -> Result<multipart_index::MultipartIndexPage> {
+        Err(DiskError::MethodNotAllowed)
+    }
     async fn get_disk_id(&self) -> Result<Option<Uuid>>;
     async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()>;
 
@@ -1690,6 +1764,35 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
     ) -> Result<DeleteRecoveryPage> {
         Err(DiskError::MethodNotAllowed)
     }
+    /// Bounded foreground inventory owns all traversal state until return or cancellation.
+    /// An implementation without the bounded protocol is an unsupported participant
+    /// (the same verdict a remote peer without it receives); callers must never fall
+    /// back to paginated discovery.
+    async fn discover_delete_transactions_bounded(
+        &self,
+        _volume: &str,
+        _object: &str,
+        _max_entries: u32,
+    ) -> Result<BoundedDeleteRecoveryDiscovery> {
+        Err(DiskError::UnsupportedDisk)
+    }
+    async fn inspect_delete_transaction_bounded(
+        &self,
+        _volume: &str,
+        _object: &str,
+        _transaction: Uuid,
+        _max_entries: u32,
+    ) -> Result<BoundedDeleteRecoveryInspection> {
+        Err(DiskError::UnsupportedDisk)
+    }
+    async fn settle_delete_transaction_bounded(
+        &self,
+        _volume: &str,
+        _request: BoundedDeleteRecoverySettlement,
+        _namespace_owner: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<BoundedDeleteRecoverySettlementReply> {
+        Err(DiskError::UnsupportedDisk)
+    }
     /// A read-only capability check; unsupported implementations must fail closed.
     async fn supports_guarded_delete_versions(&self) -> Result<()> {
         Err(DiskError::GuardedDeleteUnsupported)
@@ -2022,10 +2125,19 @@ pub struct GuardedFileInfoVersions {
 pub struct GuardedDeleteVersionsPayload {
     pub versions: Vec<GuardedFileInfoVersions>,
     pub opts: DeleteOptions,
+    // The strict legacy decoder rejects this field before applying any change.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub bounded_recovery_protocol: Option<u32>,
 }
 
 impl GuardedDeleteVersionsPayload {
     pub fn validate(&self) -> Result<()> {
+        if self
+            .bounded_recovery_protocol
+            .is_some_and(|version| version != BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION)
+        {
+            return Err(DiskError::UnsupportedDisk);
+        }
         validate_delete_recovery_options(&self.opts)?;
         if self.versions.is_empty()
             || self.versions.len() > 1000
@@ -2075,6 +2187,13 @@ pub struct WalkDirOptions {
     #[serde(default)]
     pub skip_hidden_prefix_check: bool,
 
+    /// Bound hidden metadata read while proving a non-recursive prefix visible.
+    #[serde(default)]
+    pub hidden_entry_budget: usize,
+    /// A prefix whose visibility was not established before the preceding cut.
+    #[serde(default)]
+    pub pending_prefix: Option<String>,
+
     // ReportNotFound will return errFileNotFound if all disks reports the BaseDir cannot be found.
     pub report_notfound: bool,
 
@@ -2196,6 +2315,9 @@ pub struct DeleteOptions {
     pub delete_recovery_bucket_incarnation: Option<Uuid>,
     #[serde(default)]
     pub recoverable_delete: bool,
+    /// Transport requirement for preparing a delete that will use bounded settlement.
+    #[serde(skip)]
+    pub requires_bounded_delete_recovery: bool,
     /// Reclaim only plain erasure-part leaves, preserving every other entry.
     /// Keep `recursive` false so an older peer cannot widen this operation.
     #[serde(default)]
@@ -2203,6 +2325,8 @@ pub struct DeleteOptions {
 }
 
 pub const DELETE_RECOVERY_PROTOCOL_VERSION: u32 = 1;
+/// Accepted only for prepare RPCs. Legacy peers must refuse before staging a delete.
+pub const BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION: u32 = 2;
 pub(crate) fn get_delete_recovery_timeout() -> Result<std::time::Duration> {
     let value = std::env::var("RUSTFS_DELETE_RECOVERY_TIMEOUT_SECS");
     match value {
@@ -2407,9 +2531,62 @@ pub enum DeleteRecoveryAction {
     AdoptCommit { authority: Box<DeleteRecoveryJournal> },
 }
 
+/// A complete inventory or a work-limit refusal. No public discovery cursor is allocated.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct BoundedDeleteRecoveryDiscovery {
+    pub transactions: Vec<DeleteTransactionId>,
+    pub visited: u32,
+    pub complete: bool,
+}
+
+pub const DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES: u32 = 4096;
+pub const DELETE_RECOVERY_MAX_INSPECTION_ENTRIES: u32 = 65536;
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct BoundedDeleteRecoverySettlement {
+    pub object: String,
+    pub transaction: Uuid,
+    pub expected_fingerprint: DeleteRecoveryDigest,
+    pub action: DeleteRecoveryAction,
+    pub max_entries: u32,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub enum DeleteRecoverySettlementOutcome {
+    Inspected(Box<DeleteRecoveryInspection>),
+    ReceiptPublishedInspectionDeferred,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct BoundedDeleteRecoverySettlementReply {
+    pub outcome: DeleteRecoverySettlementOutcome,
+    pub visited: u32,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct BoundedDeleteRecoveryInspection {
+    pub inspection: DeleteRecoveryInspection,
+    pub visited: u32,
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
 pub enum DeleteRecoveryRpcOperation {
+    SettleBounded(BoundedDeleteRecoverySettlement),
+    InspectBounded {
+        object: String,
+        transaction: Uuid,
+        max_entries: u32,
+    },
+    DiscoverBounded {
+        object: String,
+        max_entries: u32,
+    },
     InspectOrphanDataDir {
         path: String,
     },
@@ -2447,7 +2624,11 @@ impl DeleteRecoveryRpcOperation {
     /// Full metadata application and adoption share the existing mutation envelope.
     pub fn max_wire_payload_bytes(&self) -> usize {
         match self {
-            Self::ApplyVersion { .. }
+            Self::SettleBounded(BoundedDeleteRecoverySettlement {
+                action: DeleteRecoveryAction::AdoptCommit { .. },
+                ..
+            })
+            | Self::ApplyVersion { .. }
             | Self::ApplyVersions { .. }
             | Self::Settle {
                 action: DeleteRecoveryAction::AdoptCommit { .. },
@@ -2463,8 +2644,11 @@ impl DeleteRecoveryRpcOperation {
 pub enum DeleteRecoveryRpcReply {
     OrphanDataDir { plain: bool },
     Discovered(DeleteRecoveryPage),
+    DiscoveredBounded(BoundedDeleteRecoveryDiscovery),
     Inspected(DeleteRecoveryInspection),
+    InspectedBounded(BoundedDeleteRecoveryInspection),
     Settled(DeleteRecoveryInspection),
+    SettledBounded(BoundedDeleteRecoverySettlementReply),
     AppliedVersion,
     AppliedVersions(Vec<Option<u32>>),
     Evidence { present: bool },
@@ -2739,6 +2923,7 @@ mod tests {
         insert_str(&mut fi.metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID, "ab".repeat(32));
         let generation = CurrentDeleteGeneration::from_file_info(&fi).expect("capture complete remote identity");
         let payload = GuardedDeleteVersionsPayload {
+            bounded_recovery_protocol: None,
             versions: vec![GuardedFileInfoVersions {
                 group: FileInfoVersions {
                     name: "object".into(),
@@ -2793,6 +2978,7 @@ mod tests {
             expected_current: CurrentDeleteGeneration::Absent,
         };
         let mut payload = GuardedDeleteVersionsPayload {
+            bounded_recovery_protocol: None,
             versions: vec![group],
             opts: DeleteOptions {
                 old_data_dir: Some(Uuid::new_v4()),
@@ -2804,6 +2990,25 @@ mod tests {
         let decoded: GuardedDeleteVersionsPayload =
             rustfs_protos::decode_delete_recovery_payload(&encoded).expect("decode guarded batch");
         decoded.validate().expect("round-trip guarded batch remains valid");
+        #[derive(Debug, Deserialize)]
+        #[serde(deny_unknown_fields)]
+        struct LegacyGuardedDeletePayload {
+            #[serde(rename = "versions")]
+            _versions: Vec<GuardedFileInfoVersions>,
+            #[serde(rename = "opts")]
+            _opts: DeleteOptions,
+        }
+        rustfs_protos::decode_delete_recovery_payload::<LegacyGuardedDeletePayload>(&encoded)
+            .expect("legacy reader accepts legacy prepare");
+        payload.bounded_recovery_protocol = Some(BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION);
+        let bounded = rmp_serde::to_vec_named(&payload).expect("serialize bounded guarded prepare");
+        assert!(rustfs_protos::decode_delete_recovery_payload::<LegacyGuardedDeletePayload>(&bounded).is_err());
+        let bounded: GuardedDeleteVersionsPayload =
+            rustfs_protos::decode_delete_recovery_payload(&bounded).expect("current reader accepts bounded prepare");
+        bounded.validate().expect("valid bounded guarded prepare");
+        payload.bounded_recovery_protocol = Some(BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION + 1);
+        assert_eq!(payload.validate(), Err(DiskError::UnsupportedDisk));
+        payload.bounded_recovery_protocol = None;
         let mut trailing = encoded;
         trailing.push(0xc0);
         assert!(rustfs_protos::decode_delete_recovery_payload::<GuardedDeleteVersionsPayload>(&trailing).is_err());
@@ -2942,6 +3147,8 @@ mod tests {
             recursive: true,
             incl_deleted: false,
             skip_hidden_prefix_check: false,
+            hidden_entry_budget: 0,
+            pending_prefix: None,
             report_notfound: false,
             filter_prefix: Some("prefix_".to_string()),
             forward_to: Some("object/path".to_string()),
@@ -2996,9 +3203,15 @@ mod tests {
             expected_delete_marker: None,
             delete_recovery_bucket_incarnation: None,
             recoverable_delete: false,
+            requires_bounded_delete_recovery: true,
             orphan_data_dir: false,
         };
 
+        let wire = serde_json::to_value(&opts).expect("encode legacy wire options");
+        assert!(wire.get("requires_bounded_delete_recovery").is_none());
+        let decoded: DeleteOptions = serde_json::from_value(wire).expect("decode legacy wire options");
+        assert!(!decoded.requires_bounded_delete_recovery);
+
         assert!(opts.recursive);
         assert!(!opts.immediate);
         assert!(opts.undo_write);

crates/ecstore/src/disk/multipart_index.rs

@@ -0,0 +1,359 @@
+// Copyright 2026 RustFS Team
+// SPDX-License-Identifier: Apache-2.0
+
+//! A candidate index never replaces multipart metadata or quorum validation.
+
+use serde::{Deserialize, Serialize};
+use uuid::Uuid;
+
+pub(crate) const RUSTFS_MULTIPART_BUCKET_KEY: &str = "x-rustfs-internal-multipart-bucket";
+pub(crate) const RUSTFS_MULTIPART_OBJECT_KEY: &str = "x-rustfs-internal-multipart-object";
+
+pub(crate) const MULTIPART_INDEX_PAGE_SIZE: usize = 256;
+/// Pages one listing accepts from one disk. Pruned pages may carry few roots, so
+/// the cursor alone no longer bounds a peer's answer; a disk that keeps
+/// continuing past this many pages contributes no coverage.
+pub(crate) const MULTIPART_INDEX_MAX_PAGES_PER_DISK: usize = 4096;
+
+/// A bounded page from one disk's continuously maintained, boot-local index.
+/// `epoch` changes whenever coverage must be rebuilt; callers must not combine
+/// pages from different epochs or physical disk identities.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct MultipartIndexPage {
+    pub bucket: String,
+    pub disk_id: Uuid,
+    pub epoch: Uuid,
+    pub roots: Vec<String>,
+    pub next_marker: Option<String>,
+}
+
+pub const MULTIPART_INDEX_RPC_MAX_BYTES: usize = 64 * 1024;
+pub const MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES: usize = 4096;
+
+/// Additive operation inside the legacy read-only DiskInfo JSON envelope.
+/// Old peers may ignore this field, but their ordinary response cannot decode
+/// as the required signed, challenge-bound MultipartIndexRpcReply.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct MultipartIndexRpcRequest {
+    pub disk_id: String,
+    pub metrics: bool,
+    pub noop: bool,
+    pub multipart_index: MultipartIndexQuery,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct MultipartIndexQuery {
+    pub operation: String,
+    pub version: u32,
+    pub challenge: Uuid,
+    pub bucket: String,
+    pub marker: Option<String>,
+    pub page_cap: usize,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct MultipartIndexRpcReply {
+    pub operation: String,
+    pub version: u32,
+    pub challenge: Uuid,
+    pub page: MultipartIndexPage,
+    pub proof: Vec<u8>,
+}
+
+impl MultipartIndexRpcRequest {
+    pub fn new(bucket: &str, marker: Option<&str>) -> Self {
+        Self {
+            disk_id: String::new(),
+            metrics: false,
+            noop: true,
+            multipart_index: MultipartIndexQuery {
+                operation: "list_multipart_index_roots".into(),
+                version: 1,
+                challenge: Uuid::new_v4(),
+                bucket: bucket.to_owned(),
+                marker: marker.map(str::to_owned),
+                page_cap: MULTIPART_INDEX_PAGE_SIZE,
+            },
+        }
+    }
+
+    pub fn validate(&self) -> crate::disk::error::Result<()> {
+        let query = &self.multipart_index;
+        if !self.disk_id.is_empty()
+            || self.metrics
+            || !self.noop
+            || query.operation != "list_multipart_index_roots"
+            || query.version != 1
+            || query.challenge.is_nil()
+            || query.bucket.is_empty()
+            || query.bucket.len() > 255
+            || query.bucket.contains('/')
+            || matches!(query.bucket.as_str(), "." | "..")
+            || query.page_cap != MULTIPART_INDEX_PAGE_SIZE
+            || query.marker.as_ref().is_some_and(|marker| !valid_root(marker))
+        {
+            return Err(crate::disk::error::DiskError::MethodNotAllowed);
+        }
+        Ok(())
+    }
+
+    pub fn canonical_request(&self, disk: &str) -> crate::disk::error::Result<Vec<u8>> {
+        self.validate()?;
+        if disk.len() > MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES {
+            return Err(crate::disk::error::DiskError::FileCorrupt);
+        }
+        Ok(serde_json::to_vec(&("rustfs-multipart-index-request-v1", disk, self))?)
+    }
+
+    pub fn canonical_response(&self, disk: &str, page: &MultipartIndexPage) -> crate::disk::error::Result<Vec<u8>> {
+        self.validate()?;
+        Ok(serde_json::to_vec(&("rustfs-multipart-index-response-v1", disk, self, page))?)
+    }
+}
+
+fn valid_root(root: &str) -> bool {
+    root.len() == 64 && root.bytes().all(|byte| byte.is_ascii_hexdigit())
+}
+
+impl MultipartIndexPage {
+    pub fn validate(&self, bucket: &str, marker: Option<&str>, expected_disk: Uuid) -> crate::disk::error::Result<()> {
+        if expected_disk.is_nil()
+            || self.disk_id != expected_disk
+            || self.epoch.is_nil()
+            || self.bucket != bucket
+            || self.roots.len() > MULTIPART_INDEX_PAGE_SIZE
+            || self
+                .roots
+                .iter()
+                .any(|root| !valid_root(root) || marker.is_some_and(|marker| root.as_str() <= marker))
+            || self.roots.windows(2).any(|pair| pair[0] >= pair[1])
+            // The cursor names the last examined marker; pruning may leave fewer
+            // live roots than examined names, so it only has to bound them.
+            || self.next_marker.as_ref().is_some_and(|next| {
+                !valid_root(next)
+                    || marker.is_some_and(|marker| next.as_str() <= marker)
+                    || self.roots.last().is_some_and(|last| last > next)
+            })
+        {
+            return Err(crate::disk::error::DiskError::FileCorrupt);
+        }
+        Ok(())
+    }
+}
+
+impl MultipartIndexRpcReply {
+    /// This is structural validation only; the caller must also verify `proof`.
+    pub fn validate(&self, request: &MultipartIndexRpcRequest, expected_disk: Uuid) -> crate::disk::error::Result<()> {
+        let query = &request.multipart_index;
+        request.validate()?;
+        if self.operation != query.operation
+            || self.version != query.version
+            || self.challenge != query.challenge
+            || self.proof.len() != 32
+        {
+            return Err(crate::disk::error::DiskError::FileCorrupt);
+        }
+        self.page.validate(&query.bucket, query.marker.as_deref(), expected_disk)
+    }
+}
+
+/// A disk can attest coverage only after every page belongs to the same physical
+/// identity and index generation. An interrupted/changed stream contributes no
+/// partial candidates to the erasure set's coverage count.
+pub(crate) async fn collect_pages<F, Fut>(bucket: &str, identity: Uuid, mut fetch: F) -> crate::disk::error::Result<Vec<String>>
+where
+    F: FnMut(Option<String>) -> Fut,
+    Fut: std::future::Future<Output = crate::disk::error::Result<MultipartIndexPage>>,
+{
+    let mut epoch = None;
+    let mut marker = None;
+    let mut roots = Vec::new();
+    let mut pages = 0_usize;
+    loop {
+        if pages >= MULTIPART_INDEX_MAX_PAGES_PER_DISK {
+            return Err(crate::disk::error::DiskError::FileCorrupt);
+        }
+        pages += 1;
+        let page = fetch(marker.clone()).await?;
+        page.validate(bucket, marker.as_deref(), identity)?;
+        if epoch.is_some_and(|epoch| epoch != page.epoch) {
+            return Err(crate::disk::error::DiskError::DiskNotFound);
+        }
+        epoch = Some(page.epoch);
+        roots.extend(page.roots);
+        marker = page.next_marker;
+        if marker.is_none() {
+            return Ok(roots);
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn multipart_index_reply_rejects_challenge_disk_cursor_and_unbounded_pages() {
+        let request = MultipartIndexRpcRequest::new("bucket", None);
+        let id = Uuid::new_v4();
+        let mut reply = MultipartIndexRpcReply {
+            operation: request.multipart_index.operation.clone(),
+            version: 1,
+            challenge: request.multipart_index.challenge,
+            page: MultipartIndexPage {
+                bucket: "bucket".into(),
+                disk_id: id,
+                epoch: Uuid::new_v4(),
+                roots: vec!["a".repeat(64)],
+                next_marker: None,
+            },
+            proof: vec![0; 32],
+        };
+        reply.validate(&request, id).expect("valid structural envelope");
+        assert!(
+            reply.validate(&MultipartIndexRpcRequest::new("bucket", None), id).is_err(),
+            "old challenge cannot replay"
+        );
+        assert!(reply.validate(&request, Uuid::new_v4()).is_err(), "physical disk identity cannot change");
+        reply.page.next_marker = Some("9".repeat(64));
+        assert!(reply.validate(&request, id).is_err(), "a cursor below the last root would re-list it");
+        reply.page.next_marker = Some("a".repeat(64));
+        reply
+            .validate(&request, id)
+            .expect("a pruned short page continues from its last examined name");
+        reply.page.next_marker = None;
+        reply.page.roots = vec!["a".repeat(64); MULTIPART_INDEX_PAGE_SIZE + 1];
+        assert!(reply.validate(&request, id).is_err(), "peer cannot enlarge the fixed page cap");
+    }
+
+    #[test]
+    fn multipart_index_response_proof_binds_the_complete_query_and_page() {
+        crate::runtime::sources::ensure_test_rpc_secret();
+        let mut request = MultipartIndexRpcRequest::new("bucket", None);
+        let mut page = MultipartIndexPage {
+            bucket: "bucket".into(),
+            disk_id: Uuid::new_v4(),
+            epoch: Uuid::new_v4(),
+            roots: vec![],
+            next_marker: None,
+        };
+        let canonical = request.canonical_response("disk", &page).expect("canonical response");
+        let proof = crate::cluster::rpc::sign_tonic_rpc_response_proof(&canonical).expect("signed response");
+        let verify = |request: &MultipartIndexRpcRequest, page: &MultipartIndexPage| {
+            crate::cluster::rpc::verify_tonic_rpc_response_proof(
+                &request.canonical_response("disk", page).expect("canonical"),
+                &proof,
+            )
+        };
+        verify(&request, &page).expect("exact response");
+        request.multipart_index.marker = Some("0".repeat(64));
+        assert!(verify(&request, &page).is_err(), "cursor is signed");
+        request.multipart_index.marker = None;
+        page.epoch = Uuid::new_v4();
+        assert!(verify(&request, &page).is_err(), "boot-local coverage epoch is signed");
+    }
+    #[test]
+    fn multipart_index_page_cursor_bounds_roots_without_requiring_a_full_page() {
+        let disk = Uuid::new_v4();
+        let page = |roots: Vec<String>, next_marker: Option<String>| MultipartIndexPage {
+            bucket: "bucket".into(),
+            disk_id: disk,
+            epoch: Uuid::new_v4(),
+            roots,
+            next_marker,
+        };
+        // Pruning leaves fewer live roots than examined names; the cursor still advances.
+        page(vec![format!("{:064x}", 1)], Some(format!("{:064x}", 5)))
+            .validate("bucket", None, disk)
+            .expect("short page with a cursor");
+        page(vec![], Some(format!("{:064x}", 5)))
+            .validate("bucket", Some(&format!("{:064x}", 2)), disk)
+            .expect("all examined names pruned");
+        assert!(
+            page(vec![format!("{:064x}", 6)], Some(format!("{:064x}", 5)))
+                .validate("bucket", None, disk)
+                .is_err(),
+            "a root beyond the cursor would be re-listed"
+        );
+        assert!(
+            page(vec![], Some(format!("{:064x}", 2)))
+                .validate("bucket", Some(&format!("{:064x}", 2)), disk)
+                .is_err(),
+            "the cursor must advance past the request marker"
+        );
+        assert!(
+            page(vec![], Some("not-a-sha".to_owned()))
+                .validate("bucket", None, disk)
+                .is_err(),
+            "the cursor is a marker name"
+        );
+    }
+
+    #[tokio::test]
+    async fn multipart_index_page_collection_bounds_the_pages_one_disk_may_answer() {
+        let id = Uuid::new_v4();
+        let epoch = Uuid::new_v4();
+        let mut served = 0_usize;
+        let result = collect_pages("bucket", id, |marker| {
+            served += 1;
+            let next = marker.map_or(0_u64, |marker| u64::from_str_radix(&marker[48..], 16).expect("hex cursor") + 1);
+            std::future::ready(Ok(MultipartIndexPage {
+                bucket: "bucket".into(),
+                disk_id: id,
+                epoch,
+                roots: vec![],
+                next_marker: Some(format!("{next:064x}")),
+            }))
+        })
+        .await;
+        assert!(matches!(result, Err(crate::disk::error::DiskError::FileCorrupt)), "{result:?}");
+        assert_eq!(served, MULTIPART_INDEX_MAX_PAGES_PER_DISK, "the peer is cut off at the page cap");
+    }
+
+    #[tokio::test]
+    async fn multipart_index_page_collection_rejects_replaced_disk_or_restarted_epoch() {
+        let id = Uuid::new_v4();
+        let epoch = Uuid::new_v4();
+        for replace_disk in [false, true] {
+            let first = MultipartIndexPage {
+                bucket: "bucket".into(),
+                disk_id: id,
+                epoch,
+                roots: (0..MULTIPART_INDEX_PAGE_SIZE).map(|index| format!("{index:064x}")).collect(),
+                next_marker: Some(format!("{:064x}", MULTIPART_INDEX_PAGE_SIZE - 1)),
+            };
+            let second = MultipartIndexPage {
+                bucket: "bucket".into(),
+                disk_id: if replace_disk { Uuid::new_v4() } else { id },
+                epoch: if replace_disk { epoch } else { Uuid::new_v4() },
+                roots: vec![],
+                next_marker: None,
+            };
+            let mut replies = std::collections::VecDeque::from([first, second]);
+            let result = collect_pages("bucket", id, |marker| {
+                let page = replies.pop_front().expect("bounded fixture pages");
+                if marker.is_some() {
+                    assert!(page.roots.is_empty());
+                }
+                std::future::ready(Ok(page))
+            })
+            .await;
+            // A replaced disk fails the page's identity check; a restarted
+            // epoch fails the cross-page continuity check.
+            let expected_corrupt = replace_disk;
+            assert!(
+                match &result {
+                    Err(crate::disk::error::DiskError::FileCorrupt) => expected_corrupt,
+                    Err(crate::disk::error::DiskError::DiskNotFound) => !expected_corrupt,
+                    _ => false,
+                },
+                "changed physical source cannot contribute partial coverage: {result:?}"
+            );
+            assert!(replies.is_empty(), "exercise the continuation response");
+        }
+    }
+}

crates/ecstore/src/disk/multipart_source.rs

@@ -1149,6 +1149,8 @@ fn normalized_info(fi: &FileInfo, completion: bool) -> FileInfo {
             part.checksums = None;
             part.index = None;
             part.integrity = None;
+            // The completed object carries the aggregate proof in metadata.
+            part.encrypted_etag = false;
         }
     }
     fi
@@ -1739,6 +1741,12 @@ mod multipart_proof_authority_tests {
         completed.mod_time = Some(original.mod_time.unwrap() + time::Duration::seconds(1));
         completed.size = 15;
         completed.add_object_part(1, "etag".into(), 15, completed.mod_time, 15, None, None);
+        completed.parts[0].encrypted_etag = true;
+        rustfs_utils::http::metadata_compat::insert_str(
+            &mut completed.metadata,
+            rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG,
+            "v1".to_owned(),
+        );
         let publication = condition.attach(&mut completed).unwrap();
         assert_eq!(publication.version_id, None);
         assert_eq!(
@@ -1748,6 +1756,11 @@ mod multipart_proof_authority_tests {
         );
         let mut read_completed = persist_null_fixture(&completed);
         assert_eq!(read_completed.version_id, None);
+        assert!(!read_completed.parts[0].encrypted_etag, "xl.meta stores the aggregate proof");
+        assert_eq!(
+            rustfs_utils::http::get_consistent_str(&read_completed.metadata, rustfs_utils::http::SUFFIX_OPAQUE_ENCRYPTED_ETAG),
+            Some("v1")
+        );
         let reread_publication = reread_condition.attach(&mut read_completed).unwrap();
         assert_eq!(
             publication, reread_publication,

crates/ecstore/src/erasure/coding/encode.rs

@@ -326,6 +326,40 @@ impl Default for WriteProgressPolicy {
     }
 }
 
+/// Caller-specific completion behavior for an erasure write.
+///
+/// A multipart upload must not discard a different healthy shard from each
+/// part merely because another quorum finished first: the final object needs
+/// a quorum of disks carrying every part. Pending writers still obey the
+/// existing progress and absolute deadlines and never count as completed.
+#[derive(Debug, Clone, Copy)]
+pub(crate) struct ShardWritePolicy {
+    quorum: usize,
+    post_quorum: PostQuorumPolicy,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum PostQuorumPolicy {
+    Grace,
+    ExistingBudget,
+}
+
+impl ShardWritePolicy {
+    pub(crate) fn object(quorum: usize) -> Self {
+        Self {
+            quorum,
+            post_quorum: PostQuorumPolicy::Grace,
+        }
+    }
+
+    pub(crate) fn multipart(quorum: usize) -> Self {
+        Self {
+            quorum,
+            post_quorum: PostQuorumPolicy::ExistingBudget,
+        }
+    }
+}
+
 /// After write quorum is already in hand, how long a slower shard may still
 /// finish before it is dropped. The per-shard stall budget (30s by default)
 /// is what a black-hole peer is allowed to consume *before* quorum exists.
@@ -338,13 +372,15 @@ fn post_quorum_straggler_grace(stall_timeout: Option<std::time::Duration>) -> Op
     stall_timeout.map(|stall| stall.min(WRITE_QUORUM_STRAGGLER_GRACE))
 }
 
-/// Drive shard ops until write quorum has succeeded and stragglers have had
-/// [`post_quorum_straggler_grace`] to finish. Returns `true` for each slot
-/// whose future completed; `false` means the caller must fail that slot.
+/// Drive shard operations within their existing deadlines. Ordinary object
+/// writes additionally stop after the post-quorum grace; multipart writes keep
+/// polling all candidates within the same operation budget. A `false` slot
+/// never completed and must be excluded from commit.
 async fn await_write_quorum<F>(
     write_quorum: usize,
     stall_timeout: Option<std::time::Duration>,
     slot_count: usize,
+    post_quorum: PostQuorumPolicy,
     mut futures: FuturesUnordered<F>,
 ) -> Vec<bool>
 where
@@ -376,6 +412,7 @@ where
         if succeeded {
             successes = successes.saturating_add(1);
             if successes >= write_quorum
+                && post_quorum == PostQuorumPolicy::Grace
                 && grace.is_none()
                 && let Some(delay) = post_quorum_straggler_grace(stall_timeout)
             {
@@ -404,6 +441,7 @@ pub(crate) struct MultiWriter<'a> {
     writers: &'a mut [Option<BitrotWriterWrapper>],
     integrity: Option<&'a mut IntegrityBuilder>,
     write_quorum: usize,
+    post_quorum: PostQuorumPolicy,
     errs: Vec<Option<Error>>,
     policy: WriteProgressPolicy,
     /// Absolute deadline for the whole object, armed lazily on the first
@@ -412,16 +450,21 @@ pub(crate) struct MultiWriter<'a> {
 }
 
 impl<'a> MultiWriter<'a> {
-    pub fn new(writers: &'a mut [Option<BitrotWriterWrapper>], write_quorum: usize) -> Self {
-        Self::with_policy(writers, write_quorum, WriteProgressPolicy::default())
+    pub fn new(writers: &'a mut [Option<BitrotWriterWrapper>], write_policy: ShardWritePolicy) -> Self {
+        Self::with_policy(writers, write_policy, WriteProgressPolicy::default())
     }
 
-    pub fn with_policy(writers: &'a mut [Option<BitrotWriterWrapper>], write_quorum: usize, policy: WriteProgressPolicy) -> Self {
+    pub fn with_policy(
+        writers: &'a mut [Option<BitrotWriterWrapper>],
+        write_policy: ShardWritePolicy,
+        policy: WriteProgressPolicy,
+    ) -> Self {
         let length = writers.len();
         MultiWriter {
             writers,
             integrity: None,
-            write_quorum,
+            write_quorum: write_policy.quorum,
+            post_quorum: write_policy.post_quorum,
             errs: vec![None; length],
             policy,
             absolute_deadline: None,
@@ -506,9 +549,8 @@ impl<'a> MultiWriter<'a> {
                 // failed and its disk dropped, so a stalled peer cannot pin an
                 // otherwise-healthy write quorum (rustfs/backlog#1319). `budget`
                 // is recomputed per block, so it bounds a stall — not the total
-                // transfer time — and a slow-but-honest writer is never killed
-                // while quorum is still open. Once quorum has succeeded, a
-                // straggler only gets `WRITE_QUORUM_STRAGGLER_GRACE` more time.
+                // transfer time. Only ordinary object writes add the shorter
+                // post-quorum grace; multipart preserves this operation budget.
                 futures.push(async move {
                     match budget {
                         Some(budget) => match tokio::time::timeout(budget, Self::write_shard(writer_opt, err, shard)).await {
@@ -524,7 +566,7 @@ impl<'a> MultiWriter<'a> {
                     (index, succeeded)
                 });
             }
-            await_write_quorum(write_quorum, stall_timeout, slot_count, futures).await
+            await_write_quorum(write_quorum, stall_timeout, slot_count, self.post_quorum, futures).await
         };
         abandon_unfinished_writers(self.writers, &mut self.errs, &saw);
 
@@ -587,8 +629,8 @@ impl<'a> MultiWriter<'a> {
                 // here forever for a small object whose bytes were fully buffered
                 // (so `write` never blocked). Bound it with the same progress
                 // budget and drop the stalled writer before the quorum check.
-                // After quorum has shut down, do not keep waiting out the rest of
-                // that budget for a peer that has stopped answering.
+                // Ordinary object writes also apply their post-quorum grace;
+                // multipart waits for all candidates within this same budget.
                 futures.push(async move {
                     match budget {
                         Some(budget) => match tokio::time::timeout(budget, Self::shutdown_writer(writer_opt, err)).await {
@@ -604,7 +646,7 @@ impl<'a> MultiWriter<'a> {
                     (index, succeeded)
                 });
             }
-            await_write_quorum(write_quorum, stall_timeout, slot_count, futures).await
+            await_write_quorum(write_quorum, stall_timeout, slot_count, self.post_quorum, futures).await
         };
         abandon_unfinished_writers(self.writers, &mut self.errs, &saw);
 
@@ -639,7 +681,7 @@ impl Erasure {
         self: Arc<Self>,
         reader: R,
         writers: &mut [Option<BitrotWriterWrapper>],
-        quorum: usize,
+        write_policy: ShardWritePolicy,
         part_number: usize,
         mode: IntegrityEncodeMode,
         protected: bool,
@@ -668,19 +710,19 @@ impl Erasure {
             }
             IntegrityEncodeMode::SingleBlock(hint) => {
                 let (reader, size) = self
-                    .encode_small_direct(reader, writers, quorum, true, hint, integrity.as_mut())
+                    .encode_small_direct(reader, writers, write_policy, true, hint, integrity.as_mut())
                     .await?;
                 (reader, size, None)
             }
             IntegrityEncodeMode::Streaming => {
                 let (reader, size) = self
-                    .encode_with_ingest_mode(reader, writers, quorum, use_bytesmut_ingest(), integrity.as_mut())
+                    .encode_with_ingest_mode(reader, writers, write_policy, use_bytesmut_ingest(), integrity.as_mut())
                     .await?;
                 (reader, size, None)
             }
             IntegrityEncodeMode::Batched => {
                 let (reader, size) = self
-                    .encode_batched_with_integrity(reader, writers, quorum, integrity.as_mut())
+                    .encode_batched_with_integrity(reader, writers, write_policy, integrity.as_mut())
                     .await?;
                 (reader, size, None)
             }
@@ -745,7 +787,7 @@ impl Erasure {
         self: Arc<Self>,
         mut reader: R,
         writers: &mut [Option<BitrotWriterWrapper>],
-        quorum: usize,
+        write_policy: ShardWritePolicy,
         require_single_block: bool,
         size_hint: usize,
         integrity: Option<&mut IntegrityBuilder>,
@@ -780,7 +822,7 @@ impl Erasure {
         }
 
         let block = self.encode_data_owned_block(buf)?;
-        let mut mw = MultiWriter::new(writers, quorum);
+        let mut mw = MultiWriter::new(writers, write_policy);
         mw.integrity = integrity;
         mw.write_block(&block).await?;
         mw.shutdown().await?;
@@ -847,7 +889,7 @@ impl Erasure {
         R: AsyncRead + Send + Sync + Unpin + 'static,
     {
         let use_bytesmut_ingest = use_bytesmut_ingest();
-        self.encode_with_ingest_mode(reader, writers, quorum, use_bytesmut_ingest, None)
+        self.encode_with_ingest_mode(reader, writers, ShardWritePolicy::object(quorum), use_bytesmut_ingest, None)
             .await
     }
 
@@ -859,7 +901,7 @@ impl Erasure {
         self: Arc<Self>,
         mut reader: R,
         writers: &mut [Option<BitrotWriterWrapper>],
-        quorum: usize,
+        write_policy: ShardWritePolicy,
         use_bytesmut_ingest: bool,
         integrity: Option<&mut IntegrityBuilder>,
     ) -> std::io::Result<(R, usize)>
@@ -961,7 +1003,7 @@ impl Erasure {
             Ok((reader, total))
         }));
 
-        let mut writers = MultiWriter::new(writers, quorum);
+        let mut writers = MultiWriter::new(writers, write_policy);
         writers.integrity = integrity;
 
         let mut write_err = None;
@@ -1017,14 +1059,15 @@ impl Erasure {
     where
         R: AsyncRead + Send + Sync + Unpin + 'static,
     {
-        self.encode_batched_with_integrity(reader, writers, quorum, None).await
+        self.encode_batched_with_integrity(reader, writers, ShardWritePolicy::object(quorum), None)
+            .await
     }
 
     async fn encode_batched_with_integrity<R>(
         self: Arc<Self>,
         mut reader: R,
         writers: &mut [Option<BitrotWriterWrapper>],
-        quorum: usize,
+        write_policy: ShardWritePolicy,
         integrity: Option<&mut IntegrityBuilder>,
     ) -> std::io::Result<(R, usize)>
     where
@@ -1105,7 +1148,7 @@ impl Erasure {
             Ok((reader, total))
         }));
 
-        let mut writers = MultiWriter::new(writers, quorum);
+        let mut writers = MultiWriter::new(writers, write_policy);
         writers.integrity = integrity;
         let mut write_err = None;
 
@@ -1165,7 +1208,7 @@ impl Erasure {
         R: AsyncRead + Send + Sync + Unpin,
     {
         let size_hint = self.block_size;
-        self.encode_small_direct(reader, writers, quorum, false, size_hint, None)
+        self.encode_small_direct(reader, writers, ShardWritePolicy::object(quorum), false, size_hint, None)
             .await
     }
 
@@ -1182,7 +1225,7 @@ impl Erasure {
     where
         R: AsyncRead + Send + Sync + Unpin,
     {
-        self.encode_small_direct(reader, writers, quorum, false, size_hint, None)
+        self.encode_small_direct(reader, writers, ShardWritePolicy::object(quorum), false, size_hint, None)
             .await
     }
 
@@ -1199,7 +1242,8 @@ impl Erasure {
         R: AsyncRead + Send + Sync + Unpin,
     {
         let size_hint = self.block_size;
-        self.encode_small_direct(reader, writers, quorum, true, size_hint, None).await
+        self.encode_small_direct(reader, writers, ShardWritePolicy::object(quorum), true, size_hint, None)
+            .await
     }
 
     /// Size-aware single-block fast path. `size_hint` only controls the bounded
@@ -1215,7 +1259,8 @@ impl Erasure {
     where
         R: AsyncRead + Send + Sync + Unpin,
     {
-        self.encode_small_direct(reader, writers, quorum, true, size_hint, None).await
+        self.encode_small_direct(reader, writers, ShardWritePolicy::object(quorum), true, size_hint, None)
+            .await
     }
 }
 
@@ -1632,6 +1677,219 @@ mod tests {
         }
     }
 
+    #[derive(Clone, Copy, Debug)]
+    enum DelayedPhase {
+        Write,
+        Flush,
+        Shutdown,
+    }
+
+    struct PhaseDelayedWriter {
+        inner: DeferredCommitWriter,
+        phase: DelayedPhase,
+        delay: Duration,
+        timer: Option<Pin<Box<tokio::time::Sleep>>>,
+        ready: bool,
+    }
+
+    impl PhaseDelayedWriter {
+        fn poll_delay(&mut self, cx: &mut Context<'_>) -> Poll<()> {
+            if self.ready {
+                return Poll::Ready(());
+            }
+            let timer = self.timer.get_or_insert_with(|| Box::pin(tokio::time::sleep(self.delay)));
+            match timer.as_mut().poll(cx) {
+                Poll::Pending => Poll::Pending,
+                Poll::Ready(()) => {
+                    self.ready = true;
+                    Poll::Ready(())
+                }
+            }
+        }
+    }
+
+    impl AsyncWrite for PhaseDelayedWriter {
+        fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, bytes: &[u8]) -> Poll<std::io::Result<usize>> {
+            if matches!(self.phase, DelayedPhase::Write) && self.poll_delay(cx).is_pending() {
+                return Poll::Pending;
+            }
+            Pin::new(&mut self.inner).poll_write(cx, bytes)
+        }
+
+        fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+            if matches!(self.phase, DelayedPhase::Flush) && self.poll_delay(cx).is_pending() {
+                return Poll::Pending;
+            }
+            Pin::new(&mut self.inner).poll_flush(cx)
+        }
+
+        fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+            if matches!(self.phase, DelayedPhase::Shutdown) && self.poll_delay(cx).is_pending() {
+                return Poll::Pending;
+            }
+            Pin::new(&mut self.inner).poll_shutdown(cx)
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn multipart_budget_keeps_slow_writers_in_every_encode_mode() {
+        const BLOCK: usize = 64;
+        for make_mode in [
+            || IntegrityEncodeMode::SingleBlock(BLOCK),
+            || IntegrityEncodeMode::Streaming,
+            || IntegrityEncodeMode::Batched,
+        ] {
+            for phase in [DelayedPhase::Write, DelayedPhase::Flush, DelayedPhase::Shutdown] {
+                for delay in [Duration::from_secs(2), Duration::from_secs(5)] {
+                    let committed: Vec<_> = (0..4).map(|_| Arc::new(Mutex::new(Vec::new()))).collect();
+                    let mut writers = committed
+                        .iter()
+                        .enumerate()
+                        .map(|(slot, data)| {
+                            if slot == 0 {
+                                Some(bitrot_writer(
+                                    PhaseDelayedWriter {
+                                        inner: DeferredCommitWriter::new(data.clone()),
+                                        phase,
+                                        delay,
+                                        timer: None,
+                                        ready: false,
+                                    },
+                                    BLOCK / 2,
+                                ))
+                            } else {
+                                Some(bitrot_writer(DeferredCommitWriter::new(data.clone()), BLOCK / 2))
+                            }
+                        })
+                        .collect::<Vec<_>>();
+                    let payload = vec![0x5a; BLOCK];
+                    let erasure = Arc::new(Erasure::new(2, 2, BLOCK));
+                    let started = tokio::time::Instant::now();
+                    erasure
+                        .encode_with_shard_integrity(
+                            Cursor::new(payload.clone()),
+                            &mut writers,
+                            ShardWritePolicy::multipart(3),
+                            1,
+                            make_mode(),
+                            false,
+                        )
+                        .await
+                        .expect("multipart must keep finite slow writers within the existing budget");
+                    assert!(started.elapsed() >= delay, "must await the slow phase: {phase:?}");
+                    assert!(writers.iter().all(Option::is_some), "all four completed shards must survive");
+                    for data in &committed {
+                        let bytes = data.lock().expect("committed bytes").clone();
+                        crate::erasure::coding::bitrot_verify(
+                            Cursor::new(bytes.clone()),
+                            bytes.len(),
+                            BLOCK / 2,
+                            HashAlgorithm::HighwayHash256S,
+                            BLOCK / 2,
+                        )
+                        .await
+                        .expect("each completed shard must retain valid bitrot framing");
+                    }
+                }
+            }
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn multipart_budget_expires_blackholes_without_restarting_deadlines() {
+        for shutdown in [false, true] {
+            for blackholes in [1, 2] {
+                for absolute in [Duration::ZERO, Duration::from_secs(3)] {
+                    let mut writers = (0..4)
+                        .map(|slot| {
+                            if slot < blackholes {
+                                Some(bitrot_writer_plain(
+                                    if shutdown {
+                                        StallingWriter::stalls_on_shutdown()
+                                    } else {
+                                        StallingWriter::stalls_on_write(0)
+                                    },
+                                    64,
+                                ))
+                            } else {
+                                Some(bitrot_writer_plain(DeferredCommitWriter::new(Arc::new(Mutex::new(Vec::new()))), 64))
+                            }
+                        })
+                        .collect::<Vec<_>>();
+                    let started = tokio::time::Instant::now();
+                    let result = {
+                        let mut writer = MultiWriter::with_policy(
+                            &mut writers,
+                            ShardWritePolicy::multipart(3),
+                            WriteProgressPolicy::new(Duration::from_secs(7), absolute),
+                        );
+                        if shutdown {
+                            writer.write(four_shards()).await.expect("buffered writes should complete");
+                            tokio::time::sleep(Duration::from_secs(1)).await;
+                            writer.shutdown().await
+                        } else {
+                            writer.write(four_shards()).await
+                        }
+                    };
+                    assert_eq!(result.is_ok(), blackholes == 1, "only a confirmed quorum may succeed");
+                    let expected = if absolute.is_zero() {
+                        Duration::from_secs(7 + u64::from(shutdown))
+                    } else {
+                        absolute
+                    };
+                    assert_eq!(started.elapsed(), expected, "budget must not restart at quorum or shutdown");
+                    assert!(writers[..blackholes].iter().all(Option::is_none));
+                    assert!(writers[blackholes..].iter().all(Option::is_some));
+                }
+            }
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn multipart_budget_cancellation_drops_pending_writers() {
+        struct DropSignal(Option<oneshot::Sender<()>>);
+        impl Drop for DropSignal {
+            fn drop(&mut self) {
+                if let Some(tx) = self.0.take() {
+                    let _ = tx.send(());
+                }
+            }
+        }
+        impl AsyncWrite for DropSignal {
+            fn poll_write(self: Pin<&mut Self>, _: &mut Context<'_>, bytes: &[u8]) -> Poll<std::io::Result<usize>> {
+                Poll::Ready(Ok(bytes.len()))
+            }
+            fn poll_flush(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+                Poll::Ready(Ok(()))
+            }
+            fn poll_shutdown(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+                Poll::Pending
+            }
+        }
+        let (tx, rx) = oneshot::channel();
+        let (entered_tx, entered) = oneshot::channel();
+        let task = tokio::spawn(async move {
+            let mut writers = vec![Some(bitrot_writer_plain(DropSignal(Some(tx)), 64))];
+            writers.extend(
+                (0..3).map(|_| Some(bitrot_writer_plain(DeferredCommitWriter::new(Arc::new(Mutex::new(Vec::new()))), 64))),
+            );
+            let mut writer = MultiWriter::new(&mut writers, ShardWritePolicy::multipart(3));
+            writer.write(four_shards()).await.expect("buffered writes should succeed");
+            let _ = entered_tx.send(());
+            writer.shutdown().await
+        });
+        entered.await.expect("write must reach shutdown");
+        tokio::task::yield_now().await;
+        assert!(!task.is_finished());
+        task.abort();
+        assert!(
+            task.await
+                .expect_err("caller cancellation should cancel the task")
+                .is_cancelled()
+        );
+        rx.await.expect("pending writer must be dropped with its owning task");
+    }
+
     /// A writer that always makes forward progress but each `poll_write` first
     /// waits `delay` (on the runtime timer). Under a paused virtual clock this
     /// deterministically models a slow-but-honest peer: it must never be killed
@@ -1704,12 +1962,16 @@ mod tests {
         let erasure = Arc::new(Erasure::new(1, 0, BLOCK_SIZE));
 
         let encode = match pipeline {
-            EncodePipeline::Vec => {
-                tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, false, None).await })
-            }
-            EncodePipeline::BytesMut => {
-                tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, true, None).await })
-            }
+            EncodePipeline::Vec => tokio::spawn(async move {
+                erasure
+                    .encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), false, None)
+                    .await
+            }),
+            EncodePipeline::BytesMut => tokio::spawn(async move {
+                erasure
+                    .encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), true, None)
+                    .await
+            }),
             EncodePipeline::Batched => tokio::spawn(async move { erasure.encode_batched(reader, &mut writers, 1).await }),
         };
 
@@ -1767,8 +2029,16 @@ mod tests {
         ))];
 
         let result = match pipeline {
-            EncodePipeline::Vec => erasure.encode_with_ingest_mode(reader, &mut writers, 1, false, None).await,
-            EncodePipeline::BytesMut => erasure.encode_with_ingest_mode(reader, &mut writers, 1, true, None).await,
+            EncodePipeline::Vec => {
+                erasure
+                    .encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), false, None)
+                    .await
+            }
+            EncodePipeline::BytesMut => {
+                erasure
+                    .encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), true, None)
+                    .await
+            }
             EncodePipeline::Batched => erasure.encode_batched(reader, &mut writers, 1).await,
         };
 
@@ -1841,8 +2111,16 @@ mod tests {
         let mut writers = vec![Some(bitrot_writer(FailAfterReaderErrorWriter { reader_failed }, BLOCK_SIZE))];
 
         let result = match pipeline {
-            EncodePipeline::Vec => erasure.encode_with_ingest_mode(reader, &mut writers, 1, false, None).await,
-            EncodePipeline::BytesMut => erasure.encode_with_ingest_mode(reader, &mut writers, 1, true, None).await,
+            EncodePipeline::Vec => {
+                erasure
+                    .encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), false, None)
+                    .await
+            }
+            EncodePipeline::BytesMut => {
+                erasure
+                    .encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), true, None)
+                    .await
+            }
             EncodePipeline::Batched => erasure.encode_batched(reader, &mut writers, 1).await,
         };
 
@@ -1879,7 +2157,7 @@ mod tests {
         let erasure_for_task = erasure.clone();
         let encode = tokio::spawn(async move {
             erasure_for_task
-                .encode_with_ingest_mode(reader, &mut writers, 1, false, None)
+                .encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), false, None)
                 .await
         });
 
@@ -2048,8 +2326,16 @@ mod tests {
             let erasure = Arc::new(Erasure::new(2, 2, 64));
             let result = match path {
                 "direct" => erasure.encode_single_block_non_inline(reader, &mut writers, 2).await,
-                "vec" => erasure.encode_with_ingest_mode(reader, &mut writers, 2, false, None).await,
-                "bytesmut" => erasure.encode_with_ingest_mode(reader, &mut writers, 2, true, None).await,
+                "vec" => {
+                    erasure
+                        .encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(2), false, None)
+                        .await
+                }
+                "bytesmut" => {
+                    erasure
+                        .encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(2), true, None)
+                        .await
+                }
                 "batched" => erasure.encode_batched(reader, &mut writers, 2).await,
                 _ => unreachable!(),
             };
@@ -2094,7 +2380,7 @@ mod tests {
     async fn multi_writer_short_write_fails_before_shutdown() {
         let mut writers = vec![Some(bitrot_writer(ShortWriteWriter, 32))];
         let err = {
-            let mut writer = MultiWriter::new(&mut writers, 1);
+            let mut writer = MultiWriter::new(&mut writers, ShardWritePolicy::object(1));
             writer
                 .write(vec![Bytes::from_static(b"short-write payload")])
                 .await
@@ -2111,7 +2397,7 @@ mod tests {
     async fn multi_writer_reports_fallback_summary_when_only_offline_writers_remain() {
         let mut writers = vec![None, None];
         let (err, summary) = {
-            let mut writer = MultiWriter::new(&mut writers, 1);
+            let mut writer = MultiWriter::new(&mut writers, ShardWritePolicy::object(1));
             let err = writer
                 .write(vec![Bytes::from_static(b"offline-a"), Bytes::from_static(b"offline-b")])
                 .await
@@ -2131,7 +2417,7 @@ mod tests {
         assert!(summary.contains("required=1"));
 
         let (shutdown_err, summary) = {
-            let mut writer = MultiWriter::new(&mut writers, 1);
+            let mut writer = MultiWriter::new(&mut writers, ShardWritePolicy::object(1));
             let err = writer
                 .shutdown()
                 .await
@@ -2155,7 +2441,7 @@ mod tests {
     async fn multi_writer_reports_quorum_failure_when_quorum_exceeds_writer_count() {
         let committed = Arc::new(Mutex::new(Vec::new()));
         let mut writers = vec![Some(bitrot_writer(DeferredCommitWriter::new(committed), 32))];
-        let mut writer = MultiWriter::new(&mut writers, 2);
+        let mut writer = MultiWriter::new(&mut writers, ShardWritePolicy::object(2));
 
         let err = writer
             .write(vec![Bytes::from_static(b"quorum impossible")])
@@ -2244,7 +2530,7 @@ mod tests {
 
         {
             let policy = WriteProgressPolicy::new(Duration::from_secs(5), Duration::ZERO);
-            let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
+            let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
             mw.write(four_shards())
                 .await
                 .expect("one stalled writer must still satisfy a 3/4 write quorum without hanging");
@@ -2271,7 +2557,7 @@ mod tests {
         let started = tokio::time::Instant::now();
         {
             let policy = WriteProgressPolicy::new(Duration::from_secs(5), Duration::ZERO);
-            let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
+            let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
             mw.write(four_shards())
                 .await
                 .expect("quorum must succeed without waiting out the black-hole stall");
@@ -2304,7 +2590,7 @@ mod tests {
 
         {
             let policy = WriteProgressPolicy::new(Duration::from_secs(5), Duration::ZERO);
-            let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
+            let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
             mw.write(four_shards())
                 .await
                 .expect("a straggler inside the grace window must still satisfy the write");
@@ -2329,7 +2615,7 @@ mod tests {
         ];
 
         let policy = WriteProgressPolicy::new(Duration::from_secs(5), Duration::ZERO);
-        let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
+        let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
         let err = mw
             .write(four_shards())
             .await
@@ -2358,7 +2644,7 @@ mod tests {
 
         {
             let policy = WriteProgressPolicy::new(Duration::from_secs(5), Duration::ZERO);
-            let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
+            let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
             mw.write(four_shards()).await.expect("all writers accept the buffered shard");
             mw.shutdown()
                 .await
@@ -2379,7 +2665,7 @@ mod tests {
         ];
 
         let policy = WriteProgressPolicy::new(Duration::from_secs(5), Duration::ZERO);
-        let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
+        let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
         mw.write(four_shards()).await.expect("all writers accept the buffered shard");
         let err = mw
             .shutdown()
@@ -2408,7 +2694,7 @@ mod tests {
 
         {
             let policy = WriteProgressPolicy::new(Duration::from_secs(5), Duration::ZERO);
-            let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
+            let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
             for _ in 0..8 {
                 mw.write(four_full_shards(64))
                     .await
@@ -2438,7 +2724,7 @@ mod tests {
             // stall (10s) never fires for an 8s-per-block writer, but the 15s
             // absolute cap does once its aggregate engagement crosses the cap.
             let policy = WriteProgressPolicy::new(Duration::from_secs(10), Duration::from_secs(15));
-            let mut mw = MultiWriter::with_policy(&mut writers, 3, policy);
+            let mut mw = MultiWriter::with_policy(&mut writers, ShardWritePolicy::object(3), policy);
             for _ in 0..4 {
                 mw.write(four_full_shards(64))
                     .await
@@ -2583,7 +2869,7 @@ mod tests {
         let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE));
         let reader = tokio::io::BufReader::new(Cursor::new(payload.clone()));
         let (_reader, written) = erasure
-            .encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, true, None)
+            .encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(DATA_SHARDS), true, None)
             .await
             .expect("BytesMut ingest path should encode the streaming payload");
 
@@ -2612,7 +2898,9 @@ mod tests {
         let reader = tokio::io::BufReader::new(Cursor::new(vec![0x5a; block_size * 2]));
 
         rustfs_io_metrics::set_put_stage_metrics_enabled(true);
-        let result = erasure.encode_with_ingest_mode(reader, &mut writers, 1, true, None).await;
+        let result = erasure
+            .encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(1), true, None)
+            .await;
         rustfs_io_metrics::set_put_stage_metrics_enabled(false);
 
         let (_reader, written) = result.expect("bytesmut streaming encode should complete");
@@ -3230,12 +3518,12 @@ mod tests {
         let (_reader, total) = match pipeline {
             EncodePipeline::Vec => {
                 erasure
-                    .encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, false, None)
+                    .encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(DATA_SHARDS), false, None)
                     .await
             }
             EncodePipeline::BytesMut => {
                 erasure
-                    .encode_with_ingest_mode(reader, &mut writers, DATA_SHARDS, true, None)
+                    .encode_with_ingest_mode(reader, &mut writers, ShardWritePolicy::object(DATA_SHARDS), true, None)
                     .await
             }
             EncodePipeline::Batched => erasure.encode_batched(reader, &mut writers, DATA_SHARDS).await,
@@ -3348,4 +3636,73 @@ mod tests {
         assert_eq!(encode_channel_capacity(16 * 1024 * 1024, 32 * 1024 * 1024), 2);
         assert_eq!(encode_channel_capacity(1, usize::MAX), DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BLOCKS);
     }
+
+    #[tokio::test]
+    async fn read_full_buf_or_eof_preserves_invalid_data_and_wraps_truncated_io_errors() {
+        use std::pin::Pin;
+        use std::task::{Context, Poll};
+        use tokio::io::{AsyncRead, ReadBuf};
+
+        struct StepReader {
+            prefix: Vec<u8>,
+            offset: usize,
+            err_kind: Option<std::io::ErrorKind>,
+            err_msg: &'static str,
+        }
+
+        impl AsyncRead for StepReader {
+            fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
+                if self.offset < self.prefix.len() {
+                    let take = (self.prefix.len() - self.offset).min(buf.remaining());
+                    buf.put_slice(&self.prefix[self.offset..self.offset + take]);
+                    self.offset += take;
+                    return Poll::Ready(Ok(()));
+                }
+                if let Some(kind) = self.err_kind.take() {
+                    return Poll::Ready(Err(std::io::Error::new(kind, self.err_msg)));
+                }
+                Poll::Ready(Ok(()))
+            }
+        }
+
+        // 1. Error on the very first read (total == 0) preserves the original ErrorKind.
+        let mut r1 = StepReader {
+            prefix: Vec::new(),
+            offset: 0,
+            err_kind: Some(std::io::ErrorKind::BrokenPipe),
+            err_msg: "immediate broken pipe",
+        };
+        let mut buf = BytesMut::with_capacity(16);
+        let err = read_full_buf_or_eof(&mut r1, &mut buf, 16)
+            .await
+            .expect_err("immediate read error must fail");
+        assert_eq!(err.kind(), std::io::ErrorKind::BrokenPipe);
+
+        // 2. InvalidData (e.g. HashReader checksum/trailer mismatch) after partial fill preserves InvalidData.
+        let mut r2 = StepReader {
+            prefix: b"partial".to_vec(),
+            offset: 0,
+            err_kind: Some(std::io::ErrorKind::InvalidData),
+            err_msg: "Checksum mismatch",
+        };
+        let mut buf2 = BytesMut::with_capacity(16);
+        let err = read_full_buf_or_eof(&mut r2, &mut buf2, 16)
+            .await
+            .expect_err("InvalidData after partial fill must fail");
+        assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
+        assert!(err.to_string().contains("Checksum mismatch"));
+
+        // 3. Non-InvalidData error after partial fill is normalized to UnexpectedEof.
+        let mut r3 = StepReader {
+            prefix: b"partial".to_vec(),
+            offset: 0,
+            err_kind: Some(std::io::ErrorKind::BrokenPipe),
+            err_msg: "mid-stream reset",
+        };
+        let mut buf3 = BytesMut::with_capacity(16);
+        let err = read_full_buf_or_eof(&mut r3, &mut buf3, 16)
+            .await
+            .expect_err("truncated stream error must fail");
+        assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof);
+    }
 }

crates/ecstore/src/erasure/coding/heal.rs

@@ -205,7 +205,7 @@ impl super::Erasure {
         // the survivors. Reconstruction correctness is still guaranteed by the
         // read-side quorum check and parity cross-checks below, which are unaffected.
         let write_quorum = 1;
-        let mut writers = MultiWriter::new(writers, write_quorum);
+        let mut writers = MultiWriter::new(writers, super::encode::ShardWritePolicy::object(write_quorum));
         let read_timeout = get_object_disk_read_timeout();
         let shard_file_size = self.shard_file_size(total_length as i64) as usize;