Merge remote-tracking branch 'upstream-https/main' into fix/ant-storage-layer-batch

# Conflicts:
#	crates/ecstore/src/disk/local/delete_recovery.rs

.agents/references/implementation.md

@@ -18,6 +18,7 @@ These implementation, style, and quality rules apply to all contributions, regar
 - Do not include agent, tool, contributor, account, or organization names in branch names.
 - Push to the user-requested remote or the repository's configured push remote. Do not hard-code or infer a remote from an account name.
 - Check free space before artifact-heavy builds, tests, coverage, or downloads. Re-check before a broad gate when space is tight.
+- Before handoff, remove the Cargo target directories and temporary worktrees the task created unless they stay intentionally or hold unmerged work. Identify build output by Cargo's markers (`.rustc_info.json`, `CACHEDIR.TAG`), not by directory name: `target` is also a source directory here.
 - Remove only task-owned temporary/build artifacts. Never delete another task's worktree or uncommitted data.
 - At handoff, mention disk or cleanup details only when they affected execution or artifacts/worktrees remain intentionally.
 - For Docker verification, ensure the tested image contains the intended binary. `buildx build` without `--load` leaves the image in the build cache; use the Makefile's loaded image and rebuild without cache only when stale artifacts are evidenced.

.config/ci-plan.json

@@ -0,0 +1,16 @@
+{
+  "schema_version": 1,
+  "rollout": "enforced",
+  "core_packages": ["rustfs", "rustfs-ecstore", "rustfs-filemeta", "rustfs-iam", "rustfs-policy"],
+  "connect_packages": [
+    "rustfs-credentials", "rustfs-config", "rustfs-common", "rustfs-crypto",
+    "rustfs-iam", "rustfs-policy", "rustfs-kms", "rustfs-license", "rustfs-security-governance",
+    "rustfs-utils", "rustfs-tls-runtime", "rustfs-trusted-proxies", "rustfs-filemeta",
+    "rustfs-lock", "rustfs-obs", "rustfs-signer", "rustfs-io-metrics", "rustfs-s3-ops"
+  ],
+  "full_paths": [
+    "Cargo.toml", "Cargo.lock", "*/Cargo.toml", "rust-toolchain*", "rustfmt.toml",
+    "clippy.toml", ".cargo/*", ".config/*", ".github/*", "Makefile",
+    "scripts/*", "build.rs", "*.proto"
+  ]
+}

.config/e2e-distributed-selection.txt

@@ -1,2 +1,2 @@
-sha256-linux=eb33067d90ee66c0a97e8f3658e46c152e47f897d2fbfdc6cb5ad93cfa55414b
-sha256-darwin=eb33067d90ee66c0a97e8f3658e46c152e47f897d2fbfdc6cb5ad93cfa55414b
+sha256-linux=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6
+sha256-darwin=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=67251cb7d9978d38e899268cac4d7898297ff7e5b5c48c9c1546dc760b93568d
-sha256-linux=673da76c2e5710c4622fe2fa32d2b297730895769487fcf1152ee1c2366f32f4
+sha256-darwin=7c705942d3a9e583ebaa85845daa21c4f81fdbf36def24db931aaef38636c53e
+sha256-linux=5c33dc395233671cb77452f8fb12900ecdbeb368632222b96835d310bce1aab8

.config/e2e-smoke-selection.txt

@@ -1 +1 @@
-sha256=e9dc896774122a304e2b8b90f28c82d4d4e0af8ded36a82b147b30945ec994e3
+sha256=00f8a56b75990b487a594b97c36610b94476a6944ba00473e37187ea654d42bf

.config/make/lint-fmt.mak

@@ -90,6 +90,7 @@ test-wiring-check: ## Check tests stay registered and selected by their intended
 	@echo "🧪 Checking test wiring..."
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/ci_gate.py --check-workflow
+	$(RUSTFS_PYTHON_BIN) ./scripts/validation_registry.py check
 
 .PHONY: log-analyzer-rules-check
 log-analyzer-rules-check: core-deps ## Check log-analyzer rule anchors still exist verbatim in source

.config/make/tests.mak

@@ -59,12 +59,18 @@ script-tests: ## Run shell script tests
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_upgrade_workflow.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_migration_gate_evidence.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/ci_gate.py --self-test
+	$(RUSTFS_PYTHON_BIN) ./scripts/check_connect_release_boundary.py --self-test
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_test_artifact.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_test_shards.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_merge_queue.py
+	$(RUSTFS_PYTHON_BIN) -m unittest discover -s scripts/ci -p 'test_*.py'
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain_health.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_validation_registry.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_validation_runner.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_retain_validation_evidence.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_timing_report.py

.config/nextest.toml

@@ -234,7 +234,7 @@ test-group = 'embedded-test-ports'
 # processes cannot exhaust those budgets. Keep deadlines and assertions intact.
 # The short HTTP fixtures are tracked in rustfs/backlog#2714.
 [[profile.default.overrides]]
-filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
+filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
 threads-required = "num-test-threads"
 
 # This cancellation probe hashes the serving executable before acquiring its
@@ -548,7 +548,7 @@ filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnost
 test-group = 'embedded-test-ports'
 
 [[profile.ci.overrides]]
-filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
+filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
 threads-required = "num-test-threads"
 
 # Preserve the cancellation probe's startup budget and assertions in CI too.
@@ -827,7 +827,7 @@ slow-timeout = { period = "120s", terminate-after = 12, grace-period = "10s" }
 # ---------------------------------------------------------------------------
 # e2e-distributed profile — 4-node 4-disk Actions suite
 # ---------------------------------------------------------------------------
-# Storage-sensitive PR / nightly / dispatch lane owned by
+# Nightly and manual-dispatch lane owned by
 # .github/workflows/e2e-distributed.yml.
 # Each case starts four rustfs processes (and for site replication, two
 # clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
@@ -960,6 +960,12 @@ test-group = 'e2e-inline-boundaries'
 filter = 'package(e2e_test) & test(/^heal_erasure_disk_rebuild_test::/)'
 test-group = 'e2e-cluster-nightly'
 
+# The multi-pool crash fixture seeds three nodes and 36 drives. Reserve this
+# nextest run's workers so other multi-node tests do not starve its deadline.
+[[profile.e2e-full.overrides]]
+filter = 'package(e2e_test) & test(=heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_across_multi_pool_after_background_target_crash)'
+threads-required = "num-test-threads"
+
 [[profile.e2e-full.overrides]]
 filter = 'package(e2e_test) & (test(/^kms::kms_vault_test::/) | test(/^kms::kms_rekey_sweep_test::/) | test(/^kms::configured_roundtrip_test::test_configured_vault_kms_admin_and_versioned_cleanup$/))'
 test-group = 'e2e-vault'
@@ -969,3 +975,23 @@ test-group = 'e2e-vault'
 [[profile.e2e-full.overrides]]
 filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
 threads-required = "num-test-threads"
+
+# The mc fixture uploads hundreds of objects concurrently before measuring listing.
+# Reserve the runner so other server fixtures do not starve its disk operations.
+[[profile.e2e-full.overrides]]
+filter = 'package(e2e_test) & test(=mc_mirror_small_bucket_test::test_mc_mirror_small_bucket_completes_without_list_timeout)'
+threads-required = "num-test-threads"
+
+# Fixed, small contract for every executable server change. These existing
+# scenarios exercise write/read, anonymous denial and retained bytes on restart.
+[profile.core-contract]
+default-filter = '''
+    package(e2e_test) & test(/^(anonymous_access_test::test_anonymous_access_denied_when_restrict_public_buckets_enabled|head_object_consistency_test::head_object_consistency_after_write_and_multipart_and_presigned_head|storage_class_capability_test::tests::historical_label_only_metadata_is_standard_without_hiding_a_real_transition_tier)$/)
+'''
+retries = 0
+fail-fast = false
+test-threads = 1
+slow-timeout = { period = "60s", terminate-after = 3 }
+
+[profile.core-contract.junit]
+path = "junit.xml"

.config/release-validation-plan.json

@@ -0,0 +1,77 @@
+{
+  "schema_version": 1,
+  "plan_id": "rustfs-release-validation",
+  "description": "Versioned certification plan for every public RustFS release exit (rustfs/backlog#2662, R01). preview, prerelease and stable all require this explicit plan. Scripts/release/release_gate.py fail-closes publication on any missing, duplicate, stale, substituted or non-PASS required item.",
+  "release_types": ["preview", "prerelease", "stable"],
+  "targets": [
+    {"os": "linux", "arch": "x86_64", "abi": "musl"},
+    {"os": "linux", "arch": "aarch64", "abi": "musl"},
+    {"os": "linux", "arch": "x86_64", "abi": "gnu"},
+    {"os": "linux", "arch": "aarch64", "abi": "gnu"},
+    {"os": "macos", "arch": "aarch64", "abi": "none"},
+    {"os": "windows", "arch": "x86_64", "abi": "msvc"}
+  ],
+  "targets_basis": "Platform matrix of .github/workflows/build.yml (prepare-platform-matrix). gnu and macOS targets build with the pyroscope feature; musl and Windows targets build with no extra features.",
+  "features": ["pyroscope"],
+  "trusted_workflows": [
+    {"repository": "rustfs/s3chaos", "workflow_id": "release-gate.yml"}
+  ],
+  "trusted_workflows_basis": "The shared evidence envelope producer lives in rustfs/s3chaos (G01, fault/release_gate.rs) and its plan/live split workflow is .github/workflows/release-gate.yml there. Only results from these identities may enter a release certification; PR-uploaded JSON never gains release authority. R02 must extend this list when additional verifier workflows land, never widen it to untrusted sources.",
+  "lanes": [
+    {
+      "lane": "identity",
+      "harness_repository": "rustfs/s3chaos",
+      "harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
+      "harness_basis": "Fixed G01 evidence-envelope producer revision (rustfs/s3chaos PR #102 head recorded in rustfs/backlog#2658). R02 re-pins this when the live verifier ships.",
+      "timeout_seconds": 1800,
+      "environment_capabilities": ["process-exec"],
+      "cases": ["artifact-identity", "startup"],
+      "applies_to": {
+        "release_types": ["preview", "prerelease", "stable"],
+        "platforms": "all"
+      },
+      "applies_basis": "Artifact identity and startup checks are mandatory for every release type, preview included, on every shipped target (rustfs/backlog#2662 frozen design item 5)."
+    },
+    {
+      "lane": "upgrade",
+      "harness_repository": "rustfs/s3chaos",
+      "harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
+      "harness_basis": "Same fixed envelope producer as the identity lane; the workflow SHA and the product build SHA are bound separately and need not be equal.",
+      "timeout_seconds": 5400,
+      "environment_capabilities": ["process-exec", "previous-release-artifact"],
+      "cases": [
+        "direct-upgrade",
+        "mixed-version-upgrade",
+        "bucket-config-upgrade",
+        "bucket-config-rollback",
+        "multipart-layout-upgrade",
+        "multipart-layout-baseline"
+      ],
+      "cases_basis": "Case IDs of the pinned previous-release matrix in .github/workflows/e2e-upgrade.yml (source.version/asset/sha256 and the case list mapped there).",
+      "applies_to": {
+        "release_types": ["prerelease", "stable"],
+        "platforms": [{"os": "linux", "arch": "x86_64", "abi": "gnu"}]
+      },
+      "applies_basis": "preview releases use the pre-declared smaller set (identity lane only). The upgrade lane is only exercised on linux-x86_64-gnu, the platform of the pinned source assets."
+    }
+  ],
+  "upgrade_sources": {
+    "basis": "Exactly the two previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
+    "sources": [
+      {
+        "version": "1.0.1",
+        "role": "N-1",
+        "artifact_name": "rustfs-linux-x86_64-gnu-v1.0.1.zip",
+        "artifact_sha256": "9742e183cb0dfb0d48fa909797af572fe47b7f827569036baa4156e0d3958ee5"
+      },
+      {
+        "version": "1.0.0-rc.5",
+        "role": "N-2",
+        "artifact_name": "rustfs-linux-x86_64-gnu-v1.0.0-rc.5.zip",
+        "artifact_sha256": "3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d"
+      }
+    ],
+    "rollback_supported": false,
+    "rollback_basis": "Only the bucket-config-rollback case (rollback_to_previous_release_reads_current_bucket_metadata) is exercised. Full data-plane rollback is not certified and is never a default claim."
+  }
+}

.github/actions/quick-checks/action.yml

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 name: Quick Checks
-description: Run the shared compile-free RustFS quality checks.
+description: Run the shared RustFS quality checks.
 
 runs:
   using: composite
@@ -24,6 +24,7 @@ runs:
         tool: |
           ripgrep@15.2.0
           shellcheck@0.11.0
+          cargo-nextest@0.9.138
 
     - name: Install actionlint
       shell: bash
@@ -37,6 +38,15 @@ runs:
         rm "$actionlint_dir/actionlint.tar.gz"
         echo "$actionlint_dir" >> "$GITHUB_PATH"
 
+    - name: Install Python
+      uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+      with:
+        python-version: "3.12"
+
+    - name: Install workflow parser
+      shell: bash
+      run: python3 -m pip install --disable-pip-version-check PyYAML==6.0.3
+
     - name: Install Rust toolchain
       uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
       with:
@@ -104,7 +114,10 @@ runs:
 
     - name: Check test wiring
       shell: bash
-      run: python3 ./scripts/check_test_wiring.py
+      run: |
+        python3 ./scripts/check_test_wiring.py
+        python3 ./scripts/ci_gate.py --check-workflow
+        python3 ./scripts/validation_registry.py check
 
     - name: Check no planning docs committed
       shell: bash

.github/actions/runner-pilot/action.yml

@@ -0,0 +1,107 @@
+name: Runner capability and build pilot
+description: Inspect an existing runner and measure isolated fixed-source cold/warm builds
+inputs:
+  workload:
+    description: probe, server or workspace-tests
+    required: true
+  source-sha:
+    description: Exact merged main commit for builds
+    default: ""
+  toolchain:
+    description: Exact Rust version for builds
+    default: ""
+  scratch-root:
+    description: Existing absolute scratch root
+    default: ""
+  build-jobs:
+    description: Cargo concurrency
+    default: "2"
+  local-ssd:
+    description: Require operator storage attestation
+    default: "false"
+runs:
+  using: composite
+  steps:
+    - name: Validate pilot input and inspect existing capabilities
+      shell: bash
+      env:
+        PILOT_WORKLOAD: ${{ inputs.workload }}
+        PILOT_SHA: ${{ inputs.source-sha }}
+        PILOT_TOOLCHAIN: ${{ inputs.toolchain }}
+        PILOT_SCRATCH: ${{ inputs.scratch-root }}
+        PILOT_JOBS: ${{ inputs.build-jobs }}
+        PILOT_SSD: ${{ inputs.local-ssd }}
+      run: |
+        [[ "$PILOT_WORKLOAD" =~ ^(probe|server|workspace-tests)$ ]]
+        [[ "$PILOT_JOBS" =~ ^([1-9]|1[0-6])$ ]]
+        if [[ "$PILOT_WORKLOAD" != probe ]]; then
+          [[ "$PILOT_SHA" =~ ^[a-f0-9]{40}$ ]]
+          [[ "$PILOT_TOOLCHAIN" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]
+        fi
+        scratch="${PILOT_SCRATCH:-$RUNNER_TEMP}"
+        [[ "$scratch" == /* && -d "$scratch" ]]
+        minimum_disk=1
+        minimum_memory=0
+        if [[ "$PILOT_WORKLOAD" == server ]]; then minimum_disk=24; minimum_memory=8; fi
+        if [[ "$PILOT_WORKLOAD" == workspace-tests ]]; then minimum_disk=90; minimum_memory=16; fi
+        extra=()
+        if [[ "$PILOT_SSD" == true ]]; then extra+=(--local-ssd-attestation /etc/rustfs-runner/storage.json); fi
+        python3 scripts/ci/runner_pilot.py --root "$scratch" \
+          --output "$RUNNER_TEMP/runner-pilot-evidence" \
+          --min-free-gib "$minimum_disk" --min-memory-gib "$minimum_memory" --architecture x86_64 "${extra[@]}"
+    - name: Checkout the trusted baseline history
+      if: inputs.workload != 'probe'
+      uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+      with:
+        ref: main
+        path: .runner-pilot-source
+        fetch-depth: 0
+        persist-credentials: false
+    - name: Bind the build to a merged main commit
+      if: inputs.workload != 'probe'
+      shell: bash
+      env:
+        PILOT_SHA: ${{ inputs.source-sha }}
+      run: |
+        git -C .runner-pilot-source merge-base --is-ancestor "$PILOT_SHA" HEAD
+        git -C .runner-pilot-source checkout --detach "$PILOT_SHA"
+    - name: Install build dependencies
+      if: inputs.workload != 'probe'
+      shell: bash
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y build-essential pkg-config libssl-dev cmake clang time
+    - name: Install the fixed protobuf compiler
+      if: inputs.workload != 'probe'
+      uses: rustfs/setup-protoc@a3705324d8f9bf5b6c3573fb6cf8ae421db55dd6 # v3.0.1
+      with:
+        version: "35.1"
+        repo-token: ${{ github.token }}
+    - name: Install the fixed flatbuffer compiler
+      if: inputs.workload != 'probe'
+      uses: Nugine/setup-flatc@698800de72a96bfb22cf60431dc21a2ff9a7e07b # v1
+      with:
+        version: "25.12.19"
+    - name: Install the requested exact Rust toolchain
+      if: inputs.workload != 'probe'
+      uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
+      with:
+        toolchain: ${{ inputs.toolchain }}
+    - name: Measure cold and warm builds without shared target caches
+      if: inputs.workload != 'probe'
+      shell: bash
+      env:
+        PILOT_WORKLOAD: ${{ inputs.workload }}
+        PILOT_SHA: ${{ inputs.source-sha }}
+        PILOT_TOOLCHAIN: ${{ inputs.toolchain }}
+        PILOT_SCRATCH: ${{ inputs.scratch-root }}
+        PILOT_JOBS: ${{ inputs.build-jobs }}
+        PILOT_SSD: ${{ inputs.local-ssd }}
+      run: |
+        extra=()
+        if [[ "$PILOT_SSD" == true ]]; then extra+=(--local-ssd-attestation /etc/rustfs-runner/storage.json); fi
+        python3 scripts/ci/runner_pilot.py --root "${PILOT_SCRATCH:-$RUNNER_TEMP}" \
+          --output "$RUNNER_TEMP/runner-pilot-evidence" \
+          --source .runner-pilot-source --source-sha "$PILOT_SHA" \
+          --toolchain "$PILOT_TOOLCHAIN" --benchmark "$PILOT_WORKLOAD" \
+          --jobs "$PILOT_JOBS" --timeout-seconds 3600 "${extra[@]}"

.github/actions/test-runtime/action.yml

@@ -0,0 +1,16 @@
+name: Verification runtime
+description: Install the runtime for prebuilt test archives without a target cache
+runs:
+  using: composite
+  steps:
+    - name: Install runtime tools
+      shell: bash
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y libssl-dev ripgrep
+    - name: Install Rust toolchain
+      uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
+    - name: Install cargo-nextest
+      uses: taiki-e/install-action@96c7780c1d8a2b8723e12031def873a434d39d8d # nextest
+      with:
+        tool: cargo-nextest@0.9.138

.github/dependabot.yml

@@ -32,6 +32,8 @@ updates:
       - "overtrue"
       - "majinghe"
     ignore:
+      - dependency-name: "pulsar"
+        versions: [ "6.9.x" ]
       - dependency-name: "object_store"
         versions: [ "0.13.x" ]
       - dependency-name: "libunftp"

.github/workflows/README.md

@@ -32,7 +32,7 @@ A job runs on `ubuntu-latest` unless it has one of these reasons, and the reason
 
 Jobs that do only the following have no reason to be self-hosted: shell or Python scripts, `cargo fmt`, `cargo deny`, typos, actionlint, API calls (`gh`, dependency review, issue and stale bots, CLA checks), downloading an artifact and running a prebuilt binary, or a `cargo build -p rustfs` with its own cache key. `e2e-distributed.yml` and `e2e-upgrade.yml` build the server and the `e2e_test` harness on `ubuntu-latest` inside their budgets, so "it compiles something" alone is not a reason.
 
-The functional chain's `prepare` and `complete-chain` jobs, the performance runner availability check, and runner hygiene use `ubuntu-latest`: they inspect APIs or artifacts, not the lab hosts. The actual functional and performance suites remain on their lab runners. Functional-chain health collection stays on the operator-managed `smoke-testing` runner as part of the automation environment. Docker image packaging also uses `ubuntu-latest` for both musl and glibc variants; it downloads release binaries rather than compiling the workspace, and still sets up QEMU and Buildx explicitly.
+The functional chain's `prepare` and `complete-chain` jobs, the performance runner availability check, and runner hygiene use `ubuntu-latest`: they inspect APIs or artifacts, not the lab hosts. The actual functional and performance suites remain on their lab runners. Functional-chain health collection uses `ubuntu-latest` so a lab outage cannot also stop its independent monitor. Docker image packaging also uses `ubuntu-latest` for both musl and glibc variants; it downloads release binaries rather than compiling the workspace, and still sets up QEMU and Buildx explicitly.
 
 Moving these jobs off AKS reduces capacity demand, not necessarily the Azure bill immediately. VM costs fall only when nodes scale down or smaller capacity replaces them.
 

.github/workflows/build.yml

@@ -814,13 +814,18 @@ jobs:
           export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
           export AWS_DEFAULT_REGION="auto"
 
-          # Determine upload path based on build type
+          # Determine upload path based on build type. Release and prerelease
+          # builds only stage their bytes here: the official
+          # artifacts/rustfs/release/ path is written solely by the
+          # certification-gated promote-release-artifacts job, from these same
+          # staged objects. Development builds keep the uncertified dev channel
+          # and never touch official or latest channels.
           if [[ "$BUILD_TYPE" == "development" ]]; then
             R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/dev/"
             echo "📤 Uploading development build to R2 dev directory"
           else
-            R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
-            echo "📤 Uploading release build to R2 release directory"
+            R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
+            echo "📤 Staging release build for certification-gated promotion"
           fi
 
           # Upload all rustfs zip files to R2 using S3-compatible API
@@ -869,6 +874,221 @@ jobs:
             echo "This run does not trigger Docker publishing."
           fi
 
+  # Aggregate the parent candidate batch. This job always collects results and
+  # retains its evidence, including on failure, and every publication job
+  # below requires its success. Missing, stale, substituted or non-PASS
+  # evidence fails closed here: only the trusted verification sources
+  # (rustfs/s3chaos release-gate envelopes, rustfs/backlog#2658) can satisfy
+  # it, so a PR-uploaded JSON set never gains release authority.
+  release-certification:
+    name: Release Certification
+    needs: [ build-check, build-rustfs ]
+    if: ${{ !cancelled() && needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') }}
+    runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      actions: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Download all build artifacts
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          path: ./artifacts
+          pattern: rustfs-*
+          merge-multiple: true
+
+      - name: Write candidate and collect trusted validation results
+        env:
+          # Cross-repo read of the trusted verification run. Falls back to the
+          # workflow token, which cannot read rustfs/s3chaos; a missing token
+          # therefore fail-closes instead of certifying.
+          GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
+          PRODUCT_COMMIT: ${{ github.sha }}
+          VERSION: ${{ needs.build-check.outputs.version }}
+          BUILD_TYPE: ${{ needs.build-check.outputs.build_type }}
+          FEATURES: ${{ needs.build-check.outputs.build_type == 'development' && '' || 'pyroscope' }}
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          mkdir -p results release-gate-output
+          PLAN_SHA256=$(python3 -c "import hashlib,json,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('.config/release-validation-plan.json'))))")
+
+          python3 - "$PLAN_SHA256" > candidate.json <<'PY'
+          import hashlib, json, os, sys, zipfile
+
+          plan_sha256 = sys.argv[1]
+          profile = {"release": "stable", "prerelease": "prerelease", "preview": "preview"}.get(os.environ["BUILD_TYPE"])
+          if profile is None:
+              raise SystemExit("release certification only covers preview, prerelease and stable builds")
+
+          def sha256(path):
+              digest = hashlib.sha256()
+              with open(path, "rb") as handle:
+                  for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+                      digest.update(chunk)
+              return digest.hexdigest()
+
+          deliverables = []
+          for name in sorted(os.listdir("artifacts")):
+              if not name.startswith("rustfs-") or not name.endswith(".zip"):
+                  continue
+              path = os.path.join("artifacts", name)
+              members = zipfile.ZipFile(path).namelist()
+              binary_name = "rustfs.exe" if "rustfs.exe" in members else "rustfs"
+              with zipfile.ZipFile(path) as archive:
+                  binary_bytes = archive.read(binary_name)
+              import tempfile
+              with tempfile.NamedTemporaryFile() as handle:
+                  handle.write(binary_bytes)
+                  handle.flush()
+                  binary_sha = sha256(handle.name)
+              deliverables.append({
+                  "kind": "zip",
+                  "name": name,
+                  "location": "https://github.com/rustfs/rustfs/releases/download/{}/{}".format(os.environ["VERSION"], name),
+                  "sha256": sha256(path),
+                  "size": os.path.getsize(path),
+                  "binary_sha256": binary_sha,
+              })
+          if not deliverables:
+              raise SystemExit("no built deliverables found to certify")
+          candidate = {
+              "schema_version": 1,
+              "product_commit": os.environ["PRODUCT_COMMIT"],
+              "version": os.environ["VERSION"],
+              "release_profile": profile,
+              "features": [f for f in os.environ.get("FEATURES", "").split(",") if f],
+              "plan_sha256": plan_sha256,
+              "validation_batch": {"run_id": os.environ["GITHUB_RUN_ID"], "run_attempt": int(os.environ["GITHUB_RUN_ATTEMPT"])},
+              "deliverables": deliverables,
+          }
+          json.dump(candidate, sys.stdout, indent=2, sort_keys=True)
+          print()
+          PY
+
+          # Trusted verification evidence only: the release-gate envelopes of
+          # rustfs/s3chaos for this exact product commit. Absence blocks below.
+          TRUSTED_REPO="rustfs/s3chaos"
+          TRUSTED_WORKFLOW="release-gate.yml"
+          TRUSTED_RUN=$(gh api "repos/${TRUSTED_REPO}/actions/workflows/${TRUSTED_WORKFLOW}/runs?head_sha=${PRODUCT_COMMIT}&status=success&per_page=1" --jq '.workflow_runs[0].id // empty' || true)
+          if [[ -z "$TRUSTED_RUN" ]]; then
+            echo "❌ No trusted verification run of ${TRUSTED_REPO}/${TRUSTED_WORKFLOW} for ${PRODUCT_COMMIT}; refusing to certify" >&2
+          else
+            gh run download "$TRUSTED_RUN" -R "$TRUSTED_REPO" -n release-validation-evidence -D results || \
+              echo "❌ Trusted verification run ${TRUSTED_RUN} published no release-validation-evidence artifact" >&2
+          fi
+
+          python3 scripts/release/release_gate.py validate \
+            --plan .config/release-validation-plan.json \
+            --candidate candidate.json \
+            --results results \
+            --output release-gate-output
+
+      - name: Upload certification evidence
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: release-certification
+          path: |
+            candidate.json
+            results/
+            release-gate-output/
+          if-no-files-found: warn
+          retention-days: 30
+
+  # The only writer of the official R2 release path. Promotion re-checks the
+  # exact bytes about to be published against the passed certification list
+  # and copies the staged objects byte-for-byte: no rebuild-after-verify.
+  promote-release-artifacts:
+    name: Promote Release Artifacts
+    needs: [ build-check, build-rustfs, release-certification ]
+    # Target-channel lock for the official R2 release path.
+    concurrency:
+      group: release-channel-r2-release
+      cancel-in-progress: false
+    if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      actions: read
+    env:
+      R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
+      R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
+      R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
+      R2_BUCKET: ${{ secrets.R2_BUCKET }}
+      AWS_EC2_METADATA_DISABLED: true
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Download build and certification artifacts
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          path: ./artifacts
+          pattern: rustfs-*
+          merge-multiple: true
+
+      - name: Download certification evidence
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: release-certification
+          path: ./certification
+
+      - name: Verify and promote staged release artifacts
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
+            echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
+            exit 1
+          fi
+
+          for zip_file in artifacts/rustfs-*.zip; do
+            [[ -f "$zip_file" ]] || continue
+            name="$(basename "$zip_file")"
+            python3 scripts/release/release_gate.py verify-promotion \
+              --plan .config/release-validation-plan.json \
+              --candidate certification/candidate.json \
+              --summary certification/release-gate-output \
+              --artifact "$zip_file" \
+              --artifact-name "$name"
+          done
+
+          if ! command -v aws >/dev/null 2>&1; then
+            sudo apt-get update && sudo apt-get install -y awscli
+          fi
+          export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
+          export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
+          export AWS_DEFAULT_REGION="auto"
+
+          STAGING_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
+          RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
+          for zip_file in artifacts/rustfs-*.zip; do
+            [[ -f "$zip_file" ]] || continue
+            name="$(basename "$zip_file")"
+            staged="$(mktemp)"
+            aws s3 cp "${STAGING_PATH}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+            staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
+            built_sha="$(sha256sum "$zip_file" | awk '{print $1}')"
+            if [[ "$staged_sha" != "$built_sha" ]]; then
+              echo "❌ Staged object ${name} does not match the verified bytes" >&2
+              exit 1
+            fi
+            aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+            rm -f "$staged"
+            echo "✅ Promoted ${name} (${built_sha})"
+          done
+
   # Create GitHub Release for every valid release tag, including previews
   create-release:
     name: Create GitHub Release
@@ -1028,6 +1248,10 @@ jobs:
   update-latest-version:
     name: Update Latest Version
     needs: [ build-check, publish-release ]
+    # Target-channel lock for the latest.json version pointer.
+    concurrency:
+      group: release-channel-oss-latest
+      cancel-in-progress: false
     if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
     runs-on: ubuntu-latest
     timeout-minutes: 30
@@ -1068,6 +1292,45 @@ jobs:
           rm -rf "$OSSUTIL_DIR" "$OSSUTIL_ZIP"
           chmod +x "$OSSUTIL_BIN"
 
+          # Candidate/pointer ordering: an older candidate must never overwrite
+          # a newer pointer, even if its promotion job finishes later.
+          CURRENT_LATEST="${RUNNER_TEMP}/current-latest.json"
+          if "$OSSUTIL_BIN" cp oss://rustfs-version/latest.json "$CURRENT_LATEST" 2>/dev/null; then
+            python3 - "$CURRENT_LATEST" "$VERSION" <<'PY'
+          import json
+          import re
+          import sys
+
+          def key(version):
+              match = re.match(r"^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$", version)
+              if not match:
+                  raise SystemExit(3)
+              core = tuple(int(part) for part in match.groups()[:3])
+              prerelease = match.group(4)
+              if prerelease is None:
+                  return core + (1,)
+              return core + (0, tuple(int(p) if p.isdigit() else p for p in re.split(r"[.-]", prerelease)))
+
+          try:
+              current = json.load(open(sys.argv[1], encoding="utf-8")).get("version", "")
+              proposed = sys.argv[2]
+              if current and key(current) > key(proposed):
+                  raise SystemExit(1)
+          except SystemExit:
+              raise
+          except (OSError, ValueError, TypeError):
+              raise SystemExit(3)
+          PY
+            guard_status=$?
+            if [[ "$guard_status" -eq 1 ]]; then
+              echo "Current latest.json points at a newer version; refusing to downgrade" >&2
+              exit 0
+            elif [[ "$guard_status" -ne 0 ]]; then
+              echo "Existing latest.json is unparsable; refusing to overwrite it blindly" >&2
+              exit 1
+            fi
+          fi
+
           # Create latest.json
           cat > latest.json << EOF
           {
@@ -1084,16 +1347,59 @@ jobs:
 
           echo "✅ Updated latest.json for ${RELEASE_TYPE} release $VERSION"
 
-  # Publish release (remove draft status)
+  # Publish release (remove draft status). Requires the release-certification
+  # aggregate: without a complete legal success there is no public write.
   publish-release:
     name: Publish Release
-    needs: [ build-check, create-release, upload-release-assets ]
+    needs: [ build-check, create-release, upload-release-assets, release-certification ]
+    # Target-channel lock: concurrent releases serialize on this channel so an
+    # older candidate cannot interleave its publish with a newer one.
+    concurrency:
+      group: release-channel-github-release
+      cancel-in-progress: false
     if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
     runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
     timeout-minutes: 30
     permissions:
       contents: write
+      actions: read
     steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Download certification evidence
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: release-certification
+          path: ./certification
+
+      - name: Verify release assets against certification
+        env:
+          GH_TOKEN: ${{ github.token }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          TAG="${{ needs.build-check.outputs.version }}"
+          mkdir -p "$RUNNER_TEMP/promotion-assets"
+          gh release download "$TAG" --dir "$RUNNER_TEMP/promotion-assets" --pattern '*.zip'
+          found=0
+          for zip_file in "$RUNNER_TEMP"/promotion-assets/*.zip; do
+            [[ -f "$zip_file" ]] || continue
+            found=1
+            python3 scripts/release/release_gate.py verify-promotion \
+              --plan .config/release-validation-plan.json \
+              --candidate certification/candidate.json \
+              --summary certification/release-gate-output \
+              --artifact "$zip_file" \
+              --artifact-name "$(basename "$zip_file")"
+          done
+          if [[ "$found" -ne 1 ]]; then
+            echo "No release zip assets to verify against the certification" >&2
+            exit 1
+          fi
+
       - name: Publish release
         env:
           GH_TOKEN: ${{ github.token }}

.github/workflows/cache-warm.yml

@@ -96,7 +96,7 @@ env:
   RUST_BACKTRACE: 1
 
 jobs:
-  # Readers: test-and-lint (workspace and ILM), build-rustfs-debug-binary,
+  # Readers: workspace-lint-and-build, workspace-tests (workspace and ILM), build-rustfs-debug-binary,
   # e2e-tests, e2e-full.
   warm-ci-dev:
     name: Warm ci-dev
@@ -146,7 +146,7 @@ jobs:
           if-no-files-found: error
 
       # --all-targets covers the test binaries nextest builds, including
-      # e2e_test, which test-and-lint's own run excludes. The second build adds
+      # e2e_test, which the workspace run excludes. The second build adds
       # the e2e-test-hooks feature resolution that build-rustfs-debug-binary uses
       # and that no lint lane enables.
       - name: Build ci-dev superset

.github/workflows/ci-resource-accounting.yml

@@ -0,0 +1,60 @@
+name: CI Resource Accounting
+
+on:
+  workflow_dispatch:
+    inputs:
+      since:
+        description: "Inclusive run creation timestamp, for example 2026-10-01T00:00:00Z"
+        required: true
+        type: string
+      until:
+        description: "Inclusive end timestamp; keep the window below 1000 runs"
+        required: true
+        type: string
+
+permissions:
+  contents: read
+  actions: read
+  pull-requests: read
+
+concurrency:
+  group: ci-resource-accounting
+  cancel-in-progress: false
+
+jobs:
+  collect:
+    if: github.repository == 'rustfs/rustfs'
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+      - name: Collect all workflow attempts in the window
+        env:
+          GH_TOKEN: ${{ github.token }}
+          ACCOUNTING_SINCE: ${{ inputs.since }}
+          ACCOUNTING_UNTIL: ${{ inputs.until }}
+          ACCOUNTING_REPOSITORY: ${{ github.repository }}
+        run: |
+          python3 scripts/ci/actions_cost_report.py \
+            --repository "$ACCOUNTING_REPOSITORY" \
+            --since "$ACCOUNTING_SINCE" --until "$ACCOUNTING_UNTIL" \
+            --output "$RUNNER_TEMP/ci-accounting.json" \
+            --summary "$RUNNER_TEMP/ci-accounting.md"
+      - name: Show available accounting summary
+        if: always()
+        run: |
+          if [[ -f "$RUNNER_TEMP/ci-accounting.md" ]]; then
+            cat "$RUNNER_TEMP/ci-accounting.md" >> "$GITHUB_STEP_SUMMARY"
+          fi
+      - name: Retain recomputable accounting evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: ci-accounting-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            ${{ runner.temp }}/ci-accounting.json
+            ${{ runner.temp }}/ci-accounting.md
+          if-no-files-found: error
+          retention-days: 30

.github/workflows/ci-runner-pilot.yml

@@ -0,0 +1,92 @@
+name: CI Runner Pilot
+
+on:
+  workflow_dispatch:
+    inputs:
+      pool:
+        description: "Existing runner pool; no infrastructure is created"
+        required: true
+        type: choice
+        options: [hosted, arc]
+        default: hosted
+      workload:
+        description: "Probe only, or perform cold/warm fixed-source builds"
+        required: true
+        type: choice
+        options: [probe, server, workspace-tests]
+        default: probe
+      source_sha:
+        description: "Exact 40-character merged main SHA; required for builds"
+        type: string
+      toolchain:
+        description: "Exact Rust version such as 1.92.0; required for builds"
+        type: string
+      scratch_root:
+        description: "Existing absolute scratch directory; empty uses runner.temp"
+        type: string
+      build_jobs:
+        description: "Fixed Cargo concurrency for both comparisons (1-16)"
+        type: string
+        default: "2"
+      local_ssd:
+        description: "Require operator /etc/rustfs-runner/storage.json attestation"
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ci-runner-pilot
+  cancel-in-progress: false
+
+jobs:
+  hosted:
+    if: github.repository == 'rustfs/rustfs' && github.ref == 'refs/heads/main' && inputs.pool == 'hosted'
+    runs-on: ubuntu-latest
+    timeout-minutes: 150
+    steps:
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+      - uses: ./.github/actions/runner-pilot
+        with:
+          workload: ${{ inputs.workload }}
+          source-sha: ${{ inputs.source_sha }}
+          toolchain: ${{ inputs.toolchain }}
+          scratch-root: ${{ inputs.scratch_root }}
+          build-jobs: ${{ inputs.build_jobs }}
+          local-ssd: ${{ inputs.local_ssd }}
+      - name: Retain hosted pilot evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: runner-pilot-hosted-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/runner-pilot-evidence
+          if-no-files-found: error
+          retention-days: 30
+  arc:
+    if: github.repository == 'rustfs/rustfs' && github.ref == 'refs/heads/main' && inputs.pool == 'arc'
+    # Disk and hardware: measure the existing workspace build pool's storage.
+    runs-on: sm-standard-4
+    timeout-minutes: 150
+    steps:
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+      - uses: ./.github/actions/runner-pilot
+        with:
+          workload: ${{ inputs.workload }}
+          source-sha: ${{ inputs.source_sha }}
+          toolchain: ${{ inputs.toolchain }}
+          scratch-root: ${{ inputs.scratch_root }}
+          build-jobs: ${{ inputs.build_jobs }}
+          local-ssd: ${{ inputs.local_ssd }}
+      - name: Retain ARC pilot evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: runner-pilot-arc-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/runner-pilot-evidence
+          if-no-files-found: error
+          retention-days: 30

.github/workflows/ci.yml

@@ -44,6 +44,11 @@ on:
   schedule:
     - cron: "11 0 * * *" # Nightly 00:11 UTC, after the 22:41 UTC cache-warm run
   workflow_dispatch:
+    inputs:
+      rehearse_workspace_shards:
+        description: Compare one full execution with at most two resource-safe shards of the same archive
+        type: boolean
+        default: false
 
 permissions:
   contents: read
@@ -83,6 +88,11 @@ jobs:
     timeout-minutes: 10
     outputs:
       mode: ${{ steps.scope.outputs.mode }}
+      plan: ${{ steps.scope.outputs.plan }}
+      connect: ${{ steps.scope.outputs.connect || 'true' }}
+      packages: ${{ steps.scope.outputs.packages }}
+      full_validation: ${{ steps.scope.outputs.full_validation || 'true' }}
+      rehearse_shards: ${{ steps.scope.outputs.rehearse_shards || 'false' }}
     steps:
       - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
@@ -91,16 +101,37 @@ jobs:
       - name: Select scope using the base revision's policy
         id: scope
         env:
-          CI_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          CI_BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }}
+          CI_EVENT_ACTION: ${{ github.event.action }}
+          CI_REHEARSE_SHARDS: ${{ inputs.rehearse_workspace_shards && 'true' || 'false' }}
         run: |
           if [[ "$GITHUB_EVENT_NAME" != "pull_request" ]]; then
-            printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
+            python3 -I scripts/ci_gate.py select
           elif [[ "$CI_BASE_SHA" =~ ^[0-9a-f]{40}$ ]] && git show "$CI_BASE_SHA:scripts/ci_gate.py" > "$RUNNER_TEMP/ci-gate-base.py"; then
             python3 -I "$RUNNER_TEMP/ci-gate-base.py" select
           else
             printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
             echo "Base CI policy unavailable; running the full matrix."
           fi
+          if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=3$' "$GITHUB_OUTPUT"; then
+            printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
+            echo "Base policy does not cover this workflow contract; retaining the full matrix."
+          fi
+      - name: Retain verification plan
+        if: steps.scope.outputs.plan != ''
+        env:
+          CI_PLAN: ${{ steps.scope.outputs.plan }}
+        run: |
+          mkdir -p artifacts/ci-plan
+          printf '%s\n' "$CI_PLAN" > artifacts/ci-plan/plan.json
+      - name: Upload verification plan
+        if: steps.scope.outputs.plan != ''
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: verification-plan-${{ github.run_attempt }}
+          path: artifacts/ci-plan/plan.json
+          if-no-files-found: error
+          retention-days: 7
 
   typos:
     name: Typos
@@ -129,14 +160,15 @@ jobs:
       - name: Run shared quick checks
         uses: ./.github/actions/quick-checks
 
-  test-and-lint:
-    name: Workspace Test and Lint
+  workspace-lint-and-build:
+    name: Workspace Lint and Build
     if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
     needs: [ quick-checks, classify-changes ]
     runs-on: sm-standard-4
     timeout-minutes: 90
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+      CI_PACKAGES: ${{ needs.classify-changes.outputs.packages }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -161,13 +193,13 @@ jobs:
 
       - name: Prepare test evidence
         run: |
-          mkdir -p artifacts/test-and-lint
+          mkdir -p artifacts/workspace
           {
             echo "run_id=${GITHUB_RUN_ID}"
             echo "job=${GITHUB_JOB}"
             echo "runner=${RUNNER_NAME}"
             echo "started_at=$(date --utc --iso-8601=seconds)"
-          } > artifacts/test-and-lint/run-metadata.txt
+          } > artifacts/workspace/run-metadata.txt
 
       # Clippy gates every event, pull requests included. When it ran only after
       # merge, a lint regression merged freely and then failed every main run
@@ -179,73 +211,33 @@ jobs:
         run: |
           ./scripts/ci/resource_sampler.sh start clippy
           trap './scripts/ci/resource_sampler.sh stop' EXIT
-          cargo clippy --all-targets -- -D warnings
+          python3 scripts/ci_test_artifact.py cargo lint --configuration workspace
 
-      - name: Run nextest tests
+      - name: Archive workspace test binaries
         env:
-          # #5394 mitigation, settled by the backlog#1601 experiment.
-          #
-          # 2 was chosen when three concurrent workspace test links were believed
-          # to saturate the runner's overlay I/O and wedge Cargo until the 75m
-          # timeout. cgroup v2 readings from the sampler show the pod actually
-          # has 14 CPUs and 28GB (peak use 2.1GB): the label name "sm-standard-4"
-          # had led everyone, including the original mitigation, to assume 4
-          # cores, and memory was never the constraint.
-          #
-          # Measured on the same ci-dev cache, the test-profile compile took
-          # 28m21s at 2 on a pull request and 16-23 min at 3 across three
-          # successful main runs. The first pull-request run at 3 that met a
-          # busy node (run 39193827281) then hit the 75m timeout before any
-          # test started: the sampler shows cargo spinning for 70 minutes with
-          # the pod's io.pressure "full" at 86-88% and CPU pressure at zero,
-          # while node-level IO pressure was already 36% from co-tenants when
-          # the step began. The constraint is the pod's shared storage, not
-          # CPU or memory, and the limit that storage sustains on a contended
-          # node is 2. Main pushes and dispatches keep 3 (rarer, and three
-          # successes); pull requests stay at 2 until the pods get local disk
-          # for target/, which is the lever that would unlock the 14 cores.
-          #
-          # Must stay step-level: rust-cache hashes CARGO/CC/CFLAGS/CXX/CMAKE/RUST
-          # prefixed variables from process.env into the cache key, so promoting
-          # this to job level would rotate every key on this lane.
           CARGO_BUILD_JOBS: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && '3' || '2' }}
         run: |
-          mkdir -p artifacts/test-and-lint
-          rm -f target/nextest/ci/junit.xml
-          ./scripts/ci/resource_sampler.sh start nextest
+          ./scripts/ci/resource_sampler.sh start workspace-archive
           trap './scripts/ci/resource_sampler.sh stop' EXIT
-          set +e
-          NEXTEST_HIDE_PROGRESS_BAR=1 timeout --verbose --signal=TERM --kill-after=30s 75m \
-            cargo nextest run --profile ci --all --exclude e2e_test \
-            --status-level all --final-status-level all \
-            2>&1 | tee artifacts/test-and-lint/nextest.log
-          status=${PIPESTATUS[0]}
-          if [[ "${status}" -eq 0 ]]; then
-            cargo nextest list --profile ci --all --exclude e2e_test --message-format json \
-              > artifacts/test-and-lint/core-test-listing.json \
-              && python3 scripts/check_test_wiring.py --check-core artifacts/test-and-lint/core-test-listing.json \
-              && test -s target/nextest/ci/junit.xml || status=$?
-          fi
-          {
-            echo "command=cargo nextest run --profile ci --all --exclude e2e_test"
-            echo "exit_status=${status}"
-            echo "finished_at=$(date --utc --iso-8601=seconds)"
-            echo
-            echo "Remaining test-related processes:"
-            pgrep -af 'cargo|nextest|target/.*/deps/' || true
-            echo
-            echo "Kernel OOM / kill events:"
-            dmesg -T 2>/dev/null | grep -iE 'oom|out of memory|killed process' | tail -20 || true
-          } > artifacts/test-and-lint/nextest-diagnostics.txt
-          exit "${status}"
+          python3 scripts/ci_test_artifact.py build --configuration workspace --archive "$RUNNER_TEMP/workspace-tests.tar.zst"
+
+      - name: Upload workspace test binaries
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: workspace-test-binaries
+          overwrite: true
+          path: ${{ runner.temp }}/workspace-tests.tar.zst*
+          compression-level: 0
+          if-no-files-found: error
+          retention-days: 1
 
       - name: Run documentation tests
         run: |
-          mkdir -p artifacts/test-and-lint
+          mkdir -p artifacts/workspace
           set +e
           timeout --verbose --signal=TERM --kill-after=30s 15m \
-            cargo test --all --exclude e2e_test --doc \
-            2>&1 | tee artifacts/test-and-lint/doctest.log
+            python3 scripts/ci_test_artifact.py cargo doc --configuration workspace \
+            2>&1 | tee artifacts/workspace/doctest.log
           status=${PIPESTATUS[0]}
           {
             echo "command=cargo test --all --exclude e2e_test --doc"
@@ -254,17 +246,16 @@ jobs:
             echo
             echo "Remaining test-related processes:"
             pgrep -af 'cargo|rustdoc|target/.*/deps/' || true
-          } > artifacts/test-and-lint/doctest-diagnostics.txt
+          } > artifacts/workspace/doctest-diagnostics.txt
           exit "${status}"
 
       - name: Upload test reports and diagnostics
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: junit-test-and-lint-${{ github.run_number }}
+          name: junit-test-and-lint-${{ github.run_number }}-${{ github.run_attempt }}
           path: |
-            target/nextest/ci/junit.xml
-            artifacts/test-and-lint
+            artifacts/workspace
           retention-days: 3
           if-no-files-found: error
 
@@ -275,33 +266,78 @@ jobs:
       - name: Check log-analyzer rule anchors
         run: ./scripts/check_log_analyzer_rules.sh
 
-      # Preserve the migration floor and require successful execution evidence
-      # from the workspace run, without compiling or executing its subset again.
-      - name: Verify rebalance/decommission migration proofs
-        run: ./scripts/check_migration_gate_count.sh evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
+  workspace-tests:
+    name: Workspace tests
+    needs: [ workspace-lint-and-build, classify-changes ]
+    # The archive retains compile-time fixture paths and the large test set
+    # requires the same runner image until the hosted-runtime pilot passes.
+    runs-on: sm-standard-4
+    timeout-minutes: 90
+    outputs:
+      workspace_shards: ${{ steps.shards.outputs.workspace_shards || '[]' }}
+    env:
+      CI_PACKAGES: ${{ needs.classify-changes.outputs.packages }}
+    steps:
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+      - uses: ./.github/actions/test-runtime
+      - name: Protect Connect test home
+        run: chmod go-w "$(realpath "$HOME")"
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: workspace-test-binaries
+          path: ${{ runner.temp }}
+      - name: Verify workspace archive identity
+        run: python3 scripts/ci_test_artifact.py verify --configuration workspace --archive "$RUNNER_TEMP/workspace-tests.tar.zst"
+
+      - name: Extract and list workspace tests
+        run: |
+          mkdir -p artifacts/workspace
+          cargo nextest list --profile ci --archive-file "$RUNNER_TEMP/workspace-tests.tar.zst" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json \
+            > artifacts/workspace/core-test-listing.json
+          python3 scripts/check_test_wiring.py --check-core artifacts/workspace/core-test-listing.json
+
+      - name: Run nextest tests
+        run: |
+          mkdir -p artifacts/workspace
+          rm -f target/nextest/ci/junit.xml
+          ./scripts/ci/resource_sampler.sh start nextest
+          trap './scripts/ci/resource_sampler.sh stop' EXIT
+          set +e
+          NEXTEST_HIDE_PROGRESS_BAR=1 timeout --verbose --signal=TERM --kill-after=30s 75m \
+            cargo nextest run --profile ci --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json \
+            --workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" \
+            --status-level all --final-status-level all \
+            2>&1 | tee artifacts/workspace/nextest.log
+          status=${PIPESTATUS[0]}
+          {
+            echo "command=workspace archive execution"
+            echo "exit_status=${status}"
+            echo "finished_at=$(date --utc --iso-8601=seconds)"
+            echo
+            echo "Remaining test-related processes:"
+            pgrep -af 'cargo|nextest|target/.*/deps/' || true
+            echo
+            echo "Kernel OOM / kill events:"
+            dmesg -T 2>/dev/null | grep -iE 'oom|out of memory|killed process' | tail -20 || true
+          } > artifacts/workspace/nextest-diagnostics.txt
+          exit "${status}"
+
+      - name: Verify workspace execution completeness
+        run: |
+          python3 scripts/ci_test_artifact.py evidence artifacts/workspace/core-test-listing.json target/nextest/ci/junit.xml
+          ./scripts/check_migration_gate_count.sh evidence artifacts/workspace/core-test-listing.json target/nextest/ci/junit.xml
 
-      # The workspace already compiled these ignored tests. Keep the same Cargo
-      # package selection to reuse their binaries, then run them serially in
-      # separate processes because they share fixed ports and global state.
-      # test_transition_and_restore_flows was re-enabled by rustfs/backlog#1303:
-      # its "missing xl.meta on disk2" was a test-util bug (open_disk hardcoded
-      # disk_index 0), not an EC metadata-distribution issue.
-      # restore_object_usecase_reports_ongoing_conflict_and_completion was
-      # re-enabled by backlog#1304 (restore accepts serialize on a short CAS
-      # guard; the copy-back no longer holds the #4877 whole-copy-back lock,
-      # so the mid-restore ongoing read and fast 409 rejection it asserts are
-      # the implemented contract).
       - name: Run ignored ILM integration tests serially
         if: needs.classify-changes.outputs.mode == 'full'
-        env:
-          # Match the preceding workspace build and reuse its feature graph.
-          CARGO_BUILD_JOBS: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && '3' || '2' }}
         run: |
           mkdir -p artifacts/ilm-integration
           set +e
           NEXTEST_HIDE_PROGRESS_BAR=1 timeout --verbose --signal=TERM --kill-after=30s 80m \
             cargo nextest run --profile default -j1 --run-ignored ignored-only \
-            --all --exclude e2e_test \
+            --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json \
+            --workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" \
             -E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))' \
             --status-level all --final-status-level all \
             2>&1 | tee artifacts/ilm-integration/nextest.log
@@ -315,21 +351,105 @@ jobs:
           } > artifacts/ilm-integration/diagnostics.txt
           exit "${status}"
 
-      - name: Upload ILM test diagnostics
-        if: always() && needs.classify-changes.outputs.mode == 'full'
+      - name: Upload workspace test evidence
+        if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: ilm-integration-${{ github.run_number }}-${{ github.run_attempt }}
-          path: artifacts/ilm-integration
+          name: workspace-test-evidence-${{ github.run_attempt }}
+          path: |
+            target/nextest/ci/junit.xml
+            artifacts/workspace
+            artifacts/ilm-integration
           retention-days: 3
+          if-no-files-found: error
+
+      - name: Prepare bounded shards from this candidate's observed durations
+        id: shards
+        if: needs.classify-changes.outputs.rehearse_shards == 'true'
+        run: |
+          python3 scripts/ci_test_shards.py prepare \
+            --listing artifacts/workspace/core-test-listing.json \
+            --junit target/nextest/ci/junit.xml --output artifacts/workspace-shard-plan
+      - name: Upload same-run shard plan
+        if: needs.classify-changes.outputs.rehearse_shards == 'true'
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: workspace-shard-plan-${{ github.run_attempt }}
+          path: artifacts/workspace-shard-plan
+          retention-days: 3
+          if-no-files-found: error
+
+  workspace-shards:
+    name: Workspace shards (${{ matrix.shard }})
+    if: needs.classify-changes.outputs.rehearse_shards == 'true'
+    needs: [classify-changes, workspace-tests]
+    # Reuse the producer image and checkout path for the full RustFS archive.
+    runs-on: sm-standard-4
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      max-parallel: 2
+      matrix:
+        shard: ${{ fromJSON(needs.workspace-tests.outputs.workspace_shards || '[]') }}
+    env:
+      CI_PACKAGES: ${{ needs.classify-changes.outputs.packages }}
+    steps:
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+      - uses: ./.github/actions/test-runtime
+      - name: Protect Connect test home
+        run: chmod go-w "$(realpath "$HOME")"
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: workspace-test-binaries-${{ github.run_attempt }}
+          path: ${{ runner.temp }}
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: workspace-shard-plan-${{ github.run_attempt }}
+          path: artifacts/workspace-shard-plan
+      - name: Verify and execute this shard without compilation
+        env:
+          SHARD_INDEX: ${{ matrix.shard }}
+        run: |
+          python3 scripts/ci_test_artifact.py verify --configuration workspace --archive "$RUNNER_TEMP/workspace-tests.tar.zst"
+          python3 scripts/ci_test_shards.py execute --plan artifacts/workspace-shard-plan \
+            --archive "$RUNNER_TEMP/workspace-tests.tar.zst" --index "$SHARD_INDEX" --output artifacts/workspace-shard-result
+      - name: Retain shard evidence including failures
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: workspace-shard-result-${{ matrix.shard }}-${{ github.run_attempt }}
+          path: artifacts/workspace-shard-result
+          retention-days: 3
+          if-no-files-found: error
+
+  workspace-shard-evidence:
+    name: Workspace shard completeness
+    if: always() && needs.classify-changes.outputs.rehearse_shards == 'true'
+    needs: [classify-changes, workspace-tests, workspace-shards]
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: workspace-shard-plan-${{ github.run_attempt }}
+          path: artifacts/workspace-shard-plan
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          pattern: workspace-shard-result-*-${{ github.run_attempt }}
+          path: artifacts/workspace-shards
+      - name: Require the exact union of successful shard executions
+        run: |
+          python3 scripts/ci_test_shards.py verify --plan artifacts/workspace-shard-plan --evidence artifacts/workspace-shards
+          ./scripts/check_migration_gate_count.sh evidence artifacts/workspace-shard-plan/listing.json artifacts/workspace-shards/combined.xml
 
-  # Both Connect boundaries rebuild only the rustfs crate on top of the restored
-  # ci-dev cache: the two e2e features are empty, so every dependency keeps its
-  # cached dev-profile artifact and each feature set recompiles rustfs once.
-  # The budget covers a ci-dev miss, where the first step builds the whole graph.
   connect-feature-boundaries:
     name: Connect Feature Boundaries
-    if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
+    if: needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect != 'false' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
     needs: [ quick-checks, classify-changes ]
     runs-on: sm-standard-4
     timeout-minutes: 90
@@ -357,12 +477,10 @@ jobs:
         env:
           CARGO_BUILD_JOBS: "2"
         run: |
-          cargo test -p rustfs --test connect_registration \
-            --features connect-e2e-short-credentials \
-            registration_enforces_build_profile_credential_lifetime -- --exact
-          cargo test -p rustfs --test connect_registration \
-            --features connect-e2e-short-credentials \
-            rotation_waits_for_threshold_and_stops_on_revocation -- --exact
+          cargo test --locked -p rustfs --test connect_registration \
+            --features connect-e2e-short-credentials -- --exact --test-threads=1 \
+            registration_enforces_build_profile_credential_lifetime \
+            rotation_waits_for_threshold_and_stops_on_revocation
 
       # The guard in rustfs/src/connect/registration.rs is cfg(not(debug_assertions)),
       # and it must be tripped by a profile that turns debug assertions off for
@@ -374,37 +492,77 @@ jobs:
       # link-concurrency limit; it is cold because ci-dev holds no release
       # artifacts.
       - name: Reject short credentials in release builds
-        run: |
-          log="$(mktemp)"
-          set +e
-          CARGO_TERM_COLOR=never cargo check -p rustfs --release \
-            --features connect-e2e-short-credentials >"$log" 2>&1
-          status=$?
-          set -e
-          cat "$log"
-          expected='error: connect-e2e-short-credentials is restricted to debug builds'
-          summary="error: could not compile \`rustfs\` (lib) due to 1 previous error"
-          expected_count="$(grep -Fxc "$expected" "$log" || true)"
-          summary_count="$(grep -Fc "$summary" "$log" || true)"
-          error_count="$(grep -Ec '^error(:|\[)' "$log" || true)"
-          if [ "$status" -ne 101 ] || [ "$expected_count" -ne 1 ] \
-            || [ "$summary_count" -ne 1 ] || [ "$error_count" -ne 2 ]; then
-            echo "feature gate did not fail solely at the expected compile_error" >&2
-            rm -f "$log"
-            exit 1
-          fi
-          rm -f "$log"
+        run: python3 scripts/check_connect_release_boundary.py
 
       - name: Check offline enrollment E2E root boundary
         run: ./scripts/check_offline_enrollment_e2e.sh
 
+  fips-provider:
+    name: FIPS provider boundaries
+    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
+    needs: [ quick-checks, classify-changes ]
+    # This builds the RustFS library test target, which exceeds hosted-runner disk capacity.
+    runs-on: sm-standard-4
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        mode: [default, fips]
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+      CARGO_BUILD_JOBS: "2"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-fips-provider
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'
+
+      - name: Install FIPS build tools
+        if: matrix.mode == 'fips'
+        run: sudo apt-get install -y cmake golang clang libclang-dev
+
+      - name: Verify provider installation
+        shell: bash
+        run: |
+          args=()
+          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
+            args+=(--features fips)
+          fi
+          cargo nextest run --locked -p rustfs-tls-runtime --lib "${args[@]}" --no-tests fail -E 'test(provider_policy_isolated)'
+          cargo nextest run --locked -p rustfs-s3-client --lib "${args[@]}" --no-tests fail -E 'test(overflowing_closed_range_is_rejected_before_network_io)'
+          cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(validate_pulsar)'
+          cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(webhook_provider_policy_isolated)'
+          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
+            cargo nextest run --locked -p rustfs-protocols --lib --features ftps,webdav,fips --no-tests fail -E 'test(protocols_reject_incompatible_host_provider)'
+            cargo nextest run --locked -p rustfs-targets --lib --features fips --no-tests fail -E 'test(fips_postgres_rejects_incompatible_host_provider)'
+            cargo nextest run --locked -p rustfs-s3-client --lib --features fips --no-tests fail -E 'test(fips_transition_rejects_incompatible_host_provider)'
+          fi
+          tests=(provider_installation_isolated prepared_generation_shares_crypto_and_serves_sni_with_mtls)
+          if [[ '${{ matrix.mode }}' == 'fips' ]]; then
+            tests+=(fips_provider_builds_fips_tls_configs fips_provider_tls_handshake_accepts_and_rejects_algorithms)
+          fi
+          for test_name in "${tests[@]}"; do
+            cargo nextest run --locked -p rustfs --lib "${args[@]}" --no-tests fail -E "test($test_name)"
+          done
+          cargo clippy --locked -p rustfs --lib "${args[@]}" -- -D warnings
+
   test-and-lint-rio-v2:
     name: Test and Lint (rio-v2)
-    # Alternate feature configurations run on the nightly schedule, merge
-    # groups and manual dispatch. The default workspace remains the PR gate and
-    # the per-push main baseline: repeating four feature builds on every merge
-    # bought nothing the nightly run does not report within a day.
-    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["merge_group", "schedule", "workflow_dispatch"]'), github.event_name)
+    # Specialized boundary, hardware and alternate-feature lanes run only on
+    # the nightly schedule and manual dispatch. They do not gate a merge: the
+    # nightly run reports a regression within a day, and repeating these builds
+    # on every PR revision and merge group bought no earlier signal worth the
+    # runner occupancy. Workspace lint and tests, the core contract, the debug
+    # build, E2E smoke and the S3 lanes remain the pull-request gate.
+    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
     needs: [ quick-checks, classify-changes ]
     runs-on: sm-standard-4
     timeout-minutes: 90
@@ -441,29 +599,29 @@ jobs:
           # --profile ci so the quarantine list (and its junit flaky markers)
           # covers this leg too; the default profile is the local no-retry
           # profile and silently ignored quarantined flakes here (rustfs#6703).
-          mkdir -p artifacts/test-and-lint
+          mkdir -p artifacts/workspace
           rm -f target/nextest/ci/junit.xml
           ./scripts/ci/resource_sampler.sh start nextest-rio-v2
           trap './scripts/ci/resource_sampler.sh stop' EXIT
           cargo nextest run --profile ci -p rustfs -p rustfs-ecstore --features rio-v2 \
-            2>&1 | tee artifacts/test-and-lint/nextest.log
+            2>&1 | tee artifacts/workspace/nextest.log
           cargo test -p rustfs --doc --features rio-v2
 
       - name: Upload rio-v2 test reports and diagnostics
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: junit-test-and-lint-rio-v2-${{ github.run_number }}
+          name: junit-test-and-lint-rio-v2-${{ github.run_number }}-${{ github.run_attempt }}
           path: |
             target/nextest/ci/junit.xml
-            artifacts/test-and-lint
+            artifacts/workspace
           retention-days: 3
           if-no-files-found: error
 
   test-and-lint-protocols:
     name: "Test and Lint (${{ matrix.features.name }})"
     # Same cadence as test-and-lint-rio-v2 above.
-    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["merge_group", "schedule", "workflow_dispatch"]'), github.event_name)
+    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
     needs: [ quick-checks, classify-changes ]
     runs-on: sm-standard-4
     timeout-minutes: 90
@@ -532,7 +690,7 @@ jobs:
           path: |
             target/nextest/ci/junit.xml
             artifacts/protocol-tests
-            artifacts/test-and-lint
+            artifacts/workspace
           retention-days: 3
           if-no-files-found: warn
 
@@ -541,7 +699,7 @@ jobs:
     if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
     needs: [ quick-checks, classify-changes ]
     runs-on: sm-standard-4
-    timeout-minutes: 30
+    timeout-minutes: 60
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
@@ -609,6 +767,7 @@ jobs:
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: rustfs-debug-binary
+          overwrite: true
           path: |
             target/debug/rustfs
             target/debug/rustfs.e2e.json
@@ -616,6 +775,24 @@ jobs:
           if-no-files-found: error
           retention-days: 1
 
+      - name: Archive E2E test binaries
+        env:
+          CARGO_BUILD_JOBS: "2"
+        run: python3 scripts/ci_test_artifact.py build --configuration e2e --archive "$RUNNER_TEMP/e2e-tests.tar.zst"
+
+      - name: Run E2E documentation tests
+        run: cargo test --locked -p e2e_test --doc
+
+      - name: Upload E2E test binaries
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: e2e-test-binaries
+          overwrite: true
+          path: ${{ runner.temp }}/e2e-tests.tar.zst*
+          compression-level: 0
+          if-no-files-found: error
+          retention-days: 1
+
   build-rustfs-debug-binary-rio-v2:
     name: Build RustFS Debug Binary (rio-v2)
     # Dormant rio-v2 variant (rustfs/backlog#1835): the feature ships in no
@@ -650,6 +827,7 @@ jobs:
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: rustfs-debug-binary-rio-v2
+          overwrite: true
           path: |
             target/debug/rustfs
             target/debug/rustfs.e2e.json
@@ -664,7 +842,7 @@ jobs:
     # job had neither, so each closed/merged PR really ran the whole io_uring
     # suite (measured 4m17s / 7m19s / 7m31s on runs 30678272341 / 30678117601 /
     # 30662728539) and kept the cancellation run in progress for minutes.
-    if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
+    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
     needs: [ quick-checks, classify-changes ]
     # GitHub-hosted ubuntu-latest runs a recent kernel with io_uring and, unlike
     # a container, applies no seccomp filter that would block io_uring_setup — so
@@ -732,10 +910,51 @@ jobs:
         # scripts/check_uring_lane_lib_only.sh guards the --lib precondition.
         run: cargo test -p rustfs-ecstore --lib uring_ -- --test-threads=1 --nocapture
 
+  core-contract:
+    name: Core contract
+    needs: [ build-rustfs-debug-binary ]
+    # Compile-time fixture paths must match the producer until the pilot passes.
+    runs-on: sm-standard-4
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+      - uses: ./.github/actions/test-runtime
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: rustfs-debug-binary
+          path: target/debug
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: e2e-test-binaries
+          path: ${{ runner.temp }}
+      - name: Run core storage and authorization contract
+        env:
+          RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/core-contract-logs
+        run: |
+          chmod +x target/debug/rustfs
+          python3 scripts/ci_test_artifact.py verify --configuration e2e --archive "$RUNNER_TEMP/e2e-tests.tar.zst"
+          cargo nextest list --profile core-contract --archive-file "$RUNNER_TEMP/e2e-tests.tar.zst" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json > "$RUNNER_TEMP/core-contract-list.json"
+          python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile core-contract --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json --workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" --no-tests fail
+          python3 scripts/ci_test_artifact.py evidence --core "$RUNNER_TEMP/core-contract-list.json" target/nextest/core-contract/junit.xml
+      - name: Upload core contract evidence
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: core-contract-${{ github.run_attempt }}
+          path: |
+            target/nextest/core-contract/junit.xml
+            ${{ runner.temp }}/core-contract-list.json
+            ${{ runner.temp }}/core-contract-logs
+          if-no-files-found: error
+          retention-days: 3
+
   e2e-tests:
     name: End-to-End Tests
     needs: [ build-rustfs-debug-binary ]
-    runs-on: sm-standard-2
+    # Retain the producer image and fixture paths until the runtime pilot passes.
+    runs-on: sm-standard-4
     timeout-minutes: 30
     steps:
       - name: Checkout repository
@@ -743,16 +962,17 @@ jobs:
         with:
           persist-credentials: false
 
-      # Full setup with dependency caching: the smoke-suite step below
-      # compiles the e2e_test crate, which pulls in most of the workspace.
-      # Without a restored cache this was a cold multi-GB build on every run.
-      - name: Setup Rust environment
-        uses: ./.github/actions/setup
+      - name: Install verification runtime
+        uses: ./.github/actions/test-runtime
+
+      - name: Download E2E test binaries
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
-          rust-version: stable
-          cache-shared-key: ci-dev
-          cache-save-if: 'false'
-          install-build-packaging-tools: 'false'
+          name: e2e-test-binaries
+          path: ${{ runner.temp }}
+
+      - name: Verify E2E archive identity
+        run: python3 scripts/ci_test_artifact.py verify --configuration e2e --archive "$RUNNER_TEMP/e2e-tests.tar.zst"
 
       - name: Set up Python
         uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
@@ -781,20 +1001,15 @@ jobs:
       # Build the e2e test graph once. The archive is reused by the smoke
       # selection guard, security exact-count check, and run below, avoiding a
       # second compile of the same e2e_test target on cold runners (backlog#1645).
-      - name: Archive e2e smoke test binaries
+      - name: Verify e2e smoke test membership
         env:
-          NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
+          NEXTEST_ARCHIVE: ${{ runner.temp }}/e2e-tests.tar.zst
           NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-smoke-list.json
         run: |
-          cargo nextest archive --profile e2e-smoke -p e2e_test --archive-file "${NEXTEST_ARCHIVE}"
-          cargo nextest list --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" --message-format json > "${NEXTEST_LISTING}"
+          cargo nextest list --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json > "${NEXTEST_LISTING}"
           python3 ./scripts/check_test_wiring.py --check-profile e2e-smoke "${NEXTEST_LISTING}"
           ./scripts/check_security_smoke_count.sh check "${NEXTEST_LISTING}"
 
-      # Keep E2E documentation coverage with its already-built package graph.
-      - name: Run E2E documentation tests
-        run: cargo test -p e2e_test --doc
-
       # PR smoke subset of the in-repo e2e suite (backlog#1149 ci-4). The
       # profile.e2e-smoke default-filter in .config/nextest.toml is the single
       # wiring mechanism for e2e tests in CI — extend that filter instead of
@@ -802,34 +1017,40 @@ jobs:
       # random port and reuses the downloaded debug binary above.
       - name: Run e2e smoke suite
         env:
-          NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
+          NEXTEST_ARCHIVE: ${{ runner.temp }}/e2e-tests.tar.zst
           RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-smoke-logs
           SAMPLER_INTERVAL_SECS: "15"
         run: |
           ./scripts/ci/resource_sampler.sh start e2e-smoke
           trap './scripts/ci/resource_sampler.sh stop' EXIT
-          python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" \
+          python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json --workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" \
             --status-level all --final-status-level all --failure-output final
 
+      - name: Verify e2e smoke execution completeness
+        run: python3 scripts/ci_test_artifact.py evidence "$RUNNER_TEMP/rustfs-e2e-smoke-list.json" target/nextest/e2e-smoke/junit.xml
+
       - name: Upload e2e smoke diagnostics
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: e2e-smoke-diagnostics-${{ github.run_number }}
+          name: e2e-smoke-diagnostics-${{ github.run_number }}-${{ github.run_attempt }}
           path: |
             ${{ runner.temp }}/rustfs-e2e-smoke-logs/
             ${{ runner.temp }}/rustfs-e2e-smoke-list.json
-            artifacts/test-and-lint
+            artifacts/workspace
           if-no-files-found: warn
 
       - name: Upload e2e smoke JUnit report
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: e2e-smoke-junit-${{ github.run_number }}
+          name: e2e-smoke-junit-${{ github.run_number }}-${{ github.run_attempt }}
           path: target/nextest/e2e-smoke/junit.xml
           if-no-files-found: warn
 
+      - name: Install tool build dependencies
+        run: sudo apt-get install -y build-essential cmake pkg-config
+
       - name: Install s3s-e2e test tool
         uses: taiki-e/cache-cargo-install-action@7447f04c51f2ba27ca35e7f1e28fab848c5b3ba7 # v2
         with:
@@ -851,7 +1072,7 @@ jobs:
         if: failure()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: e2e-test-logs-${{ github.run_number }}
+          name: e2e-test-logs-${{ github.run_number }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/rustfs-e2e-*/rustfs.log
           retention-days: 3
 
@@ -868,7 +1089,8 @@ jobs:
       github.event_name == 'merge_group' ||
       (github.event_name == 'push' && github.ref == 'refs/heads/main')
     needs: [ build-rustfs-debug-binary ]
-    runs-on: sm-standard-2
+    # Retain the producer image and fixture paths until the runtime pilot passes.
+    runs-on: sm-standard-4
     timeout-minutes: 55
     steps:
       - name: Checkout repository
@@ -876,13 +1098,17 @@ jobs:
         with:
           persist-credentials: false
 
-      - name: Setup Rust environment
-        uses: ./.github/actions/setup
+      - name: Install verification runtime
+        uses: ./.github/actions/test-runtime
+
+      - name: Download E2E test binaries
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
-          rust-version: stable
-          cache-shared-key: ci-dev
-          cache-save-if: 'false'
-          install-build-packaging-tools: 'false'
+          name: e2e-test-binaries
+          path: ${{ runner.temp }}
+
+      - name: Verify E2E archive identity
+        run: python3 scripts/ci_test_artifact.py verify --configuration e2e --archive "$RUNNER_TEMP/e2e-tests.tar.zst"
 
       - name: Install network fault-injection tools
         run: |
@@ -981,7 +1207,7 @@ jobs:
         env:
           NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-full-list.json
         run: |
-          cargo nextest list --profile e2e-full -p e2e_test --message-format json > "${NEXTEST_LISTING}"
+          cargo nextest list --profile e2e-full --archive-file "$RUNNER_TEMP/e2e-tests.tar.zst" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json > "${NEXTEST_LISTING}"
           python3 ./scripts/check_test_wiring.py --check-profile e2e-full "${NEXTEST_LISTING}"
 
       # Full single-node e2e lane (backlog#1149 ci-5). The e2e-full
@@ -995,13 +1221,16 @@ jobs:
           RUSTFS_E2E_STARTUP_CAS_BUILD_MANIFEST: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json
           RUSTFS_E2E_STARTUP_CAS_ARTIFACT_DIR: ${{ runner.temp }}/rustfs-startup-cas-evidence
           RUSTFS_HEAL_CHAOS_LOG_ROOT: ${{ runner.temp }}/rustfs-heal-chaos-logs
-        run: python3 scripts/e2e_binary.py run --binary "$RUSTFS_E2E_STARTUP_CAS_BINARY" --features e2e-test-hooks -- cargo nextest run --profile e2e-full -p e2e_test
+        run: python3 scripts/e2e_binary.py run --binary "$RUSTFS_E2E_STARTUP_CAS_BINARY" --features e2e-test-hooks -- cargo nextest run --profile e2e-full --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json --workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target"
+
+      - name: Verify e2e full execution completeness
+        run: python3 scripts/ci_test_artifact.py evidence "$RUNNER_TEMP/rustfs-e2e-full-list.json" target/nextest/e2e-full/junit.xml
 
       - name: Upload junit
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: e2e-full-junit-${{ github.run_number }}
+          name: e2e-full-junit-${{ github.run_number }}-${{ github.run_attempt }}
           path: |
             target/nextest/e2e-full/junit.xml
             ${{ runner.temp }}/rustfs-e2e-full-list.json
@@ -1011,7 +1240,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: fresh-startup-cas-evidence-${{ github.run_number }}
+          name: fresh-startup-cas-evidence-${{ github.run_number }}-${{ github.run_attempt }}
           path: |
             ${{ runner.temp }}/rustfs-startup-cas-evidence
             ${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json
@@ -1022,7 +1251,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: e2e-full-heal-logs-${{ github.run_number }}
+          name: e2e-full-heal-logs-${{ github.run_number }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/rustfs-heal-chaos-logs
           if-no-files-found: warn
           retention-days: 7
@@ -1082,13 +1311,13 @@ jobs:
         if: failure()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: e2e-test-logs-rio-v2-${{ github.run_number }}
+          name: e2e-test-logs-rio-v2-${{ github.run_number }}-${{ github.run_attempt }}
           path: /tmp/rustfs.log
           retention-days: 3
 
   s3-implemented-tests:
     name: S3 Implemented Tests
-    needs: [ build-rustfs-debug-binary, e2e-tests ]
+    needs: [ build-rustfs-debug-binary ]
     runs-on: ubuntu-latest
     timeout-minutes: 60
     steps:
@@ -1125,7 +1354,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: s3tests-implemented-${{ github.run_number }}
+          name: s3tests-implemented-${{ github.run_number }}-${{ github.run_attempt }}
           path: artifacts/s3tests-single/**
           if-no-files-found: ignore
           retention-days: 3
@@ -1162,7 +1391,7 @@ jobs:
     # prebuilt debug binary (no cargo build), and s3-implemented-tests — which
     # already waits on e2e-tests — finishes later anyway, so a green PR's total
     # wall clock is unchanged.
-    needs: [ build-rustfs-debug-binary, e2e-tests ]
+    needs: [ build-rustfs-debug-binary ]
     runs-on: ubuntu-latest
     timeout-minutes: 30
     steps:
@@ -1207,7 +1436,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: s3tests-lifecycle-behavior-${{ github.run_number }}
+          name: s3tests-lifecycle-behavior-${{ github.run_number }}-${{ github.run_attempt }}
           path: artifacts/s3tests-single/**
           if-no-files-found: ignore
           retention-days: 3
@@ -1219,8 +1448,13 @@ jobs:
       - classify-changes
       - typos
       - quick-checks
-      - test-and-lint
+      - workspace-lint-and-build
+      - workspace-tests
+      - workspace-shards
+      - workspace-shard-evidence
+      - core-contract
       - connect-feature-boundaries
+      - fips-provider
       - test-and-lint-rio-v2
       - test-and-lint-protocols
       - build-rustfs-debug-binary
@@ -1248,10 +1482,13 @@ jobs:
     needs:
       - classify-changes
       - connect-feature-boundaries
+      - fips-provider
       - required-checks
       - typos
       - quick-checks
-      - test-and-lint
+      - workspace-lint-and-build
+      - workspace-tests
+      - core-contract
       - test-and-lint-rio-v2
       - test-and-lint-protocols
       - build-rustfs-debug-binary

.github/workflows/docker.yml

@@ -329,13 +329,75 @@ jobs:
           echo "  - Create latest: $create_latest"
           echo "  - Source ref: $source_ref"
 
+  # Consume the trusted release certification for the exact version this
+  # workflow wraps and verify the certified input digests. A manual dispatch
+  # without that certification fails here and pushes no official tag; only
+  # results from the trusted verification sources carry release authority.
+  release-certification:
+    name: Release Certification
+    needs: build-check
+    if: needs.build-check.outputs.should_build == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      actions: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+          ref: ${{ needs.build-check.outputs.source_ref }}
+
+      - name: Download and verify release certification
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
+          TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
+          VERSION: ${{ needs.build-check.outputs.version }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p certification
+          run_id="$TRIGGERING_RUN_ID"
+          if [[ -z "$run_id" ]]; then
+            run_id="$(gh api --method GET \
+              "repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
+              -f branch="$VERSION" -f status=success -F per_page=1 \
+              --jq '.workflow_runs[0].id // empty' || true)"
+          fi
+          if [[ -z "$run_id" ]]; then
+            echo "No certified Build and Release run found for ${VERSION}; failing closed" >&2
+            exit 1
+          fi
+          if ! gh run download "$run_id" -n release-certification -D certification; then
+            echo "Build run ${run_id} published no release-certification artifact; failing closed" >&2
+            exit 1
+          fi
+          python3 scripts/release/release_gate.py verify-promotion \
+            --plan .config/release-validation-plan.json \
+            --candidate certification/candidate.json \
+            --summary certification/release-gate-output \
+            --artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
+            --artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
+
+          # Input digest binding: every zip this image wraps must be the
+          # certified byte-for-byte content, never a mutable latest pointer.
+          while IFS=$'\t' read -r name location expected; do
+            echo "Verifying certified input ${name}"
+            actual="$(curl -fsSL "$location" | sha256sum | awk '{print $1}')"
+            if [[ "$actual" != "$expected" ]]; then
+              echo "Certified input ${name} does not match its recorded digest" >&2
+              exit 1
+            fi
+          done < <(jq -r '.deliverables[] | select(.name | contains("linux")) | [.name, .location, .sha256] | @tsv' certification/candidate.json)
+
   # Build multi-arch Docker images
   # Strategy: Build images using pre-built binaries from dl.rustfs.com
   # Supports both release and dev channel binaries based on build context
   # Only runs when should_build is true (which includes workflow success check)
   build-docker:
     name: Build Docker Images
-    needs: build-check
+    needs: [ build-check, release-certification ]
     if: needs.build-check.outputs.should_build == 'true'
     runs-on: ubuntu-latest
     timeout-minutes: 60
@@ -443,6 +505,14 @@ jobs:
           # Output tags
           echo "tags=$TAGS" >> "$GITHUB_OUTPUT"
 
+          # Staging identity for this run's promotion: official tags are only
+          # created after the staged image verifies against the certification,
+          # and they point at this same digest (no rebuild-after-verify).
+          STAGING_TAG="staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${VARIANT_SUFFIX}"
+          STAGING_TAGS="${{ env.REGISTRY_DOCKERHUB }}:${STAGING_TAG},${{ env.REGISTRY_GHCR }}:${STAGING_TAG},${{ env.REGISTRY_QUAY }}:${STAGING_TAG}"
+          echo "staging_tags=$STAGING_TAGS" >> "$GITHUB_OUTPUT"
+          echo "staging_tag=$STAGING_TAG" >> "$GITHUB_OUTPUT"
+
           # Generate labels
           LABELS="org.opencontainers.image.title=RustFS"
           LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system"
@@ -467,7 +537,9 @@ jobs:
           file: ${{ matrix.file }}
           platforms: ${{ env.DOCKER_PLATFORMS }}
           push: ${{ needs.build-check.outputs.should_push == 'true' }}
-          tags: ${{ steps.meta.outputs.tags }}
+          # Push to staging tags only: official tags are created by the
+          # certification-gated promotion step below from this same digest.
+          tags: ${{ steps.meta.outputs.staging_tags }}
           labels: ${{ steps.meta.outputs.labels }}
           # No layer cache. This build compiles nothing — it downloads a
           # release zip and runs apk/apt — so the cache could only save the
@@ -493,6 +565,114 @@ jobs:
           no-cache: false
           pull: true
 
+      # A later-built wrapper: the staged image must complete this platform's
+      # own binary identity verification before any official tag is written.
+      # It may not inherit an unverified package's conclusion.
+      - name: Verify staged image against certification
+        if: needs.build-check.outputs.should_push == 'true'
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
+          TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
+          VERSION: ${{ needs.build-check.outputs.version }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p certification
+          run_id="$TRIGGERING_RUN_ID"
+          if [[ -z "$run_id" ]]; then
+            run_id="$(gh api --method GET \
+              "repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
+              -f branch="$VERSION" -f status=success -F per_page=1 \
+              --jq '.workflow_runs[0].id // empty' || true)"
+          fi
+          gh run download "$run_id" -n release-certification -D certification
+
+          STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
+          case "${{ matrix.variant }}" in
+            musl) libc="musl" ;;
+            glibc) libc="gnu" ;;
+          esac
+          for platform in linux/amd64 linux/arm64; do
+            case "$platform" in
+              linux/amd64) arch_sub="x86_64" ;;
+              linux/arm64) arch_sub="aarch64" ;;
+            esac
+            docker pull --platform "$platform" "$STAGING_REF"
+            container="$(docker create --platform "$platform" "$STAGING_REF")"
+            docker cp "$container:/usr/bin/rustfs" "$RUNNER_TEMP/staged-rustfs"
+            docker rm "$container" >/dev/null
+            actual="$(sha256sum "$RUNNER_TEMP/staged-rustfs" | awk '{print $1}')"
+            expected="$(jq -r --arg needle "linux-${arch_sub}-${libc}" \
+              '[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
+              certification/candidate.json)"
+            if [[ -z "$expected" || "$actual" != "$expected" ]]; then
+              echo "Staged image binary for ${platform} does not match the certified binary digest" >&2
+              exit 1
+            fi
+            python3 scripts/release/release_gate.py verify-promotion \
+              --plan .config/release-validation-plan.json \
+              --candidate certification/candidate.json \
+              --summary certification/release-gate-output \
+              --artifact "$RUNNER_TEMP/staged-rustfs" \
+              --artifact-role binary
+          done
+
+      - name: Promote staged image to official tags
+        if: needs.build-check.outputs.should_push == 'true'
+        shell: bash
+        run: |
+          set -euo pipefail
+          STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
+          IFS=',' read -r -a official_tags <<< "${{ steps.meta.outputs.tags }}"
+          for tag in "${official_tags[@]}"; do
+            docker buildx imagetools create -t "$tag" "$STAGING_REF"
+            echo "Promoted $tag -> $STAGING_REF"
+          done
+
+      # Downstream consumers (Helm) verify this record instead of inheriting
+      # an unverified wrapper's conclusion.
+      - name: Record image promotion
+        if: needs.build-check.outputs.should_push == 'true'
+        shell: bash
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
+          TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
+          VERSION: ${{ needs.build-check.outputs.version }}
+        run: |
+          set -euo pipefail
+          mkdir -p certification
+          run_id="$TRIGGERING_RUN_ID"
+          if [[ -z "$run_id" ]]; then
+            run_id="$(gh api --method GET \
+              "repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
+              -f branch="$VERSION" -f status=success -F per_page=1 \
+              --jq '.workflow_runs[0].id // empty' || true)"
+          fi
+          gh run download "$run_id" -n release-certification -D certification
+          STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
+          digest="$(docker buildx imagetools inspect "$STAGING_REF" --format '{{json .Manifest}}' | jq -r '.digest')"
+          candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
+          jq -n \
+            --arg version "$VERSION" \
+            --arg digest "$digest" \
+            --arg staging_ref "$STAGING_REF" \
+            --arg candidate_sha256 "$candidate_sha" \
+            --arg tags "${{ steps.meta.outputs.tags }}" \
+            --slurpfile candidate certification/candidate.json \
+            '{schema_version: 1, version: $version, image_digest: $digest, staging_ref: $staging_ref,
+              promoted_tags: ($tags | split(",")), candidate_sha256: $candidate_sha256,
+              binary_sha256s: ($candidate[0].deliverables | map(.binary_sha256) | unique)}' \
+            > image-promotion-record.json
+
+      - name: Upload image promotion record
+        if: needs.build-check.outputs.should_push == 'true'
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: image-promotion-record${{ matrix.suffix }}
+          path: image-promotion-record.json
+          if-no-files-found: error
+          retention-days: 30
+
   # Note: Manifest creation is no longer needed as we only build one variant
   # Multi-arch manifests are automatically created by docker/build-push-action
 

.github/workflows/e2e-distributed.yml

@@ -17,8 +17,8 @@
 # Each selected test starts a real localhost cluster via
 # `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
 # case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
-# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
-# nightly runs, and manual dispatches all execute the same fail-closed suite.
+# `[profile.e2e-distributed]` in `.config/nextest.toml`. The nightly schedule
+# and manual dispatches execute the same fail-closed suite.
 # Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
 #
 # Isolated pool filesystems: expand/decommission/rebalance cases require
@@ -33,32 +33,6 @@
 name: e2e-distributed
 
 on:
-  pull_request:
-    paths:
-      - "Cargo.lock"
-      - "Cargo.toml"
-      - ".config/nextest.toml"
-      - ".config/e2e-distributed-selection.txt"
-      - ".github/workflows/e2e-distributed.yml"
-      - "crates/audit/**"
-      - "crates/common/**"
-      - "crates/config/**"
-      - "crates/e2e_test/**"
-      - "crates/ecstore/**"
-      - "crates/filemeta/**"
-      - "crates/heal/**"
-      - "crates/iam/**"
-      - "crates/lock/**"
-      - "crates/madmin/**"
-      - "crates/notify/**"
-      - "crates/replication/**"
-      - "crates/s3-client/**"
-      - "crates/s3-ops/**"
-      - "crates/s3-types/**"
-      - "crates/scanner/**"
-      - "crates/storage-api/**"
-      - "crates/utils/**"
-      - "rustfs/**"
   workflow_dispatch:
     inputs:
       filter:

.github/workflows/e2e-upgrade.yml

@@ -12,30 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Single-node upgrade and rollback lane.
+#
+# The weekly Monday schedule, version tags, and workflow_dispatch run this
+# suite. Distributed upgrade cases stay on the nightly e2e-distributed lane.
+
 name: Upgrade Compatibility
 
 on:
-  pull_request:
-    paths:
-      - ".github/workflows/e2e-upgrade.yml"
-      - "crates/e2e_test/src/common.rs"
-      - "crates/e2e_test/src/distributed/**"
-      - "crates/e2e_test/src/fake_s3_target/**"
-      - "crates/e2e_test/src/lib.rs"
-      - "crates/e2e_test/src/replication_extension_test.rs"
-      - "crates/e2e_test/src/rpc_fault_proxy.rs"
-      - "crates/e2e_test/src/upgrade_compatibility_test.rs"
-      - "crates/ecstore/**"
-      - "crates/filemeta/**"
-      - "crates/heal/**"
-      - "crates/kms/**"
-      - "crates/storage-api/**"
-      - "rustfs/**"
-      - "Cargo.lock"
   push:
     tags:
       - "[0-9]*.[0-9]*.[0-9]*"
   schedule:
+    # 03:17 UTC every Monday.
     - cron: "17 3 * * 1"
   workflow_dispatch:
 
@@ -44,7 +33,7 @@ permissions:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+  cancel-in-progress: false
 
 env:
   CARGO_TERM_COLOR: always

.github/workflows/functional-chain-health.yml

@@ -12,44 +12,13 @@ concurrency:
 jobs:
   collect:
     if: github.repository == 'rustfs/rustfs'
-    runs-on: smoke-testing
+    # Health remains available when either laboratory fleet is offline.
+    runs-on: ubuntu-latest
     timeout-minutes: 15
     steps:
       - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
-      - name: Ensure GitHub CLI
-        shell: bash
-        run: |
-          if command -v gh >/dev/null 2>&1; then
-            gh --version
-            exit 0
-          fi
-
-          gh_version="2.101.0"
-          case "$(uname -m)" in
-            x86_64) gh_arch="amd64" ;;
-            aarch64|arm64) gh_arch="arm64" ;;
-            *)
-              echo "Unsupported Linux architecture for GitHub CLI: $(uname -m)" >&2
-              exit 1
-              ;;
-          esac
-
-          gh_install_root="${RUNNER_TEMP:-$PWD}/gh-cli"
-          rm -rf "$gh_install_root"
-          mkdir -p "$gh_install_root/bin"
-          archive="$gh_install_root/gh.tar.gz"
-          curl --fail --location --retry 3 \
-            "https://github.com/cli/cli/releases/download/v${gh_version}/gh_${gh_version}_linux_${gh_arch}.tar.gz" \
-            --output "$archive"
-          tar -xzf "$archive" -C "$gh_install_root"
-          install -m 0755 \
-            "$gh_install_root/gh_${gh_version}_linux_${gh_arch}/bin/gh" \
-            "$gh_install_root/bin/gh"
-          echo "$gh_install_root/bin" >> "$GITHUB_PATH"
-          export PATH="$gh_install_root/bin:$PATH"
-          gh --version
       - name: Collect and publish verified chain health
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}

.github/workflows/helm-package.yml

@@ -30,9 +30,102 @@ permissions:
   contents: read
 
 jobs:
+  # Consume the same trusted release certification as every other publication
+  # exit and verify the image promotion record of the Docker workflow. The
+  # chart wraps no binary of its own, so it may only reference an image whose
+  # platform verification already passed; it must not inherit an unverified
+  # package's conclusion. A run without that evidence fails closed here and
+  # never reaches the chart repository.
+  release-certification:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      actions: read
+    outputs:
+      version: ${{ steps.version.outputs.raw_tag }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Normalize release version
+        id: version
+        env:
+          RAW_INPUT: ${{ github.event.inputs.version }}
+          RAW_BRANCH: ${{ github.event.workflow_run.head_branch }}
+        run: |
+          set -euo pipefail
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            RAW="$RAW_INPUT"
+          else
+            RAW="$RAW_BRANCH"
+          fi
+          case "$RAW" in
+            refs/tags/*) RAW_TAG="${RAW#refs/tags/}" ;;
+            *) RAW_TAG="$RAW" ;;
+          esac
+          echo "raw_tag=$RAW_TAG" >> "$GITHUB_OUTPUT"
+
+      - name: Download and verify release certification and image record
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
+          TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
+          RAW_TAG: ${{ steps.version.outputs.raw_tag }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p certification image-record
+
+          build_run="$TRIGGERING_RUN_ID"
+          if [[ -z "$build_run" ]]; then
+            build_run="$(gh api --method GET \
+              "repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
+              -f branch="$RAW_TAG" -f status=success -F per_page=1 \
+              --jq '.workflow_runs[0].id // empty' || true)"
+          fi
+          if [[ -z "$build_run" ]]; then
+            echo "No certified Build and Release run found for ${RAW_TAG}; failing closed" >&2
+            exit 1
+          fi
+          if ! gh run download "$build_run" -n release-certification -D certification; then
+            echo "Build run ${build_run} published no release-certification artifact; failing closed" >&2
+            exit 1
+          fi
+          python3 scripts/release/release_gate.py verify-promotion \
+            --plan .config/release-validation-plan.json \
+            --candidate certification/candidate.json \
+            --summary certification/release-gate-output \
+            --artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
+            --artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
+
+          docker_run="$(gh api --method GET \
+            "repos/${GITHUB_REPOSITORY}/actions/workflows/docker.yml/runs" \
+            -f branch="$RAW_TAG" -f status=success -F per_page=1 \
+            --jq '.workflow_runs[0].id // empty' || true)"
+          if [[ -z "$docker_run" ]] || ! gh run download "$docker_run" -n image-promotion-record -D image-record; then
+            echo "No image promotion record for ${RAW_TAG}; a chart may not wrap an unverified image" >&2
+            exit 1
+          fi
+          candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
+          record_sha="$(jq -r '.candidate_sha256 // empty' image-record/image-promotion-record.json)"
+          if [[ "$record_sha" != "$candidate_sha" ]]; then
+            echo "Image promotion record does not belong to this certification" >&2
+            exit 1
+          fi
+          overlap="$(jq -r --slurpfile candidate certification/candidate.json \
+            '[.binary_sha256s[]] - ($candidate[0].deliverables | map(.binary_sha256)) | length' \
+            image-record/image-promotion-record.json)"
+          if [[ "$overlap" -ne 0 ]]; then
+            echo "Image promotion record carries binary digests outside the certification" >&2
+            exit 1
+          fi
+
   build-helm-package:
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    needs: [ release-certification ]
     if: |
       (github.event_name == 'workflow_dispatch' && !contains(github.event.inputs.version, '-preview')) ||
       (

.github/workflows/nightly-gnu.yml

@@ -87,18 +87,206 @@ jobs:
           install-build-packaging-tools: 'false'
           install-test-tools: 'false'
 
+      - name: Download static console assets
+        shell: bash
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          # Packaging tools are off on this lane, so setup does not install unzip.
+          if ! command -v unzip >/dev/null 2>&1; then
+            sudo apt-get update -qq
+            sudo apt-get install -y -qq unzip
+          fi
+          if ! command -v unzip >/dev/null 2>&1; then
+            echo "unzip is required to extract console assets" >&2
+            exit 1
+          fi
+
+          mkdir -p ./rustfs/static
+
+          verify_sha256() {
+            local expected="$1"
+            local file="$2"
+            local actual=""
+            local actual_line=""
+
+            if command -v sha256sum >/dev/null 2>&1; then
+              actual_line=$(sha256sum -- "$file") || return 1
+            elif command -v shasum >/dev/null 2>&1; then
+              actual_line=$(shasum -a 256 -- "$file") || return 1
+            else
+              echo "No SHA-256 verification tool found" >&2
+              return 1
+            fi
+            actual="${actual_line%%[[:space:]]*}"
+
+            if [ "$actual" = "$expected" ]; then
+              echo "SHA256 verified OK: $actual"
+              return 0
+            else
+              echo "SHA256 mismatch: expected=$expected actual=$actual" >&2
+              return 1
+            fi
+          }
+
+          download_console_assets() {
+            local curl_bin="curl"
+            local python_bin="python3"
+            local console_api="https://api.github.com/repos/rustfs/console/releases/latest"
+            local console_json="console-release.json"
+            local console_url
+            local console_sha256
+            local curl_auth_args=()
+            console_tag=""
+
+            chmod +w ./rustfs/static/LICENSE || true
+
+            if ! command -v "$python_bin" >/dev/null 2>&1; then
+              python_bin="python"
+            fi
+            if ! command -v "$python_bin" >/dev/null 2>&1; then
+              echo "No Python interpreter found for release metadata parsing" >&2
+              return 1
+            fi
+
+            if [[ -n "${GITHUB_TOKEN:-}" ]]; then
+              curl_auth_args=(-H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28")
+            fi
+
+            "$curl_bin" "${curl_auth_args[@]}" --fail -L "$console_api" \
+              -o "$console_json" --retry 3 --retry-delay 5 --max-time 300 || return 1
+
+            read -r console_tag console_url console_sha256 < <("$python_bin" - "$console_json" <<'PY'
+          import json
+          import re
+          import sys
+
+          with open(sys.argv[1], encoding="utf-8") as handle:
+              release = json.load(handle)
+
+          tag = release.get("tag_name", "") or "unknown"
+
+          for asset in release.get("assets", []):
+              name = asset.get("name", "")
+              digest = asset.get("digest", "")
+              url = asset.get("browser_download_url", "")
+              if name.startswith("rustfs-console-") and name.endswith(".zip") and digest.startswith("sha256:") and url:
+                  sha256 = digest.split(":", 1)[1]
+                  if not re.fullmatch(r"[0-9a-fA-F]{64}", sha256):
+                      raise SystemExit(f"console zip asset has invalid sha256 digest: {sha256}")
+                  sys.stdout.buffer.write(f"{tag} {url} {sha256}\n".encode("utf-8"))
+                  break
+          else:
+              raise SystemExit("no console zip asset with sha256 digest found")
+          PY
+            ) || return 1
+
+            if [[ -z "$console_url" || -z "$console_sha256" ]]; then
+              echo "Console release metadata is missing URL or SHA-256 digest" >&2
+              return 1
+            fi
+
+            echo "Console release: ${console_tag}"
+            echo "Downloading console asset: ${console_url}"
+            "$curl_bin" --fail -L "$console_url" -o console.zip --retry 3 --retry-delay 5 --max-time 300 || return 1
+            verify_sha256 "$console_sha256" console.zip || return 2
+            unzip -o console.zip -d ./rustfs/static || return 2
+          }
+
+          if download_console_assets; then
+            rm -f console.zip console-release.json
+          else
+            status=$?
+            rm -f console.zip console-release.json
+            if [[ "$status" -eq 2 ]]; then
+              echo "Console asset integrity verification failed" >&2
+            fi
+            echo "Failed to download verified console assets" >&2
+            exit 1
+          fi
+
+          if [[ ! -s ./rustfs/static/index.html ]]; then
+            echo "Console asset archive is missing static/index.html" >&2
+            exit 1
+          fi
+
+          asset_count=$(find ./rustfs/static -type f | wc -l | tr -d '[:space:]')
+          echo "Console assets ready: version=${console_tag:-unknown}, ${asset_count} files extracted to ./rustfs/static"
+
       - name: Build RustFS
         # Pin the glibc baseline (`<target>.<glibc>` for cargo-zigbuild) so
         # nightlies run on older distros such as RHEL/Rocky 9 (glibc 2.34); #8062.
-        run: cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bins
+        shell: bash
+        run: |
+          # Force rebuild by touching build.rs so a restored cache cannot
+          # reuse a rustfs artifact compiled before these console assets existed.
+          touch rustfs/build.rs
+          cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bin rustfs
 
       - name: Verify glibc symbol baseline
         run: |
           python3 scripts/release/check_glibc.py 2.31 \
-            target/x86_64-unknown-linux-gnu/release/rustfs \
-            target/x86_64-unknown-linux-gnu/release/rustfs-cli
+            target/x86_64-unknown-linux-gnu/release/rustfs
           target/x86_64-unknown-linux-gnu/release/rustfs --version | grep -Fx 'build target : x86_64-unknown-linux-gnu'
 
+      - name: Verify console
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          find_free_port() {
+            python3 - <<'PY'
+          import socket
+
+          with socket.socket() as sock:
+              sock.bind(("127.0.0.1", 0))
+              print(sock.getsockname()[1])
+          PY
+          }
+
+          binary="target/x86_64-unknown-linux-gnu/release/rustfs"
+          data_dir="$(mktemp -d)"
+          log_file="${RUNNER_TEMP}/rustfs-console-smoke.log"
+          server_pid=""
+          trap '
+            if [[ -n "${server_pid:-}" ]]; then
+              kill "$server_pid" 2>/dev/null || true
+              wait "$server_pid" 2>/dev/null || true
+            fi
+            rm -rf "$data_dir"
+          ' EXIT
+
+          if [[ ! -x "$binary" ]]; then
+            echo "rustfs binary not found: ${binary}" >&2
+            exit 1
+          fi
+
+          api_port="$(find_free_port)"
+          console_port="$(find_free_port)"
+          console_url="http://127.0.0.1:${console_port}/rustfs/console/"
+
+          "$binary" server \
+            --address "127.0.0.1:${api_port}" \
+            --console-enable \
+            --console-address "127.0.0.1:${console_port}" \
+            --access-key console-smoke \
+            --secret-key console-smoke-secret \
+            "$data_dir" >"$log_file" 2>&1 &
+          server_pid=$!
+
+          response=""
+          for _ in {1..40}; do
+            response="$(curl --silent --output /dev/null --write-out '%{http_code} %{content_type}' "$console_url" || true)"
+            if [[ "$response" == 200\ text/html* ]]; then
+              exit 0
+            fi
+            sleep 0.25
+          done
+
+          echo "Console endpoint did not return 200 text/html: $response" >&2
+          cat "$log_file"
+          exit 1
+
       - name: Build DEB package
         id: deb
         shell: bash
@@ -346,12 +534,12 @@ jobs:
           path: ${{ steps.publish.outputs.candidate_file }}
           if-no-files-found: error
 
-      # Publish the deb/rpm pair to the auto-testing repo's `assets` branch so
-      # engineers can download and install the nightly directly. The branch is
-      # a single-commit orphan rewritten on every build, which keeps the repo
-      # small while the latest files stay reachable at stable raw URLs.
       # Publish the deb/rpm pair as assets of the rolling `nightly` release on
       # rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
+      # The dispatch below runs only after this upload and the immutable R2
+      # candidate both succeed. It does not replace rustfs-functional-chain.yml,
+      # which still starts from this workflow's workflow_run completion.
+      # kms-vault-lane and kms-vault-ha-failover stay parallel with this job.
       - name: Publish packages to auto-testing release assets
         env:
           ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
@@ -360,6 +548,90 @@ jobs:
           DEB_DATE: ${{ steps.deb.outputs.deb_date }}
           BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
         run: bash scripts/release/publish_nightly_assets.sh
+
+      # Cross-repo repository_dispatch cannot use GITHUB_TOKEN. The secret is a
+      # PAT or GitHub App token for rustfs/auto-testing: classic `repo` scope,
+      # or fine-grained Contents write (the permission GitHub checks for
+      # POST /repos/{owner}/{repo}/dispatches). A dispatch failure fails this
+      # step, and therefore this job, so the scheduled failure alert fires.
+      # GitHub allows at most 10 top-level client_payload properties, so the
+      # schema 2 candidate object is nested under `candidate` and the release
+      # fields sit beside it.
+      - name: Dispatch nightly candidate to auto-testing
+        if: ${{ steps.publish.outputs.candidate_file != '' }}
+        env:
+          DISPATCH_TOKEN: ${{ secrets.AUTO_TESTING_DISPATCH_TOKEN }}
+          CANDIDATE_FILE: ${{ steps.publish.outputs.candidate_file }}
+          DEB_DATE: ${{ steps.deb.outputs.deb_date }}
+          SOURCE_REPO: ${{ github.repository }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          : "${DISPATCH_TOKEN:?AUTO_TESTING_DISPATCH_TOKEN is required to dispatch rustfs/auto-testing}"
+          : "${CANDIDATE_FILE:?nightly candidate manifest is required}"
+          : "${DEB_DATE:?DEB_DATE is required}"
+          [[ "${DEB_DATE}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]
+          [[ "${SOURCE_REPO}" == "rustfs/rustfs" ]]
+
+          payload="${RUNNER_TEMP}/nightly-candidate-dispatch-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
+          jq -n --slurpfile candidate "${CANDIDATE_FILE}" \
+            --arg event_type "nightly-candidate-ready" \
+            --arg source_repo "${SOURCE_REPO}" \
+            --arg latest_alias_url "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb" \
+            --arg auto_testing_release "https://github.com/rustfs/auto-testing/releases/tag/nightly" \
+            --arg channel "nightly" \
+            --arg deb_date "${DEB_DATE}" \
+            --arg nightly_workflow ".github/workflows/nightly-gnu.yml" \
+            '{
+              event_type: $event_type,
+              source_repo: $source_repo,
+              latest_alias_url: $latest_alias_url,
+              auto_testing_release: $auto_testing_release,
+              channel: $channel,
+              deb_date: $deb_date,
+              nightly_workflow: $nightly_workflow,
+              candidate: $candidate[0]
+            }' > "${payload}"
+          if ! jq -e '
+            .event_type == "nightly-candidate-ready"
+            and .source_repo == "rustfs/rustfs"
+            and .channel == "nightly"
+            and .deb_date == env.DEB_DATE
+            and .nightly_workflow == ".github/workflows/nightly-gnu.yml"
+            and .latest_alias_url == "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb"
+            and .auto_testing_release == "https://github.com/rustfs/auto-testing/releases/tag/nightly"
+            and .candidate.schema == 2
+            and ((.candidate | keys | sort) == ["build_run_attempt","build_run_id","package_sha256","package_url","schema","source_ref","source_sha","workflow_sha"])
+            and ((keys | sort) == ["auto_testing_release","candidate","channel","deb_date","event_type","latest_alias_url","nightly_workflow","source_repo"])
+          ' "${payload}" >/dev/null; then
+            echo "nightly dispatch payload does not match the nightly-candidate-ready contract" >&2
+            exit 1
+          fi
+
+          body="${RUNNER_TEMP}/nightly-candidate-dispatch-body-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
+          jq -n --slurpfile client_payload "${payload}" \
+            '{event_type: "nightly-candidate-ready", client_payload: $client_payload[0]}' > "${body}"
+          response="${RUNNER_TEMP}/nightly-candidate-dispatch-response-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.txt"
+          status="$(curl -sS --retry 3 --connect-timeout 15 --max-time 60 \
+            -o "${response}" -w '%{http_code}' \
+            -X POST \
+            -H "Authorization: Bearer ${DISPATCH_TOKEN}" \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            -H "Content-Type: application/json" \
+            --data-binary @"${body}" \
+            https://api.github.com/repos/rustfs/auto-testing/dispatches)" || {
+            echo "repository_dispatch request to rustfs/auto-testing failed before an HTTP status" >&2
+            exit 1
+          }
+          if [[ "${status}" != "204" ]]; then
+            echo "repository_dispatch to rustfs/auto-testing failed: HTTP ${status}" >&2
+            if [[ -s "${response}" ]]; then
+              cat "${response}" >&2
+            fi
+            exit 1
+          fi
+          echo "dispatched nightly-candidate-ready to rustfs/auto-testing (deb_date=${DEB_DATE}, run=${GITHUB_RUN_ID}, attempt=${GITHUB_RUN_ATTEMPT})"
   # Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
   #
   # RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and

.github/workflows/package.yml

@@ -225,10 +225,57 @@ jobs:
           echo "  Build run ID: $BUILD_RUN_ID"
           echo "  Build run number: $RUN_NUMBER"
 
+  # Consume the trusted release certification of the resolved build run and
+  # verify the certified input digests before any package is wrapped. A run
+  # without that certification fails closed here; packages built later must
+  # complete their own verification and cannot inherit an unverified
+  # conclusion.
+  release-certification:
+    name: Release Certification
+    needs: resolve
+    if: needs.resolve.result == 'success'
+    runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      actions: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Download and verify release certification
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
+          BUILD_RUN_ID: ${{ needs.resolve.outputs.build_run_id }}
+          BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p certification
+          if [[ "$BUILD_TYPE" == "development" ]]; then
+            # Development artifacts are explicitly uncertified and never touch
+            # official or latest channels; only the staging/dev paths remain.
+            jq -n '{schema_version: 1, certified: false, reason: "development artifacts are explicitly uncertified"}' \
+              > certification/development-uncertified.json
+            exit 0
+          fi
+          if ! gh run download "$BUILD_RUN_ID" -n release-certification -D certification; then
+            echo "Build run ${BUILD_RUN_ID} published no release-certification artifact; failing closed" >&2
+            exit 1
+          fi
+          python3 scripts/release/release_gate.py verify-promotion \
+            --plan .config/release-validation-plan.json \
+            --candidate certification/candidate.json \
+            --summary certification/release-gate-output \
+            --artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
+            --artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
+
   # Build DEB and RPM packages for each architecture and libc variant
   package:
     name: Package (${{ matrix.arch }}-${{ matrix.libc }})
-    needs: resolve
+    needs: [ resolve, release-certification ]
     runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
     timeout-minutes: 30
     strategy:
@@ -494,6 +541,55 @@ jobs:
           echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
           echo "✅ RPM built: $RPM_FILE"
 
+      - name: Download certification evidence
+        if: needs.resolve.outputs.build_type != 'development'
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: release-certification
+          path: ./certification
+
+      # Later-built wrapper verification: the binary inside each package must
+      # match the certified binary digest for this platform before anything is
+      # uploaded, and the package itself stays on staging until promotion.
+      - name: Verify wrapped binary against certification
+        if: needs.resolve.outputs.build_type != 'development'
+        shell: bash
+        env:
+          DEB_FILE: ${{ steps.deb.outputs.deb_file }}
+          RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
+          ARTIFACT_NAME: ${{ matrix.artifact_name }}
+        run: |
+          set -euo pipefail
+          expected="$(jq -r --arg needle "$ARTIFACT_NAME" \
+            '[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
+            certification/candidate.json)"
+          if [[ -z "$expected" ]]; then
+            echo "No certified binary digest recorded for ${ARTIFACT_NAME}" >&2
+            exit 1
+          fi
+          if ! command -v rpm2cpio >/dev/null 2>&1; then
+            sudo apt-get update && sudo apt-get install -y rpm2cpio
+          fi
+          if ! command -v cpio >/dev/null 2>&1; then
+            sudo apt-get update && sudo apt-get install -y cpio
+          fi
+          for package_file in "$DEB_FILE" "$RPM_FILE"; do
+            case "$package_file" in
+              *.deb)
+                dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
+                ;;
+              *.rpm)
+                rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
+                ;;
+            esac
+            python3 scripts/release/release_gate.py verify-promotion \
+              --plan .config/release-validation-plan.json \
+              --candidate certification/candidate.json \
+              --summary certification/release-gate-output \
+              --artifact "$RUNNER_TEMP/wrapped-rustfs" \
+              --artifact-role binary
+          done
+
       - name: Upload packages to artifacts
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
@@ -531,10 +627,14 @@ jobs:
           export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
           export AWS_DEFAULT_REGION="auto"
 
+          # Build-phase writes stay on staging for release types: the official
+          # packages/release and packages/latest paths are written only by the
+          # certification-gated promote-packages job, from these same staged
+          # objects. Development packages keep the uncertified dev channel.
           if [[ "$BUILD_TYPE" == "development" ]]; then
             R2_PREFIX="artifacts/rustfs/packages/dev"
           else
-            R2_PREFIX="artifacts/rustfs/packages/release"
+            R2_PREFIX="artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}"
           fi
           R2_PATH="s3://${R2_BUCKET}/${R2_PREFIX}/"
 
@@ -549,18 +649,6 @@ jobs:
 
           echo "✅ Upload complete"
 
-          # Also upload as latest for release/prerelease
-          if [[ "$BUILD_TYPE" == "release" || "$BUILD_TYPE" == "prerelease" ]]; then
-            LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
-            for f in "$DEB_FILE" "$RPM_FILE"; do
-              if [[ -n "$f" && -f "$f" ]]; then
-                echo "Uploading latest: $(basename "$f")"
-                aws s3 cp "$f" "$LATEST_PATH" --endpoint-url "$R2_ENDPOINT" --only-show-errors
-              fi
-            done
-            echo "✅ Latest packages updated"
-          fi
-
       - name: Upload packages to GitHub Release
         if: needs.resolve.outputs.tag != ''
         env:
@@ -614,6 +702,114 @@ jobs:
 
           echo "✅ GitHub release assets updated"
 
+  # The only writer of the official packages/release and packages/latest R2
+  # paths. Promotion re-checks each package's wrapped binary against the
+  # passed certification list and copies the staged objects byte-for-byte:
+  # no rebuild-after-verify, no mutable latest resolution downstream.
+  promote-packages:
+    name: Promote Packages
+    needs: [ resolve, package, release-certification ]
+    # Target-channel lock for the official package paths.
+    concurrency:
+      group: release-channel-r2-packages
+      cancel-in-progress: false
+    if: needs.resolve.outputs.tag != '' && (needs.resolve.outputs.build_type == 'release' || needs.resolve.outputs.build_type == 'prerelease')
+    runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      actions: read
+    env:
+      R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
+      R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
+      R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
+      R2_BUCKET: ${{ secrets.R2_BUCKET }}
+      AWS_EC2_METADATA_DISABLED: true
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Download packages and certification
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          path: ./packages
+          pattern: packages-*
+          merge-multiple: true
+
+      - name: Download certification evidence
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: release-certification
+          path: ./certification
+
+      - name: Verify and promote staged packages
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
+            echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
+            exit 1
+          fi
+
+          STAGING_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
+          RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/release/"
+          LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
+
+          if ! command -v aws >/dev/null 2>&1; then
+            sudo apt-get update && sudo apt-get install -y awscli
+          fi
+          export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
+          export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
+          export AWS_DEFAULT_REGION="auto"
+
+          if ! command -v rpm2cpio >/dev/null 2>&1; then
+            sudo apt-get install -y rpm2cpio
+          fi
+          if ! command -v cpio >/dev/null 2>&1; then
+            sudo apt-get install -y cpio
+          fi
+
+          promoted=0
+          for package_file in packages/*.deb packages/*.rpm; do
+            [[ -f "$package_file" ]] || continue
+            name="$(basename "$package_file")"
+            case "$package_file" in
+              *.deb)
+                dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
+                ;;
+              *.rpm)
+                rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
+                ;;
+            esac
+            python3 scripts/release/release_gate.py verify-promotion \
+              --plan .config/release-validation-plan.json \
+              --candidate certification/candidate.json \
+              --summary certification/release-gate-output \
+              --artifact "$RUNNER_TEMP/wrapped-rustfs" \
+              --artifact-role binary
+
+            staged="$(mktemp)"
+            aws s3 cp "${STAGING_PREFIX}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+            staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
+            built_sha="$(sha256sum "$package_file" | awk '{print $1}')"
+            if [[ "$staged_sha" != "$built_sha" ]]; then
+              echo "❌ Staged package ${name} does not match the verified bytes" >&2
+              exit 1
+            fi
+            aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+            aws s3 cp "$staged" "${LATEST_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+            rm -f "$staged"
+            promoted=$((promoted + 1))
+            echo "✅ Promoted ${name} (${built_sha})"
+          done
+          if [[ "$promoted" -eq 0 ]]; then
+            echo "No staged packages found to promote" >&2
+            exit 1
+          fi
+
   # Summary
   summary:
     name: Summary

.github/workflows/rustfs-fault-tolerance-matrix.yml

@@ -93,6 +93,12 @@ jobs:
             printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
           } >> "${GITHUB_ENV}"
 
+      - name: Checkout environment guard
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+          path: rustfs-tools
+
       - name: Checkout auto-testing scripts
         timeout-minutes: 5
         uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -118,6 +124,13 @@ jobs:
           echo "auto-testing: ${AUTO_TESTING_SHA}"
           df -h /data | tail -1
 
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-matrix.json
+        run: python3 rustfs-tools/scripts/functional_environment.py enter --suite fault-tolerance
+
       - name: Cleanup environment (before)
         timeout-minutes: 5
         if: ${{ inputs.cleanup_before != 'false' }}
@@ -215,7 +228,24 @@ jobs:
           if-no-files-found: warn
 
       - name: Cleanup environment (after)
+        id: environment_cleanup
         timeout-minutes: 5
-        if: ${{ always() && inputs.cleanup_after != 'false' }}
+        if: ${{ always() && inputs.cleanup_after != 'false' && steps.environment.outcome == 'success' }}
         run: |
-          ./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}" || true
+          ./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}"
+
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-matrix.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 rustfs-tools/scripts/functional_environment.py finish --suite fault-tolerance --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Retain environment ownership and cleanup receipt
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: functional-environment-matrix-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/environment-matrix.json
+          if-no-files-found: error

.github/workflows/rustfs-fault-tolerance-test.yml

@@ -50,10 +50,6 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
-  repository_dispatch:
-    # Chain handoff: dispatched when the replication suite finishes, ahead of
-    # the performance suite.
-    types: [rustfs-chain-fault-tolerance]
 
 permissions:
   contents: read
@@ -80,7 +76,7 @@ jobs:
   fault-tolerance-test:
     runs-on: smoke-testing
     timeout-minutes: 60
-    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
         uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -136,6 +132,13 @@ jobs:
           echo "auto-testing: ${AUTO_TESTING_SHA}"
           df -h /data | tail -1
 
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-fault-tolerance.json
+        run: python3 scripts/functional_environment.py enter --suite fault-tolerance
+
       - name: Cleanup environment (before)
         timeout-minutes: 5
         if: ${{ inputs.cleanup_before != 'false' }}
@@ -194,21 +197,21 @@ jobs:
           jq -e --arg testing_sha "${AUTO_TESTING_SHA}" '
             .schema_version == 1 and
             .auto_testing_sha == $testing_sha and
-            .selected_scenarios == ["A", "B", "C", "C2", "D", "E"] and
+            .selected_scenarios == ["A", "B", "C", "C2", "D", "E", "F"] and
             .complete == true and
-            .counts.expected == 44 and
-            .counts.seen == 44 and
+            .counts.expected == 54 and
+            .counts.seen == 54 and
             .counts.harness_errors == 0 and
-            (.expected_cases | length) == 44 and
-            (.expected_cases | unique | length) == 44 and
-            (.seen_cases | length) == 44 and
-            (.seen_cases | unique | length) == 44 and
+            (.expected_cases | length) == 54 and
+            (.expected_cases | unique | length) == 54 and
+            (.seen_cases | length) == 54 and
+            (.seen_cases | unique | length) == 54 and
             (.seen_cases | sort) == (.expected_cases | sort) and
             (.missing_cases | length) == 0 and
             (.harness_errors | length) == 0 and
-            (.cases | length) == 44 and
+            (.cases | length) == 54 and
             ([.cases[].case_id] | sort) == (.expected_cases | sort) and
-            ([.cases[].case_id] | unique | length) == 44 and
+            ([.cases[].case_id] | unique | length) == 54 and
             all(.cases[]; .verdict == "pass" or .verdict == "known-divergence" or .verdict == "UNEXPECTED") and
             .counts.unexpected == ([.cases[] | select(.verdict == "UNEXPECTED")] | length) and
             .counts.known_divergence == ([.cases[] | select(.verdict == "known-divergence")] | length)
@@ -305,58 +308,27 @@ jobs:
           if-no-files-found: warn
 
       - name: Cleanup environment (after)
+        id: environment_cleanup
         timeout-minutes: 5
-        if: ${{ always() && inputs.cleanup_after != 'false' }}
+        if: ${{ always() && inputs.cleanup_after != 'false' && steps.environment.outcome == 'success' }}
         run: |
-          ./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}" || true
+          ./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}"
 
-      - name: "Continue functional chain (next: Performance)"
-        # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup. A failed
-        # handoff retries, then files an alert issue in rustfs/backlog.
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        continue-on-error: true
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          DISPATCHED=0
-          for attempt in 1 2 3; do
-            if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-performance' \
-              -F 'client_payload[from_suite]=fault-tolerance'; then
-              echo "dispatched next suite Performance (attempt ${attempt})"
-              DISPATCHED=1
-              break
-            fi
-            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
-            sleep "${attempt}0"
-          done
-          if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch Performance after 3 attempts" >&2
-            TITLE="[functional][chain] stalled after fault-tolerance (run ${GITHUB_RUN_ID})"
-            BODY_FILE="$(mktemp)"
-            {
-              echo "The functional chain could not hand off from **fault-tolerance** to **Performance** after 3 attempts."
-              echo ""
-              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-performance'"
-              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
-              echo "- Recovery: re-dispatch manually with"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-performance'"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-            } > "${BODY_FILE}"
-            gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test \
-              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
-              || echo "could not file the stall alert issue either; check the token" >&2
-            exit 1
-          fi
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-fault-tolerance.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 scripts/functional_environment.py finish --suite fault-tolerance --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Retain environment ownership and cleanup receipt
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: functional-environment-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/environment-fault-tolerance.json
+          if-no-files-found: error
 
       - name: Notify on failure
         if: failure()
@@ -391,3 +363,13 @@ jobs:
           name: functional-chain-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}/fault-tolerance.json
           if-no-files-found: error
+
+      - name: Enforce suite and report result
+        if: ${{ always() }}
+        env:
+          SUITE_OUTCOME: ${{ steps.test.outcome }}
+          REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: |
+          test "$SUITE_OUTCOME" = success
+          test "$REPORT_OUTCOME" = success
+          python3 scripts/functional_chain_evidence.py verify-report --suite fault-tolerance --report "${FUNCTIONAL_ARTIFACTS_DIR}/suite.log"

.github/workflows/rustfs-functional-chain.yml

@@ -13,12 +13,23 @@
 # limitations under the License.
 
 # Reusable workflows run from this driver's commit in one Actions run. Each
-# suite still runs after an earlier suite fails; the final job requires all twelve.
+# selected suite still runs after an earlier failure; the final job checks its plan.
 name: RustFS Functional Chain
 
 on:
   workflow_dispatch:
     inputs:
+      validation_plan:
+        description: Registered cadence plan (existing suite frequencies are preserved)
+        type: choice
+        options: [nightly, weekly, monthly]
+        required: false
+        default: nightly
+      suites:
+        description: Comma-separated suite IDs, or empty for the complete registered plan
+        type: string
+        required: false
+        default: ''
       build_run_id:
         description: Successful nightly or validation package run; empty for package-only build
         type: string
@@ -216,10 +227,25 @@ jobs:
     outputs:
       manifest: ${{ steps.candidate.outputs.manifest }}
       performance_ready: ${{ steps.perf_probe.outputs.performance_ready }}
+      plan: ${{ steps.selection.outputs.plan }}
+      suites: ${{ steps.selection.outputs.suites }}
+      optional_suites: ${{ steps.selection.outputs.optional_suites }}
     steps:
       - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
+      - name: Select registered suites
+        id: selection
+        env:
+          VALIDATION_PLAN: ${{ inputs.validation_plan || 'nightly' }}
+          VALIDATION_SUITES: ${{ inputs.suites }}
+        run: python3 scripts/validation_registry.py select --output "${RUNNER_TEMP}/suite-plan.json"
+      - name: Retain selected responsibilities
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: functional-plan-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/suite-plan.json
+          if-no-files-found: error
       - name: Resolve published candidate
         id: candidate
         env:
@@ -239,6 +265,7 @@ jobs:
           if-no-files-found: error
 
       - name: Check shared functional fleet runner before scheduling suites
+        if: ${{ contains(fromJSON(steps.selection.outputs.environments), 'functional-fleet') }}
         # The eleven functional suites run on the smoke-testing fleet; a hard
         # requirement. The performance lane runs on pf-testing, which is a
         # separate fleet - probe it without failing the whole chain so an
@@ -248,15 +275,26 @@ jobs:
         run: python3 scripts/check_functional_runners.py smoke-testing
 
       - name: Probe performance fleet runner
+        if: ${{ contains(fromJSON(steps.selection.outputs.suites), 'performance') }}
         id: perf_probe
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          PERFORMANCE_OPTIONAL: ${{ contains(fromJSON(steps.selection.outputs.optional_suites), 'performance') }}
         run: |
-          if python3 scripts/check_functional_runners.py pf-testing; then
-            echo "performance_ready=online" >> "$GITHUB_OUTPUT"
-          else
-            echo "performance_ready=offline" >> "$GITHUB_OUTPUT"
-          fi
+          # TEMPORARY: the pf-testing environment is broken, so the
+          # performance lane is disabled until it is repaired. Forcing
+          # offline keeps the chain on its designed skip path: the lane job
+          # is skipped, aggregate receives --allow-skipped performance
+          # (required=false in scheduled-validations.json), and the report
+          # records the optional-lane skip with its reason.
+          # Restore the probe below once the environment is healthy again.
+          # if python3 scripts/check_functional_runners.py pf-testing; then
+          #   echo "performance_ready=online" >> "$GITHUB_OUTPUT"
+          # else
+          #   echo "performance_ready=offline" >> "$GITHUB_OUTPUT"
+          #   test "$PERFORMANCE_OPTIONAL" = true
+          # fi
+          echo "performance_ready=offline" >> "$GITHUB_OUTPUT"
 
   validation:
     if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha == '' && (inputs.validation_case != '' || inputs.validation_tools_only || inputs.validation_native_tools || inputs.validation_shared_tools || inputs.validation_manager_tools || inputs.validation_cleanup_run_id != '' || inputs.validation_cleanup_retained_evidence) }}
@@ -580,6 +618,10 @@ jobs:
             > "${VALIDATION_ROOT}/shared-fixture-manifest.json"
           printf 'VALIDATION_HEAL_SHARED_DIRECTORY=%s\n' "${SHARED_DIRECTORY}" >> "${GITHUB_ENV}"
 
+      - name: Reject quarantined suite environment before canonical validation
+        if: ${{ !inputs.validation_tools_only && inputs.validation_cleanup_run_id == '' && inputs.validation_case != 'environment.inspect' }}
+        run: python3 scripts/functional_environment.py check --suite heal
+
       - name: Run selected canonical validation case
         if: ${{ !inputs.validation_tools_only && inputs.validation_cleanup_run_id == '' && inputs.validation_case != 'environment.inspect' }}
         env:
@@ -665,7 +707,7 @@ jobs:
 
   upgrade:
     needs: [prepare]
-    if: ${{ always() && needs.prepare.result == 'success' }}
+    if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'upgrade') }}
     uses: ./.github/workflows/rustfs-upgrade-test.yml
     with:
       chain_manifest: ${{ needs.prepare.outputs.manifest }}
@@ -673,7 +715,7 @@ jobs:
 
   s3:
     needs: [prepare, upgrade]
-    if: ${{ always() && needs.prepare.result == 'success' }}
+    if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 's3') }}
     uses: ./.github/workflows/rustfs-s3-compat-test.yml
     with:
       chain_manifest: ${{ needs.prepare.outputs.manifest }}
@@ -681,7 +723,7 @@ jobs:
 
   kms:
     needs: [prepare, s3]
-    if: ${{ always() && needs.prepare.result == 'success' }}
+    if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'kms') }}
     uses: ./.github/workflows/rustfs-kms-test.yml
     with:
       chain_manifest: ${{ needs.prepare.outputs.manifest }}
@@ -689,7 +731,7 @@ jobs:
 
   tier:
     needs: [prepare, kms]
-    if: ${{ always() && needs.prepare.result == 'success' }}
+    if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'tier') }}
     uses: ./.github/workflows/rustfs-tier-test.yml
     with:
       chain_manifest: ${{ needs.prepare.outputs.manifest }}
@@ -697,7 +739,7 @@ jobs:
 
   storage:
     needs: [prepare, tier]
-    if: ${{ always() && needs.prepare.result == 'success' }}
+    if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'storage') }}
     uses: ./.github/workflows/rustfs-storage-test.yml
     with:
       chain_manifest: ${{ needs.prepare.outputs.manifest }}
@@ -705,7 +747,7 @@ jobs:
 
   heal:
     needs: [prepare, storage]
-    if: ${{ always() && needs.prepare.result == 'success' }}
+    if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'heal') }}
     uses: ./.github/workflows/rustfs-heal-test.yml
     with:
       chain_manifest: ${{ needs.prepare.outputs.manifest }}
@@ -713,7 +755,7 @@ jobs:
 
   pool:
     needs: [prepare, heal]
-    if: ${{ always() && needs.prepare.result == 'success' }}
+    if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'pool') }}
     uses: ./.github/workflows/rustfs-pool-expand-test.yml
     with:
       chain_manifest: ${{ needs.prepare.outputs.manifest }}
@@ -721,7 +763,7 @@ jobs:
 
   security:
     needs: [prepare, pool]
-    if: ${{ always() && needs.prepare.result == 'success' }}
+    if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'security') }}
     uses: ./.github/workflows/rustfs-security-test.yml
     with:
       chain_manifest: ${{ needs.prepare.outputs.manifest }}
@@ -729,7 +771,7 @@ jobs:
 
   replication:
     needs: [prepare, security]
-    if: ${{ always() && needs.prepare.result == 'success' }}
+    if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'replication') }}
     uses: ./.github/workflows/rustfs-replication-test.yml
     with:
       chain_manifest: ${{ needs.prepare.outputs.manifest }}
@@ -737,7 +779,7 @@ jobs:
 
   fault-tolerance:
     needs: [prepare, replication]
-    if: ${{ always() && needs.prepare.result == 'success' }}
+    if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'fault-tolerance') }}
     uses: ./.github/workflows/rustfs-fault-tolerance-test.yml
     with:
       chain_manifest: ${{ needs.prepare.outputs.manifest }}
@@ -745,15 +787,15 @@ jobs:
 
   table:
     needs: [prepare, fault-tolerance]
-    if: ${{ always() && needs.prepare.result == 'success' }}
+    if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'table') }}
     uses: ./.github/workflows/rustfs-table-test.yml
     with:
       chain_manifest: ${{ needs.prepare.outputs.manifest }}
     secrets: inherit
 
   performance:
-    needs: [prepare, table]
-    if: ${{ always() && needs.prepare.result == 'success' && needs.prepare.outputs.performance_ready == 'online' }}
+    needs: [prepare]
+    if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'performance') && needs.prepare.outputs.performance_ready == 'online' }}
     uses: ./.github/workflows/rustfs-performance-test.yml
     with:
       chain_manifest: ${{ needs.prepare.outputs.manifest }}
@@ -781,8 +823,10 @@ jobs:
         env:
           CHAIN_MANIFEST: ${{ needs.prepare.outputs.manifest }}
           CHAIN_NEEDS: ${{ toJSON(needs) }}
+          CHAIN_PLAN: ${{ needs.prepare.outputs.plan }}
         run: >-
           python3 scripts/functional_chain_evidence.py summarize
+          --allow-skipped "${{ contains(fromJSON(needs.prepare.outputs.optional_suites || '[]'), 'performance') && needs.prepare.outputs.performance_ready != 'online' && 'performance' || '' }}"
           --directory "${RUNNER_TEMP}/chain-evidence"
           --output "${RUNNER_TEMP}/chain-report.json"
       - name: Retain chain report regardless of test verdict
@@ -799,9 +843,10 @@ jobs:
         env:
           CHAIN_MANIFEST: ${{ needs.prepare.outputs.manifest }}
           CHAIN_NEEDS: ${{ toJSON(needs) }}
+          CHAIN_PLAN: ${{ needs.prepare.outputs.plan }}
         run: >-
           python3 scripts/functional_chain_evidence.py aggregate
-          --allow-skipped "${{ needs.prepare.outputs.performance_ready == 'online' && '' || 'performance' }}"
+          --allow-skipped "${{ contains(fromJSON(needs.prepare.outputs.optional_suites || '[]'), 'performance') && needs.prepare.outputs.performance_ready != 'online' && 'performance' || '' }}"
           --directory "${RUNNER_TEMP}/chain-evidence"
           --output "${RUNNER_TEMP}/chain-complete.json"
       - name: Upload complete-chain evidence

.github/workflows/rustfs-heal-test.yml

@@ -29,11 +29,6 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
-  repository_dispatch:
-    # Chain handoff: dispatched when the storage suite finishes. Heal runs
-    # exactly once per chain; the pool expansion workflow no longer embeds
-    # its own heal pass.
-    types: [rustfs-chain-heal]
 
 permissions:
   contents: read
@@ -60,10 +55,11 @@ env:
 jobs:
   heal-test:
     runs-on: smoke-testing
-    timeout-minutes: 60
+    # 90-minute heal run (see the step below) plus setup, report and cleanup.
+    timeout-minutes: 120
     # Standalone manual run, or one link of the nightly functional chain
     # (storage -> heal -> pool). Pool expansion no longer re-runs heal.
-    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
     steps:
       - name: Checkout chain tooling
         uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -109,6 +105,13 @@ jobs:
           warp --version || true
           df -h /data | tail -1
 
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-heal.json
+        run: python3 scripts/functional_environment.py enter --suite heal
+
       - name: Cleanup environment (before)
         timeout-minutes: 5
         if: ${{ inputs.cleanup_before != 'false' }}
@@ -152,10 +155,9 @@ jobs:
           ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
 
       - name: Run heal test (write -> outage -> heal -> verify)
-        timeout-minutes: 45
+        timeout-minutes: 90
         id: test
-        # Case failures keep the run green: the report and the backlog
-        # issue manager carry the product signal.
+        # Collect reports and backlog evidence before the final suite verdict.
         continue-on-error: true
         run: |
           ./auto-testing/rustfs_heal_test.sh \
@@ -241,9 +243,8 @@ jobs:
             RESULT=success
           fi
 
-          # Product gate: failing steps keep the run green — they are reported
-          # below and tracked in rustfs/backlog. Only harness/environment
-          # breakdowns turn the workflow red.
+          # Preserve harness completion separately; the final verdict checks
+          # every case after retaining reports and cleanup evidence.
           STEPS_TOTAL="$(grep -cE '^\[HEAL-STEP\]' "${LOG_FILE}" 2>/dev/null || true)"
           STEPS_FAIL="$(grep -cE '^\[HEAL-STEP\].* FAIL$' "${LOG_FILE}" 2>/dev/null || true)"
           STEPS_TOTAL=$(( ${STEPS_TOTAL:-0} + 0 )); STEPS_FAIL=$(( ${STEPS_FAIL:-0} + 0 ))
@@ -281,7 +282,7 @@ jobs:
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          # Red only for harness/environment breakdowns; step failures stay green.
+          # Preserve the harness verdict separately from the final case verdict.
           [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
@@ -377,8 +378,9 @@ jobs:
           if-no-files-found: error
 
       - name: Cleanup environment (after)
+        id: environment_cleanup
         timeout-minutes: 5
-        if: ${{ always() && inputs.cleanup_after != 'false' }}
+        if: ${{ always() && inputs.cleanup_after != 'false' && steps.environment.outcome == 'success' }}
         run: |
           set -euo pipefail
           read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
@@ -396,54 +398,21 @@ jobs:
             '
           done
 
-      - name: "Continue functional chain (next: Pool expansion)"
-        # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup. A failed
-        # handoff must never pass silently: it retries, then files an alert
-        # issue in rustfs/backlog so a stalled chain is visible.
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        continue-on-error: true
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          DISPATCHED=0
-          for attempt in 1 2 3; do
-            if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-pool' \
-              -F 'client_payload[from_suite]=heal'; then
-              echo "dispatched next suite Pool expansion (attempt ${attempt})"
-              DISPATCHED=1
-              break
-            fi
-            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
-            sleep "${attempt}0"
-          done
-          if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch Pool expansion after 3 attempts" >&2
-            TITLE="[functional][chain] stalled after heal (run ${GITHUB_RUN_ID})"
-            BODY_FILE="$(mktemp)"
-            {
-              echo "The functional chain could not hand off from **heal** to **Pool expansion** after 3 attempts."
-              echo ""
-              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-pool'"
-              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
-              echo "- Recovery: re-dispatch manually with"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-pool'"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-            } > "${BODY_FILE}"
-            gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test \
-              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
-              || echo "could not file the stall alert issue either; check the token" >&2
-            exit 1
-          fi
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-heal.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 scripts/functional_environment.py finish --suite heal --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Retain environment ownership and cleanup receipt
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: functional-environment-heal-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/environment-heal.json
+          if-no-files-found: error
 
       - name: Notify on failure
         if: failure()
@@ -472,3 +441,13 @@ jobs:
           name: functional-chain-heal-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-heal-${{ github.run_id }}-${{ github.run_attempt }}/heal.json
           if-no-files-found: error
+
+      - name: Enforce suite and report result
+        if: ${{ always() }}
+        env:
+          SUITE_OUTCOME: ${{ steps.test.outcome }}
+          REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: |
+          test "$SUITE_OUTCOME" = success
+          test "$REPORT_OUTCOME" = success
+          python3 scripts/functional_chain_evidence.py verify-report --suite heal --report "${FUNCTIONAL_ARTIFACTS_DIR}/steps.md"

.github/workflows/rustfs-kms-test.yml

@@ -29,9 +29,6 @@ on:
         description: 'Set RUSTFS_KMS_CONFIG_SECRET (runs KMS-107 config sealing)'
         required: false
         type: string
-  repository_dispatch:
-    # Chain handoff: dispatched when the S3 compatibility suite finishes.
-    types: [rustfs-chain-kms]
 
 permissions:
   contents: read
@@ -56,7 +53,7 @@ jobs:
   kms-test:
     runs-on: smoke-testing
     timeout-minutes: 60
-    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
         uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -100,6 +97,13 @@ jobs:
           openssl version
           docker --version || true
 
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-kms.json
+        run: python3 scripts/functional_environment.py enter --suite kms
+
       - name: Cleanup environment (before)
         timeout-minutes: 5
         run: |
@@ -131,8 +135,7 @@ jobs:
       - name: Run KMS suite
         id: test
         timeout-minutes: 45
-        # Case failures keep the run green: the report and the backlog
-        # issue manager carry the product signal.
+        # Collect reports and backlog evidence before the final suite verdict.
         continue-on-error: true
         run: |
           set -euo pipefail
@@ -185,10 +188,8 @@ jobs:
             RESULT=success
           fi
 
-          # Product gate: failing cases keep the run green — they are reported
-          # below and tracked in rustfs/backlog. Only harness/environment
-          # breakdowns (the suite never reached case level, or a wholesale
-          # failure with zero passes) turn the workflow red.
+          # Report harness completion here; the final verdict checks every
+          # case result after diagnostics and cleanup have been retained.
           CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
           CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
           CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
@@ -226,7 +227,7 @@ jobs:
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          # Red only for harness/environment breakdowns; case failures stay green.
+          # Preserve the harness verdict separately from the final case verdict.
           [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
@@ -325,8 +326,9 @@ jobs:
           if-no-files-found: error
 
       - name: Cleanup environment (after)
+        id: environment_cleanup
         timeout-minutes: 5
-        if: always()
+        if: ${{ always() && steps.environment.outcome == 'success' }}
         run: |
           set -euo pipefail
           read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
@@ -344,54 +346,21 @@ jobs:
             '
           done
 
-      - name: "Continue functional chain (next: Tier)"
-        # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup. A failed
-        # handoff must never pass silently: it retries, then files an alert
-        # issue in rustfs/backlog so a stalled chain is visible.
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        continue-on-error: true
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          DISPATCHED=0
-          for attempt in 1 2 3; do
-            if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-tier' \
-              -F 'client_payload[from_suite]=kms'; then
-              echo "dispatched next suite Tier (attempt ${attempt})"
-              DISPATCHED=1
-              break
-            fi
-            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
-            sleep "${attempt}0"
-          done
-          if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch Tier after 3 attempts" >&2
-            TITLE="[functional][chain] stalled after kms (run ${GITHUB_RUN_ID})"
-            BODY_FILE="$(mktemp)"
-            {
-              echo "The functional chain could not hand off from **kms** to **Tier** after 3 attempts."
-              echo ""
-              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-tier'"
-              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
-              echo "- Recovery: re-dispatch manually with"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-tier'"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-            } > "${BODY_FILE}"
-            gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test \
-              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
-              || echo "could not file the stall alert issue either; check the token" >&2
-            exit 1
-          fi
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-kms.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 scripts/functional_environment.py finish --suite kms --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Retain environment ownership and cleanup receipt
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: functional-environment-kms-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/environment-kms.json
+          if-no-files-found: error
 
       - name: Notify on failure
         if: failure()
@@ -419,3 +388,13 @@ jobs:
           name: functional-chain-kms-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-kms-${{ github.run_id }}-${{ github.run_attempt }}/kms.json
           if-no-files-found: error
+
+      - name: Enforce suite and report result
+        if: ${{ always() }}
+        env:
+          SUITE_OUTCOME: ${{ steps.test.outcome }}
+          REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: |
+          test "$SUITE_OUTCOME" = success
+          test "$REPORT_OUTCOME" = success
+          python3 scripts/functional_chain_evidence.py verify-report --suite kms --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"

.github/workflows/rustfs-performance-test.yml

@@ -54,9 +54,6 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
-  repository_dispatch:
-    # Chain handoff: dispatched when the replication suite finishes.
-    types: [rustfs-chain-performance]
 
 permissions:
   contents: read
@@ -89,7 +86,7 @@ env:
 
 jobs:
   runner-check:
-    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:
@@ -113,7 +110,7 @@ jobs:
       RUSTFS_WARP_CONCURRENCY: ${{ inputs.warp_concurrency || '64' }}
     # Run on manual dispatch, or when the nightly build completed successfully.
     # Skipped when nightly failed.
-    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
     steps:
       - name: Checkout chain tooling
         uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -159,6 +156,13 @@ jobs:
           warp --version || true
           df -h /data | tail -1
 
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-performance.json
+        run: python3 scripts/functional_environment.py enter --suite performance
+
       - name: Reset test environment (before)
         if: ${{ inputs.cleanup_before != 'false' }}
         run: |
@@ -323,10 +327,27 @@ jobs:
           if-no-files-found: error
 
       - name: Reset test environment (after)
-        if: ${{ always() && inputs.cleanup_after != 'false' }}
+        id: environment_cleanup
+        if: ${{ always() && inputs.cleanup_after != 'false' && steps.environment.outcome == 'success' }}
         run: |
           ./auto-testing/rustfs_performance_test.sh --step 7 -y --log-file "${LOG_FILE:-/dev/null}"
 
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-performance.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 scripts/functional_environment.py finish --suite performance --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Retain environment ownership and cleanup receipt
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: functional-environment-performance-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/environment-performance.json
+          if-no-files-found: error
+
       - name: Notify on failure
         if: failure()
         run: |
@@ -354,3 +375,13 @@ jobs:
           name: functional-chain-performance-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-performance-${{ github.run_id }}-${{ github.run_attempt }}/performance.json
           if-no-files-found: error
+
+      - name: Enforce suite and report result
+        if: ${{ always() }}
+        env:
+          SUITE_OUTCOME: ${{ steps.benchmark.outcome }}
+          REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: |
+          test "$SUITE_OUTCOME" = success
+          test "$REPORT_OUTCOME" = success
+          python3 scripts/functional_chain_evidence.py verify-report --suite performance --report "${RUSTFS_RESULT_DIR}/summary.tsv"

.github/workflows/rustfs-pool-expand-test.yml

@@ -56,9 +56,6 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
-  repository_dispatch:
-    # Chain handoff: dispatched when the heal suite finishes.
-    types: [rustfs-chain-pool]
 
 permissions:
   contents: read
@@ -86,14 +83,13 @@ env:
   RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
 
 jobs:
-  # Pool expansion: dispatched by the heal suite's chain handoff. Heal
-  # itself lives in rustfs-heal-test.yml and runs exactly once per chain.
+  # Pool expansion owns an independent reusable suite.
   pool-expansion-test:
     name: Pool expansion / decommission test
     runs-on: smoke-testing
     # Leave setup and finalizer headroom beyond the default 240-minute test.
     timeout-minutes: 360
-    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
     env:
       RUSTFS_POOL_ADMIN_ENDPOINT: ${{ secrets.RUSTFS_POOL_ADMIN_ENDPOINT || vars.RUSTFS_POOL_ADMIN_ENDPOINT || 'http://rustfs-node1:9000' }}
       RUSTFS_POOL_PROXY_ENDPOINT: http://127.0.0.1:19000
@@ -149,6 +145,13 @@ jobs:
             exit 1
           fi
 
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-pool.json
+        run: python3 scripts/functional_environment.py enter --suite pool
+
       - name: Cleanup environment (before)
         timeout-minutes: 5
         if: ${{ inputs.cleanup_before != 'false' }}
@@ -270,8 +273,7 @@ jobs:
         # timeout-minutes requires; the bare || form yielded the string '240'
         # and the runner rejected the template (Unexpected value '240').
         timeout-minutes: ${{ fromJSON(inputs.pool_timeout_minutes || '240') }}
-        # Case failures keep the run green: the report and the backlog
-        # issue manager carry the product signal.
+        # Collect reports and backlog evidence before the final suite verdict.
         continue-on-error: true
         run: |
           set -o pipefail
@@ -698,8 +700,9 @@ jobs:
             ./auto-testing/rustfs_pool_nginx_stage.sh cleanup
 
       - name: Cleanup environment (after)
+        id: environment_cleanup
         timeout-minutes: 5
-        if: ${{ always() && steps.topology.outcome == 'success' && inputs.cleanup_after != 'false' }}
+        if: ${{ always() && steps.topology.outcome == 'success' && inputs.cleanup_after != 'false' && steps.environment.outcome == 'success' }}
         run: |
           set -euo pipefail
           read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
@@ -717,54 +720,21 @@ jobs:
             '
           done
 
-      - name: "Continue functional chain (next: Security)"
-        # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup. A failed
-        # handoff must never pass silently: it retries, then files an alert
-        # issue in rustfs/backlog so a stalled chain is visible.
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        continue-on-error: true
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          DISPATCHED=0
-          for attempt in 1 2 3; do
-            if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-security' \
-              -F 'client_payload[from_suite]=pool'; then
-              echo "dispatched next suite Security (attempt ${attempt})"
-              DISPATCHED=1
-              break
-            fi
-            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
-            sleep "${attempt}0"
-          done
-          if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch Security after 3 attempts" >&2
-            TITLE="[functional][chain] stalled after pool (run ${GITHUB_RUN_ID})"
-            BODY_FILE="$(mktemp)"
-            {
-              echo "The functional chain could not hand off from **pool** to **Security** after 3 attempts."
-              echo ""
-              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-security'"
-              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
-              echo "- Recovery: re-dispatch manually with"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-security'"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-            } > "${BODY_FILE}"
-            gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test \
-              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
-              || echo "could not file the stall alert issue either; check the token" >&2
-            exit 1
-          fi
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-pool.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 scripts/functional_environment.py finish --suite pool --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Retain environment ownership and cleanup receipt
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: functional-environment-pool-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/environment-pool.json
+          if-no-files-found: error
 
       - name: Notify on failure
         if: failure()
@@ -793,3 +763,13 @@ jobs:
           name: functional-chain-pool-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-pool-${{ github.run_id }}-${{ github.run_attempt }}/pool.json
           if-no-files-found: error
+
+      - name: Enforce suite and report result
+        if: ${{ always() }}
+        env:
+          SUITE_OUTCOME: ${{ steps.pool_test.outcome }}
+          REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: |
+          test "$SUITE_OUTCOME" = success
+          test "$REPORT_OUTCOME" = success
+          python3 scripts/functional_chain_evidence.py verify-report --suite pool --report "${POOL_ARTIFACT_DIR}/pool-steps.md"

.github/workflows/rustfs-replication-test.yml

@@ -39,9 +39,6 @@ on:
           - bucket
           - site
         default: all
-  repository_dispatch:
-    # Chain handoff: dispatched when the security suite finishes.
-    types: [rustfs-chain-replication]
 
 permissions:
   contents: read
@@ -68,7 +65,7 @@ jobs:
   replication-test:
     runs-on: smoke-testing
     timeout-minutes: 60
-    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
         uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -113,6 +110,13 @@ jobs:
           aws --version
           df -h /data | tail -1 || true
 
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-replication.json
+        run: python3 scripts/functional_environment.py enter --suite replication
+
       - name: Cleanup environment (before)
         timeout-minutes: 5
         run: |
@@ -135,8 +139,7 @@ jobs:
       - name: Run replication suite
         id: test
         timeout-minutes: 45
-        # Case failures keep the run green: the report and the backlog
-        # issue manager carry the product signal.
+        # Collect reports and backlog evidence before the final suite verdict.
         continue-on-error: true
         run: |
           set -euo pipefail
@@ -192,10 +195,8 @@ jobs:
             RESULT=success
           fi
 
-          # Product gate: failing cases keep the run green — they are reported
-          # below and tracked in rustfs/backlog. Only harness/environment
-          # breakdowns (the suite never reached case level, or a wholesale
-          # failure with zero passes) turn the workflow red.
+          # Report harness completion here; the final verdict checks every
+          # case result after diagnostics and cleanup have been retained.
           CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
           CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
           CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
@@ -234,7 +235,7 @@ jobs:
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          # Red only for harness/environment breakdowns; case failures stay green.
+          # Preserve the harness verdict separately from the final case verdict.
           [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
@@ -333,8 +334,9 @@ jobs:
           if-no-files-found: error
 
       - name: Cleanup environment (after)
+        id: environment_cleanup
         timeout-minutes: 5
-        if: always()
+        if: ${{ always() && steps.environment.outcome == 'success' }}
         run: |
           set -euo pipefail
           read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
@@ -352,50 +354,21 @@ jobs:
             '
           done
 
-      - name: "Continue functional chain (next: Fault tolerance)"
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          DISPATCHED=0
-          for attempt in 1 2 3; do
-            if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-fault-tolerance' \
-              -F 'client_payload[from_suite]=replication'; then
-              echo "dispatched next suite Fault tolerance (attempt ${attempt})"
-              DISPATCHED=1
-              break
-            fi
-            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
-            sleep "${attempt}0"
-          done
-          if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch Fault tolerance after 3 attempts" >&2
-            TITLE="[functional][chain] stalled after replication (run ${GITHUB_RUN_ID})"
-            BODY_FILE="$(mktemp)"
-            trap 'rm -f "${BODY_FILE}"' EXIT
-            {
-              echo "The functional chain could not hand off from **replication** to **Fault tolerance** after 3 attempts."
-              echo ""
-              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-fault-tolerance'"
-              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
-              echo "- Recovery: re-dispatch manually with"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-fault-tolerance'"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-            } > "${BODY_FILE}"
-            gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test \
-              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
-              || echo "could not file the stall alert issue either; check the token" >&2
-            exit 1
-          fi
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-replication.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 scripts/functional_environment.py finish --suite replication --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Retain environment ownership and cleanup receipt
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: functional-environment-replication-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/environment-replication.json
+          if-no-files-found: error
 
       - name: Notify on failure
         if: failure()
@@ -424,3 +397,13 @@ jobs:
           name: functional-chain-replication-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-replication-${{ github.run_id }}-${{ github.run_attempt }}/replication.json
           if-no-files-found: error
+
+      - name: Enforce suite and report result
+        if: ${{ always() }}
+        env:
+          SUITE_OUTCOME: ${{ steps.test.outcome }}
+          REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: |
+          test "$SUITE_OUTCOME" = success
+          test "$REPORT_OUTCOME" = success
+          python3 scripts/functional_chain_evidence.py verify-report --suite replication --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"

.github/workflows/rustfs-s3-compat-test.yml

@@ -17,9 +17,6 @@ on:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false
         type: string
-  repository_dispatch:
-    # Chain handoff: dispatched when the upgrade suite finishes.
-    types: [rustfs-chain-s3]
 
 permissions:
   contents: read
@@ -44,7 +41,7 @@ jobs:
   s3-compat-test:
     runs-on: smoke-testing
     timeout-minutes: 60
-    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
         uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -88,6 +85,13 @@ jobs:
           openssl version
           df -h /data | tail -1
 
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-s3.json
+        run: python3 scripts/functional_environment.py enter --suite s3
+
       - name: Cleanup environment (before)
         timeout-minutes: 5
         run: |
@@ -110,8 +114,7 @@ jobs:
       - name: Run S3 compatibility suite
         id: test
         timeout-minutes: 45
-        # Case failures keep the run green: the report and the backlog
-        # issue manager carry the product signal.
+        # Collect reports and backlog evidence before the final suite verdict.
         continue-on-error: true
         run: |
           set -euo pipefail
@@ -161,10 +164,8 @@ jobs:
             RESULT=success
           fi
 
-          # Product gate: failing cases keep the run green — they are reported
-          # below and tracked in rustfs/backlog. Only harness/environment
-          # breakdowns (the suite never reached case level, or a wholesale
-          # failure with zero passes) turn the workflow red.
+          # Report harness completion here; the final verdict checks every
+          # case result after diagnostics and cleanup have been retained.
           CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
           CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
           CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
@@ -203,7 +204,7 @@ jobs:
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          # Red only for harness/environment breakdowns; case failures stay green.
+          # Preserve the harness verdict separately from the final case verdict.
           [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
@@ -302,8 +303,9 @@ jobs:
           if-no-files-found: error
 
       - name: Cleanup environment (after)
+        id: environment_cleanup
         timeout-minutes: 5
-        if: always()
+        if: ${{ always() && steps.environment.outcome == 'success' }}
         run: |
           set -euo pipefail
           read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
@@ -321,54 +323,21 @@ jobs:
             '
           done
 
-      - name: "Continue functional chain (next: KMS)"
-        # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup. A failed
-        # handoff must never pass silently: it retries, then files an alert
-        # issue in rustfs/backlog so a stalled chain is visible.
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        continue-on-error: true
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          DISPATCHED=0
-          for attempt in 1 2 3; do
-            if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-kms' \
-              -F 'client_payload[from_suite]=s3'; then
-              echo "dispatched next suite KMS (attempt ${attempt})"
-              DISPATCHED=1
-              break
-            fi
-            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
-            sleep "${attempt}0"
-          done
-          if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch KMS after 3 attempts" >&2
-            TITLE="[functional][chain] stalled after s3 (run ${GITHUB_RUN_ID})"
-            BODY_FILE="$(mktemp)"
-            {
-              echo "The functional chain could not hand off from **s3** to **KMS** after 3 attempts."
-              echo ""
-              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-kms'"
-              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
-              echo "- Recovery: re-dispatch manually with"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-kms'"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-            } > "${BODY_FILE}"
-            gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test \
-              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
-              || echo "could not file the stall alert issue either; check the token" >&2
-            exit 1
-          fi
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-s3.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 scripts/functional_environment.py finish --suite s3 --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Retain environment ownership and cleanup receipt
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: functional-environment-s3-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/environment-s3.json
+          if-no-files-found: error
 
       - name: Notify on failure
         if: failure()
@@ -396,3 +365,13 @@ jobs:
           name: functional-chain-s3-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-s3-${{ github.run_id }}-${{ github.run_attempt }}/s3.json
           if-no-files-found: error
+
+      - name: Enforce suite and report result
+        if: ${{ always() }}
+        env:
+          SUITE_OUTCOME: ${{ steps.test.outcome }}
+          REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: |
+          test "$SUITE_OUTCOME" = success
+          test "$REPORT_OUTCOME" = success
+          python3 scripts/functional_chain_evidence.py verify-report --suite s3 --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"

.github/workflows/rustfs-security-test.yml

@@ -52,9 +52,6 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
-  repository_dispatch:
-    # Chain handoff: dispatched when the pool expansion suite finishes.
-    types: [rustfs-chain-security]
 
 permissions:
   contents: read
@@ -81,7 +78,7 @@ jobs:
   security-test:
     runs-on: smoke-testing
     timeout-minutes: 60
-    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
     steps:
       # Checkout the repository into its own subdirectory. Checking out at
       # the workspace root would wipe the auto-testing clone above (that is
@@ -126,6 +123,13 @@ jobs:
           docker --version || true
           df -h /data | tail -1
 
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-security.json
+        run: python3 rustfs-repo/scripts/functional_environment.py enter --suite security
+
       - name: Cleanup environment (before)
         timeout-minutes: 5
         if: ${{ inputs.cleanup_before != 'false' || github.event_name != 'workflow_dispatch' }}
@@ -149,8 +153,7 @@ jobs:
       - name: Run security suite
         id: test
         timeout-minutes: 45
-        # Case failures keep the run green: the report and the backlog
-        # issue manager carry the product signal.
+        # Collect reports and backlog evidence before the final suite verdict.
         continue-on-error: true
         env:
           REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
@@ -194,9 +197,8 @@ jobs:
             RESULT=success
           fi
 
-          # Product gate: failing cases keep the run green — they are reported
-          # below and tracked in rustfs/backlog. Only harness/environment
-          # breakdowns turn the workflow red. The suite log lines carry ANSI
+          # The final verdict checks cases after diagnostics and cleanup.
+          # The suite log lines carry ANSI
           # color escapes both before and after the verdict tag
           # (e.g. "\e[1;31m[FAIL]\e[0m STS-105 ..."), so the pattern allows
           # any number of escapes on each side of it.
@@ -236,7 +238,7 @@ jobs:
             fi
           } > "${SECURITY_ARTIFACTS_DIR}/report.md"
           cat "${SECURITY_ARTIFACTS_DIR}/report.md" >> "${GITHUB_STEP_SUMMARY}"
-          # Red only for harness/environment breakdowns; case failures stay green.
+          # Preserve the harness verdict separately from the final case verdict.
           [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
@@ -336,8 +338,9 @@ jobs:
           retention-days: 3
 
       - name: Cleanup environment (after)
+        id: environment_cleanup
         timeout-minutes: 5
-        if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}
+        if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') && steps.environment.outcome == 'success' }}
         run: |
           set -euo pipefail
           read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
@@ -355,23 +358,21 @@ jobs:
             '
           done
 
-      - name: "Continue functional chain (next: Replication)"
-        # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup.
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        continue-on-error: true
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          echo "Dispatching next functional suite: Replication"
-          gh api --method POST repos/rustfs/rustfs/dispatches \
-            -f event_type='rustfs-chain-replication' \
-            -F 'client_payload[from_suite]=security'
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-security.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 rustfs-repo/scripts/functional_environment.py finish --suite security --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Retain environment ownership and cleanup receipt
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: functional-environment-security-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/environment-security.json
+          if-no-files-found: error
 
       - name: Notify on failure
         if: failure()
@@ -400,3 +401,13 @@ jobs:
           name: functional-chain-security-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-security-${{ github.run_id }}-${{ github.run_attempt }}/security.json
           if-no-files-found: error
+
+      - name: Enforce suite and report result
+        if: ${{ always() }}
+        env:
+          SUITE_OUTCOME: ${{ steps.test.outcome }}
+          REPORT_OUTCOME: ${{ steps.report.outcome }}
+        run: |
+          test "$SUITE_OUTCOME" = success
+          test "$REPORT_OUTCOME" = success
+          python3 rustfs-repo/scripts/functional_chain_evidence.py verify-report --suite security --report "${SECURITY_ARTIFACTS_DIR}/suite-report.md"

.github/workflows/rustfs-storage-test.yml

@@ -26,9 +26,6 @@ on:
           - single-multi
           - multi-multi
         default: all
-  repository_dispatch:
-    # Chain handoff: dispatched when the tier suite finishes.
-    types: [rustfs-chain-storage]
 
 permissions:
   contents: read
@@ -53,7 +50,7 @@ jobs:
   storage-test:
     runs-on: smoke-testing
     timeout-minutes: 60
-    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
         uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -97,6 +94,13 @@ jobs:
           openssl version
           df -h /data | tail -1
 
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-storage.json
+        run: python3 scripts/functional_environment.py enter --suite storage
+
       - name: Cleanup environment (before)
         timeout-minutes: 5
         run: |
@@ -119,8 +123,7 @@ jobs:
       - name: Run storage engine suite
         id: test
         timeout-minutes: 45
-        # Case failures keep the run green: the report and the backlog
-        # issue manager carry the product signal.
+        # Collect reports and backlog evidence before the final suite verdict.
         continue-on-error: true
         run: |
           set -euo pipefail
@@ -176,10 +179,8 @@ jobs:
             RESULT=success
           fi
 
-          # Product gate: failing cases keep the run green — they are reported
-          # below and tracked in rustfs/backlog. Only harness/environment
-          # breakdowns (the suite never reached case level, or a wholesale
-          # failure with zero passes) turn the workflow red.
+          # Report harness completion here; the final verdict checks every
+          # case result after diagnostics and cleanup have been retained.
           CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
           CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
           CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
@@ -218,7 +219,7 @@ jobs:
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          # Red only for harness/environment breakdowns; case failures stay green.
+          # Preserve the harness verdict separately from the final case verdict.
           [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
@@ -317,8 +318,9 @@ jobs:
           if-no-files-found: error
 
       - name: Cleanup environment (after)
+        id: environment_cleanup
         timeout-minutes: 5
-        if: always()
+        if: ${{ always() && steps.environment.outcome == 'success' }}
         run: |
           set -euo pipefail
           read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
@@ -336,54 +338,21 @@ jobs:
             '
           done
 
-      - name: "Continue functional chain (next: Heal)"
-        # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup. A failed
-        # handoff must never pass silently: it retries, then files an alert
-        # issue in rustfs/backlog so a stalled chain is visible.
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        continue-on-error: true
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          DISPATCHED=0
-          for attempt in 1 2 3; do
-            if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-heal' \
-              -F 'client_payload[from_suite]=storage'; then
-              echo "dispatched next suite Heal (attempt ${attempt})"
-              DISPATCHED=1
-              break
-            fi
-            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
-            sleep "${attempt}0"
-          done
-          if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch Heal after 3 attempts" >&2
-            TITLE="[functional][chain] stalled after storage (run ${GITHUB_RUN_ID})"
-            BODY_FILE="$(mktemp)"
-            {
-              echo "The functional chain could not hand off from **storage** to **Heal** after 3 attempts."
-              echo ""
-              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-heal'"
-              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
-              echo "- Recovery: re-dispatch manually with"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-heal'"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-            } > "${BODY_FILE}"
-            gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test \
-              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
-              || echo "could not file the stall alert issue either; check the token" >&2
-            exit 1
-          fi
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-storage.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 scripts/functional_environment.py finish --suite storage --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Retain environment ownership and cleanup receipt
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: functional-environment-storage-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/environment-storage.json
+          if-no-files-found: error
 
       - name: Notify on failure
         if: failure()
@@ -411,3 +380,13 @@ jobs:
           name: functional-chain-storage-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-storage-${{ github.run_id }}-${{ github.run_attempt }}/storage.json
           if-no-files-found: error
+
+      - name: Enforce suite and report result
+        if: ${{ always() }}
+        env:
+          SUITE_OUTCOME: ${{ steps.test.outcome }}
+          REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: |
+          test "$SUITE_OUTCOME" = success
+          test "$REPORT_OUTCOME" = success
+          python3 scripts/functional_chain_evidence.py verify-report --suite storage --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"

.github/workflows/rustfs-table-test.yml

@@ -89,6 +89,13 @@ jobs:
           aws --version
           python3 --version
 
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-table.json
+        run: python3 scripts/functional_environment.py enter --suite table
+
       - name: Cleanup environment (before)
         timeout-minutes: 5
         continue-on-error: true
@@ -162,10 +169,8 @@ jobs:
             RESULT=success
           fi
 
-          # Product gate: failing cases keep the run green — they are reported
-          # below and tracked in rustfs/backlog. Only harness/environment
-          # breakdowns (the suite never reached case level, or a wholesale
-          # failure with zero passes) turn the workflow red.
+          # Report harness completion here; the final verdict checks every
+          # case result after diagnostics and cleanup have been retained.
           CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
           CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
           CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
@@ -203,7 +208,7 @@ jobs:
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          # Red only for harness/environment breakdowns; case failures stay green.
+          # Preserve the harness verdict separately from the final case verdict.
           [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
@@ -292,11 +297,12 @@ jobs:
           retention-days: 30
 
       - name: Cleanup environment (after)
+        id: environment_cleanup
         timeout-minutes: 5
-        if: always()
-        continue-on-error: true
+        if: ${{ always() && steps.environment.outcome == 'success' }}
         run: |
-          set -uo pipefail
+          set -euo pipefail
+          cleanup_status=0
           for node in ${RUSTFS_NODES}; do
             ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
               set -euo pipefail
@@ -307,26 +313,25 @@ jobs:
               fi
               for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
               ${SUDO} rm -f /etc/default/rustfs
-            ' || echo "cleanup failed on ${node} (continuing)"
+            ' || cleanup_status=$?
           done
+          exit "${cleanup_status}"
 
-      - name: "Continue functional chain (next: Performance)"
-        # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup.
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        continue-on-error: true
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          echo "Dispatching next functional suite: Performance"
-          gh api --method POST repos/rustfs/rustfs/dispatches \
-            -f event_type='rustfs-chain-performance' \
-            -F 'client_payload[from_suite]=table'
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-table.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 scripts/functional_environment.py finish --suite table --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Retain environment ownership and cleanup receipt
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: functional-environment-table-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/environment-table.json
+          if-no-files-found: error
 
       - name: Notify on failure
         if: failure()
@@ -358,3 +363,13 @@ jobs:
           name: functional-chain-table-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-table-${{ github.run_id }}-${{ github.run_attempt }}/table.json
           if-no-files-found: error
+
+      - name: Enforce suite and report result
+        if: ${{ always() }}
+        env:
+          SUITE_OUTCOME: ${{ steps.test.outcome }}
+          REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: |
+          test "$SUITE_OUTCOME" = success
+          test "$REPORT_OUTCOME" = success
+          python3 scripts/functional_chain_evidence.py verify-report --suite table --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"

.github/workflows/rustfs-tier-test.yml

@@ -37,9 +37,6 @@ on:
         required: false
         default: false
         type: boolean
-  repository_dispatch:
-    # Chain handoff: dispatched when the KMS suite finishes.
-    types: [rustfs-chain-tier]
 
 permissions:
   contents: read
@@ -68,7 +65,7 @@ jobs:
   tier-test:
     runs-on: smoke-testing
     timeout-minutes: 60
-    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
     steps:
       - name: Checkout chain tooling
         uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -216,6 +213,13 @@ jobs:
           openssl version
           df -h /data | tail -1
 
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-tier.json
+        run: python3 scripts/functional_environment.py enter --suite tier
+
       - name: Cleanup environment (before)
         timeout-minutes: 5
         run: |
@@ -269,8 +273,7 @@ jobs:
       - name: Run tier suite
         id: test
         timeout-minutes: 45
-        # Case failures keep the run green: the report and the backlog
-        # issue manager carry the product signal.
+        # Collect reports and backlog evidence before the final suite verdict.
         continue-on-error: true
         env:
           PACKAGE_URL_INPUT: ${{ inputs.package_url }}
@@ -467,8 +470,9 @@ jobs:
           if-no-files-found: error
 
       - name: Cleanup environment (after)
+        id: environment_cleanup
         timeout-minutes: 5
-        if: always()
+        if: ${{ always() && steps.environment.outcome == 'success' }}
         run: |
           set -euo pipefail
           sudo docker rm -f rustfs-test-mqtt >/dev/null 2>&1 || true
@@ -488,6 +492,22 @@ jobs:
             '
           done
 
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-tier.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 scripts/functional_environment.py finish --suite tier --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Retain environment ownership and cleanup receipt
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: functional-environment-tier-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/environment-tier.json
+          if-no-files-found: error
+
       - name: Cleanup pinned RustFS CLI
         timeout-minutes: 2
         if: always()
@@ -664,55 +684,6 @@ jobs:
             --package-source "${PACKAGE_SOURCE}" \
             --date "$(date -u +%Y-%m-%d)"
 
-      - name: "Continue functional chain (next: Storage engine)"
-        # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup. A failed
-        # handoff must never pass silently: it retries, then files an alert
-        # issue in rustfs/backlog so a stalled chain is visible.
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          DISPATCHED=0
-          for attempt in 1 2 3; do
-            if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-storage' \
-              -F 'client_payload[from_suite]=tier'; then
-              echo "dispatched next suite Storage engine (attempt ${attempt})"
-              DISPATCHED=1
-              break
-            fi
-            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
-            sleep "${attempt}0"
-          done
-          if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch Storage engine after 3 attempts" >&2
-            TITLE="[functional][chain] stalled after tier (run ${GITHUB_RUN_ID})"
-            BODY_FILE="$(mktemp)"
-            {
-              echo "The functional chain could not hand off from **tier** to **Storage engine** after 3 attempts."
-              echo ""
-              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-storage'"
-              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
-              echo "- Recovery: re-dispatch manually with"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-storage'"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-            } > "${BODY_FILE}"
-            gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test \
-              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
-              || echo "could not file the stall alert issue either; check the token" >&2
-            exit 1
-          fi
-
       - name: Notify on failure
         if: failure()
         run: |
@@ -739,3 +710,13 @@ jobs:
           name: functional-chain-tier-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-tier-${{ github.run_id }}-${{ github.run_attempt }}/tier.json
           if-no-files-found: error
+
+      - name: Enforce suite and report result
+        if: ${{ always() }}
+        env:
+          SUITE_OUTCOME: ${{ steps.test.outcome }}
+          REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: |
+          test "$SUITE_OUTCOME" = success
+          test "$REPORT_OUTCOME" = success
+          python3 scripts/functional_chain_evidence.py verify-report --suite tier --report "${TIER_ARTIFACTS_DIR}/rustfs-tier-cases.md"

.github/workflows/rustfs-upgrade-test.yml

@@ -59,9 +59,6 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
-  repository_dispatch:
-    # Functional-chain entry: dispatched by rustfs-functional-chain.yml.
-    types: [rustfs-chain-upgrade]
 
 permissions:
   contents: read
@@ -86,7 +83,7 @@ jobs:
   upgrade-test:
     runs-on: smoke-testing
     timeout-minutes: 60
-    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
         uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -132,6 +129,13 @@ jobs:
           docker --version || true
           df -h /data | tail -1
 
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-upgrade.json
+        run: python3 scripts/functional_environment.py enter --suite upgrade
+
       - name: Cleanup environment (before)
         timeout-minutes: 5
         if: ${{ inputs.cleanup_before != 'false' || github.event_name != 'workflow_dispatch' }}
@@ -165,8 +169,7 @@ jobs:
       - name: Run upgrade compatibility suite
         id: test
         timeout-minutes: 45
-        # Case failures keep the run green: the report and the backlog
-        # issue manager carry the product signal.
+        # Collect reports and backlog evidence before the final suite verdict.
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
@@ -258,10 +261,8 @@ jobs:
             RESULT=success
           fi
 
-          # Product gate: failing cases keep the run green — they are reported
-          # below and tracked in rustfs/backlog. Only harness/environment
-          # breakdowns (the suite never reached case level, or a wholesale
-          # failure with zero passes) turn the workflow red.
+          # Report harness completion here; the final verdict checks every
+          # case result after diagnostics and cleanup have been retained.
           CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
           CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
           CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
@@ -304,7 +305,7 @@ jobs:
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          # Red only for harness/environment breakdowns; case failures stay green.
+          # Preserve the harness verdict separately from the final case verdict.
           [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
@@ -412,8 +413,9 @@ jobs:
           retention-days: 3
 
       - name: Cleanup environment (after)
+        id: environment_cleanup
         timeout-minutes: 5
-        if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}
+        if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') && steps.environment.outcome == 'success' }}
         run: |
           set -euo pipefail
           read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
@@ -431,54 +433,21 @@ jobs:
             '
           done
 
-      - name: "Continue functional chain (next: S3 compatibility)"
-        # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup. A failed
-        # handoff must never pass silently: it retries, then files an alert
-        # issue in rustfs/backlog so a stalled chain is visible.
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        continue-on-error: true
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          DISPATCHED=0
-          for attempt in 1 2 3; do
-            if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-s3' \
-              -F 'client_payload[from_suite]=upgrade'; then
-              echo "dispatched next suite S3 compatibility (attempt ${attempt})"
-              DISPATCHED=1
-              break
-            fi
-            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
-            sleep "${attempt}0"
-          done
-          if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch S3 compatibility after 3 attempts" >&2
-            TITLE="[functional][chain] stalled after upgrade (run ${GITHUB_RUN_ID})"
-            BODY_FILE="$(mktemp)"
-            {
-              echo "The functional chain could not hand off from **upgrade** to **S3 compatibility** after 3 attempts."
-              echo ""
-              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-s3'"
-              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
-              echo "- Recovery: re-dispatch manually with"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-s3'"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-            } > "${BODY_FILE}"
-            gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test \
-              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
-              || echo "could not file the stall alert issue either; check the token" >&2
-            exit 1
-          fi
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-upgrade.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 scripts/functional_environment.py finish --suite upgrade --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Retain environment ownership and cleanup receipt
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: functional-environment-upgrade-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/environment-upgrade.json
+          if-no-files-found: error
 
       - name: Notify on failure
         if: failure()
@@ -508,3 +477,13 @@ jobs:
           name: functional-chain-upgrade-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-upgrade-${{ github.run_id }}-${{ github.run_attempt }}/upgrade.json
           if-no-files-found: error
+
+      - name: Enforce suite and report result
+        if: ${{ always() }}
+        env:
+          SUITE_OUTCOME: ${{ steps.test.outcome }}
+          REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: |
+          test "$SUITE_OUTCOME" = success
+          test "$REPORT_OUTCOME" = success
+          python3 scripts/functional_chain_evidence.py verify-report --suite upgrade --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"

.github/workflows/rustfs-validation-sweep.yml

@@ -0,0 +1,457 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: RustFS Validation Sweep
+
+# Executes the migrated canonical validation cases (rustfs/auto-testing#113)
+# on the shared functional fleet. Everything with a lane executor is swept in
+# rotating nightly batches so the whole set is covered roughly weekly; a
+# single case or one module can be dispatched manually. That is the
+# implemented catalog (66 today) plus the eight planned cases whose dedicated
+# executors already exist and have live PASS evidence - fresh nightly
+# evidence feeds their acceptance review. heal.* cases stay on the
+# interactive validation flow: they require reviewed native/tool collectors
+# built by separate validation-tools runs before fleet execution.
+
+on:
+  schedule:
+    # After the nightly functional chain settles (chain starts ~16:48 UTC and
+    # runs for about two hours).
+    - cron: "30 19 * * *"
+  workflow_dispatch:
+    inputs:
+      case:
+        description: 'Single canonical validation case id (overrides module and rotation), e.g. pool.decommission'
+        type: string
+        required: false
+        default: ''
+      module:
+        description: 'Restrict the sweep to one catalog module, e.g. bucket-replication'
+        type: string
+        required: false
+        default: ''
+      limit:
+        description: 'Maximum cases per sweep (0 = run everything selected)'
+        type: number
+        required: false
+        default: 12
+      build_run_id:
+        description: 'Successful nightly package run; empty resolves the latest'
+        type: string
+        required: false
+        default: ''
+      build_run_attempt:
+        description: 'Attempt of that package run'
+        type: string
+        required: false
+        default: '1'
+
+permissions:
+  contents: read
+
+concurrency:
+  group: rustfs-validation-sweep
+  cancel-in-progress: false
+
+env:
+  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES || 'vm000 vm001 vm002 vm003' }}
+  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER || 'azureuser' }}
+  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
+  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
+  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
+  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+jobs:
+  sweep:
+    if: github.repository == 'rustfs/rustfs'
+    runs-on: smoke-testing
+    timeout-minutes: 150
+    steps:
+      - name: Checkout chain tooling
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Resolve nightly candidate
+        id: candidate
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
+          CHAIN_OUTPUT: ${{ runner.temp }}/validation-sweep-candidate.json
+        run: |
+          set -euo pipefail
+          # Scheduled runs carry no inputs and the resolver only derives a
+          # build from a workflow_run payload (which a schedule cannot
+          # provide), so resolve the newest successful nightly build
+          # explicitly before invoking it.
+          if [ -z "${BUILD_RUN_ID:-}" ]; then
+            LATEST="$(gh api "repos/rustfs/rustfs/actions/workflows/nightly-gnu.yml/runs?branch=main&status=success&per_page=1" --jq '.workflow_runs[0]')"
+            BUILD_RUN_ID="$(printf '%s' "${LATEST}" | jq -r '.id')"
+            BUILD_RUN_ATTEMPT="$(printf '%s' "${LATEST}" | jq -r '.run_attempt')"
+            export BUILD_RUN_ID BUILD_RUN_ATTEMPT
+            echo "scheduled sweep pinned to nightly ${BUILD_RUN_ID} attempt ${BUILD_RUN_ATTEMPT}"
+          fi
+          python3 scripts/resolve_functional_candidate.py
+          python3 - <<'PY'
+          import json
+          import os
+          from pathlib import Path
+          root = Path(os.environ["RUNNER_TEMP"]) / "validation-sweep"
+          root.mkdir(parents=True, exist_ok=True)
+          chain = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
+          manifest = chain["candidate"]["manifest"]
+          (root / "candidate.json").write_text(json.dumps({
+              "package_url": manifest["package_url"],
+              "package_sha256": manifest["package_sha256"],
+              "source_sha": manifest["source_sha"],
+          }, sort_keys=True) + "\n")
+          with open(os.environ["GITHUB_OUTPUT"], "a") as output:
+              output.write("testing_sha=" + chain["testing_sha"] + "\n")
+              output.write("package_url=" + manifest["package_url"] + "\n")
+              output.write("source_sha=" + manifest["source_sha"] + "\n")
+          PY
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.candidate.outputs.testing_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
+
+      - name: Prepare validation Python runtime
+        # The pool / ilm / replication validation runners require a managed
+        # Python, exactly like the chain's validation job provides.
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.12"
+
+      - name: Install validation Python requirements
+        # Case scripts call python3 and import boto3 at startup. Install the
+        # functional chain's client into a per-run env so that interpreter
+        # sees it without changing the runner's shared Python.
+        run: |
+          set -euo pipefail
+          validation_python="${RUNNER_TEMP}/rustfs-validation-python-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          test ! -e "${validation_python}"
+          python3 -m venv "${validation_python}"
+          "${validation_python}/bin/python" -m pip install \
+            -r auto-testing/scripts/validation_ilm/requirements.txt
+          printf '%s\n' "${validation_python}/bin" >> "${GITHUB_PATH}"
+
+      - name: Select sweep cases
+        id: select
+        env:
+          SELECTED_CASE: ${{ inputs.case }}
+          SELECTED_MODULE: ${{ inputs.module }}
+          CASE_LIMIT: ${{ inputs.limit }}
+          SWEEP_EVENT: ${{ github.event_name }}
+          SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
+        run: |
+          set -euo pipefail
+          python3 - <<'PY'
+          import datetime
+          import json
+          import os
+          from pathlib import Path
+          sweep_root = Path(os.environ["SWEEP_ROOT"])
+
+          catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
+          def sweepable(case):
+              status = case.get("source_status")
+              if status == "implemented":
+                  return True
+              # The remaining planned non-heal cases route to the lane
+              # executors and already have live PASS evidence; sweeping them
+              # nightly produces fresh evidence for their acceptance review.
+              return status == "planned" and not case["id"].startswith("heal.")
+
+          selected = [(case["id"], module["id"]) for module in catalog["modules"]
+                      for case in module["cases"]
+                      if sweepable(case)]
+          if not selected:
+              raise SystemExit("catalog has no sweepable cases")
+
+          one_case = os.environ.get("SELECTED_CASE", "").strip()
+          module_filter = os.environ.get("SELECTED_MODULE", "").strip()
+          known = {case_id for case_id, _ in selected}
+          modules = {module_id for _, module_id in selected}
+          if one_case:
+              if one_case not in known:
+                  raise SystemExit(f"unknown case or no lane executor for it: {one_case}")
+              selected = [(one_case, dict(selected)[one_case])]
+          elif module_filter:
+              if module_filter not in modules:
+                  raise SystemExit(f"unknown module or no implemented cases: {module_filter}")
+              selected = [pair for pair in selected if pair[1] == module_filter]
+          elif os.environ.get("SWEEP_EVENT") == "schedule":
+              # Stateless rotation: every night take one window of the catalog
+              # so a full sweep needs len(selected)/limit nights and no state
+              # has to be carried between runs.
+              limit = max(1, int(os.environ.get("CASE_LIMIT") or 12))
+              day = datetime.datetime.now(datetime.timezone.utc).timetuple().tm_yday
+              start = (day * limit) % len(selected)
+              selected = (selected[start:] + selected[:start])[:limit]
+          limit = int(os.environ.get("CASE_LIMIT") or 0)
+          if one_case or module_filter or (not os.environ.get("SWEEP_EVENT") == "schedule"):
+              if limit > 0:
+                  selected = selected[:limit]
+
+          (sweep_root / "sweep-cases.txt").write_text("".join(f"{case_id}\n" for case_id, _ in selected))
+          with open(os.environ["GITHUB_OUTPUT"], "a") as output:
+              output.write(f"count={len(selected)}\n")
+          print(f"selected {len(selected)} case(s):", ", ".join(c for c, _ in selected))
+          PY
+
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
+        run: python3 scripts/functional_environment.py enter --suite heal
+
+      - name: Run validation cases
+        id: run_cases
+        timeout-minutes: 120
+        env:
+          AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
+          SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
+        run: |
+          set -uo pipefail
+          mkdir -p "${SWEEP_ROOT}/results"
+          : > "${SWEEP_ROOT}/results.txt"
+          while read -r CASE_ID; do
+            [ -n "${CASE_ID}" ] || continue
+            echo "::group::case ${CASE_ID}"
+            rm -rf "${SWEEP_ROOT}/run"
+            if ./auto-testing/scripts/run_validation_case.sh \
+                 "${CASE_ID}" "${SWEEP_ROOT}/candidate.json" \
+                 "${AUTO_TESTING_SHA}" "${SWEEP_ROOT}/run" \
+                 > "${SWEEP_ROOT}/results/${CASE_ID//\//_}.log" 2>&1; then
+              printf '%s\tPASS\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
+              echo "case ${CASE_ID}: PASS"
+            else
+              printf '%s\tFAIL\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
+              echo "case ${CASE_ID}: FAIL (log retained in the sweep artifact)"
+            fi
+            if [ -f "${SWEEP_ROOT}/run/report.md" ]; then
+              cp "${SWEEP_ROOT}/run/report.md" "${SWEEP_ROOT}/results/${CASE_ID//\//_}.report.md" 2>/dev/null || true
+            fi
+            echo "::endgroup::"
+            # Fleet hygiene between cases: the case runners set up their own
+            # topology, this purge guarantees the next case starts clean even
+            # when the previous one died mid-run.
+            for node in ${RUSTFS_NODES}; do
+              # -n keeps ssh from consuming the case list this loop reads from stdin
+              ssh -n -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
+                SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+                ${SUDO} systemctl stop rustfs 2>/dev/null || true
+                if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                  ${SUDO} dpkg -P rustfs
+                fi
+                for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+                ${SUDO} rm -f /etc/default/rustfs
+              ' || echo "inter-case cleanup failed on ${node} (continuing)"
+            done
+          done < "${SWEEP_ROOT}/sweep-cases.txt"
+          echo "sweep finished: $(wc -l < "${SWEEP_ROOT}/results.txt") case(s) executed"
+
+      - name: Generate sweep report
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        env:
+          SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
+          PACKAGE_URL: ${{ steps.candidate.outputs.package_url }}
+          SOURCE_SHA: ${{ steps.candidate.outputs.source_sha }}
+          AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
+        run: |
+          set -euo pipefail
+          python3 - <<'PY'
+          import json
+          import os
+          from pathlib import Path
+
+          root = Path(os.environ["SWEEP_ROOT"])
+          catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
+          module_of = {case["id"]: module["id"] for module in catalog["modules"] for case in module["cases"]}
+          rows = []
+          for line in (root / "results.txt").read_text().splitlines():
+            if not line:
+                continue
+            case_id, status = line.split("\t")
+            rows.append((case_id, module_of.get(case_id, "?"), status))
+          passed = sum(1 for _, _, s in rows if s == "PASS")
+          failed = len(rows) - passed
+          run_url = (f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/"
+                     f"{os.environ.get('GITHUB_REPOSITORY', 'rustfs/rustfs')}/actions/runs/"
+                     f"{os.environ.get('GITHUB_RUN_ID', '')}")
+
+          report = [
+              "# RustFS validation sweep report", "",
+              f"- Run: {run_url}",
+              f"- Attempt: {os.environ.get('GITHUB_RUN_ATTEMPT', '')}",
+              f"- Trigger: {os.environ.get('GITHUB_EVENT_NAME', '')}",
+              f"- Package: {os.environ.get('PACKAGE_URL', 'nightly (R2 latest)')}",
+              f"- Source SHA: {os.environ.get('SOURCE_SHA', '')}",
+              f"- auto-testing: {os.environ.get('AUTO_TESTING_SHA', '')}", "",
+              "Canonical validation cases (rustfs/auto-testing#113) executed by the sweep, grouped by catalog module.", "",
+          ]
+          # Group the executed cases by catalog module (the major category:
+          # pool lifecycle, heal, ILM expiration, bucket/site replication, kms,
+          # data services, identity, s3 compatibility). Per-module subtotals
+          # deliberately use "- Subtotal:" so the dashboard overview regexes
+          # (which anchor on "- Total:") keep reading the grand totals below.
+          by_module = {}
+          module_order = []
+          for case_id, module, status in rows:
+              if module not in by_module:
+                  by_module[module] = []
+                  module_order.append(module)
+              by_module[module].append((case_id, status))
+          report += ["## Case Summary", ""]
+          for module in module_order:
+              cases = by_module[module]
+              mod_pass = sum(1 for _, st in cases if st == "PASS")
+              mod_fail = len(cases) - mod_pass
+              report += [f"### {module} ({len(cases)})", "",
+                         "| Case | Status |", "| --- | --- |"]
+              report += [f"| {case_id} | {status} |" for case_id, status in cases]
+              report += ["", f"- Subtotal: {mod_pass} passed, {mod_fail} failed", ""]
+          report += [
+              "## Totals", "",
+              f"- Total: {len(rows)}",
+              f"- PASS: {passed}",
+              f"- FAIL: {failed}",
+              "- UNSUPPORTED: 0", "",
+              "Failing cases keep this workflow red only as a report; the sweep artifact",
+              "retains every case log and per-case report for diagnosis.", "",
+          ]
+          (root / "report.md").write_text("\n".join(report))
+          (root / "cases.md").write_text("\n".join(f"| {c} | {m} | {s} |" for c, m, s in rows) + "\n")
+          PY
+          cat "${SWEEP_ROOT}/report.md" >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Upload functional report to dashboard
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 2
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: ${{ runner.temp }}/validation-sweep/report.md
+          SUITE: validation
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
+            exit 0
+          fi
+          DATE="$(date -u +%Y-%m-%d)"
+          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
+          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${SHA}" ]; then
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          else
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          fi
+          echo "dashboard updated: ${REPORT_PATH}"
+
+      - name: Retain sweep evidence
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: rustfs-validation-sweep-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            ${{ runner.temp }}/validation-sweep/report.md
+            ${{ runner.temp }}/validation-sweep/results.txt
+            ${{ runner.temp }}/validation-sweep/sweep-cases.txt
+            ${{ runner.temp }}/validation-sweep/candidate.json
+            ${{ runner.temp }}/validation-sweep/results/
+            ${{ runner.temp }}/validation-sweep/run/
+          # warn (not error): an early infra failure legitimately leaves no
+          # per-case evidence, and the failing step is already red.
+          if-no-files-found: warn
+          retention-days: 30
+
+      - name: Cleanup environment (after)
+        id: environment_cleanup
+        timeout-minutes: 5
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        run: |
+          set -uo pipefail
+          cleanup_status=0
+          for node in ${RUSTFS_NODES}; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -f /etc/default/rustfs
+            ' || cleanup_status=$?
+          done
+          exit "${cleanup_status}"
+
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 scripts/functional_environment.py finish --suite heal --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Notify on failure
+        if: failure()
+        run: echo "RustFS validation sweep failed (harness/environment breakdown); see the sweep artifact for case logs."
+
+      - name: Enforce sweep result
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        env:
+          SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
+        run: |
+          set -euo pipefail
+          if grep -q $'\tFAIL$' "${SWEEP_ROOT}/results.txt"; then
+            echo "validation sweep has failing cases:"
+            grep $'\tFAIL$' "${SWEEP_ROOT}/results.txt"
+            exit 1
+          fi
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    # A scheduled sweep failure usually means a canonical validation case
+    # regressed or the fleet environment broke; surface it as a tracking
+    # issue instead of leaving it in the Actions tab.
+    needs: [sweep]
+    if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/scheduled-validation-freshness.yml

@@ -34,7 +34,7 @@ jobs:
     # self-hosted sm-standard-* pool is down, which is exactly when the
     # scheduled validations it watches stall.
     runs-on: ubuntu-latest
-    timeout-minutes: 10
+    timeout-minutes: 20
     permissions:
       actions: read
       contents: read
@@ -47,6 +47,7 @@ jobs:
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           RUSTFS_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+          NIGHTLY_SOURCE_REF: ${{ vars.NIGHTLY_BRANCH || 'main' }}
         run: |
           set +e
           python3 scripts/check_scheduled_validation_freshness.py \

.github/workflows/scheduled-validation-watchdog.yml

@@ -16,19 +16,8 @@ name: Scheduled Validation Watchdog
 
 on:
   workflow_run:
-    workflows:
-      - "Security Audit"
-      - "Build and Release"
-      - "Continuous Integration"
-      - "coverage"
-      - "e2e-nightly"
-      - "e2e-distributed"
-      - "e2e-s3tests"
-      - "Fuzz"
-      - "mint"
-      - "minio-interop"
-      - "Nightly GNU Build"
-      - "Runner Hygiene"
+    # The registry filters this broad subscription; names are not duplicated here.
+    workflows: ["*"]
     types: [completed]
 
 permissions:
@@ -37,11 +26,7 @@ permissions:
 jobs:
   alert-on-incomplete-run:
     name: Alert on incomplete scheduled run
-    if: >-
-      github.repository == 'rustfs/rustfs' &&
-      github.event.workflow_run.event == 'schedule' &&
-      github.event.workflow_run.conclusion != 'success' &&
-      github.event.workflow_run.conclusion != 'failure'
+    if: github.repository == 'rustfs/rustfs'
     # Stay on GitHub-hosted runners: this monitor must keep running when the
     # self-hosted sm-standard-* pool is down, which is exactly when the
     # scheduled validations it watches stall.
@@ -55,7 +40,11 @@ jobs:
       - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
+      - name: Select expected completion from registry
+        id: registry
+        run: python3 scripts/validation_registry.py watchdog
       - name: Open or update incomplete-run issue
+        if: steps.registry.outputs.selected == 'true'
         uses: ./.github/actions/schedule-failure-issue
         with:
           github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

CHANGELOG.md

@@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - IAM snapshot/deletion replay, target-assigned delete-marker purges, timestamp ordering, and best-effort peer broadcast now close the control-plane gaps found by the R6 review ([#7195](https://github.com/rustfs/rustfs/pull/7195)).
 - A `set-remote-target?update=true&creds=true` request on a site-replication peer's bucket target is now rejected with `400 InvalidRequest`, naming the target and why it counts as peer-owned. It used to return success while keeping the stored endpoint, credentials, and TLS settings, so an endpoint edit from the Console's bucket replication form appeared to apply but did not. Change a peer's address with a site-replication edit instead; see "Changing a site's address" in `docs/operations/site-replication-operations.md`. The `sync`, `bandwidth`, `proxy`, and `path` update groups are unchanged.
 - Site-replication edits that omit the site name, such as `mc admin replicate update --deployment-id <ID> --endpoint <URL>`, now keep the stored name on the editing site and on every receiver instead of renaming the site after the new endpoint host. No persisted format changes.
+- The persisted replication MRF journal is now replayed every 5 minutes while the server runs, not only at startup. Delete-marker purges whose targets were unreachable, and replications that overflowed the worker queues, now converge within one replay interval after the target returns instead of waiting for a restart. A target that stays down is re-attempted once per retained entry per interval across the cluster: the start time of the last pass is shared through a small `config/replication/mrf.bin.replay` object, so nodes do not each replay once per interval. Older versions ignore that object; existing persisted formats are unchanged.
 - Upgrade and rollback: upgrade every node in one site consecutively and verify reconciliation before moving to the next site; do not intentionally run a site mixed-version. Target-version ledger keys are harmless on rollback, although old code cannot use their routing. Before rolling back past [#7307](https://github.com/rustfs/rustfs/pull/7307), drain or repair every pending version purge: older code can free a retained version's data directory before its remote purge is acknowledged. See `docs/operations/site-replication-operations.md`.
 
 ### Security

Cargo.toml

@@ -147,7 +147,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
 async-channel = "2.5.0"
 async_zip = { default-features = false, version = "0.0.19" }
 mysql_async = { default-features = false, version = "0.37.1" }
-async-compression = { version = "0.4.51" }
+async-compression = { version = "0.4.52" }
 async-trait = "0.1.92"
 async-nats = { version = "0.51.0", default-features = false }
 axum = "0.8.9"
@@ -168,16 +168,16 @@ http-body-util = "0.1.5"
 memchr = "2.8.3"
 minlz = "1.2.3"
 reqwest = "0.13.5"
-rustfs-kafka-async = { version = "1.4.0" }
+rustfs-kafka-async = { version = "1.4.1" }
 socket2 = { version = "0.6.5" }
 tokio = { version = "1.54.1" }
 tokio-rustls = { default-features = false, version = "0.26.6" }
 tokio-stream = { version = "0.1.19" }
 tokio-test = "0.4.6"
 tokio-util = { version = "0.7.20" }
-tonic = { version = "0.14.8" }
-tonic-prost = { version = "0.14.8" }
-tonic-prost-build = { version = "0.14.8" }
+tonic = { version = "0.14.9" }
+tonic-prost = { version = "0.14.9" }
+tonic-prost-build = { version = "0.14.9" }
 tower = { version = "0.5.3" }
 tower-http = { version = "0.7.1" }
 h3 = "0.0.8"
@@ -191,6 +191,7 @@ quinn = { version = "0.11.12", default-features = false, features = [
 # Serialization and Data Formats
 apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
 bytes = { version = "1.12.1" }
+csv-core = "0.1.13"
 bytesize = "2.7.0"
 byteorder = "1.5.0"
 flatbuffers = "25.12.19"
@@ -212,12 +213,13 @@ serde_urlencoded = "0.7.1"
 # releases.
 aes-gcm = { version = "0.11.1" }
 argon2 = { version = "0.6.0" }
+aws-lc-rs = { version = "1.19.0", default-features = false }
 blake2 = "0.11.0"
 chacha20poly1305 = { version = "0.11.0" }
 crc-fast = "1.10.0"
 hmac = { version = "0.13.0" }
 jsonwebtoken = { version = "11.1.0" }
-openidconnect = { default-features = false, version = "4.0" }
+openidconnect = { default-features = false, version = "4.1.0" }
 pbkdf2 = "0.13.0"
 p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
 rsa = { version = "=0.10.0-rc.18" }
@@ -234,7 +236,7 @@ proptest = "1"
 # Time and Date
 chrono = { version = "0.4.45" }
 humantime = "2.4.0"
-jiff = { version = "0.2.38" }
+jiff = { version = "0.2.39" }
 time = { version = "0.3.55" }
 
 # Database
@@ -253,9 +255,9 @@ atoi = "3.1.1"
 atomic_enum = "0.3.0"
 aws-config = { version = "1.13.0" }
 aws-credential-types = { version = "1.3.1" }
-aws-sdk-kms = { default-features = false, version = "1.128.0" }
-aws-sdk-s3 = { default-features = false, version = "1.156.0" }
-aws-sdk-sts = { default-features = false, version = "1.123.0" }
+aws-sdk-kms = { default-features = false, version = "1.129.0" }
+aws-sdk-s3 = { default-features = false, version = "1.157.0" }
+aws-sdk-sts = { default-features = false, version = "1.124.0" }
 aws-smithy-async = { version = "1.3.1" }
 aws-smithy-http-client = { default-features = false, version = "1.5.2" }
 aws-smithy-runtime-api = { version = "1.19.2" }
@@ -319,7 +321,7 @@ rustfs-erasure-codec = { version = "9.0.0" }
 reed-solomon-simd = "3.1.0"
 regex = { version = "1.13.1" }
 rumqttc = { package = "rumqttc-next", version = "0.34.0" }
-redis = { version = "1.8.0" }
+redis = { version = "1.8.1" }
 rustify = { version = "0.7", default-features = false }
 rustix = { version = "1.1.6" }
 rust-embed = { version = "8.13.0" }
@@ -334,10 +336,10 @@ compact_str = "0.10.0"
 snap = "1.1.2"
 starshard = { version = "2.4.0" }
 strum = { version = "0.28.0" }
-sysinfo = "0.40.0"
+sysinfo = "0.40.1"
 temp-env = "0.3.6"
 tempfile = "3.27.0"
-test-case = "3.4.0"
+test-case = "3.4.1"
 thiserror = "2.0.21"
 tracing = { version = "0.1.44" }
 tracing-appender = "0.2.5"
@@ -356,13 +358,13 @@ winapi-util = "0.1.11"
 windows = { version = "0.62.2" }
 windows-sys = "0.61.2"
 xxhash-rust = { version = "0.8.19" }
-zip = "9.1.3"
+zip = "9.3.0"
 zstd = "0.14.3"
 
 # Observability and Metrics
 metrics = "0.24.6"
 metrics-util = "0.20"
-dial9 = { version = "0.5.6", default-features = false }
+dial9 = { version = "0.5.7", default-features = false }
 opentelemetry = { version = "0.33.1" }
 opentelemetry-appender-tracing = { version = "0.33.1" }
 opentelemetry-otlp = { version = "0.33.1" }
@@ -377,16 +379,16 @@ libunftp = { version = "0.24.0" }
 unftp-core = "0.1.0"
 suppaftp = { version = "12.2.1" }
 rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
-russh = { version = "0.65.0" }
+russh = { version = "0.65.1" }
 russh-sftp = "3.0.1"
 
 # WebDAV
 dav-server = "0.11.0"
 
 # Performance Analysis and Memory Profiling
-rustfs-mimalloc = { version = "0.7.2" }
+rustfs-mimalloc = { version = "0.7.3" }
 # Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
-hotpath = { version = "0.30.1", default-features = false }
+hotpath = { version = "0.30.3", default-features = false }
 # Snapshot testing for output format regression detection
 insta = { version = "1.49" }
 
@@ -427,7 +429,3 @@ strip = "none"
 # Pin upstream fixes until compatible crates.io releases are available.
 [patch.crates-io]
 zstd = { path = "crates/zstd-compat" }
-# Preserve RustFS OIDC flows while migrating the old elliptic-curve dependency chain.
-# Upstream: openidconnect-rs#244 and oauth2-rs#352.
-openidconnect = { git = "https://github.com/ramosbugs/openidconnect-rs", rev = "2f77ee73bc4554e1cb6279adb386005429fb4e41" }
-oauth2 = { git = "https://github.com/ramosbugs/oauth2-rs", rev = "89585f4a280afd7b285eaf1c9be0068874e03d22" }

crates/concurrency/src/adaptive.rs

@@ -10,6 +10,10 @@ use std::time::Duration;
 use tokio::sync::{Notify, Semaphore};
 use tokio::time::Instant;
 
+pub mod permit_set;
+
+const MAX_PERMIT_REFERENCES: usize = usize::MAX / 2;
+
 /// A resource owner publishes a capacity; running units retain their permits
 /// after a reduction. Lease slots are reserved once, with no per-unit Arc
 /// allocation. Tokio wait queues may lazily initialize platform mutex storage
@@ -36,6 +40,7 @@ pub struct AdaptiveAdmission {
     maximum: u32,
     slots: Box<[PermitSlots]>,
     next_chunk: AtomicUsize,
+    completed: AtomicU64,
 }
 
 struct PermitSlots {
@@ -61,10 +66,12 @@ pub struct AdaptivePermit {
 
 impl Clone for AdaptivePermit {
     fn clone(&self) -> Self {
-        // Every slot reference owns a strong owner reference. Clone the Arc
-        // first, so its overflow protection also bounds the slot refcount.
+        // Subset carriers can retain slot references through a shared owner,
+        // so the slot count needs its own bound in addition to Arc's guard.
         let owner = Arc::clone(&self.owner);
-        owner.slots[self.slot / 64].references[self.slot % 64].fetch_add(1, Ordering::Relaxed);
+        if owner.slots[self.slot / 64].references[self.slot % 64].fetch_add(1, Ordering::Relaxed) >= MAX_PERMIT_REFERENCES {
+            std::process::abort();
+        }
         Self { owner, slot: self.slot }
     }
 }
@@ -114,11 +121,22 @@ impl NodeAdmission {
 
     /// Publish feedback with this owner's actual completed-work counter.
     pub fn update(&self, controller: &mut FeedbackController, observation: Option<FeedbackObservation>) -> FeedbackDecision {
+        self.update_with_probe(controller, observation, true)
+    }
+
+    /// Coordinate growth with other resource controllers in the same window.
+    /// Protective reductions never depend on permission to probe.
+    pub fn update_with_probe(
+        &self,
+        controller: &mut FeedbackController,
+        observation: Option<FeedbackObservation>,
+        allow_probe: bool,
+    ) -> FeedbackDecision {
         let observation = observation.map(|mut observation| {
             observation.completed_units = self.completed();
             observation
         });
-        controller.update(&self.capacity, observation)
+        controller.update_with_probe(&self.capacity, observation, allow_probe)
     }
 
     /// Current leased scheduling cap and held async units.
@@ -249,16 +267,31 @@ impl FeedbackController {
     /// Consume one sample and publish a three-second lease to the same gate.
     /// If this function stops being called, admission expires independently.
     pub fn update(&mut self, gate: &AdaptiveAdmission, observation: Option<FeedbackObservation>) -> FeedbackDecision {
+        self.update_with_probe(gate, observation, true)
+    }
+
+    /// Allow at most the sampling owner's selected controller to probe upward.
+    pub fn update_with_probe(
+        &mut self,
+        gate: &AdaptiveAdmission,
+        observation: Option<FeedbackObservation>,
+        allow_probe: bool,
+    ) -> FeedbackDecision {
         let observation = observation.map(|mut observation| {
             observation.max_concurrency = observation.max_concurrency.min(gate.maximum);
             observation
         });
-        let decision = self.decide(Instant::now(), observation);
+        let decision = self.decide_with_probe(Instant::now(), observation, allow_probe);
         gate.set_sampled_limit(decision.limit, Duration::from_secs(3));
         decision
     }
 
-    fn decide(&mut self, now: Instant, observation: Option<FeedbackObservation>) -> FeedbackDecision {
+    fn decide_with_probe(
+        &mut self,
+        now: Instant,
+        observation: Option<FeedbackObservation>,
+        allow_probe: bool,
+    ) -> FeedbackDecision {
         let Some(observation) = observation else {
             self.previous = None;
             self.stable_since = None;
@@ -303,7 +336,8 @@ impl FeedbackController {
             FeedbackReason::NewBaseline
         } else {
             let since = *self.stable_since.get_or_insert(now);
-            if advanced
+            if allow_probe
+                && advanced
                 && now.saturating_duration_since(since) >= Duration::from_secs(8)
                 && now.saturating_duration_since(self.last_change) >= Duration::from_secs(8)
                 && self.limit < observation.max_concurrency
@@ -325,6 +359,13 @@ impl FeedbackController {
 }
 
 impl AdaptiveAdmission {
+    /// Successful async operations using this physical owner. Native clones
+    /// retain capacity but never count another completion.
+    pub fn completed(&self) -> Option<u64> {
+        let value = self.completed.load(Ordering::Acquire);
+        (value != u64::MAX).then_some(value)
+    }
+
     /// Construct a bounded owner, with zero initial capacity allowed for unknown
     /// input. The immutable maximum reserves at most 65,535 lease slots once;
     /// acquiring or cloning a permit does not allocate a per-unit Arc.
@@ -351,6 +392,7 @@ impl AdaptiveAdmission {
             maximum: maximum_u32,
             slots,
             next_chunk: AtomicUsize::new(0),
+            completed: AtomicU64::new(0),
         })
     }
 
@@ -466,6 +508,21 @@ impl AdaptiveAdmission {
         self.slots[slot / 64].free.fetch_or(1_u64 << (slot % 64), Ordering::Release);
     }
 
+    fn release_reference(&self, slot: usize) {
+        self.release_reference_with(slot, || {});
+    }
+
+    fn release_reference_with(&self, slot: usize, before_release: impl FnOnce()) {
+        if self.slots[slot / 64].references[slot % 64].fetch_sub(1, Ordering::AcqRel) != 1 {
+            return;
+        }
+        before_release();
+        // Publish the free slot before exposing another admission capacity.
+        self.release_slot(slot);
+        self.state.fetch_sub(1, Ordering::AcqRel);
+        self.changed.notify_one();
+    }
+
     /// Wait for capacity. The future owns no permit until it returns; dropping
     /// a pending future leaves capacity untouched. Wrap in the caller's existing
     /// cancellation/deadline select before acquiring any storage commit locks.
@@ -493,13 +550,7 @@ impl AdaptiveAdmission {
 
 impl Drop for AdaptivePermit {
     fn drop(&mut self) {
-        if self.owner.slots[self.slot / 64].references[self.slot % 64].fetch_sub(1, Ordering::AcqRel) != 1 {
-            return;
-        }
-        // Publish the free slot before exposing another admission capacity.
-        self.owner.release_slot(self.slot);
-        self.owner.state.fetch_sub(1, Ordering::AcqRel);
-        self.owner.changed.notify_one();
+        self.owner.release_reference(self.slot);
     }
 }
 
@@ -861,6 +912,24 @@ mod tests {
         }
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn shared_probe_permission_blocks_growth_but_never_protective_cuts() {
+        let gate = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bound"), 0);
+        let mut controller = FeedbackController::new();
+        controller.update_with_probe(&gate, Some(healthy(0)), false);
+        for completed in 1..=20 {
+            tokio::time::advance(Duration::from_secs(1)).await;
+            assert_eq!(controller.update_with_probe(&gate, Some(healthy(completed)), false).limit, 1);
+        }
+        tokio::time::advance(Duration::from_secs(1)).await;
+        assert_eq!(controller.update_with_probe(&gate, Some(healthy(21)), true).reason, FeedbackReason::Probe);
+        assert_eq!(gate.snapshot().limit, 2);
+        let mut busy = healthy(22);
+        busy.pressure = ResourcePressure::Busy;
+        assert_eq!(controller.update_with_probe(&gate, Some(busy), false).limit, 1);
+        assert_eq!(controller.update_with_probe(&gate, None, false).limit, 0);
+    }
+
     #[tokio::test(start_paused = true)]
     async fn controller_grows_one_unit_per_cooldown_and_cuts_pressure_immediately() {
         let gate = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded test owner"), 0);

crates/config/src/constants/env.rs

@@ -30,6 +30,16 @@ pub const COMMENT_KEY: &str = "comment";
 pub const ENV_AUDIT_ENABLE: &str = "RUSTFS_AUDIT_ENABLE";
 /// Global switch for enabling the notify module.
 pub const ENV_NOTIFY_ENABLE: &str = "RUSTFS_NOTIFY_ENABLE";
+/// Maximum audit entries per second for authorization denials of authenticated callers.
+/// `0` disables denial auditing for that class of caller.
+pub const ENV_AUDIT_DENIAL_RATE: &str = "RUSTFS_AUDIT_DENIAL_RATE";
+/// Maximum audit entries per second for authorization denials of anonymous callers.
+/// `0` disables denial auditing for that class of caller.
+pub const ENV_AUDIT_DENIAL_ANONYMOUS_RATE: &str = "RUSTFS_AUDIT_DENIAL_ANONYMOUS_RATE";
+/// Default denial audit rate for authenticated callers (entries per second).
+pub const DEFAULT_AUDIT_DENIAL_RATE: u64 = 50;
+/// Default denial audit rate for anonymous callers (entries per second).
+pub const DEFAULT_AUDIT_DENIAL_ANONYMOUS_RATE: u64 = 5;
 /// Default global audit switch (disabled by default).
 pub const DEFAULT_AUDIT_ENABLE: bool = false;
 /// Default global notify switch (disabled by default).
@@ -328,5 +338,7 @@ mod tests {
     fn test_global_audit_notify_switch_constants() {
         assert_eq!(ENV_AUDIT_ENABLE, "RUSTFS_AUDIT_ENABLE");
         assert_eq!(ENV_NOTIFY_ENABLE, "RUSTFS_NOTIFY_ENABLE");
+        assert_eq!(ENV_AUDIT_DENIAL_RATE, "RUSTFS_AUDIT_DENIAL_RATE");
+        assert_eq!(ENV_AUDIT_DENIAL_ANONYMOUS_RATE, "RUSTFS_AUDIT_DENIAL_ANONYMOUS_RATE");
     }
 }

crates/config/src/constants/proxy.rs

@@ -30,10 +30,10 @@ pub const ENV_TRUSTED_PROXY_VALIDATION_MODE: &str = "RUSTFS_TRUSTED_PROXY_VALIDA
 /// Default validation mode is "hop_by_hop".
 pub const DEFAULT_TRUSTED_PROXY_VALIDATION_MODE: &str = "hop_by_hop";
 
-/// Environment variable to enable RFC 7239 "Forwarded" header support.
+/// Select RFC 7239 "Forwarded" instead of "X-Forwarded-For" in legacy mode.
 pub const ENV_TRUSTED_PROXY_ENABLE_RFC7239: &str = "RUSTFS_TRUSTED_PROXY_ENABLE_RFC7239";
-/// RFC 7239 support is enabled by default.
-pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = true;
+/// Legacy mode uses X-Forwarded-For unless RFC 7239 is explicitly selected.
+pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = false;
 
 /// Environment variable for the maximum allowed proxy hops.
 pub const ENV_TRUSTED_PROXY_MAX_HOPS: &str = "RUSTFS_TRUSTED_PROXY_MAX_HOPS";

crates/config/src/constants/targets.rs

@@ -146,3 +146,9 @@ pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";
 
 /// Queue-store compression is enabled by default to reduce disk footprint.
 pub const DEFAULT_TARGET_STORE_COMPRESS: bool = true;
+
+/// Maximum on-disk bytes in each target queue, including failed entries and pending writes.
+pub const ENV_TARGET_STORE_MAX_BYTES: &str = "RUSTFS_TARGET_STORE_MAX_BYTES";
+
+/// Queue writes fail at one GiB unless the operator sets another positive byte budget.
+pub const DEFAULT_TARGET_STORE_MAX_BYTES: u64 = 1024 * 1024 * 1024;

crates/data-usage/src/data_usage.rs

@@ -24,6 +24,14 @@ use std::{
     time::{Duration, SystemTime},
 };
 
+/// Copy an existing cache-model value without including surrounding work.
+#[inline]
+pub fn clone_cache_value<T: ToOwned + ?Sized>(source: &T) -> T::Owned {
+    #[cfg(feature = "hotpath-alloc")]
+    let _copy_scope = hotpath::MeasurementGuardSync::new("cache_model.owned_copy", false, false);
+    source.to_owned()
+}
+
 /// Maximum amount a persisted `last_update` may lead the local wall clock before the
 /// persisted timestamp is treated as untrustworthy.
 ///
@@ -342,7 +350,7 @@ impl UnknownTierStats {
                     self.diagnostics_dropped = checked_saturating_add(1, self.diagnostics_dropped, &mut self.counter_overflowed);
                     continue;
                 }
-                self.diagnostics.push(diagnostic.clone());
+                self.diagnostics.push(clone_cache_value(diagnostic));
             }
         }
         if !self.diagnostics.is_empty() {
@@ -367,7 +375,7 @@ impl UnknownTierStats {
         if !self.fits_add(other) {
             return None;
         }
-        let mut merged = self.clone();
+        let mut merged = clone_cache_value(self);
         merged.merge(other);
         Some(merged)
     }
@@ -407,7 +415,7 @@ impl AllTierStats {
             if st.is_empty() {
                 continue;
             }
-            let entry = self.tiers.entry(tier.clone()).or_default();
+            let entry = self.tiers.entry(clone_cache_value(tier)).or_default();
             *entry = entry.add(st);
         }
     }
@@ -437,7 +445,7 @@ impl AllTierStats {
             .tiers
             .keys()
             .filter(|tier| tier.as_str() != "STANDARD" && tier.as_str() != "REDUCED_REDUNDANCY" && !known.contains(tier.as_str()))
-            .cloned()
+            .map(clone_cache_value)
             .collect();
         for tier in retired_tiers {
             if let Some(stats) = self.tiers.remove(&tier) {
@@ -1237,10 +1245,10 @@ impl DataUsageEntry {
     }
 
     pub fn add_child(&mut self, hash: &DataUsageHash) {
-        if self.children.contains(&hash.key()) {
+        if self.children.contains(&clone_cache_value(&hash.0)) {
             return;
         }
-        self.children.insert(hash.key());
+        self.children.insert(clone_cache_value(&hash.0));
     }
 
     pub fn merge(&mut self, other: &DataUsageEntry) {
@@ -1257,7 +1265,7 @@ impl DataUsageEntry {
             s_rep.replica_size += o_rep.replica_size;
             s_rep.replica_count += o_rep.replica_count;
             for (arn, stat) in o_rep.targets.iter() {
-                let st = s_rep.targets.entry(arn.clone()).or_default();
+                let st = s_rep.targets.entry(clone_cache_value(arn)).or_default();
                 st.pending_size += stat.pending_size;
                 st.replicated_size += stat.replicated_size;
                 st.failed_size += stat.failed_size;
@@ -1451,7 +1459,7 @@ pub fn prefix_usage_in_cache(
     } else {
         format!("{bucket}/{prefix}")
     };
-    let entry = cache.get(&hash_path(&root).key())?.clone();
+    let entry = clone_cache_value(cache.get(&clone_cache_value(&hash_path(&root).0))?);
 
     let usage = PrefixUsageSummary::from_entry(&flatten_entry(cache, &entry, 0)?);
 
@@ -1594,38 +1602,38 @@ pub struct DataUsageCache {
 impl DataUsageCache {
     pub fn replace(&mut self, path: &str, parent: &str, e: DataUsageEntry) {
         let hash = hash_path(path);
-        self.cache.insert(hash.key(), e);
+        self.cache.insert(clone_cache_value(&hash.0), e);
         if !parent.is_empty() {
             let phash = hash_path(parent);
             let p = {
-                let p = self.cache.entry(phash.key()).or_default();
+                let p = self.cache.entry(clone_cache_value(&phash.0)).or_default();
                 p.add_child(&hash);
-                p.clone()
+                clone_cache_value(p)
             };
-            self.cache.insert(phash.key(), p);
+            self.cache.insert(clone_cache_value(&phash.0), p);
         }
     }
 
     pub fn replace_hashed(&mut self, hash: &DataUsageHash, parent: &Option<DataUsageHash>, e: &DataUsageEntry) {
-        self.cache.insert(hash.key(), e.clone());
+        self.cache.insert(clone_cache_value(&hash.0), clone_cache_value(e));
         if let Some(parent) = parent {
-            self.cache.entry(parent.key()).or_default().add_child(hash);
+            self.cache.entry(clone_cache_value(&parent.0)).or_default().add_child(hash);
         }
     }
 
     pub fn find(&self, path: &str) -> Option<DataUsageEntry> {
-        self.cache.get(&hash_path(path).key()).cloned()
+        self.cache.get(&clone_cache_value(&hash_path(path).0)).map(clone_cache_value)
     }
 
     pub fn find_children_copy(&mut self, h: DataUsageHash) -> DataUsageHashMap {
-        self.cache.entry(h.string()).or_default().children.clone()
+        clone_cache_value(&self.cache.entry(clone_cache_value(&h.0)).or_default().children)
     }
 
     pub fn flatten(&self, root: &DataUsageEntry) -> DataUsageEntry {
-        let mut root = root.clone();
-        for id in root.children.clone().iter() {
+        let mut root = clone_cache_value(root);
+        for id in clone_cache_value(&root.children).iter() {
             if let Some(e) = self.cache.get(id) {
-                let mut e = e.clone();
+                let mut e = clone_cache_value(e);
                 if !e.children.is_empty() {
                     e = self.flatten(&e);
                 }
@@ -1637,16 +1645,16 @@ impl DataUsageCache {
     }
 
     pub fn copy_with_children(&mut self, src: &DataUsageCache, hash: &DataUsageHash, parent: &Option<DataUsageHash>) {
-        if let Some(e) = src.cache.get(&hash.string()) {
-            self.cache.insert(hash.key(), e.clone());
+        if let Some(e) = src.cache.get(&clone_cache_value(&hash.0)) {
+            self.cache.insert(clone_cache_value(&hash.0), clone_cache_value(e));
             for ch in e.children.iter() {
-                if *ch == hash.key() {
+                if *ch == clone_cache_value(&hash.0) {
                     return;
                 }
-                self.copy_with_children(src, &DataUsageHash(ch.to_string()), &Some(hash.clone()));
+                self.copy_with_children(src, &DataUsageHash(clone_cache_value(ch)), &Some(clone_cache_value(hash)));
             }
             if let Some(parent) = parent {
-                let p = self.cache.entry(parent.key()).or_default();
+                let p = self.cache.entry(clone_cache_value(&parent.0)).or_default();
                 p.add_child(hash);
             }
         }
@@ -1654,14 +1662,14 @@ impl DataUsageCache {
 
     pub fn delete_recursive(&mut self, hash: &DataUsageHash) {
         let mut need_remove = Vec::new();
-        if let Some(v) = self.cache.get(&hash.string()) {
+        if let Some(v) = self.cache.get(&clone_cache_value(&hash.0)) {
             for child in v.children.iter() {
-                need_remove.push(child.clone());
+                need_remove.push(clone_cache_value(child));
             }
         }
-        self.cache.remove(&hash.string());
+        self.cache.remove(&clone_cache_value(&hash.0));
         need_remove.iter().for_each(|child| {
-            self.delete_recursive(&DataUsageHash(child.to_string()));
+            self.delete_recursive(&DataUsageHash(clone_cache_value(child)));
         });
     }
 
@@ -1682,7 +1690,7 @@ impl DataUsageCache {
     }
 
     pub fn search_parent(&self, hash: &DataUsageHash) -> Option<DataUsageHash> {
-        let want = hash.key();
+        let want = clone_cache_value(&hash.0);
         if let Some(last_index) = want.rfind('/')
             && let Some(v) = self.find(&want[0..last_index])
             && v.children.contains(&want)
@@ -1693,7 +1701,7 @@ impl DataUsageCache {
 
         for (k, v) in self.cache.iter() {
             if v.children.contains(&want) {
-                let found = DataUsageHash(k.clone());
+                let found = DataUsageHash(clone_cache_value(k));
                 return Some(found);
             }
         }
@@ -1701,7 +1709,7 @@ impl DataUsageCache {
     }
 
     pub fn is_compacted(&self, hash: &DataUsageHash) -> bool {
-        match self.cache.get(&hash.key()) {
+        match self.cache.get(&clone_cache_value(&hash.0)) {
             Some(due) => due.compacted,
             None => false,
         }
@@ -1726,7 +1734,7 @@ impl DataUsageCache {
         if self.cache.len() < limit {
             return;
         }
-        let top = hash_path(&self.info.name).key();
+        let top = clone_cache_value(&hash_path(&self.info.name).0);
         let top_e = match self.find(&top) {
             Some(e) => e,
             None => return,
@@ -1753,7 +1761,7 @@ impl DataUsageCache {
     }
 
     pub fn reduce_children_of(&mut self, path: &DataUsageHash, limit: usize, compact_self: bool) {
-        let e = match self.cache.get(&path.key()) {
+        let e = match self.cache.get(&clone_cache_value(&path.0)) {
             Some(e) => e,
             None => return,
         };
@@ -1763,13 +1771,13 @@ impl DataUsageCache {
         }
 
         if e.children.len() > limit && compact_self {
-            let mut flat = self.size_recursive(&path.key()).unwrap_or_default();
+            let mut flat = self.size_recursive(&clone_cache_value(&path.0)).unwrap_or_default();
             flat.compacted = true;
             self.delete_recursive(path);
             self.replace_hashed(path, &None, &flat);
             return;
         }
-        let total = self.total_children_rec(&path.key());
+        let total = self.total_children_rec(&clone_cache_value(&path.0));
         if total < limit {
             return;
         }
@@ -1782,12 +1790,12 @@ impl DataUsageCache {
         let mut candidate_index = 0;
         while remove > 0 && candidate_index < candidates.len() {
             let e = &candidates[candidate_index];
-            let candidate = e.path.clone();
+            let candidate = clone_cache_value(&e.path);
             if candidate == *path && !compact_self {
                 break;
             }
-            let removing = self.total_children_rec(&candidate.key());
-            let mut flat = match self.size_recursive(&candidate.key()) {
+            let removing = self.total_children_rec(&clone_cache_value(&candidate.0));
+            let mut flat = match self.size_recursive(&clone_cache_value(&candidate.0)) {
                 Some(flat) => flat,
                 None => {
                     candidate_index += 1;
@@ -1824,7 +1832,7 @@ impl DataUsageCache {
             if o.root().is_none() {
                 return;
             }
-            *self = o.clone();
+            *self = clone_cache_value(o);
             return;
         };
 
@@ -1837,7 +1845,8 @@ impl DataUsageCache {
         }
 
         existing_root.merge(&other_root);
-        self.cache.insert(hash_path(&self.info.name).key(), existing_root);
+        self.cache
+            .insert(clone_cache_value(&hash_path(&self.info.name).0), existing_root);
 
         let root_hash = self.root_hash();
         for key in other_root.children.iter() {
@@ -1848,7 +1857,7 @@ impl DataUsageCache {
             if let Some(existing) = self.cache.get_mut(key) {
                 existing.merge(&flat);
             } else {
-                self.replace_hashed(&DataUsageHash(key.clone()), &Some(root_hash.clone()), &flat);
+                self.replace_hashed(&DataUsageHash(clone_cache_value(key)), &Some(clone_cache_value(&root_hash)), &flat);
             }
         }
     }
@@ -1892,7 +1901,7 @@ impl DataUsageCache {
 
                 for (arn, stat) in rs.targets.iter() {
                     bui.replication_info.insert(
-                        arn.clone(),
+                        clone_cache_value(arn),
                         BucketTargetUsageInfo {
                             replication_pending_size: stat.pending_size,
                             replicated_size: stat.replicated_size,
@@ -1905,7 +1914,7 @@ impl DataUsageCache {
                     );
                 }
             }
-            buckets_usage.insert(bucket_name.clone(), bui);
+            buckets_usage.insert(clone_cache_value(bucket_name), bui);
         }
 
         DataUsageInfo {
@@ -1937,20 +1946,20 @@ struct Inner {
 }
 
 fn add(data_usage_cache: &DataUsageCache, path: &DataUsageHash, candidates: &mut Vec<Inner>) -> usize {
-    let e = match data_usage_cache.cache.get(&path.key()) {
+    let e = match data_usage_cache.cache.get(&clone_cache_value(&path.0)) {
         Some(e) => e,
         None => return 0,
     };
     let mut objects = e.objects;
     for ch in e.children.iter() {
-        objects += add(data_usage_cache, &DataUsageHash(ch.clone()), candidates);
+        objects += add(data_usage_cache, &DataUsageHash(clone_cache_value(ch)), candidates);
     }
     // Collect internal nodes (with children) as compaction candidates.
     // Leaf nodes have no children to remove, so compacting them is a no-op.
     if !e.children.is_empty() {
         candidates.push(Inner {
             objects,
-            path: path.clone(),
+            path: clone_cache_value(path),
         });
     }
     objects
@@ -1958,7 +1967,7 @@ fn add(data_usage_cache: &DataUsageCache, path: &DataUsageHash, candidates: &mut
 
 fn mark(duc: &DataUsageCache, entry: &DataUsageEntry, found: &mut HashSet<String>) {
     for k in entry.children.iter() {
-        found.insert(k.to_string());
+        found.insert(clone_cache_value(k));
         if let Some(ch) = duc.cache.get(k) {
             mark(duc, ch, found);
         }
@@ -2408,11 +2417,11 @@ impl SizeSummary {
         }
 
         for entry in &other.size_reconciliation {
-            self.record_size_reconciliation(entry.clone());
+            self.record_size_reconciliation(clone_cache_value(entry));
         }
         self.size_reconciliation_truncated |= other.size_reconciliation_truncated;
         for scope in &other.reconciliation_scopes {
-            self.record_reconciliation_scope(&scope.bucket, &scope.object);
+            self.record_reconciliation_scope_with(&scope.bucket, &scope.object, || clone_cache_value(scope));
         }
     }
 
@@ -2437,6 +2446,13 @@ impl SizeSummary {
     /// Mark one object scope as refreshed. Duplicate scopes are suppressed so
     /// merging summaries remains bounded and deterministic.
     pub fn record_reconciliation_scope(&mut self, bucket: &str, object: &str) {
+        self.record_reconciliation_scope_with(bucket, object, || SizeReconciliationScope {
+            bucket: bucket.to_string(),
+            object: object.to_string(),
+        });
+    }
+
+    fn record_reconciliation_scope_with(&mut self, bucket: &str, object: &str, build: impl FnOnce() -> SizeReconciliationScope) {
         if !self
             .reconciliation_scopes
             .iter()
@@ -2446,10 +2462,7 @@ impl SizeSummary {
                 self.size_reconciliation_truncated = true;
                 return;
             }
-            self.reconciliation_scopes.push(SizeReconciliationScope {
-                bucket: bucket.to_string(),
-                object: object.to_string(),
-            });
+            self.reconciliation_scopes.push(build());
         }
     }
 }
@@ -3404,6 +3417,42 @@ mod tests {
         assert_eq!(summary1.versions, 15);
     }
 
+    #[test]
+    fn test_reconciliation_scope_builds_only_after_admission() {
+        let mut summary = SizeSummary::new();
+        let builds = std::cell::Cell::new(0);
+        let build = || {
+            builds.set(builds.get() + 1);
+            SizeReconciliationScope {
+                bucket: "bucket".to_string(),
+                object: "object".to_string(),
+            }
+        };
+
+        summary.record_reconciliation_scope_with("bucket", "object", build);
+        assert_eq!(builds.get(), 1, "an admitted scope is copied once");
+        assert_eq!(summary.reconciliation_scopes.len(), 1);
+        assert_eq!(summary.reconciliation_scopes[0].bucket, "bucket");
+        assert_eq!(summary.reconciliation_scopes[0].object, "object");
+
+        summary.record_reconciliation_scope_with("bucket", "object", build);
+        assert_eq!(builds.get(), 1, "duplicate scopes must not be copied");
+        assert_eq!(summary.reconciliation_scopes.len(), 1);
+        assert!(!summary.size_reconciliation_truncated);
+
+        for index in 1..1024 {
+            summary.record_reconciliation_scope("bucket", &index.to_string());
+        }
+        summary.record_reconciliation_scope_with("bucket", "object", build);
+        assert_eq!(builds.get(), 1, "a duplicate at capacity must not be copied");
+        assert!(!summary.size_reconciliation_truncated);
+
+        summary.record_reconciliation_scope_with("bucket", "overflow", build);
+        assert_eq!(builds.get(), 1, "a rejected scope must not be copied");
+        assert_eq!(summary.reconciliation_scopes.len(), 1024);
+        assert!(summary.size_reconciliation_truncated);
+    }
+
     #[test]
     fn test_size_histogram_compat_rollup_sums_all_sub_buckets() {
         let mut hist = SizeHistogram::default();

crates/e2e_test/Cargo.toml

@@ -25,6 +25,7 @@ workspace = true
 
 [features]
 default = []
+scanner-peer-mixed-version = []
 hotpath = [
     "hotpath/hotpath",
     "hotpath/tokio",

crates/e2e_test/README.md

@@ -26,7 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
 | **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
 | **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
 | **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
-| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
+| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
 | **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
 | **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
 
@@ -64,6 +64,25 @@ Each build/run holds an exclusive `rustfs.e2e.lock` marker beside the binary; co
 
 The protocols suite has its own fixed-port and single-worker contract in [`src/protocols/README.md`](src/protocols/README.md). Use its command under [Troubleshooting](#troubleshooting).
 
+### Scanner mixed-version gate
+
+The Linux N/O scanner gate needs a separately qualified `33482bb` server, which supports scoped ACKs. Set `RUSTFS_SCANNER_COMPAT_OLD_BINARY` to that executable and `RUSTFS_SCANNER_COMPAT_OLD_SHA256` to its independently verified digest. The release binary used by ordinary upgrade CI is a different baseline. Missing or mismatched inputs fail the test.
+
+Enable the harness-only `scanner-peer-mixed-version` feature explicitly; keep the server wrapper features identical between build and run:
+
+```bash
+python3 scripts/e2e_binary.py build --features e2e-test-hooks
+python3 scripts/e2e_binary.py run --features e2e-test-hooks -- \
+  cargo nextest run --profile e2e-distributed -p e2e_test \
+  --features scanner-peer-mixed-version --ignore-default-filter \
+  -E 'test(=distributed::scanner_peer_bucket_test::scanner_peer_bucket_new_coordinator_certifies_old_peers_and_scopes_next_cycle)' \
+  --test-threads 1 --retries 0 --no-tests=fail
+```
+
+The separate `scanner_peer_bucket_new_coordinator_certifies_old_worker_peers_and_scopes_next_cycle` case uses the same feature and qualified old binary inputs. Select its exact name in the command above. It keeps the original N process enabled and starts the three official O replacements with `RUSTFS_SCANNER_ENABLED=false`: their autonomous scanner loops are disabled, while real S3 dirty tracking, authenticated snapshots, publication leases, scoped ACKs, and N-dispatched RemoteScanner workers remain active. It records each node's actual scanner status and retains the same identity, durability, root, ACK, data-conservation, and physical-walk requirements. This fixed-role case does not validate autonomous O scanning or all-enabled election behavior. The original all-enabled N/O test remains separate and has not passed acceptance; a fixed-role result cannot satisfy that gate.
+
+For this fixed-role case, the `e2e-test-hooks` server also records the N client's complete bounded worker inventory. Requests are observed before opening the normal stream and paired only after authenticated source/plan validation and terminal EOF. Closed manifests and final health receipts must bind both cycles to their actual candidate/root; the next cycle needs exactly four sets of hot-only execution and at least one positive-progress worker mapped to a measured official O process. A zero-progress Current response, missing/extra request, unhealthy recorder, or missing physical coverage fails the gate. The recorder does not add O hooks or change any protocol or timeout. Final health reads check immutable closed-cycle evidence, not current N liveness.
+
 ### `#[ignore]` semantics
 
 Ignored tests are excluded from the default `cargo nextest run` pass because
@@ -175,9 +194,9 @@ the same profile for membership and execution with one nightly worker.
 | `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
 | ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
 | KMS suite | `e2e-full` job, merge queue + main | **Active** |
-| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
+| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, release tags + weekly schedule + manual dispatch | **Active** |
 | Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
-| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
+| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (nightly / dispatch) |
 | Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
 | Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
 | Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |

crates/e2e_test/src/delete_marker_migration_semantics_test.rs

@@ -16,8 +16,193 @@
 mod tests {
     use crate::common::{RustFSTestEnvironment, init_logging};
     use aws_sdk_s3::Client;
+    use aws_sdk_s3::config::http::HttpResponse;
+    use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
     use aws_sdk_s3::primitives::ByteStream;
-    use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
+    use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
+
+    fn assert_not_found<E: ProvideErrorMetadata + std::fmt::Debug>(error: &SdkError<E, HttpResponse>) {
+        assert_eq!(error.raw_response().map(|r| r.status().as_u16()), Some(404), "{error:?}");
+        assert_eq!(
+            error.as_service_error().and_then(ProvideErrorMetadata::code),
+            Some("NoSuchKey"),
+            "{error:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn conditional_writes_cannot_replace_delete_marker_with_if_match() {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
+        env.start_rustfs_server(vec![]).await.expect("start RustFS");
+        let client = env.create_s3_client();
+        let bucket = "conditional-delete-marker-proof";
+        create_versioned_bucket(&client, bucket).await;
+        client
+            .put_object()
+            .bucket(bucket)
+            .key("source")
+            .body(ByteStream::from_static(b"source"))
+            .send()
+            .await
+            .expect("write copy source");
+
+        for operation in ["put", "copy", "multipart"] {
+            let old = client
+                .put_object()
+                .bucket(bucket)
+                .key(operation)
+                .body(ByteStream::from_static(b"old"))
+                .send()
+                .await
+                .expect("write historical version");
+            let old_etag = old.e_tag().expect("historical ETag");
+            let marker = client
+                .delete_object()
+                .bucket(bucket)
+                .key(operation)
+                .send()
+                .await
+                .expect("create delete marker");
+            let marker_id = marker.version_id().expect("delete marker version");
+
+            let (upload_id, completed) = if operation == "multipart" {
+                let upload = client
+                    .create_multipart_upload()
+                    .bucket(bucket)
+                    .key(operation)
+                    .send()
+                    .await
+                    .expect("initiate multipart upload");
+                let upload_id = upload.upload_id().expect("upload ID").to_owned();
+                let part = client
+                    .upload_part()
+                    .bucket(bucket)
+                    .key(operation)
+                    .upload_id(&upload_id)
+                    .part_number(1)
+                    .body(ByteStream::from_static(b"new"))
+                    .send()
+                    .await
+                    .expect("upload part");
+                let completed = CompletedMultipartUpload::builder()
+                    .parts(
+                        CompletedPart::builder()
+                            .part_number(1)
+                            .set_e_tag(part.e_tag().map(str::to_owned))
+                            .build(),
+                    )
+                    .build();
+                (Some(upload_id), Some(completed))
+            } else {
+                (None, None)
+            };
+
+            for etag in [old_etag, "\"00000000000000000000000000000000\""] {
+                match operation {
+                    "put" => {
+                        let err = client
+                            .put_object()
+                            .bucket(bucket)
+                            .key(operation)
+                            .if_match(etag)
+                            .body(ByteStream::from_static(b"new"))
+                            .send()
+                            .await
+                            .expect_err("If-Match must reject marker");
+                        assert_not_found(&err);
+                    }
+                    "copy" => {
+                        let err = client
+                            .copy_object()
+                            .bucket(bucket)
+                            .key(operation)
+                            .copy_source(format!("{bucket}/source"))
+                            .if_match(etag)
+                            .send()
+                            .await
+                            .expect_err("If-Match copy must reject marker");
+                        assert_not_found(&err);
+                    }
+                    _ => {
+                        let err = client
+                            .complete_multipart_upload()
+                            .bucket(bucket)
+                            .key(operation)
+                            .upload_id(upload_id.as_deref().expect("upload ID"))
+                            .multipart_upload(completed.clone().expect("completed parts"))
+                            .if_match(etag)
+                            .send()
+                            .await
+                            .expect_err("If-Match completion must reject marker");
+                        assert_not_found(&err);
+                    }
+                }
+                let versions = client
+                    .list_object_versions()
+                    .bucket(bucket)
+                    .prefix(operation)
+                    .send()
+                    .await
+                    .expect("list versions after rejected write");
+                assert_eq!(versions.versions().len(), 1, "{operation}: failed write created a version");
+                assert_eq!(versions.delete_markers().len(), 1);
+                assert_eq!(versions.delete_markers()[0].version_id(), Some(marker_id));
+                assert_eq!(versions.delete_markers()[0].is_latest(), Some(true));
+            }
+
+            match operation {
+                "put" => {
+                    client
+                        .put_object()
+                        .bucket(bucket)
+                        .key(operation)
+                        .if_none_match("*")
+                        .body(ByteStream::from_static(b"new"))
+                        .send()
+                        .await
+                        .expect("If-None-Match PUT over marker");
+                }
+                "copy" => {
+                    client
+                        .copy_object()
+                        .bucket(bucket)
+                        .key(operation)
+                        .copy_source(format!("{bucket}/source"))
+                        .if_none_match("*")
+                        .send()
+                        .await
+                        .expect("If-None-Match copy over marker");
+                }
+                _ => {
+                    client
+                        .complete_multipart_upload()
+                        .bucket(bucket)
+                        .key(operation)
+                        .upload_id(upload_id.as_deref().expect("upload ID"))
+                        .multipart_upload(completed.expect("completed parts"))
+                        .if_none_match("*")
+                        .send()
+                        .await
+                        .expect("If-None-Match completion over marker");
+                }
+            }
+            let current = client
+                .get_object()
+                .bucket(bucket)
+                .key(operation)
+                .send()
+                .await
+                .expect("read successful conditional write")
+                .body
+                .collect()
+                .await
+                .expect("read object body")
+                .into_bytes();
+            let expected: &[u8] = if operation == "copy" { b"source" } else { b"new" };
+            assert_eq!(current.as_ref(), expected, "{operation}: unexpected current object");
+        }
+    }
 
     async fn create_versioned_bucket(client: &Client, bucket: &str) {
         client

crates/e2e_test/src/distributed/concurrency_stability_test.rs

@@ -12,11 +12,249 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
+use super::harness::{DistCluster, DistLayout, TestResult, payload_for, s3_probe_client, sha256_hex, unique_bucket};
 use crate::common::init_logging;
+use aws_sdk_s3::{
+    Client,
+    error::{ProvideErrorMetadata, SdkError},
+    primitives::ByteStream,
+};
+use std::any::TypeId;
 use std::collections::BTreeSet;
+use std::fmt::Debug;
+use std::future::Future;
 use std::sync::Arc;
+use std::time::Duration;
 use tokio::sync::Barrier;
+use tokio::time::Instant;
+
+const LOCK_ADMISSION_BUDGET: Duration = Duration::from_secs(45);
+
+async fn after_lock_contention<T, E, F, Fut>(operation: &str, mut call: F) -> TestResult<Result<T, SdkError<E>>>
+where
+    E: Debug + ProvideErrorMetadata + 'static,
+    F: FnMut() -> Fut,
+    Fut: Future<Output = Result<T, SdkError<E>>>,
+{
+    let started = Instant::now();
+    let mut attempts = 0;
+    // The workload uses unique unversioned keys. Repeat only explicitly
+    // advertised lock backpressure; every step still needs its real result.
+    let outcome = tokio::time::timeout(LOCK_ADMISSION_BUDGET, async {
+        loop {
+            attempts += 1;
+            let result = call().await;
+            let retry = result.as_ref().err().is_some_and(|error| {
+                let Some(response) = error.raw_response() else {
+                    return false;
+                };
+                let Some(service_error) = error.as_service_error() else {
+                    return false;
+                };
+                // HEAD has no XML error body, so its SDK service error can
+                // lack a code even when the server supplies retry headers.
+                let namespace_code = service_error.code() == Some("ServiceUnavailable")
+                    || (service_error.code().is_none()
+                        && TypeId::of::<E>() == TypeId::of::<aws_sdk_s3::operation::head_object::HeadObjectError>());
+                response.status().as_u16() == 503
+                    && namespace_code
+                    && response.headers().get("x-rustfs-error-cause") == Some("namespace_lock_timeout")
+                    && response.headers().get("retry-after") == Some("1")
+            });
+            if !retry {
+                return result;
+            }
+            tokio::time::sleep(Duration::from_secs(1)).await;
+        }
+    })
+    .await;
+    if attempts > 1 || !matches!(&outcome, Ok(Ok(_))) {
+        eprintln!(
+            "mixed workload request: operation={operation}, attempts={attempts}, elapsed={:?}",
+            started.elapsed()
+        );
+    }
+    outcome.map_err(|_| {
+        format!("{operation} exceeded its {LOCK_ADMISSION_BUDGET:?} request budget after {attempts} attempts").into()
+    })
+}
+
+async fn assert_workload_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
+    let output = after_lock_contention(&format!("GET {key}"), || client.get_object().bucket(bucket).key(key).send()).await??;
+    let actual = output.body.collect().await?.into_bytes();
+    if actual.as_ref() != expected {
+        return Err(format!(
+            "object {key} bytes mismatch: expected {} bytes sha256={} got {} bytes sha256={}",
+            expected.len(),
+            sha256_hex(expected),
+            actual.len(),
+            sha256_hex(&actual)
+        )
+        .into());
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use aws_smithy_http_client::test_util::infallible_client_fn;
+    use aws_smithy_types::body::SdkBody;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+
+    type Reply = (u16, &'static str, Option<&'static str>, Option<&'static str>, &'static str);
+
+    fn client(respond: impl Fn(usize) -> Reply + Send + Sync + 'static) -> (Client, Arc<AtomicUsize>) {
+        let requests = Arc::new(AtomicUsize::new(0));
+        let seen = Arc::clone(&requests);
+        let transport = infallible_client_fn(move |_| {
+            let (status, code, cause, retry_after, body) = respond(seen.fetch_add(1, Ordering::SeqCst));
+            let body = if code.is_empty() {
+                body.to_string()
+            } else {
+                format!("<Error><Code>{code}</Code></Error>")
+            };
+            let mut response = http::Response::builder()
+                .status(status)
+                .header("content-type", "application/xml")
+                .header("content-length", body.len());
+            if let Some(cause) = cause {
+                response = response.header("x-rustfs-error-cause", cause);
+            }
+            if let Some(retry_after) = retry_after {
+                response = response.header("retry-after", retry_after);
+            }
+            response.body(SdkBody::from(body)).expect("scripted S3 response")
+        });
+        let config = crate::common::build_test_s3_config(
+            "http://contention.example.invalid",
+            "synthetic-access",
+            "synthetic-secret",
+            None,
+            "contention-test",
+        );
+        let client = Client::from_conf(config.to_builder().http_client(transport).build());
+        (s3_probe_client(&client), requests)
+    }
+
+    fn contention() -> Reply {
+        (503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("1"), "")
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn advertised_contention_recovers_after_standard_sdk_attempts_are_exhausted() -> TestResult {
+        let (client, requests) = client(|attempt| {
+            if attempt < 3 {
+                contention()
+            } else {
+                (200, "", None, None, "")
+            }
+        });
+        let standard = Client::from_conf(
+            client
+                .config()
+                .to_builder()
+                .retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(3))
+                .build(),
+        );
+        standard
+            .put_object()
+            .bucket("bucket")
+            .key("key")
+            .body(ByteStream::from_static(b"body"))
+            .send()
+            .await
+            .expect_err("three advertised lock responses exhaust the standard SDK policy");
+        assert_eq!(requests.load(Ordering::SeqCst), 3);
+        requests.store(0, Ordering::SeqCst);
+        let started = Instant::now();
+        after_lock_contention("PUT recovery", || {
+            client
+                .put_object()
+                .bucket("bucket")
+                .key("key")
+                .body(ByteStream::from_static(b"body"))
+                .send()
+        })
+        .await??;
+        assert_eq!(requests.load(Ordering::SeqCst), 4);
+        assert!(started.elapsed() >= Duration::from_secs(3), "honor each Retry-After response");
+        Ok(())
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn bodyless_head_contention_uses_explicit_retry_headers() -> TestResult {
+        let (client, requests) = client(|attempt| {
+            if attempt == 0 {
+                (503, "", Some("namespace_lock_timeout"), Some("1"), "")
+            } else {
+                (200, "", None, None, "")
+            }
+        });
+        after_lock_contention("HEAD recovery", || client.head_object().bucket("bucket").key("key").send()).await??;
+        assert_eq!(requests.load(Ordering::SeqCst), 2);
+        Ok(())
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn unrelated_or_malformed_responses_are_not_retried() -> TestResult {
+        for reply in [
+            (500, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
+            (403, "AccessDenied", None, None, ""),
+            (404, "NoSuchKey", None, None, ""),
+            (503, "ServiceUnavailable", Some("storage_write_quorum"), None, ""),
+            (503, "ServiceUnavailable", Some("namespace_lock_unavailable"), Some("1"), ""),
+            (503, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
+            (503, "", Some("namespace_lock_timeout"), Some("1"), ""),
+            (503, "ServiceUnavailable", None, Some("1"), ""),
+            (503, "ServiceUnavailable", Some("namespace_lock_timeout"), None, ""),
+            (503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("2"), ""),
+        ] {
+            let (client, requests) = client(move |_| reply);
+            let result = after_lock_contention("PUT fatal response", || {
+                client
+                    .put_object()
+                    .bucket("bucket")
+                    .key("key")
+                    .body(ByteStream::from_static(b"body"))
+                    .send()
+            })
+            .await?;
+            assert!(result.is_err(), "fatal response cannot become success: {reply:?}");
+            assert_eq!(requests.load(Ordering::SeqCst), 1, "fatal response must remain visible: {reply:?}");
+        }
+        Ok(())
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn persistent_lock_contention_exhausts_the_request_budget() {
+        let (client, requests) = client(|_| contention());
+        let started = Instant::now();
+        let result = after_lock_contention("PUT unavailable", || {
+            client
+                .put_object()
+                .bucket("bucket")
+                .key("key")
+                .body(ByteStream::from_static(b"body"))
+                .send()
+        })
+        .await;
+        assert!(result.is_err(), "persistent contention cannot acknowledge an operation");
+        assert!(started.elapsed() >= LOCK_ADMISSION_BUDGET);
+        assert!(requests.load(Ordering::SeqCst) > 1);
+        assert!(requests.load(Ordering::SeqCst) <= LOCK_ADMISSION_BUDGET.as_secs() as usize + 1);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn acknowledged_wrong_bytes_fail_without_retry() {
+        let (client, requests) = client(|_| (200, "", None, None, "wrong bytes"));
+        let error = assert_workload_bytes(&client, "bucket", "key", b"expected bytes")
+            .await
+            .expect_err("a successful but incorrect read is a consistency failure");
+        assert!(error.to_string().contains("bytes mismatch"));
+        assert_eq!(requests.load(Ordering::SeqCst), 1);
+    }
+}
 
 #[tokio::test]
 async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
@@ -24,7 +262,9 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
     let dist = DistCluster::start(DistLayout::FourByFour).await?;
     let bucket = unique_bucket("conc");
     dist.create_bucket(&bucket).await?;
-    let clients = Arc::new(dist.clients()?);
+    // This consistency probe owns its admission policy; unrelated responses
+    // must not be hidden by the SDK's general-purpose retries.
+    let clients = Arc::new(dist.clients()?.iter().map(s3_probe_client).collect::<Vec<_>>());
     let worker_count = 24;
     let rounds = 4;
     let barrier = Arc::new(Barrier::new(worker_count));
@@ -44,31 +284,46 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
                 let key = format!("source/worker-{idx:02}-round-{round}.bin");
                 let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
                 let body = payload_for(&key, 64 * 1024);
-                put_object(writer, &bucket, &key, body.clone()).await?;
+                let context = format!("worker={idx} round={round}");
+                after_lock_contention(&format!("{context} PUT source"), || {
+                    writer
+                        .put_object()
+                        .bucket(&bucket)
+                        .key(&key)
+                        .body(ByteStream::from(body.clone()))
+                        .send()
+                })
+                .await??;
 
-                let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
+                let head = after_lock_contention(&format!("{context} HEAD source"), || {
+                    reader.head_object().bucket(&bucket).key(&key).send()
+                })
+                .await??;
                 if head.content_length() != Some(body.len() as i64) {
                     return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
                 }
-                assert_object_bytes(reader, &bucket, &key, &body).await?;
+                assert_workload_bytes(reader, &bucket, &key, &body).await?;
 
-                copier
-                    .copy_object()
-                    .bucket(&bucket)
-                    .key(&copy_key)
-                    .copy_source(format!("{bucket}/{key}"))
-                    .send()
-                    .await?;
-                assert_object_bytes(writer, &bucket, &copy_key, &body).await?;
+                after_lock_contention(&format!("{context} COPY retained"), || {
+                    copier
+                        .copy_object()
+                        .bucket(&bucket)
+                        .key(&copy_key)
+                        .copy_source(format!("{bucket}/{key}"))
+                        .send()
+                })
+                .await??;
+                assert_workload_bytes(writer, &bucket, &copy_key, &body).await?;
 
-                writer.delete_object().bucket(&bucket).key(&key).send().await?;
-                let missing = reader
-                    .head_object()
-                    .bucket(&bucket)
-                    .key(&key)
-                    .send()
-                    .await
-                    .expect_err("deleted source key must not remain visible");
+                after_lock_contention(&format!("{context} DELETE source"), || {
+                    writer.delete_object().bucket(&bucket).key(&key).send()
+                })
+                .await??;
+                let missing = after_lock_contention(&format!("{context} HEAD deleted source"), || {
+                    reader.head_object().bucket(&bucket).key(&key).send()
+                })
+                .await?
+                .expect_err("deleted source key must not remain visible");
                 if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
                     return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
                 }
@@ -82,14 +337,22 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
     for handle in handles {
         inventory.extend(handle.await??);
     }
+    assert_eq!(
+        inventory.len(),
+        worker_count * rounds,
+        "every worker round must retain its acknowledged copy"
+    );
 
     let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
     for (node_idx, client) in clients.iter().enumerate() {
-        let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
+        let listed = after_lock_contention(&format!("node={node_idx} LIST retained"), || {
+            client.list_objects_v2().bucket(&bucket).prefix("retained/").send()
+        })
+        .await??;
         let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
         assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
         for (key, body) in &inventory {
-            assert_object_bytes(client, &bucket, key, body)
+            assert_workload_bytes(client, &bucket, key, body)
                 .await
                 .map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
         }

crates/e2e_test/src/distributed/mod.rs

@@ -39,6 +39,8 @@ mod s3_during_data_movement_test;
 mod scanner_heal_topology_fixture;
 mod scanner_heal_topology_test;
 mod scanner_heal_topology_tier_test;
+mod scanner_peer_bucket_fixture;
+mod scanner_peer_bucket_test;
 mod site_replication_test;
 mod table_catalog_test;
 mod upgrade_test;

crates/e2e_test/src/distributed/multipart_fence_acceptance_test.rs

@@ -18,7 +18,7 @@ use super::harness::{
 };
 use crate::common::{ClusterTopology, RustFSTestClusterEnvironment};
 use crate::rpc_fault_proxy::{RpcFaultProxy, RpcPause};
-use crate::storage_api::multipart_fence_acceptance::RUSTFS_META_MULTIPART_BUCKET;
+use crate::storage_api::multipart_fence_acceptance::{LOCK_DIAGNOSTICS_FILTER, RUSTFS_META_MULTIPART_BUCKET};
 use aws_sdk_s3::{
     Client,
     config::retry::RetryConfig,
@@ -38,14 +38,16 @@ const DEADLINE: Duration = Duration::from_secs(45);
 async fn fixture(
     drives_per_node: usize,
     storage_class: &str,
+    lock_diagnostics: bool,
 ) -> TestResult<(RustFSTestClusterEnvironment, Vec<RpcFaultProxy>, Vec<Client>, String)> {
-    fixture_from_binary(drives_per_node, storage_class, None).await
+    fixture_from_binary(drives_per_node, storage_class, None, lock_diagnostics).await
 }
 
 async fn fixture_from_binary(
     drives_per_node: usize,
     storage_class: &str,
     binary: Option<&std::path::Path>,
+    lock_diagnostics: bool,
 ) -> TestResult<(RustFSTestClusterEnvironment, Vec<RpcFaultProxy>, Vec<Client>, String)> {
     let mut cluster =
         RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(4, drives_per_node)).await?;
@@ -90,6 +92,10 @@ async fn fixture_from_binary(
     ] {
         cluster.set_env(name, value);
     }
+    if lock_diagnostics {
+        cluster.set_env("RUST_LOG", LOCK_DIAGNOSTICS_FILTER);
+        cluster.set_env("RUSTFS_OBJECT_LOCK_DIAG_ENABLE", "true");
+    }
     let mut proxies = Vec::new();
     if binary.is_none() {
         for node in 0..4 {
@@ -363,26 +369,120 @@ fn upload_path_scope(bucket: &str, key: &str, upload_id: &str) -> TestResult<Str
     ))
 }
 
-async fn with_paused_peers<T: Send + 'static>(
+async fn with_paused_peers<T, E>(
     proxies: &[RpcFaultProxy],
     methods: &[&'static str],
     limits: &[usize],
     scope: &str,
-    request: impl std::future::Future<Output = T> + Send + 'static,
-) -> TestResult<T> {
+    request: impl std::future::Future<Output = Result<T, SdkError<E>>> + Send + 'static,
+) -> TestResult<Result<T, SdkError<E>>>
+where
+    T: Send + 'static,
+    E: std::error::Error + ProvideErrorMetadata + Send + Sync + 'static,
+{
+    let deadline = tokio::time::Instant::now() + DEADLINE;
     let pauses: Vec<_> = proxies[1..=limits.len()]
         .iter()
         .zip(limits)
         .map(|(proxy, limit)| proxy.pause(methods, *limit, Some(scope)))
         .collect::<Result<_, _>>()?;
-    let task = tokio::spawn(request);
-    wait_for_faults(&pauses, methods).await?;
-    let result = tokio::time::timeout(DEADLINE, task).await??;
+    let mut task = tokio::spawn(request);
+    let mut joined = None;
+    let mut deadline_expired = false;
+    let mut barrier_result = {
+        let faults = wait_for_faults(&pauses, methods);
+        tokio::pin!(faults);
+        match tokio::time::timeout_at(deadline, async {
+            tokio::select! {
+                result = &mut faults => result,
+                result = &mut task => {
+                    joined = Some(result);
+                    // A ready request is not evidence that every selected fault
+                    // was reached. Poll the original barrier once for that proof.
+                    match futures::poll!(faults.as_mut()) {
+                        std::task::Poll::Ready(result) => result,
+                        std::task::Poll::Pending => Err("SDK request completed before the selected RPC fault barrier".into()),
+                    }
+                }
+            }
+        })
+        .await
+        {
+            Ok(result) => result,
+            Err(error) => {
+                deadline_expired = true;
+                Err(std::io::Error::new(std::io::ErrorKind::TimedOut, error).into())
+            }
+        }
+    };
+    // Tokio's timeout polls an already-ready inner future before its timer.
+    // A late observation cannot certify the original absolute deadline.
+    deadline_expired |= tokio::time::Instant::now() >= deadline;
+    if deadline_expired {
+        barrier_result = Err(std::io::Error::new(
+            std::io::ErrorKind::TimedOut,
+            "selected RPC fault proof observed after the absolute MPU deadline",
+        )
+        .into());
+    }
+    if barrier_result.is_err() {
+        for pause in &pauses {
+            pause.discard_pending();
+        }
+    }
+    let mut client_task_aborted = false;
+    let request_result = match joined {
+        Some(result) => result,
+        None => match tokio::time::timeout_at(deadline, &mut task).await {
+            Ok(result) => result,
+            Err(error) => {
+                deadline_expired = true;
+                for pause in &pauses {
+                    pause.discard_pending();
+                }
+                barrier_result = Err(std::io::Error::new(std::io::ErrorKind::TimedOut, error).into());
+                // This owns only the test's SDK request, not server/native work.
+                // Retire the client task without granting another operation budget.
+                client_task_aborted = true;
+                task.abort();
+                task.await
+            }
+        },
+    };
+    deadline_expired |= tokio::time::Instant::now() >= deadline;
+    if deadline_expired {
+        barrier_result = Err(std::io::Error::new(
+            std::io::ErrorKind::TimedOut,
+            "SDK request result observed after the absolute MPU deadline",
+        )
+        .into());
+    }
+    let hits: Vec<_> = pauses.iter().map(RpcPause::hits).collect();
     for pause in &pauses {
         pause.discard_pending();
     }
     drop(pauses);
-    Ok(result)
+    if barrier_result.is_err() {
+        let sdk_error = request_result.as_ref().ok().and_then(|result| result.as_ref().err());
+        let response = sdk_error.and_then(SdkError::raw_response);
+        eprintln!(
+            "MPU fault barrier failed: methods={methods:?}, hits={hits:?}, client_task_aborted={client_task_aborted}, deadline_expired={deadline_expired}, sdk_code={:?}, cause={:?}, request_id={:?}",
+            sdk_error
+                .and_then(|error| error.as_service_error())
+                .and_then(|error| error.meta().code()),
+            response.and_then(|response| response.headers().get("x-rustfs-error-cause")),
+            response.and_then(|response| response.headers().get("x-amz-request-id")),
+        );
+    }
+    match (barrier_result, request_result) {
+        // Preserve the actual SDK source instead of replacing an early service
+        // failure with an unrelated fault-wait timeout. The outer Err stays red.
+        (Err(_), Ok(Err(error))) => Err(Box::new(error)),
+        (Err(_), Err(error)) if !client_task_aborted => Err(Box::new(error)),
+        (Err(error), _) => Err(error),
+        (Ok(()), Ok(result)) => Ok(result),
+        (Ok(()), Err(error)) => Err(Box::new(error)),
+    }
 }
 
 fn quorum_outcome<T: std::fmt::Debug, E: std::fmt::Debug + ProvideErrorMetadata>(
@@ -1318,7 +1418,7 @@ async fn concurrent_capped_parts_respect_limit_on_distinct_nodes(clients: &[Clie
 
 #[tokio::test]
 async fn metadata_delay_preserves_write_quorum_and_rejects_quorum_minus_one() -> TestResult {
-    let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
+    let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", true).await?;
     list_parts_disk_loss_after_metadata_read(&cluster, &proxies, &clients, &bucket).await?;
     concurrent_capped_parts_respect_limit_on_distinct_nodes(&clients, &bucket).await?;
     phase_quorum_matrix(&cluster, &proxies, &clients, &bucket, false, 1, 3).await
@@ -1326,13 +1426,13 @@ async fn metadata_delay_preserves_write_quorum_and_rejects_quorum_minus_one() ->
 
 #[tokio::test]
 async fn lock_delay_preserves_admission_quorum_and_reports_timeout() -> TestResult {
-    let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
+    let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
     phase_quorum_matrix(&cluster, &proxies, &clients, &bucket, true, 1, 3).await
 }
 
 #[tokio::test]
 async fn sixteen_shard_multipart_metadata_requires_twelve_votes() -> TestResult {
-    let (cluster, proxies, clients, bucket) = fixture(4, "EC:4").await?;
+    let (cluster, proxies, clients, bucket) = fixture(4, "EC:4", false).await?;
     let storage = cluster_admin_ok(&cluster, http::Method::GET, "/rustfs/admin/v3/storageinfo", None).await?;
     let storage: serde_json::Value = serde_json::from_str(&storage)?;
     let storage: rustfs_madmin::StorageInfo = serde_json::from_value(storage["info"].clone())?;
@@ -1346,7 +1446,7 @@ async fn sixteen_shard_multipart_metadata_requires_twelve_votes() -> TestResult
 #[tokio::test]
 async fn mixed_version_failed_complete_requires_current_coordinator_for_abort() -> TestResult {
     let previous = super::upgrade_test::source_binary()?;
-    let (mut cluster, mut proxies, clients, bucket) = fixture_from_binary(4, "EC:4", Some(&previous)).await?;
+    let (mut cluster, mut proxies, clients, bucket) = fixture_from_binary(4, "EC:4", Some(&previous), false).await?;
     assert!(
         proxies.is_empty(),
         "the genuine previous fleet boots and seeds through its normal direct transport"
@@ -1549,7 +1649,7 @@ async fn native_certified_partial_abort_retry_reclaims_survivors_below_metadata_
         capability_sha256: Option<[u8; 32]>,
         abort: Option<AbortRef>,
     }
-    let (mut cluster, proxies, clients, bucket) = fixture(4, "EC:4").await?;
+    let (mut cluster, proxies, clients, bucket) = fixture(4, "EC:4", false).await?;
     let key = "partial-abort";
     let committed = payload_for("committed-before-abort", 4096);
     clients[0]
@@ -1765,7 +1865,7 @@ async fn native_certified_partial_abort_retry_reclaims_survivors_below_metadata_
 #[tokio::test]
 async fn late_complete_packets_cannot_publish_after_abort_and_bucket_recreation() -> TestResult {
     for size in [0, 8 * 1024 * 1024] {
-        let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
+        let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
         let key = "late-complete";
         let upload = clients[0].create_multipart_upload().bucket(&bucket).key(key).send().await?;
         let upload_id = upload.upload_id().ok_or("missing upload ID")?;
@@ -1892,7 +1992,7 @@ async fn late_complete_packets_cannot_publish_after_abort_and_bucket_recreation(
 async fn late_part_packets_preserve_retired_and_fresh_uploads() -> TestResult {
     for (method, settles) in [("PreparePartTransaction", false), ("SettlePartTransaction", true)] {
         for recreate_bucket in [false, true] {
-            let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
+            let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
             let key = "late-part-transaction";
             let previous = payload_for("committed-before-late-part", 4096);
             clients[0]
@@ -2075,7 +2175,7 @@ async fn late_part_packets_preserve_retired_and_fresh_uploads() -> TestResult {
 
 #[tokio::test]
 async fn cancelled_metadata_wait_releases_fence_for_bucket_recreation() -> TestResult {
-    let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
+    let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
     let pauses: Vec<_> = proxies[1..]
         .iter()
         .map(|proxy| proxy.pause(&["WriteMetadata"], usize::MAX, Some(&bucket)))
@@ -2098,7 +2198,7 @@ async fn cancelled_metadata_wait_releases_fence_for_bucket_recreation() -> TestR
 async fn part_resend_owner_serializes_complete_and_abort_after_client_cancellation() -> TestResult {
     use md5::{Digest as _, Md5};
 
-    let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
+    let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
     for cancel_waiter in [false, true] {
         for complete in [false, true] {
             let key = format!("resend-{cancel_waiter}-{complete}");
@@ -2586,7 +2686,7 @@ async fn resume_at_rpc<T: Send + 'static>(
 
 #[tokio::test]
 async fn each_multipart_phase_resumes_after_metadata_and_lock_delays() -> TestResult {
-    let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
+    let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
     for locks in [false, true] {
         let key = format!("phase-{locks}");
         let client = clients[0].clone();

crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs

@@ -1700,6 +1700,15 @@ mod tests {
         );
 
         let mut cluster = RustFSTestClusterEnvironment::with_topology(topology.cluster_topology()).await?;
+        let bucket = "heal-restart-during-rebuild";
+        let mrf_fallback_scope = if !background_enabled {
+            let path = PathBuf::from(&cluster.temp_dir).join("skip-outage-mrf-fallback");
+            std::fs::write(&path, format!("{bucket}/cluster/written-while-node-down-"))?;
+            cluster.set_env("RUSTFS_E2E_SKIP_MRF_FALLBACK_PATH", path.to_string_lossy().into_owned());
+            Some(path)
+        } else {
+            None
+        };
         cluster.set_env("RUSTFS_UNSAFE_BYPASS_DISK_CHECK", "true");
         cluster.set_env("RUSTFS_HEAL_ENABLED", "true");
         if let Some(storage_class) = evidence_case.and_then(|case| case.storage_class_standard) {
@@ -1952,6 +1961,21 @@ mod tests {
             }
         };
 
+        if let Some(path) = &mrf_fallback_scope {
+            let observed = path.with_extension("observed");
+            let expected = format!("{bucket}/{outage_key}");
+            let deadline = Instant::now() + Duration::from_secs(30);
+            loop {
+                if tokio::fs::read_to_string(&observed).await.ok().as_deref() == Some(expected.as_str()) {
+                    break;
+                }
+                if Instant::now() >= deadline {
+                    return Err("outage repair did not reach the isolated best-effort fallback before target rejoin".into());
+                }
+                sleep(Duration::from_millis(50)).await;
+            }
+        }
+
         assert!(
             !outage_target_manifest_required
                 || target_set_peer_drives
@@ -2045,7 +2069,12 @@ mod tests {
             } else {
                 cluster_heal_is_idle(&recovered)
             };
-            if ready {
+            let pool_ready = if ready && background_enabled && !outage_write_deferred_until_rejoin {
+                all_nodes_pool_metadata_readiness(&cluster, recovery_deadline).await.0
+            } else {
+                true
+            };
+            if ready && pool_ready {
                 break;
             }
             if Instant::now() >= recovery_deadline {
@@ -2542,6 +2571,10 @@ mod tests {
             );
         }
 
+        if let Some(path) = &mrf_fallback_scope {
+            std::fs::remove_file(path)?;
+        }
+
         if let Some(cycle_end) = scanner_cycle_floor {
             wait_for_scanner_cycle_after(&cluster, cycle_end).await?;
         }

crates/e2e_test/src/replication_extension_test.rs

@@ -31,8 +31,8 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{
-    BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DeleteMarkerEntry, ObjectVersion, ServerSideEncryption,
-    VersioningConfiguration,
+    BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Delete, DeleteMarkerEntry, ObjectIdentifier, ObjectVersion,
+    ServerSideEncryption, VersioningConfiguration,
 };
 use base64_simd::STANDARD as BASE64_STANDARD;
 use bytes::Bytes;
@@ -3729,6 +3729,8 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
     init_logging();
 
     let mut source_env = RustFSTestEnvironment::new().await?;
+    let source_log = format!("{}/source.log", source_env.temp_dir);
+    source_env.capture_log_path = Some(source_log.clone());
     let mut source_env_vars = replication_fast_env();
     source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
     source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
@@ -3791,19 +3793,66 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
         "target did not preserve the source delete-marker version ID"
     );
 
-    source_client
-        .delete_object()
+    let wait_for_source_writeback = || async {
+        let metrics_url = format!("{}/{source_bucket}?replication-metrics", source_env.url);
+        timeout(Duration::from_secs(30), async {
+            loop {
+                let response = signed_request(
+                    http::Method::GET,
+                    &metrics_url,
+                    &source_env.access_key,
+                    &source_env.secret_key,
+                    None,
+                    None,
+                )
+                .await?;
+                let metrics: serde_json::Value = response.error_for_status()?.json().await?;
+                let queued = metrics["queued"]["curr"]["count"]
+                    .as_f64()
+                    .ok_or("missing replication queue count")?;
+                if metrics["provider_available"] == true && queued == 0.0 {
+                    return Ok::<(), Box<dyn Error + Send + Sync>>(());
+                }
+                sleep(Duration::from_millis(50)).await;
+            }
+        })
+        .await?
+    };
+    wait_for_source_writeback().await?;
+
+    let purged = source_client
+        .delete_objects()
         .bucket(source_bucket)
-        .key(object_key)
-        .version_id(&purged_version_id)
+        .delete(
+            Delete::builder()
+                .objects(
+                    ObjectIdentifier::builder()
+                        .key(object_key)
+                        .version_id(&purged_version_id)
+                        .build()?,
+                )
+                .objects(
+                    ObjectIdentifier::builder()
+                        .key(object_key)
+                        .version_id(delete_marker_version_id)
+                        .build()?,
+                )
+                .build()?,
+        )
         .send()
         .await?;
+    assert!(purged.errors().is_empty(), "batch version purge failed: {:?}", purged.errors());
+    assert_eq!(purged.deleted().len(), 2, "both the data version and marker must be purged");
     assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
     let target_state = list_replication_state(&target_client, target_bucket).await?;
     assert!(
         target_state.iter().all(|entry| entry.version_id != purged_version_id),
         "target retained the explicitly purged object version"
     );
+    assert!(
+        target_state.iter().all(|entry| entry.version_id != delete_marker_version_id),
+        "target retained the explicitly purged delete marker"
+    );
     assert!(
         target_state
             .iter()
@@ -3819,6 +3868,19 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
         .await?;
     assert_eq!(retained.body.collect().await?.into_bytes().as_ref(), b"versioned replication payload v2");
 
+    // Wait for the source workers to finish their final writeback, which runs
+    // after the remote purge has already become visible (rustfs#9121).
+    wait_for_source_writeback().await?;
+    let logs = fs::read_to_string(source_log).await?;
+    let writeback_failures: Vec<_> = logs
+        .lines()
+        .filter(|line| line.contains("apply_replication_delete_state"))
+        .collect();
+    assert!(
+        writeback_failures.is_empty(),
+        "a completed batch purge must not report a source writeback failure: {writeback_failures:?}"
+    );
+
     Ok(())
 }
 

crates/e2e_test/src/rpc_fault_proxy.rs

@@ -28,7 +28,8 @@ use hyper_util::{
 use prost::Message as _;
 use rustfs_protos::proto_gen::node_service::{
     BatchGenerallyLockRequest, DeleteRequest, GenerallyLockRequest, PreparePartTransactionResponse, ReadVersionRequest,
-    RecoverMultipartSourceRequest, RenameDataResponse, SettlePartTransactionRequest, SettlePartTransactionResponse,
+    RecoverMultipartSourceRequest, RenameDataResponse, ScannerScopedDirtyUsageAckRequest, SettlePartTransactionRequest,
+    SettlePartTransactionResponse,
 };
 use std::{
     collections::HashSet,
@@ -49,6 +50,9 @@ use tokio::{
 type ProxyBody = BoxBody<Bytes, io::Error>;
 type ProxyClient = Client<HttpConnector, ProxyBody>;
 
+mod scoped_ack;
+pub(crate) use scoped_ack::{ScopedAckFault, ScopedAckQueue, ScopedAckScope};
+
 #[derive(Clone, Copy, PartialEq, Eq)]
 enum Disposition {
     Hold,
@@ -121,7 +125,8 @@ impl RpcPause {
         self.state.blocked.load(Ordering::SeqCst)
     }
 
-    /// Selected RPCs with successful HTTP/gRPC headers, excluding discarded packets.
+    /// Selected RPCs with successful HTTP/gRPC response headers, excluding
+    /// discarded packets. This does not prove a successful gRPC trailer or ACK apply.
     pub(crate) fn forwarded(&self) -> usize {
         self.state.forwarded.load(Ordering::SeqCst)
     }
@@ -233,6 +238,7 @@ pub(crate) struct RpcFaultProxy {
     address: SocketAddr,
     slot: PauseSlot,
     lock_slot: PauseSlot,
+    scoped_ack: scoped_ack::QueueSlot,
     task: JoinHandle<()>,
     late_requests: LateRequests,
 }
@@ -245,6 +251,8 @@ impl RpcFaultProxy {
         let active = slot.clone();
         let lock_slot: PauseSlot = Arc::new(Mutex::new(None));
         let active_lock = lock_slot.clone();
+        let scoped_ack = scoped_ack::empty_slot();
+        let active_ack = scoped_ack.clone();
         let late_requests: LateRequests = Arc::new(Mutex::new(Some(Vec::new())));
         let late = late_requests.clone();
         let task = tokio::spawn(async move {
@@ -259,6 +267,7 @@ impl RpcFaultProxy {
                         let h2 = h2.clone();
                         let active = active.clone();
                         let active_lock = active_lock.clone();
+                        let active_ack = active_ack.clone();
                         let late = late.clone();
                         connections.spawn(async move {
                             let service = service_fn(move |request: Request<Incoming>| {
@@ -272,6 +281,7 @@ impl RpcFaultProxy {
                                     None
                                 };
                                 let pause = owner_pause.or_else(|| active.lock().expect("RPC pause slot").clone());
+                                let ack = active_ack.lock().expect("scoped ACK queue slot").clone();
                                 let retain = pause.as_ref().is_some_and(|pause| {
                                     pause.retain_after_disconnect
                                         && request.uri().path().strip_prefix("/node_service.NodeService/")
@@ -280,11 +290,11 @@ impl RpcFaultProxy {
                                 let late = late.clone();
                                 async move {
                                     if !retain {
-                                        return forward_request(request, client, target, pause).await;
+                                        return forward_request(request, client, target, pause, ack).await;
                                     }
                                     // This opt-in fault owns the packet after client cancellation.
                                     // Proxy teardown still aborts every retained request.
-                                    let task = tokio::spawn(forward_request(request, client, target, pause));
+                                    let task = tokio::spawn(forward_request(request, client, target, pause, ack));
                                     {
                                         let mut requests = late.lock().expect("late RPC ownership");
                                         if let Some(requests) = requests.as_mut() {
@@ -309,6 +319,7 @@ impl RpcFaultProxy {
             address,
             slot,
             lock_slot,
+            scoped_ack,
             task,
             late_requests,
         })
@@ -318,6 +329,11 @@ impl RpcFaultProxy {
         self.address
     }
 
+    /// Retain only genuine scoped ACKs for one receiver/session and bucket set.
+    pub(crate) fn scoped_ack_queue(&self, scope: ScopedAckScope, capacity: usize) -> io::Result<ScopedAckQueue> {
+        ScopedAckQueue::arm(self.scoped_ack.clone(), scope, capacity)
+    }
+
     /// Pause at most limit requests to the named unary methods; other streams
     /// on the same HTTP/2 connection continue forwarding.
     pub(crate) fn pause(&self, methods: &[&'static str], limit: usize, scope: Option<&str>) -> io::Result<RpcPause> {
@@ -469,6 +485,9 @@ fn unary_payload(body: &[u8]) -> io::Result<&[u8]> {
 }
 
 fn scope_matches(method: &str, body: &[u8], scope: &[u8]) -> io::Result<bool> {
+    if method == "ScannerScopedDirtyUsageAck" {
+        return scoped_ack_matches(body, Some(scope));
+    }
     if method == "RecoverMultipartSource" {
         let request = RecoverMultipartSourceRequest::decode(unary_payload(body)?).map_err(io::Error::other)?;
         if request.protocol_version != u32::from(MULTIPART_SOURCE_PROTOCOL_VERSION) || request.path.as_bytes() != scope {
@@ -525,7 +544,9 @@ fn scoped_lock_owners(method: &str, body: &[u8], scope: &[u8]) -> io::Result<Vec
     let args = match method {
         "Lock" => vec![GenerallyLockRequest::decode(payload).map_err(io::Error::other)?.args],
         "LockBatch" => BatchGenerallyLockRequest::decode(payload).map_err(io::Error::other)?.args,
-        _ => return Err(io::Error::new(io::ErrorKind::InvalidInput, "lock observer requires a lock method")),
+        _ => {
+            return Err(io::Error::new(io::ErrorKind::InvalidInput, "lock observer requires a lock method"));
+        }
     };
     args.into_iter()
         .filter(|args| args.as_bytes().windows(scope.len()).any(|window| window == scope))
@@ -540,18 +561,39 @@ fn scoped_lock_owners(method: &str, body: &[u8], scope: &[u8]) -> io::Result<Vec
         .collect()
 }
 
+fn scoped_ack_matches(body: &[u8], scope: Option<&[u8]>) -> io::Result<bool> {
+    let request = ScannerScopedDirtyUsageAckRequest::decode(unary_payload(body)?).map_err(io::Error::other)?;
+    Ok(!request.probe_only && scope.is_none_or(|scope| request.entries.iter().any(|entry| entry.bucket.as_bytes() == scope)))
+}
+
 async fn forward_request(
     request: Request<Incoming>,
     client: ProxyClient,
     target: SocketAddr,
     pause: Option<Arc<PauseState>>,
+    ack: Option<Arc<scoped_ack::QueueState>>,
 ) -> io::Result<hyper::Response<ProxyBody>> {
-    let (mut parts, body) = request.into_parts();
-    let mut body = body.map_err(io::Error::other).boxed();
+    let request = request.map(|body| body.map_err(io::Error::other).boxed());
+    let request = if let Some(ack) = ack
+        && request.uri().path() == "/node_service.ScannerControlService/ScannerScopedDirtyUsageAck"
+    {
+        match scoped_ack::intercept(request, client.clone(), target, ack).await? {
+            scoped_ack::Interception::Pass(request) => request,
+            scoped_ack::Interception::Handled(response) => return Ok(response),
+        }
+    } else {
+        request
+    };
+    let (parts, body) = request.into_parts();
+    let mut body = body;
     let mut observed = None;
     let mut inspection = None;
     if let Some(pause) = pause {
-        let method = parts.uri.path().strip_prefix("/node_service.NodeService/");
+        let method = parts
+            .uri
+            .path()
+            .strip_prefix("/node_service.NodeService/")
+            .or_else(|| parts.uri.path().strip_prefix("/node_service.ScannerControlService/"));
         if let Some(method) = method.filter(|name| pause.methods.contains(name)) {
             let mut route = None;
             let mut lock_owners = Vec::new();
@@ -599,6 +641,16 @@ async fn forward_request(
                     .with_trailers(async move { trailers.map(Ok::<_, io::Error>) })
                     .boxed();
                 matches
+            } else if method == "ScannerScopedDirtyUsageAck" {
+                let collected = Limited::new(body, 1024 * 1024).collect().await.map_err(io::Error::other)?;
+                let trailers = collected.trailers().cloned();
+                let bytes = collected.to_bytes();
+                let matches = scoped_ack_matches(&bytes, None)?;
+                body = Full::new(bytes)
+                    .map_err(|never| match never {})
+                    .with_trailers(async move { trailers.map(Ok::<_, io::Error>) })
+                    .boxed();
+                matches
             } else {
                 true
             };
@@ -642,24 +694,7 @@ async fn forward_request(
             }
         }
     }
-    if !parts.headers.contains_key(http::header::HOST)
-        && let Some(authority) = parts.uri.authority()
-    {
-        parts.headers.insert(
-            http::header::HOST,
-            http::HeaderValue::from_str(authority.as_str()).map_err(io::Error::other)?,
-        );
-    }
-    parts.uri = http::Uri::builder()
-        .scheme("http")
-        .authority(target.to_string())
-        .path_and_query(parts.uri.path_and_query().map_or("/", |value| value.as_str()))
-        .build()
-        .map_err(io::Error::other)?;
-    let response = client
-        .request(Request::from_parts(parts, body))
-        .await
-        .map_err(io::Error::other)?;
+    let response = forward_to_backend(Request::from_parts(parts, body), client, target).await?;
     if let Some((pause, disk)) = inspection
         && response.status().is_success()
         && response.headers().get("grpc-status").is_some_and(|status| status == "12")
@@ -712,6 +747,32 @@ async fn forward_request(
     Ok(response.map(|body| body.map_err(io::Error::other).boxed()))
 }
 
+async fn forward_to_backend(
+    request: Request<ProxyBody>,
+    client: ProxyClient,
+    target: SocketAddr,
+) -> io::Result<hyper::Response<Incoming>> {
+    let (mut parts, body) = request.into_parts();
+    if !parts.headers.contains_key(http::header::HOST)
+        && let Some(authority) = parts.uri.authority()
+    {
+        parts.headers.insert(
+            http::header::HOST,
+            http::HeaderValue::from_str(authority.as_str()).map_err(io::Error::other)?,
+        );
+    }
+    parts.uri = http::Uri::builder()
+        .scheme("http")
+        .authority(target.to_string())
+        .path_and_query(parts.uri.path_and_query().map_or("/", |value| value.as_str()))
+        .build()
+        .map_err(io::Error::other)?;
+    client
+        .request(Request::from_parts(parts, body))
+        .await
+        .map_err(io::Error::other)
+}
+
 impl Drop for RpcFaultProxy {
     fn drop(&mut self) {
         if let Some(requests) = self.late_requests.lock().expect("late RPC ownership").take() {
@@ -731,6 +792,7 @@ mod tests {
 
     struct Backend {
         received: Arc<AtomicUsize>,
+        auth_headers: Arc<Mutex<Vec<(String, String)>>>,
         address: SocketAddr,
         task: JoinHandle<()>,
     }
@@ -754,6 +816,8 @@ mod tests {
         let address = listener.local_addr()?;
         let received = Arc::new(AtomicUsize::new(0));
         let received_task = received.clone();
+        let auth_headers = Arc::new(Mutex::new(Vec::new()));
+        let backend_auth = auth_headers.clone();
         let task = tokio::spawn(async move {
             let mut children = JoinSet::new();
             loop {
@@ -761,10 +825,12 @@ mod tests {
                     accepted = listener.accept() => {
                         let Ok((socket, _)) = accepted else { break };
                         let received = received_task.clone();
+                        let backend_auth = backend_auth.clone();
                         let reply = reply.clone();
                         children.spawn(async move {
                             let service = service_fn(move |request: Request<Incoming>| {
                                 let received = received.clone();
+                                let backend_auth = backend_auth.clone();
                                 let reply = reply.clone();
                                 async move {
                                 received.fetch_add(1, Ordering::SeqCst);
@@ -773,6 +839,10 @@ mod tests {
                                         http::HeaderValue::from_str(authority.as_str()).expect("valid HTTP authority")
                                     })
                                 });
+                                if let (Some(nonce), Some(signature)) = (request.headers().get("x-rustfs-rpc-nonce"), request.headers().get("x-rustfs-rpc-signature-v2")) {
+                                    backend_auth.lock().expect("backend auth header observations").push((
+                                        nonce.to_str().expect("test nonce").to_string(), signature.to_str().expect("test signature").to_string()));
+                                }
                                 let body = request.into_body().collect().await?.to_bytes();
                                 let body = reply.unwrap_or(body);
                                 let mut trailers = http::HeaderMap::new();
@@ -795,7 +865,12 @@ mod tests {
                 }
             }
         });
-        Ok(Backend { received, address, task })
+        Ok(Backend {
+            received,
+            auth_headers,
+            address,
+            task,
+        })
     }
 
     async fn call(client: &ProxyClient, address: SocketAddr, method: &str) -> io::Result<hyper::Response<Incoming>> {
@@ -809,9 +884,14 @@ mod tests {
         bytes: Bytes,
     ) -> io::Result<hyper::Response<Incoming>> {
         let body: ProxyBody = Full::new(bytes).map_err(|never| match never {}).boxed();
+        let service = if method == "ScannerScopedDirtyUsageAck" {
+            "ScannerControlService"
+        } else {
+            "NodeService"
+        };
         let request = Request::builder()
             .method("POST")
-            .uri(format!("http://{address}/node_service.NodeService/{method}"))
+            .uri(format!("http://{address}/node_service.{service}/{method}"))
             .header(http::header::TE, "trailers")
             .body(body)
             .map_err(io::Error::other)?;
@@ -826,6 +906,353 @@ mod tests {
         frame.into()
     }
 
+    const ACK_OWNER: &str = "11111111-1111-1111-1111-111111111111";
+    const ACK_INSTANCE: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+
+    fn ack_scope() -> ScopedAckScope {
+        ScopedAckScope {
+            nonce: uuid::Uuid::new_v4().to_string(),
+            owner_id: ACK_OWNER.to_string(),
+            instance_id: ACK_INSTANCE.to_string(),
+            buckets: std::collections::BTreeSet::from(["cold-bucket".to_string()]),
+        }
+    }
+
+    fn ack_request(bucket: &str, generation: u64) -> ScannerScopedDirtyUsageAckRequest {
+        ScannerScopedDirtyUsageAckRequest {
+            challenge: vec![7; 16].into(),
+            protocol_version: 1,
+            owner_id: ACK_OWNER.to_string(),
+            instance_id: ACK_INSTANCE.to_string(),
+            scope: 1,
+            probe_only: false,
+            entries: vec![rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry {
+                bucket: bucket.to_string(),
+                bucket_incarnation: vec![4; 16].into(),
+                generation,
+            }],
+        }
+    }
+
+    fn ack_frame(request: &ScannerScopedDirtyUsageAckRequest) -> Bytes {
+        let payload = request.encode_to_vec();
+        let mut frame = vec![0];
+        frame.extend_from_slice(&u32::try_from(payload.len()).expect("bounded test frame").to_be_bytes());
+        frame.extend_from_slice(&payload);
+        Bytes::from(frame)
+    }
+
+    fn ack_hash(bytes: &[u8]) -> String {
+        use sha2::Digest as _;
+        hex_simd::encode_to_string(&sha2::Sha256::digest(bytes)[..], hex_simd::AsciiCase::Lower)
+    }
+
+    // Transport-only fixtures: the distributed tests supply production signatures.
+    async fn call_ack_frame(
+        client: &ProxyClient,
+        address: SocketAddr,
+        frame: Bytes,
+        nonce: &str,
+    ) -> io::Result<hyper::Response<Incoming>> {
+        let request = Request::builder()
+            .method("POST")
+            .uri(format!("http://{address}/node_service.ScannerControlService/ScannerScopedDirtyUsageAck"))
+            .header(http::header::TE, "trailers")
+            .header("x-rustfs-rpc-nonce", nonce)
+            .header("x-rustfs-rpc-signature-v2", format!("opaque-signature-{nonce}"))
+            .body(Full::new(frame).map_err(|never| match never {}).boxed())
+            .map_err(io::Error::other)?;
+        client.request(request).await.map_err(io::Error::other)
+    }
+
+    #[tokio::test]
+    async fn scoped_ack_queue_loss_preserves_backend_boundary() {
+        tokio::time::timeout(Duration::from_secs(10), async {
+            for fault in [ScopedAckFault::RequestLoss, ScopedAckFault::ResponseLoss] {
+                let backend = backend().await.expect("backend");
+                let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
+                    .await
+                    .expect("proxy");
+                let queue = proxy.scoped_ack_queue(ack_scope(), 1).expect("ACK queue");
+                let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
+                let frame = ack_frame(&ack_request("cold-bucket", 7));
+                let address = proxy.local_addr();
+                let body = frame.clone();
+                let caller = tokio::spawn(async move { call_ack_frame(&client, address, body, "loss").await });
+                queue.wait_for_packets(1).await.expect("captured ACK");
+                assert_eq!(backend.received.load(Ordering::SeqCst), 0, "retention must precede backend dispatch");
+                queue.settle(0, fault).expect("settle actual retained packet");
+                assert!(
+                    caller.await.expect("caller task").is_err(),
+                    "the caller must observe the injected transport failure"
+                );
+                let receipt = queue.wait_for_completion(0).await.expect("complete fault receipt");
+                let response_loss = fault == ScopedAckFault::ResponseLoss;
+                assert_eq!(receipt["forwarded"], response_loss);
+                assert_eq!(receipt["backend_complete"], response_loss);
+                assert_eq!(backend.received.load(Ordering::SeqCst), usize::from(response_loss));
+                if response_loss {
+                    assert_eq!(receipt["outcome"], "response_lost");
+                    assert_eq!(receipt["response"]["grpc_status"], "0", "status exists only in real backend trailers");
+                    assert_eq!(receipt["response"]["bytes"], frame.len());
+                    assert_eq!(receipt["response"]["sha256"], ack_hash(&frame));
+                    assert!(
+                        receipt["response"]["trailers_sha256"]
+                            .as_str()
+                            .is_some_and(|value| value.len() == 64)
+                    );
+                } else {
+                    assert_eq!(receipt["outcome"], "request_lost");
+                }
+            }
+        })
+        .await
+        .expect("bounded ACK loss checks");
+    }
+
+    #[tokio::test]
+    async fn scoped_ack_queue_keeps_cancelled_packets_and_releases_out_of_order() {
+        tokio::time::timeout(Duration::from_secs(10), async {
+            let backend = backend().await.expect("backend");
+            let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
+                .await
+                .expect("proxy");
+            let queue = proxy.scoped_ack_queue(ack_scope(), 2).expect("two-packet ACK queue");
+            let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
+            let first = ack_frame(&ack_request("cold-bucket", 7));
+            let second = ack_frame(&ack_request("cold-bucket", 9));
+            let address = proxy.local_addr();
+            let old_client = client.clone();
+            let old_body = first.clone();
+            let old_caller = tokio::spawn(async move { call_ack_frame(&old_client, address, old_body, "first").await });
+            queue.wait_for_packets(1).await.expect("first retained packet");
+            old_caller.abort();
+            assert!(old_caller.await.expect_err("cancel original caller").is_cancelled());
+            let new_body = second.clone();
+            let new_caller = tokio::spawn(async move { call_ack_frame(&client, address, new_body, "second").await });
+            queue.wait_for_packets(2).await.expect("second retained packet");
+            assert_eq!(backend.received.load(Ordering::SeqCst), 0);
+            queue.settle(1, ScopedAckFault::Forward).expect("release newer packet first");
+            let response = new_caller
+                .await
+                .expect("new caller task")
+                .expect("new response")
+                .into_body()
+                .collect()
+                .await
+                .expect("complete body");
+            assert_eq!(response.trailers().expect("response trailers")["x-proof"], "unchanged-proof");
+            assert_eq!(response.to_bytes(), second);
+            queue.wait_for_completion(1).await.expect("new packet complete");
+            queue
+                .settle(0, ScopedAckFault::Forward)
+                .expect("release old packet after its caller disappeared");
+            let old = queue.wait_for_completion(0).await.expect("detached old packet complete");
+            assert_eq!(old["backend_complete"], true);
+            assert_eq!(old["request_sha256"], ack_hash(&first));
+            assert_eq!(old["response"]["sha256"], ack_hash(&first));
+            assert_eq!(backend.received.load(Ordering::SeqCst), 2);
+            assert_eq!(
+                *backend.auth_headers.lock().expect("backend header observations"),
+                vec![
+                    ("second".to_string(), "opaque-signature-second".to_string()),
+                    ("first".to_string(), "opaque-signature-first".to_string())
+                ]
+            );
+            assert!(
+                queue.settle(0, ScopedAckFault::Forward).is_err(),
+                "one original packet may only be forwarded once"
+            );
+        })
+        .await
+        .expect("bounded detached retention check");
+    }
+
+    #[tokio::test]
+    async fn scoped_ack_queue_bounds_capacity_and_excludes_other_scopes() {
+        tokio::time::timeout(Duration::from_secs(10), async {
+            let backend = backend().await.expect("backend");
+            let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
+                .await
+                .expect("proxy");
+            assert!(proxy.scoped_ack_queue(ack_scope(), 0).is_err());
+            assert!(proxy.scoped_ack_queue(ack_scope(), 5).is_err());
+            let mut invalid = ack_scope();
+            invalid.nonce.clear();
+            assert!(proxy.scoped_ack_queue(invalid, 1).is_err());
+            let mut scope = ack_scope();
+            scope.buckets.insert("cold-bucket-z".to_string());
+            let queue = proxy.scoped_ack_queue(scope, 1).expect("bounded ACK queue");
+            let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
+            let mut probe = ack_request("cold-bucket", 7);
+            probe.probe_only = true;
+            let mut other_peer = ack_request("cold-bucket", 7);
+            other_peer.owner_id = uuid::Uuid::new_v4().to_string();
+            let mut other_instance = ack_request("cold-bucket", 7);
+            other_instance.instance_id = "b".repeat(32);
+            let mut invalid_challenge = ack_request("cold-bucket", 7);
+            invalid_challenge.challenge = vec![7; 15].into();
+            let mut noncanonical_entries = ack_request("cold-bucket-z", 7);
+            noncanonical_entries.entries.extend(ack_request("cold-bucket", 7).entries);
+            let unselected = [
+                probe,
+                other_peer,
+                other_instance,
+                ack_request("cold-bucket-other", 7),
+                ack_request("cold-bucket", u64::MAX),
+                invalid_challenge,
+                noncanonical_entries,
+            ];
+            let unselected_count = unselected.len();
+            for (index, request) in unselected.into_iter().enumerate() {
+                let body = ack_frame(&request);
+                let response = call_ack_frame(&client, proxy.local_addr(), body.clone(), "unselected")
+                    .await
+                    .expect("unselected ACK progresses");
+                assert_eq!(response.into_body().collect().await.expect("unselected body").to_bytes(), body);
+                assert!(queue.receipts().is_empty(), "unselected or protocol-invalid ACK must not consume a slot");
+                assert_eq!(backend.received.load(Ordering::SeqCst), index + 1);
+            }
+            assert!(queue.receipts().is_empty());
+            let address = proxy.local_addr();
+            let held_client = client.clone();
+            let valid_frame = ack_frame(&ack_request("cold-bucket", 7));
+            let held_body = valid_frame.clone();
+            let held = tokio::spawn(async move { call_ack_frame(&held_client, address, held_body, "held").await });
+            queue.wait_for_packets(1).await.expect("one matching packet");
+            assert!(!held.is_finished(), "the subsequent valid ACK must be retained");
+            assert_eq!(queue.receipts()[0]["request_sha256"], ack_hash(&valid_frame));
+            assert_eq!(backend.received.load(Ordering::SeqCst), unselected_count);
+            let extra = call_ack_frame(&client, address, ack_frame(&ack_request("cold-bucket", 8)), "beyond-capacity")
+                .await
+                .expect("capacity overflow passes without retention");
+            extra.into_body().collect().await.expect("overflow response");
+            assert_eq!(queue.receipts().len(), 1);
+            assert_eq!(backend.received.load(Ordering::SeqCst), unselected_count + 1);
+            drop(queue);
+            assert!(
+                held.await.expect("held caller").is_err(),
+                "dropping the session must cancel its retained packet"
+            );
+            assert_eq!(
+                backend.received.load(Ordering::SeqCst),
+                unselected_count + 1,
+                "session drop must never forward the held packet"
+            );
+        })
+        .await
+        .expect("bounded scope and lifecycle checks");
+    }
+
+    #[tokio::test]
+    async fn scoped_ack_queue_expiry_never_forwards_retained_packet() {
+        let (backend, _proxy, queue, caller) = tokio::time::timeout(Duration::from_secs(10), async {
+            let backend = backend().await.expect("backend");
+            let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
+                .await
+                .expect("proxy");
+            let queue = proxy.scoped_ack_queue(ack_scope(), 1).expect("ACK queue");
+            let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
+            let address = proxy.local_addr();
+            let caller = tokio::spawn(async move {
+                call_ack_frame(&client, address, ack_frame(&ack_request("cold-bucket", 7)), "expires").await
+            });
+            queue.wait_for_packets(1).await.expect("retained packet");
+            (backend, proxy, queue, caller)
+        })
+        .await
+        .expect("bounded queue setup");
+        tokio::time::pause();
+        tokio::time::advance(Duration::from_secs(120)).await;
+        tokio::time::resume();
+        assert!(
+            tokio::time::timeout(Duration::from_secs(10), caller)
+                .await
+                .expect("expiry completes caller")
+                .expect("caller task")
+                .is_err()
+        );
+        let receipt = queue.receipts().into_iter().next().expect("expired receipt");
+        assert_eq!(receipt["completed"], true);
+        assert_eq!(receipt["forwarded"], false);
+        assert_eq!(receipt["backend_complete"], false);
+        assert_eq!(backend.received.load(Ordering::SeqCst), 0);
+        assert_eq!(
+            queue
+                .settle(0, ScopedAckFault::Forward)
+                .expect_err("expired packet cannot be released")
+                .kind(),
+            io::ErrorKind::TimedOut
+        );
+    }
+
+    #[tokio::test]
+    async fn scanner_ack_fault_matches_exact_bucket_and_excludes_probes() {
+        use rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry;
+
+        tokio::time::timeout(Duration::from_secs(10), async {
+            let backend = backend().await.expect("backend listener");
+            let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
+                .await
+                .expect("proxy listener");
+            let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
+            let pause = proxy
+                .pause(&["ScannerScopedDirtyUsageAck"], 1, Some("cold-bucket"))
+                .expect("scoped ACK fault");
+            let frame = |bucket: &str, probe_only: bool| {
+                let payload = ScannerScopedDirtyUsageAckRequest {
+                    challenge: vec![3; 16].into(),
+                    protocol_version: 1,
+                    owner_id: "11111111-1111-1111-1111-111111111111".to_string(),
+                    instance_id: "a".repeat(32),
+                    scope: 1,
+                    probe_only,
+                    entries: vec![ScannerScopedDirtyUsageEntry {
+                        bucket: bucket.to_string(),
+                        bucket_incarnation: vec![4; 16].into(),
+                        generation: 17,
+                    }],
+                }
+                .encode_to_vec();
+                let mut frame = vec![0];
+                frame.extend_from_slice(&u32::try_from(payload.len()).expect("small ACK frame").to_be_bytes());
+                frame.extend_from_slice(&payload);
+                Bytes::from(frame)
+            };
+            for bytes in [frame("cold-bucket", true), frame("cold-bucket-other", false)] {
+                let response = call_body(&client, proxy.local_addr(), "ScannerScopedDirtyUsageAck", bytes.clone())
+                    .await
+                    .expect("probe and other bucket must progress");
+                assert_eq!(response.into_body().collect().await.expect("unselected response").to_bytes(), bytes);
+            }
+            assert_eq!(pause.hits(), 0);
+            let signed_bytes = frame("cold-bucket", false);
+            let task_bytes = signed_bytes.clone();
+            let address = proxy.local_addr();
+            let blocked =
+                tokio::spawn(async move { call_body(&client, address, "ScannerScopedDirtyUsageAck", task_bytes).await });
+            pause
+                .wait_for_hits(1)
+                .await
+                .expect("real ScannerControlService method must reach the fault");
+            assert!(!blocked.is_finished());
+            assert_eq!(backend.received.load(Ordering::SeqCst), 2, "held ACK must not apply at the backend");
+            pause.release();
+            let collected = blocked
+                .await
+                .expect("ACK task")
+                .expect("released ACK")
+                .into_body()
+                .collect()
+                .await
+                .expect("ACK body and trailers");
+            assert_eq!(collected.trailers().expect("ACK trailers")["x-proof"], "unchanged-proof");
+            assert_eq!(collected.to_bytes(), signed_bytes, "proxy must preserve the complete signed ACK frame");
+        })
+        .await
+        .expect("scanner ACK fault deadline");
+    }
+
     fn native_abort_request(path: &str) -> RecoverMultipartSourceRequest {
         use crate::storage_api::multipart_source_rpc::{
             MultipartAbortCertificate, MultipartAbortProof, MultipartAbortProofKind, MultipartAbortRound, MultipartSourcePhase,
@@ -1719,3 +2146,4 @@ mod tests {
         );
     }
 }
+// event-drill: isolated harness-only change

crates/e2e_test/src/rpc_fault_proxy/scoped_ack.rs

@@ -0,0 +1,404 @@
+// Copyright 2026 RustFS Team
+// SPDX-License-Identifier: Apache-2.0
+
+//! Bounded retention of original scoped ACK frames, independent of caller cancellation.
+
+use super::{ProxyBody, ProxyClient, forward_to_backend, unary_payload};
+use bytes::Bytes;
+use http_body_util::{BodyExt as _, Full, Limited};
+use hyper::{Request, Response};
+use prost::Message as _;
+use rustfs_protos::proto_gen::node_service::{ScannerScopedDirtyUsageAckRequest, ScannerScopedDirtyUsageAckResponse};
+use rustfs_protos::scoped_dirty_usage::validate_scoped_dirty_usage_request;
+use serde_json::{Value, json};
+use sha2::{Digest as _, Sha256};
+use std::{
+    collections::BTreeSet,
+    io,
+    net::SocketAddr,
+    sync::{
+        Arc, Mutex,
+        atomic::{AtomicBool, Ordering},
+    },
+    time::Duration,
+};
+use tokio::{
+    sync::{Notify, oneshot, watch},
+    task::AbortHandle,
+    time::Instant,
+};
+
+const MAX_RETAINED: usize = 4;
+const MAX_BYTES: usize = 1024 * 1024;
+const RETENTION: Duration = Duration::from_secs(120);
+
+pub(crate) struct ScopedAckScope {
+    pub nonce: String,
+    pub owner_id: String,
+    pub instance_id: String,
+    pub buckets: BTreeSet<String>,
+}
+
+impl ScopedAckScope {
+    fn validate(&self) -> io::Result<()> {
+        let valid_uuid = |value: &str| uuid::Uuid::parse_str(value).is_ok_and(|value| !value.is_nil());
+        if !valid_uuid(&self.nonce)
+            || !valid_uuid(&self.owner_id)
+            || self.instance_id.is_empty()
+            || self.instance_id.len() > 128
+            || self.buckets.is_empty()
+            || self.buckets.len() > 16
+            || self.buckets.iter().any(|bucket| bucket.is_empty() || bucket.len() > 63)
+        {
+            return Err(io::Error::new(
+                io::ErrorKind::InvalidInput,
+                "scoped ACK queue needs a bounded session, receiver and bucket scope",
+            ));
+        }
+        Ok(())
+    }
+
+    fn matches(&self, request: &ScannerScopedDirtyUsageAckRequest) -> bool {
+        !request.probe_only
+            && validate_scoped_dirty_usage_request(request).is_ok()
+            && request.owner_id == self.owner_id
+            && request.instance_id == self.instance_id
+            && request.entries.iter().all(|entry| self.buckets.contains(&entry.bucket))
+    }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(crate) enum ScopedAckFault {
+    Forward,
+    RequestLoss,
+    ResponseLoss,
+}
+
+struct Packet {
+    action: watch::Sender<Option<ScopedAckFault>>,
+    receipt: Mutex<Value>,
+    task: Mutex<Option<AbortHandle>>,
+}
+
+pub(super) struct QueueState {
+    scope: ScopedAckScope,
+    capacity: usize,
+    started: Instant,
+    deadline: Instant,
+    closed: AtomicBool,
+    // The packet inventory lock is released before any packet lock or await.
+    packets: Mutex<Vec<Arc<Packet>>>,
+    changed: Notify,
+}
+
+pub(super) type QueueSlot = Arc<Mutex<Option<Arc<QueueState>>>>;
+
+pub(super) fn empty_slot() -> QueueSlot {
+    Arc::new(Mutex::new(None))
+}
+
+pub(crate) struct ScopedAckQueue {
+    state: Arc<QueueState>,
+    slot: QueueSlot,
+}
+
+impl ScopedAckQueue {
+    pub(super) fn arm(slot: QueueSlot, scope: ScopedAckScope, capacity: usize) -> io::Result<Self> {
+        scope.validate()?;
+        if !(1..=MAX_RETAINED).contains(&capacity) {
+            return Err(io::Error::new(io::ErrorKind::InvalidInput, "scoped ACK capacity must be 1..=4"));
+        }
+        let mut active = slot.lock().expect("scoped ACK queue slot");
+        if active.is_some() {
+            return Err(io::Error::new(io::ErrorKind::AlreadyExists, "scoped ACK queue is already armed"));
+        }
+        let started = Instant::now();
+        let state = Arc::new(QueueState {
+            scope,
+            capacity,
+            started,
+            deadline: started + RETENTION,
+            closed: AtomicBool::new(false),
+            packets: Mutex::new(Vec::new()),
+            changed: Notify::new(),
+        });
+        *active = Some(state.clone());
+        drop(active);
+        Ok(Self { state, slot })
+    }
+
+    pub(crate) fn receipts(&self) -> Vec<Value> {
+        let packets = self.state.packets.lock().expect("scoped ACK packets").clone();
+        packets
+            .iter()
+            .map(|packet| packet.receipt.lock().expect("scoped ACK receipt").clone())
+            .collect()
+    }
+
+    pub(crate) async fn wait_for_packets(&self, count: usize) -> io::Result<()> {
+        self.wait_until(|| self.receipts().len() >= count).await
+    }
+
+    pub(crate) async fn wait_for_completion(&self, id: usize) -> io::Result<Value> {
+        self.wait_until(|| {
+            self.receipts()
+                .get(id)
+                .is_some_and(|receipt| receipt["completed"].as_bool() == Some(true))
+        })
+        .await?;
+        self.receipts()
+            .get(id)
+            .cloned()
+            .ok_or_else(|| io::Error::other("scoped ACK receipt disappeared"))
+    }
+
+    async fn wait_until(&self, ready: impl Fn() -> bool) -> io::Result<()> {
+        tokio::time::timeout_at(self.state.deadline, async {
+            loop {
+                let changed = self.state.changed.notified();
+                if ready() {
+                    return;
+                }
+                changed.await;
+            }
+        })
+        .await
+        .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "scoped ACK queue lifetime elapsed"))
+    }
+
+    pub(crate) fn settle(&self, id: usize, fault: ScopedAckFault) -> io::Result<()> {
+        if Instant::now() >= self.state.deadline {
+            return Err(io::Error::new(io::ErrorKind::TimedOut, "scoped ACK retention expired"));
+        }
+        let packet = self
+            .state
+            .packets
+            .lock()
+            .expect("scoped ACK packets")
+            .get(id)
+            .cloned()
+            .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "scoped ACK packet is not retained"))?;
+        if !packet.action.send_if_modified(|current| {
+            if current.is_none() {
+                *current = Some(fault);
+                true
+            } else {
+                false
+            }
+        }) {
+            return Err(io::Error::new(io::ErrorKind::AlreadyExists, "scoped ACK packet was already settled"));
+        }
+        Ok(())
+    }
+}
+
+impl Drop for ScopedAckQueue {
+    fn drop(&mut self) {
+        self.state.closed.store(true, Ordering::SeqCst);
+        let mut slot = self.slot.lock().expect("scoped ACK queue slot");
+        if slot.as_ref().is_some_and(|current| Arc::ptr_eq(current, &self.state)) {
+            *slot = None;
+        }
+        drop(slot);
+        let packets = self.state.packets.lock().expect("scoped ACK packets").clone();
+        for packet in packets {
+            if let Some(task) = packet.task.lock().expect("scoped ACK task").take() {
+                task.abort();
+            }
+        }
+    }
+}
+
+pub(super) enum Interception {
+    Pass(Request<ProxyBody>),
+    Handled(Response<ProxyBody>),
+}
+
+fn restored_body(bytes: Bytes, trailers: Option<http::HeaderMap>) -> ProxyBody {
+    Full::new(bytes)
+        .map_err(|never| match never {})
+        .with_trailers(async move { trailers.map(Ok::<_, io::Error>) })
+        .boxed()
+}
+
+fn hash(bytes: &[u8]) -> String {
+    hex(Sha256::digest(bytes).as_ref())
+}
+
+fn hex(bytes: &[u8]) -> String {
+    hex_simd::encode_to_string(bytes, hex_simd::AsciiCase::Lower)
+}
+
+fn trailer_receipt(trailers: Option<&http::HeaderMap>) -> Value {
+    trailers.map_or(Value::Null, |headers| {
+        json!(
+            headers
+                .iter()
+                .map(|(name, value)| { json!({"name": name.as_str(), "value_hex": hex(value.as_bytes())}) })
+                .collect::<Vec<_>>()
+        )
+    })
+}
+
+fn header_hash(headers: &http::HeaderMap) -> String {
+    let mut hasher = Sha256::new();
+    for (name, value) in headers {
+        hasher.update(name.as_str().as_bytes());
+        hasher.update([0]);
+        hasher.update(value.as_bytes());
+        hasher.update([0]);
+    }
+    hex(hasher.finalize().as_ref())
+}
+
+fn millis(state: &QueueState) -> u64 {
+    u64::try_from(state.started.elapsed().as_millis()).unwrap_or(u64::MAX)
+}
+
+pub(super) async fn intercept(
+    request: Request<ProxyBody>,
+    client: ProxyClient,
+    target: SocketAddr,
+    state: Arc<QueueState>,
+) -> io::Result<Interception> {
+    if Instant::now() >= state.deadline || state.closed.load(Ordering::SeqCst) {
+        return Ok(Interception::Pass(request));
+    }
+    let (parts, body) = request.into_parts();
+    let collected = tokio::time::timeout_at(state.deadline, Limited::new(body, MAX_BYTES).collect())
+        .await
+        .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "scoped ACK frame collection expired"))?
+        .map_err(io::Error::other)?;
+    let trailers = collected.trailers().cloned();
+    let bytes = collected.to_bytes();
+    let decoded = ScannerScopedDirtyUsageAckRequest::decode(unary_payload(&bytes)?).map_err(io::Error::other)?;
+    let request = Request::from_parts(parts, restored_body(bytes.clone(), trailers.clone()));
+    if Instant::now() >= state.deadline || !state.scope.matches(&decoded) {
+        return Ok(Interception::Pass(request));
+    }
+    let header = |name| request.headers().get(name).and_then(|value| value.to_str().ok());
+    let (action, mut selected) = watch::channel(None);
+    let packet = {
+        let mut packets = state.packets.lock().expect("scoped ACK packets");
+        if packets.len() >= state.capacity || state.closed.load(Ordering::SeqCst) {
+            return Ok(Interception::Pass(request));
+        }
+        let id = packets.len();
+        let packet = Arc::new(Packet {
+            action,
+            task: Mutex::new(None),
+            receipt: Mutex::new(json!({
+                "id": id, "session_nonce": state.scope.nonce, "receiver": target.to_string(),
+                "authority": request.uri().authority().map(|value| value.as_str()),
+                "captured_at_ms": millis(&state), "completed": false, "forwarded": false,
+                "backend_complete": false, "outcome": "held", "request_bytes": bytes.len(),
+            "request_sha256": hash(&bytes), "request_frame_hex": hex(&bytes), "request_headers_sha256": header_hash(request.headers()),
+                "request_trailers_sha256": trailers.as_ref().map(header_hash),
+            "request_trailers": trailer_receipt(trailers.as_ref()),
+                "auth_nonce": header("x-rustfs-rpc-nonce"), "replay_nonce": header("x-rustfs-rpc-replay-nonce"),
+                "signature_sha256": header("x-rustfs-rpc-signature-v2").map(|value| hash(value.as_bytes())),
+                "replay_signature_sha256": header("x-rustfs-rpc-signature-v3").map(|value| hash(value.as_bytes())),
+                "request": {"owner_id": decoded.owner_id, "instance_id": decoded.instance_id,
+                    "probe_only": decoded.probe_only, "entries": decoded.entries.iter().map(|entry| json!({
+                        "bucket": entry.bucket, "bucket_incarnation": uuid::Uuid::from_slice(&entry.bucket_incarnation).ok().map(|value| value.to_string()),
+                        "generation": entry.generation })).collect::<Vec<_>>() }
+            })),
+        });
+        packets.push(packet.clone());
+        packet
+    };
+    let (reply, response) = oneshot::channel();
+    let worker_packet = packet.clone();
+    let worker_state = state.clone();
+    // This owner survives an HTTP/2 stream reset from the original 5 s caller.
+    // Every request is settled once and is discarded at the session deadline.
+    let task = tokio::spawn(async move {
+        let result = tokio::time::timeout_at(worker_state.deadline, async {
+            let fault = loop {
+                if let Some(fault) = *selected.borrow_and_update() {
+                    break fault;
+                }
+                selected
+                    .changed()
+                    .await
+                    .map_err(|_| io::Error::other("scoped ACK queue closed"))?;
+            };
+            if fault == ScopedAckFault::RequestLoss {
+                worker_packet.receipt.lock().expect("scoped ACK receipt")["outcome"] = json!("request_lost");
+                return Err(io::Error::new(io::ErrorKind::ConnectionAborted, "injected scoped ACK request loss"));
+            }
+            if worker_state.closed.load(Ordering::SeqCst) || Instant::now() >= worker_state.deadline {
+                return Err(io::Error::new(
+                    io::ErrorKind::ConnectionAborted,
+                    "scoped ACK session ended or expired before forwarding",
+                ));
+            }
+            {
+                let mut receipt = worker_packet.receipt.lock().expect("scoped ACK receipt");
+                receipt["forwarded"] = json!(true);
+                receipt["forwarded_at_ms"] = json!(millis(&worker_state));
+            }
+            let backend = forward_to_backend(request, client, target).await?;
+            let (parts, body) = backend.into_parts();
+            let collected = Limited::new(body, MAX_BYTES).collect().await.map_err(io::Error::other)?;
+            let trailers = collected.trailers().cloned();
+            let bytes = collected.to_bytes();
+            let grpc_status = trailers
+                .as_ref()
+                .and_then(|values| values.get("grpc-status"))
+                .or_else(|| parts.headers.get("grpc-status"))
+                .and_then(|value| value.to_str().ok());
+            let response = unary_payload(&bytes)
+                .ok()
+                .and_then(|payload| ScannerScopedDirtyUsageAckResponse::decode(payload).ok())
+                .map(|response| {
+                    json!({"owner_id": response.owner_id, "instance_id": response.instance_id,
+                    "supported": response.supported, "cleared": response.cleared,
+                    "response_proof_bytes": response.response_proof.len()})
+                });
+            {
+                let mut receipt = worker_packet.receipt.lock().expect("scoped ACK receipt");
+                receipt["backend_complete"] = json!(true);
+                receipt["backend_completed_at_ms"] = json!(millis(&worker_state));
+                receipt["response"] = json!({"status": parts.status.as_u16(), "grpc_status": grpc_status,
+                    "bytes": bytes.len(), "sha256": hash(&bytes), "frame_hex": hex(&bytes), "headers_sha256": header_hash(&parts.headers),
+                    "trailers": trailer_receipt(trailers.as_ref()),
+                    "trailers_sha256": trailers.as_ref().map(header_hash), "ack": response});
+                receipt["outcome"] = json!(if fault == ScopedAckFault::ResponseLoss {
+                    "response_lost"
+                } else {
+                    "forwarded"
+                });
+            }
+            if fault == ScopedAckFault::ResponseLoss {
+                return Err(io::Error::new(
+                    io::ErrorKind::ConnectionAborted,
+                    "injected scoped ACK response loss after backend trailers",
+                ));
+            }
+            Ok(Response::from_parts(parts, restored_body(bytes, trailers)))
+        })
+        .await
+        .unwrap_or_else(|_| Err(io::Error::new(io::ErrorKind::TimedOut, "scoped ACK retention expired")));
+        {
+            let mut receipt = worker_packet.receipt.lock().expect("scoped ACK receipt");
+            if result.is_err() && receipt["outcome"] == "held" {
+                receipt["outcome"] = json!("transport_failed_or_expired");
+            }
+            receipt["completed"] = json!(true);
+            receipt["completed_at_ms"] = json!(millis(&worker_state));
+        }
+        let _ = reply.send(result);
+        worker_state.changed.notify_one();
+    });
+    *packet.task.lock().expect("scoped ACK task") = Some(task.abort_handle());
+    if state.closed.load(Ordering::SeqCst) {
+        task.abort();
+    }
+    drop(task);
+    state.changed.notify_one();
+    response
+        .await
+        .map_err(|_| io::Error::new(io::ErrorKind::ConnectionAborted, "scoped ACK queue ended"))?
+        .map(Interception::Handled)
+}

crates/e2e_test/src/storage_api.rs

@@ -70,6 +70,9 @@ pub(crate) mod multipart_rpc_delay {
 #[cfg(test)]
 pub(crate) mod multipart_fence_acceptance {
     pub(crate) use super::RUSTFS_META_MULTIPART_BUCKET;
+
+    /// Targeted lock diagnostics for the metadata-delay quorum case.
+    pub(crate) const LOCK_DIAGNOSTICS_FILTER: &str = "warn,rustfs_lock::distributed_lock=debug,rustfs_ecstore::object_lock_diag=trace,rustfs_ecstore::cluster::rpc::remote_locker=info";
 }
 
 #[cfg(test)]
@@ -80,3 +83,23 @@ pub(crate) mod multipart_source_rpc {
         PublicationIdentity, SourceGeneration, SourceGeometry, SourceLineage,
     };
 }
+
+#[cfg(test)]
+pub(crate) mod scanner_peer_bucket {
+    use rustfs_ecstore::api::disk as ecstore_disk;
+    use rustfs_ecstore::api::erasure as ecstore_erasure;
+    use rustfs_ecstore::api::rpc as ecstore_rpc;
+
+    pub(crate) use ecstore_disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
+    pub(crate) use ecstore_erasure::{ReedSolomonEncoder, calc_shard_size};
+    pub(crate) use ecstore_rpc::PeerRestClient;
+}
+
+/// Storage handles used to seed a pre-validation bucket record while the fixture server is stopped.
+#[cfg(test)]
+pub(crate) mod bucket_tag_fixture {
+    pub(crate) use rustfs_ecstore::api::bucket::metadata::load_bucket_metadata;
+    pub(crate) use rustfs_ecstore::api::layout::{DisksLayout, EndpointServerPools};
+    pub(crate) use rustfs_ecstore::api::runtime::InstanceContext;
+    pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks_with_instance_ctx};
+}

crates/e2e_test/src/sts_query_compat_test.rs

@@ -678,6 +678,44 @@ async fn test_list_buckets_opa_contract() -> TestResult {
     Ok(())
 }
 
+/// Stop and reap the directory's owning server before opening the fixture store.
+async fn seed_legacy_duplicate_bucket_tags(env: &mut RustFSTestEnvironment, bucket: &str) -> TestResult {
+    use crate::storage_api::bucket_tag_fixture::{
+        DisksLayout, ECStore, EndpointServerPools, InstanceContext, init_local_disks_with_instance_ctx, load_bucket_metadata,
+    };
+
+    let server = env
+        .process
+        .as_mut()
+        .ok_or("missing server process before offline fixture write")?;
+    server.kill()?;
+    server.wait()?;
+    env.process = None;
+    let layout = DisksLayout::from_volumes(std::slice::from_ref(&env.temp_dir))?;
+    let (pools, _) = EndpointServerPools::create_server_endpoints(&env.address, &layout).await?;
+    let context = std::sync::Arc::new(InstanceContext::new());
+    init_local_disks_with_instance_ctx(&context, pools.clone()).await?;
+    let store = ECStore::new_with_instance_ctx(env.address.parse()?, pools, Default::default(), context.clone()).await?;
+    let _cancel = context
+        .background_cancel_token()
+        .expect("store binds its shutdown token")
+        .drop_guard();
+    let mut metadata = load_bucket_metadata(store.clone(), bucket).await?;
+    let incarnation = metadata.bucket_incarnation_id;
+    let policy = metadata.policy_config_json.clone();
+    assert!(!incarnation.is_nil());
+    assert!(!policy.is_empty(), "the bucket-policy Allow must survive legacy seeding");
+    // Encode the duplicate tag set with the same bucket record and erasure writer used by the store.
+    let xml = b"<Tagging><TagSet><Tag><Key>department</Key><Value>finance</Value></Tag><Tag><Key>department</Key><Value>engineering</Value></Tag></TagSet></Tagging>";
+    metadata.tagging_config_xml = xml.to_vec();
+    metadata.save_with_store(store.clone()).await?;
+    let persisted = load_bucket_metadata(store, bucket).await?;
+    assert_eq!(persisted.bucket_incarnation_id, incarnation);
+    assert_eq!(persisted.policy_config_json, policy);
+    assert_eq!(persisted.tagging_config_xml.as_slice(), xml);
+    Ok(())
+}
+
 #[tokio::test]
 async fn test_bucket_tags_opa_contract() -> TestResult {
     use aws_sdk_s3::primitives::ByteStream;
@@ -688,12 +726,13 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
     init_logging();
     let mut opa = OpaMock::start().await?;
     let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server_without_cleanup_with_env(&[
-        ("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str()),
+    let opa_url = opa.url.clone();
+    let server_env = [
+        ("RUSTFS_POLICY_PLUGIN_URL", opa_url.as_str()),
         ("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", OPA_AUTH_TOKEN),
         ("NO_PROXY", "127.0.0.1,localhost"),
-    ])
-    .await?;
+    ];
+    env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
     let secret = uuid::Uuid::new_v4().to_string();
     create_user(&env, "opabuckettags", &secret).await?;
     let client = aws_sdk_s3::Client::from_conf(build_test_s3_config(&env.url, "opabuckettags", &secret, None, "e2e-bucket-tags"));
@@ -885,7 +924,9 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
     assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
     opa.expect_bucket_tags("s3:GetObject", destination, Some("engineering"))
         .await?;
-    admin
+    // Duplicate keys are refused on write; the ambiguous record below is seeded
+    // offline the way an upgraded deployment may still hold it on disk.
+    let invalid_write = admin
         .put_bucket_tagging()
         .bucket(destination)
         .tagging(
@@ -895,7 +936,24 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
                 .build()?,
         )
         .send()
+        .await
+        .expect_err("new duplicate tags must be rejected before publication");
+    assert_eq!(invalid_write.raw_response().map(|response| response.status().as_u16()), Some(400));
+
+    set_department(&admin, source, "finance").await?;
+    seed_legacy_duplicate_bucket_tags(&mut env, destination).await?;
+    env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
+    // Prove this process can authorize the same caller and read a healthy bucket's persisted tags.
+    let read = client
+        .get_object()
+        .bucket(source)
+        .key("report")
+        .version_id(version)
+        .send()
         .await?;
+    assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
+    opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
+
     let error = client
         .get_object()
         .bucket(destination)

crates/e2e_test/src/upgrade_compatibility_test.rs

@@ -1112,8 +1112,9 @@ async fn wait_for_upgrade_writes(
             match response {
                 Ok(Ok(_)) => break,
                 Ok(Err(error)) => {
+                    let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
                     if error.raw_response().map(|response| response.status().as_u16()) != Some(503)
-                        || error.as_service_error().and_then(ProvideErrorMetadata::code) != Some("ServiceUnavailable")
+                        || !matches!(code, Some("ServiceUnavailable" | "SlowDown" | "SlowDownRead"))
                     {
                         return Err(format!("{phase}: node {node} {probe} failed: {error:?}").into());
                     }

crates/ecstore/Cargo.toml

@@ -31,6 +31,7 @@ workspace = true
 
 [features]
 default = []
+fips = ["rustfs-s3-client/fips", "rustfs-utils/fips"]
 diagnostic-read-version-caller = ["rustfs-io-metrics/diagnostic-read-version-caller"]
 # Sampled wall/CPU diagnostics only; does not change the read backend.
 diagnostic-local-batch = ["rustix/time"]
@@ -225,9 +226,11 @@ metrics = { workspace = true }
 # The guard scripts/check_no_tokio_io_uring.sh allows an explicit
 # io-uring integration; only the tokio "io-uring" runtime feature is banned.
 [target.'cfg(target_os = "linux")'.dependencies]
-# SharedReadBudget and the bounded read APIs are available in the 0.3.0
-# registry release; keep the application on the published contract.
-rustfs-uring = "0.3.0"
+# Native completion owners keep physical I/O leases alive after cancellation.
+rustfs-uring = { git = "https://github.com/rustfs/uring.git", rev = "eb8fc160e71d804427de1e905f5c226d580f71bf" }
+
+[target.'cfg(target_os = "linux")'.dev-dependencies]
+rustfs-uring = { git = "https://github.com/rustfs/uring.git", rev = "eb8fc160e71d804427de1e905f5c226d580f71bf", features = ["fault-injection"] }
 
 [target.'cfg(windows)'.dependencies]
 winapi-util.workspace = true

crates/ecstore/src/api/mod.rs

@@ -192,6 +192,7 @@ pub mod bucket {
             table_catalog_path_hash, unreadable_config_refusal,
         };
         pub use crate::bucket::metadata::{BUCKET_DURABILITY_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
+        pub use crate::bucket::metadata::{MAX_BUCKET_TAGS, validate_tag_set_entries};
     }
 
     pub mod config_parse_mode {
@@ -331,8 +332,8 @@ pub mod bucket {
     pub mod utils {
         pub use crate::bucket::utils::{
             check_bucket_and_object_names, check_list_objs_args, check_object_name_for_length_and_slash,
-            check_valid_bucket_name_strict, deserialize, has_bad_path_component, is_meta_bucketname, is_valid_object_prefix,
-            serialize,
+            check_valid_bucket_name_strict, deserialize, has_bad_path_component, has_xml_invalid_control_characters,
+            is_meta_bucketname, is_valid_object_prefix, serialize,
         };
     }
 
@@ -447,10 +448,19 @@ pub mod data_usage {
 }
 
 pub mod disk {
+    pub use crate::cluster::rpc::background_pipe::{BackgroundPipeReader, BackgroundPipeWriter, background_pipe};
+    pub use crate::cluster::rpc::background_reader_stream::BackgroundReaderStream;
+    pub use crate::disk::background_memory::BackgroundEncodeBuffer;
     pub use crate::disk::disk_store::get_object_disk_read_timeout;
+    #[cfg(feature = "test-util")]
+    pub use crate::disk::disk_store::wait_for_multipart_index_bootstrap;
     pub use crate::disk::local::{ReplacementExecutionLease, ScanGuard};
     #[cfg(feature = "test-util")]
     pub use crate::disk::local::{local_durability_modes, set_local_rename_data_fail_after_metadata_commit};
+    pub use crate::disk::multipart_index::{
+        MULTIPART_INDEX_RPC_MAX_BYTES, MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES, MultipartIndexPage, MultipartIndexRpcReply,
+        MultipartIndexRpcRequest,
+    };
     pub use crate::disk::multipart_source::{
         MULTIPART_SOURCE_PROTOCOL_VERSION, MultipartAbortCertificate, MultipartAbortProof, MultipartAbortProofKind,
         MultipartAbortRound, MultipartSourceOperation, MultipartSourcePhase, MultipartSourceRequest, MultipartSourceResponse,
@@ -463,7 +473,9 @@ pub mod disk {
     pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage};
     pub use crate::disk::{
         BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
-        CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration, CurrentDeleteTransition,
+        BoundedDeleteRecoveryDiscovery, BoundedDeleteRecoveryInspection, BoundedDeleteRecoverySettlement,
+        BoundedDeleteRecoverySettlementReply, CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration,
+        CurrentDeleteTransition, DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES, DELETE_RECOVERY_MAX_INSPECTION_ENTRIES,
         DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest,
         DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DeleteRecoveryState,
         Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, FileInfoVersions, FileReader, FileWriter,
@@ -475,6 +487,10 @@ pub mod disk {
         UpdateMetadataOpts, VolumeInfo, WalkDirOptions, is_quota_mutation_fence_path, new_disk,
         validate_batch_read_version_item_count,
     };
+    pub use crate::disk::{
+        BackgroundMemoryBudget, PhysicalDiskBinding, PhysicalDiskPlan, PhysicalDiskSample, PhysicalDiskSampler,
+        PhysicalResourcePlan, RemoteBackgroundIo, with_background_resource_unit, with_remote_background_io,
+    };
     pub use bytes::Bytes;
     pub use endpoint::Endpoint;
     pub use error::DiskError;
@@ -610,11 +626,12 @@ pub mod rebalance {
 pub mod rio {
     pub use crate::io_support::rio::{
         DecryptReader, DynReader, EncryptReader, HardLimitReader, HashReader, ReadStream, Reader, WriteEncryption, WritePlan,
-        boxed_reader, compression_metadata_value, wrap_reader,
+        boxed_reader, compression_metadata_value, validate_compression_scheme, wrap_reader,
     };
 }
 
 pub mod rpc {
+    pub use crate::cluster::rpc::http_auth::with_rpc_background_request;
     pub use crate::cluster::rpc::{
         AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, MAX_NETWORK_PROBE_BYTES, MAX_NETWORK_PROBE_DURATION,
         NetworkPeerProbeClient, NetworkPeerProbeError, NetworkPeerProbeMeasurement, NetworkPeerTarget, PEER_RESTDRY_RUN,
@@ -624,8 +641,8 @@ pub mod rpc {
         TONIC_RPC_PREFIX, TonicInterceptor, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce,
         decode_heal_bucket_rpc_options, encode_heal_bucket_rpc_options, gen_signature_headers, gen_tonic_replay_scope_headers,
         gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
-        node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
-        sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
+        node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, rpc_background_admission,
+        set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
         sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
         tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
         verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
@@ -646,7 +663,9 @@ pub mod set_disk {
 
     #[cfg(feature = "test-util")]
     pub mod test_util {
-        pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test;
+        pub use crate::bucket::quota::reservation::{
+            fail_next_quota_ledger_save_for_test, initialize_quota_ledger_for_test, retry_quota_reconciliation_for_test,
+        };
         pub use crate::set_disk::ConditionalReadBenchmarkMetadataGuard;
         pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause};
 
@@ -675,8 +694,9 @@ pub mod storage {
     pub use crate::store::HealWalkVersion;
     pub use crate::store::{
         BootstrapLocalTarget, ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk,
-        all_local_disk_path, find_local_disk_by_ref, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients,
-        prewarm_local_disk_id_map, prewarm_local_disk_id_map_with_instance_ctx,
+        all_local_disk_path, find_local_disk_by_ref, find_local_disk_by_ref_for_rpc, init_local_disks,
+        init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
+        prewarm_local_disk_id_map_with_instance_ctx,
     };
     pub use crate::store::{HealObjectAbsenceProof, HealObjectStorageResult};
 }

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -16,7 +16,8 @@ use crate::bucket::metadata::BucketMetadata;
 use crate::bucket::metadata_sys::get_bucket_targets_config;
 use crate::bucket::metadata_sys::get_replication_config;
 use crate::bucket::remote_s3_client::{
-    PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, build_remote_s3_client,
+    PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, RemoteS3RetryPolicy,
+    UploadResponseDeadline, build_remote_s3_client,
 };
 use crate::bucket::replication::{ObjectLockIntegrity, object_lock_put_integrity, replication_etags_match};
 use crate::bucket::replication::{ReplicationStatusType, ReplicationTargetConfigBridge};
@@ -119,6 +120,7 @@ impl From<&BucketTarget> for RemoteS3EndpointSpec {
             ca_cert_pem: (!target.ca_cert_pem.trim().is_empty()).then(|| target.ca_cert_pem.clone()),
             connect_timeout: None,
             read_timeout: None,
+            operation_timeout: Some(TargetRequestTimeouts::from_env().request),
             // Replication has no retry budget of its own on the request path,
             // so it keeps the SDK's standard three attempts; stating it here
             // pins the behaviour to this line instead of an SDK default.
@@ -1191,6 +1193,7 @@ impl BucketTargetSys {
             health_check_duration: target.health_check_duration,
             replicate_sync: target.replication_sync,
             client: Arc::new(client),
+            timeouts: TargetRequestTimeouts::from_env(),
         })
     }
 
@@ -1922,6 +1925,83 @@ impl std::fmt::Display for S3ClientError {
     }
 }
 
+/// Replication request budgets, snapshotted when a target client is built.
+/// Streamed GET bodies are not subject to the upload-response budget.
+#[derive(Debug, Clone, Copy)]
+pub struct TargetRequestTimeouts {
+    request: Duration,
+    complete: Duration,
+    minimum_transfer_rate: u64,
+}
+
+impl Default for TargetRequestTimeouts {
+    fn default() -> Self {
+        Self {
+            request: Duration::from_secs(60),
+            complete: Duration::from_secs(15 * 60),
+            minimum_transfer_rate: 1024 * 1024,
+        }
+    }
+}
+
+impl TargetRequestTimeouts {
+    fn from_env() -> Self {
+        let defaults = Self::default();
+        let seconds = |name: &str, fallback: Duration| {
+            let value = rustfs_utils::get_env_u64(name, fallback.as_secs());
+            Duration::from_secs(if (1..=86400).contains(&value) {
+                value
+            } else {
+                fallback.as_secs()
+            })
+        };
+        let minimum_transfer_rate =
+            rustfs_utils::get_env_u64("RUSTFS_REPLICATION_MIN_TRANSFER_RATE_BYTES_PER_SEC", defaults.minimum_transfer_rate);
+        Self {
+            request: seconds("RUSTFS_REPLICATION_REQUEST_TIMEOUT_SECS", defaults.request),
+            complete: seconds("RUSTFS_REPLICATION_COMPLETE_TIMEOUT_SECS", defaults.complete),
+            minimum_transfer_rate: if minimum_transfer_rate == 0 {
+                defaults.minimum_transfer_rate
+            } else {
+                minimum_transfer_rate
+            },
+        }
+    }
+
+    fn upload_timeout(self, size: i64) -> Result<Duration, S3ClientError> {
+        let size = u64::try_from(size).map_err(|_| S3ClientError::new("replication upload size must be non-negative"))?;
+        // The SDK can make three attempts. Leave transmission headroom for
+        // all three; the separate response deadline still bounds a trickling
+        // target after each upload finishes.
+        let attempts = match REPLICATION_TARGET_RETRY_POLICY {
+            RemoteS3RetryPolicy::Disabled => 1,
+            RemoteS3RetryPolicy::Standard { max_attempts } => u64::from(max_attempts.max(1)),
+        };
+        let seconds = size
+            .div_ceil(self.minimum_transfer_rate)
+            .checked_mul(attempts)
+            .ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
+        let timeout = self
+            .request
+            .checked_add(Duration::from_secs(seconds))
+            .ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
+        if Instant::now().checked_add(timeout).is_none() {
+            return Err(S3ClientError::new("replication upload timeout exceeds supported duration"));
+        }
+        Ok(timeout)
+    }
+
+    fn upload_override(self, size: i64) -> Result<aws_sdk_s3::config::Builder, S3ClientError> {
+        Ok(aws_sdk_s3::config::Builder::new()
+            .timeout_config(
+                aws_smithy_types::timeout::TimeoutConfig::builder()
+                    .operation_timeout(self.upload_timeout(size)?)
+                    .build(),
+            )
+            .interceptor(UploadResponseDeadline(self.request)))
+    }
+}
+
 #[derive(Debug)]
 pub struct TargetClient {
     pub endpoint: String,
@@ -1935,6 +2015,7 @@ pub struct TargetClient {
     pub health_check_duration: Duration,
     pub replicate_sync: bool,
     pub client: Arc<S3Client>,
+    pub timeouts: TargetRequestTimeouts,
 }
 
 impl TargetClient {
@@ -2360,12 +2441,11 @@ impl TargetClient {
 
                 Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
             });
+        let mut config = self.timeouts.upload_override(size)?;
         if forwards_source_checksum {
-            operation = operation.config_override(
-                aws_sdk_s3::config::Builder::new()
-                    .request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired),
-            );
+            config = config.request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired);
         }
+        operation = operation.config_override(config);
         match operation.send().await {
             Ok(output) => {
                 // Under SSE-KMS/DSSE or SSE-C the target's ETag is not the MD5
@@ -2501,6 +2581,7 @@ impl TargetClient {
                 }
                 Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
             })
+            .config_override(self.timeouts.upload_override(size)?)
             .send()
             .await
         {
@@ -2538,6 +2619,13 @@ impl TargetClient {
                 }
                 Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
             })
+            .config_override(
+                aws_sdk_s3::config::Builder::new().timeout_config(
+                    aws_smithy_types::timeout::TimeoutConfig::builder()
+                        .operation_timeout(self.timeouts.complete)
+                        .build(),
+                ),
+            )
             .send()
             .await
         {
@@ -2966,11 +3054,241 @@ mod tests {
                 health_check_duration: Duration::from_secs(5),
                 replicate_sync: false,
                 client: Arc::new(client),
+                timeouts: TargetRequestTimeouts::default(),
             },
             request_headers,
         )
     }
 
+    #[derive(Debug, Clone)]
+    struct TimedResponseConnector {
+        upload_delay: Duration,
+        response_delay: Duration,
+        prefix_bytes: usize,
+        response: bytes::Bytes,
+        frames: Arc<std::sync::atomic::AtomicUsize>,
+        dropped: Arc<std::sync::atomic::AtomicUsize>,
+    }
+
+    struct ResponseDrop(Arc<std::sync::atomic::AtomicUsize>);
+    impl Drop for ResponseDrop {
+        fn drop(&mut self) {
+            self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+        }
+    }
+
+    impl SmithyHttpConnector for TimedResponseConnector {
+        fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
+            let settings = self.clone();
+            HttpConnectorFuture::new(async move {
+                http_body_util::BodyExt::collect(request.into_body())
+                    .await
+                    .map_err(aws_smithy_runtime_api::client::result::ConnectorError::io)?;
+                tokio::time::sleep(settings.upload_delay).await;
+                let guard = ResponseDrop(Arc::clone(&settings.dropped));
+                let stream = futures::stream::unfold(
+                    (settings.prefix_bytes, Some(settings.response), true, guard),
+                    move |(remaining, mut body, first, guard)| {
+                        let frames = Arc::clone(&settings.frames);
+                        async move {
+                            if remaining == 0 && body.is_none() {
+                                return None;
+                            }
+                            if !first {
+                                tokio::time::sleep(settings.response_delay).await;
+                            }
+                            let bytes = if remaining > 0 {
+                                bytes::Bytes::from_static(b" ")
+                            } else {
+                                body.take()?
+                            };
+                            frames.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+                            Some((
+                                Ok::<_, std::io::Error>(http_body::Frame::data(bytes)),
+                                (remaining.saturating_sub(1), body, false, guard),
+                            ))
+                        }
+                    },
+                )
+                .fuse();
+                let body = SdkBody::from_body_1_x(http_body_util::StreamBody::new(stream));
+                let mut response =
+                    HttpResponse::new(aws_smithy_runtime_api::http::StatusCode::try_from(200u16).expect("status"), body);
+                response.headers_mut().insert("etag", "\"test-etag\"");
+                Ok(response)
+            })
+        }
+    }
+
+    fn timed_response_target(
+        upload_delay: Duration,
+        response_delay: Duration,
+        prefix_bytes: usize,
+        response: &'static [u8],
+    ) -> (TargetClient, Arc<std::sync::atomic::AtomicUsize>, Arc<std::sync::atomic::AtomicUsize>) {
+        let frames = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let dropped = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let connector = SharedHttpConnector::new(TimedResponseConnector {
+            upload_delay,
+            response_delay,
+            prefix_bytes,
+            response: bytes::Bytes::from_static(response),
+            frames: Arc::clone(&frames),
+            dropped: Arc::clone(&dropped),
+        });
+        let http_client = http_client_fn(move |_settings, _components| connector.clone());
+        let (mut target, _) = header_recording_target_client(Vec::new());
+        target.timeouts = TargetRequestTimeouts {
+            request: Duration::from_secs(2),
+            complete: Duration::from_secs(10),
+            minimum_transfer_rate: 1024 * 1024,
+        };
+        target.client = Arc::new(S3Client::from_conf(
+            target
+                .client
+                .config()
+                .to_builder()
+                .http_client(http_client)
+                .retry_config(aws_sdk_s3::config::retry::RetryConfig::disabled())
+                .timeout_config(
+                    aws_smithy_types::timeout::TimeoutConfig::builder()
+                        .operation_timeout(target.timeouts.request)
+                        .build(),
+                )
+                .build(),
+        ));
+        (target, frames, dropped)
+    }
+
+    #[test]
+    fn replication_upload_budgets_preserve_large_transfers_and_reject_invalid_sizes() {
+        let policy = TargetRequestTimeouts::default();
+        assert_eq!(policy.upload_timeout(0).expect("empty upload"), Duration::from_secs(60));
+        assert_eq!(
+            policy.upload_timeout(5 * 1024 * 1024 * 1024).expect("maximum single PUT"),
+            Duration::from_secs(15420)
+        );
+        assert!(policy.upload_timeout(-1).is_err());
+        let slow = TargetRequestTimeouts {
+            minimum_transfer_rate: 1,
+            ..policy
+        };
+        assert!(slow.upload_timeout(i64::MAX).is_err());
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn replication_upload_response_deadline_is_independent_of_size_and_checksum_override() {
+        for operation in ["put", "forwarded-checksum", "part"] {
+            let (target, frames, dropped) = timed_response_target(Duration::from_secs(4), Duration::from_millis(500), 100, b"");
+            let body = vec![b'x'; 2 * 1024 * 1024];
+            let size = i64::try_from(body.len()).expect("size");
+            let started = tokio::time::Instant::now();
+            let error = if operation == "part" {
+                target
+                    .put_object_part(
+                        "bucket",
+                        "key",
+                        "upload",
+                        1,
+                        size,
+                        ByteStream::from(body),
+                        &PutObjectPartOptions::default(),
+                    )
+                    .await
+                    .expect_err("response deadline")
+            } else {
+                let mut opts = PutObjectOptions::default();
+                if operation == "forwarded-checksum" {
+                    opts.user_metadata.insert(
+                        "x-amz-checksum-sha256".to_owned(),
+                        "OoJ3yNhRwv3wwtZoGqEIPrPX9xwTnfLl+ka0wStN1g0=".to_owned(),
+                    );
+                }
+                target
+                    .put_object("bucket", "key", size, ByteStream::from(body), &opts)
+                    .await
+                    .expect_err("response deadline")
+            };
+            assert!(
+                started.elapsed() >= Duration::from_secs(6),
+                "upload gets its full independent window: {error}"
+            );
+            assert!(
+                started.elapsed() < target.timeouts.upload_timeout(size).expect("budget"),
+                "response must end before the larger total deadline"
+            );
+            assert!(
+                frames.load(std::sync::atomic::Ordering::SeqCst) >= 2,
+                "trickle delivered progress before failure"
+            );
+            assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1, "transport body released");
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn replication_slow_upload_preserves_transfer_window_and_streaming_get_body() {
+        let (target, _, _) = timed_response_target(Duration::from_secs(4), Duration::from_millis(100), 5, b"");
+        let body = vec![b'x'; 2 * 1024 * 1024];
+        let size = i64::try_from(body.len()).expect("size");
+        target
+            .put_object("bucket", "key", size, ByteStream::from(body), &PutObjectOptions::default())
+            .await
+            .expect("slow upload, prompt response");
+        let (target, _, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), 5, b"payload");
+        let response = target
+            .get_object("bucket", "key", None, None, None, HeaderMap::new())
+            .await
+            .expect("GET headers");
+        let started = tokio::time::Instant::now();
+        let body = response
+            .body
+            .collect()
+            .await
+            .expect("streamed GET remains outside upload deadline")
+            .into_bytes();
+        assert!(started.elapsed() > target.timeouts.request);
+        assert_eq!(&body[..], b"     payload");
+        assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn replication_complete_processing_whitespace_has_a_separate_bounded_window() {
+        const RESULT: &[u8] = b"<CompleteMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><ETag>\"etag\"</ETag></CompleteMultipartUploadResult>";
+        for prefix in [5, 100] {
+            let (target, frames, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), prefix, RESULT);
+            let started = tokio::time::Instant::now();
+            let result = target
+                .complete_multipart_upload(
+                    "bucket",
+                    "key",
+                    "upload",
+                    vec![CompletedPart::builder().part_number(1).e_tag("etag").build()],
+                    &PutObjectOptions::default(),
+                )
+                .await;
+            if prefix == 5 {
+                assert!(result.is_ok(), "legal processing whitespace: {result:?}");
+            } else {
+                assert!(result.is_err());
+                assert!(started.elapsed() <= Duration::from_secs(11));
+            }
+            assert!(started.elapsed() > target.timeouts.request);
+            assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
+            assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn replication_control_operation_deadline_covers_response_trickle() {
+        let (target, frames, dropped) =
+            timed_response_target(Duration::ZERO, Duration::from_millis(500), 100, b"<VersioningConfiguration/>");
+        let started = tokio::time::Instant::now();
+        assert!(target.get_bucket_versioning("bucket").await.is_err());
+        assert!(started.elapsed() <= Duration::from_secs(3));
+        assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
+        assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
+    }
+
     fn streaming_test_body(payload: &'static [u8]) -> ByteStream {
         let stream = tokio_util::io::ReaderStream::new(std::io::Cursor::new(payload));
         let body = http_body_util::StreamBody::new(futures::StreamExt::map(stream, |r| r.map(http_body::Frame::data)));
@@ -3279,6 +3597,7 @@ mod tests {
                 health_check_duration: Duration::from_secs(5),
                 replicate_sync: false,
                 client: Arc::new(client),
+                timeouts: TargetRequestTimeouts::default(),
             },
             request_uris,
         )
@@ -3591,6 +3910,7 @@ mod tests {
             health_check_duration: Duration::from_secs(5),
             replicate_sync: false,
             client: Arc::new(client),
+            timeouts: TargetRequestTimeouts::default(),
         })
     }
 

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -5487,6 +5487,14 @@ pub async fn put_restore_opts(
     if !oi.user_tags.is_empty() {
         meta.insert(AMZ_OBJECT_TAGGING.to_string(), (*oi.user_tags).clone());
     }
+    if let Some(expires) = oi.expires {
+        meta.insert(
+            "expires".to_string(),
+            expires
+                .format(&time::format_description::well_known::Rfc3339)
+                .map_err(std::io::Error::other)?,
+        );
+    }
     let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), rreq.days.unwrap_or(1));
     meta.insert(
         metadata_keys::RESTORE.to_string(),
@@ -5506,7 +5514,6 @@ pub async fn put_restore_opts(
         // computed MD5 is not the object's public plaintext ETag.
         preserve_etag: oi.etag.clone(),
         shard_integrity_write_mode: Some(oi.shard_integrity_write_mode()),
-        //expires:           oi.expires,
         ..Default::default()
     })
 }
@@ -7144,6 +7151,25 @@ mod tests {
             0,
             "tier generation lease should release after EOF"
         );
+        let unread = get_transitioned_object_reader_with_tier_manager(
+            &object_info.bucket,
+            &object_info.name,
+            &None,
+            &HeaderMap::new(),
+            &object_info,
+            &ObjectOptions::default(),
+            &manager,
+            None,
+        )
+        .await
+        .expect("open an unread transitioned response");
+        assert_eq!(TierConfigMgr::active_operation_lease_count(&manager, &tier).await, 1);
+        drop(unread);
+        assert_eq!(
+            TierConfigMgr::active_operation_lease_count(&manager, &tier).await,
+            0,
+            "canceling an unread response must release the tier lease"
+        );
     }
 
     #[cfg(feature = "test-util")]

crates/ecstore/src/bucket/metadata.rs

@@ -256,6 +256,38 @@ pub const BUCKET_POLICY_CONFIG: &str = "policy.json";
 pub const BUCKET_NOTIFICATION_CONFIG: &str = "notification.xml";
 pub const BUCKET_LIFECYCLE_CONFIG: &str = "lifecycle.xml";
 pub const BUCKET_SSECONFIG: &str = "bucket-encryption.xml";
+/// Shared tag-entry validation for object requests and persisted bucket tags.
+pub fn validate_tag_set_entries(tag_set: &[s3s::dto::Tag]) -> std::result::Result<(), &'static str> {
+    fn is_valid_tag_text(value: &str) -> bool {
+        static VALID_TAG_TEXT: std::sync::OnceLock<std::result::Result<regex::Regex, regex::Error>> = std::sync::OnceLock::new();
+        VALID_TAG_TEXT
+            .get_or_init(|| regex::Regex::new(r"^[\p{L}\p{N}\p{Z}+\-=._:/@]*$"))
+            .as_ref()
+            .is_ok_and(|pattern| pattern.is_match(value))
+    }
+    let mut keys = std::collections::HashSet::with_capacity(tag_set.len());
+    for tag in tag_set {
+        let key = tag
+            .key
+            .as_deref()
+            .filter(|key| !key.is_empty())
+            .ok_or("Tag key cannot be empty")?;
+        if key.encode_utf16().count() > 128 || !is_valid_tag_text(key) {
+            return Err("The TagKey you have provided is invalid");
+        }
+        let value = tag.value.as_deref().ok_or("Tag value cannot be null")?;
+        if value.encode_utf16().count() > 256 || !is_valid_tag_text(value) {
+            return Err("The TagValue you have provided is invalid");
+        }
+        if !keys.insert(key) {
+            return Err("Cannot provide multiple Tags with the same key");
+        }
+    }
+    Ok(())
+}
+
+pub const MAX_BUCKET_TAGS: usize = 50;
+
 pub const BUCKET_TAGGING_CONFIG: &str = "tagging.xml";
 pub const BUCKET_QUOTA_CONFIG_FILE: &str = "quota.json";
 pub const OBJECT_LOCK_CONFIG: &str = "object-lock.xml";
@@ -990,6 +1022,20 @@ impl BucketMetadata {
                 self.encryption_config_updated_at = updated;
             }
             BUCKET_TAGGING_CONFIG => {
+                if !data.is_empty() {
+                    let tagging = deserialize::<Tagging>(&data).map_err(|_| {
+                        Error::InvalidArgument(self.name.clone(), String::new(), "Invalid bucket tagging XML".to_owned())
+                    })?;
+                    if tagging.tag_set.len() > MAX_BUCKET_TAGS {
+                        return Err(Error::InvalidArgument(
+                            self.name.clone(),
+                            String::new(),
+                            "Cannot have more than 50 tags per bucket".to_owned(),
+                        ));
+                    }
+                    validate_tag_set_entries(&tagging.tag_set)
+                        .map_err(|message| Error::InvalidArgument(self.name.clone(), String::new(), message.to_owned()))?;
+                }
                 self.tagging_config_xml = data;
                 // Drop the parsed form (like lifecycle above) so clearing the
                 // payload can't leave stale parsed tags to be cached.
@@ -1503,7 +1549,11 @@ pub(crate) async fn load_bucket_metadata_parse_with_presence(
     bucket: &str,
     parse: bool,
 ) -> Result<(BucketMetadata, bool)> {
-    let (mut bm, persisted) = match read_bucket_metadata(api.clone(), bucket).await {
+    let (metadata_res, incarnation_res) = tokio::join!(
+        Box::pin(read_bucket_metadata(api.clone(), bucket)),
+        Box::pin(load_bucket_incarnation(api.clone(), bucket)),
+    );
+    let (mut bm, persisted) = match metadata_res {
         Ok(res) => (res, true),
         Err(err) => {
             if err != Error::ConfigNotFound {
@@ -1514,7 +1564,7 @@ pub(crate) async fn load_bucket_metadata_parse_with_presence(
         }
     };
 
-    let incarnation = load_bucket_incarnation(api.clone(), bucket).await?;
+    let incarnation = incarnation_res?;
     if persisted {
         if bm.name.is_empty() {
             bm.name = bucket.to_string();
@@ -1780,6 +1830,31 @@ mod test {
     /// backlog#2292: a replicated config is stamped with the source
     /// `updated_at` it was given, not the local clock, while the plain
     /// `update_config` entry keeps stamping the local clock.
+    #[test]
+    fn bucket_tag_updates_reject_unreadable_shapes_without_replacing_metadata() {
+        let mut metadata = BucketMetadata::new("validated-tags");
+        let valid = br#"<Tagging><TagSet><Tag><Key>team</Key><Value></Value></Tag></TagSet></Tagging>"#.to_vec();
+        metadata
+            .update_config(BUCKET_TAGGING_CONFIG, valid.clone())
+            .expect("empty value is valid");
+        let updated = metadata.tagging_config_updated_at;
+        for invalid in [
+            "<Tagging><TagSet><Tag><Key>team</Key></Tag></TagSet></Tagging>".to_owned(),
+            "<Tagging><TagSet><Tag><Key></Key><Value>x</Value></Tag></TagSet></Tagging>".to_owned(),
+            "<Tagging><TagSet><Tag><Key>team</Key><Value>x</Value></Tag><Tag><Key>team</Key><Value>y</Value></Tag></TagSet></Tagging>".to_owned(),
+            format!("<Tagging><TagSet>{}</TagSet></Tagging>", (0..51).map(|i| format!("<Tag><Key>k{i}</Key><Value>v</Value></Tag>")).collect::<String>()),
+        ] {
+            assert!(metadata.update_config_at(BUCKET_TAGGING_CONFIG, invalid.into_bytes(), updated + time::Duration::hours(1)).is_err());
+            assert_eq!(metadata.tagging_config_xml, valid);
+            assert_eq!(metadata.tagging_config_updated_at, updated);
+        }
+        metadata.tagging_config_xml = b"legacy corrupt tags".to_vec();
+        metadata
+            .update_config(BUCKET_TAGGING_CONFIG, Vec::new())
+            .expect("deletion repairs legacy corruption");
+        assert!(metadata.tagging_config_xml.is_empty());
+    }
+
     #[test]
     fn update_config_at_stamps_the_given_time_and_update_config_stamps_now() {
         let source_time = OffsetDateTime::now_utc() - time::Duration::hours(3);

crates/ecstore/src/bucket/metadata_sys.rs

@@ -683,6 +683,7 @@ pub(crate) async fn update_validated_in<F>(
     bucket: &str,
     config_file: &str,
     data: Vec<u8>,
+    expected_incarnation_id: Option<Uuid>,
     validate: F,
 ) -> Result<OffsetDateTime>
 where
@@ -690,7 +691,7 @@ where
 {
     Box::pin(async {
         let sys = require_bucket_metadata_sys_in(ctx)?;
-        let guard = acquire_config_write_guard(sys.clone(), bucket).await?;
+        let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?;
         validate()?;
         update_under_config_write_guard(sys, &guard, config_file, data, None).await
     })
@@ -1435,6 +1436,7 @@ pub(crate) async fn get_object_lock_config_state_in(
 
 /// Re-read the Object Lock state and bucket incarnation from the same
 /// authoritative metadata blob while the caller holds the transaction lock.
+#[cfg(all(test, feature = "test-util"))]
 pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_in(
     ctx: &crate::runtime::instance::InstanceContext,
     bucket: &str,
@@ -1453,6 +1455,27 @@ pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_with_guard_
     get_delete_config_from_disk_in(ctx, bucket, guard, None).await
 }
 
+#[allow(clippy::type_complexity)]
+pub(crate) async fn get_object_lock_and_quota_config_from_disk_with_guard_in(
+    ctx: &crate::runtime::instance::InstanceContext,
+    bucket: &str,
+    guard: Option<&rustfs_lock::NamespaceLockGuard>,
+) -> Result<(
+    ObjectLockConfigState,
+    Uuid,
+    OffsetDateTime,
+    std::result::Result<(Option<BucketQuota>, Uuid, OffsetDateTime), String>,
+)> {
+    let (state, incarnation_id, updated_at, quota_res) =
+        get_delete_and_quota_config_from_disk_in(ctx, bucket, guard, None, true).await?;
+    Ok((
+        state,
+        incarnation_id,
+        updated_at,
+        quota_res.unwrap_or(Ok((None, incarnation_id, updated_at))),
+    ))
+}
+
 /// Validate queued expiration against the same fresh metadata used for Object
 /// Lock admission. The caller holds the transaction read lock through commit,
 /// so a successful lifecycle update cannot be followed by an old-policy delete.
@@ -1462,6 +1485,24 @@ pub(crate) async fn get_delete_config_from_disk_in(
     guard: Option<&rustfs_lock::NamespaceLockGuard>,
     expiry_policy: Option<&rustfs_lifecycle::ExpiryPolicySnapshot>,
 ) -> Result<(ObjectLockConfigState, Uuid, OffsetDateTime)> {
+    let (state, incarnation_id, updated_at, _) =
+        get_delete_and_quota_config_from_disk_in(ctx, bucket, guard, expiry_policy, false).await?;
+    Ok((state, incarnation_id, updated_at))
+}
+
+#[allow(clippy::type_complexity)]
+async fn get_delete_and_quota_config_from_disk_in(
+    ctx: &crate::runtime::instance::InstanceContext,
+    bucket: &str,
+    guard: Option<&rustfs_lock::NamespaceLockGuard>,
+    expiry_policy: Option<&rustfs_lifecycle::ExpiryPolicySnapshot>,
+    include_quota: bool,
+) -> Result<(
+    ObjectLockConfigState,
+    Uuid,
+    OffsetDateTime,
+    Option<std::result::Result<(Option<BucketQuota>, Uuid, OffsetDateTime), String>>,
+)> {
     let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
     let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone();
 
@@ -1477,7 +1518,7 @@ pub(crate) async fn get_delete_config_from_disk_in(
                 .await?
         }
     };
-    match authority {
+    match &authority {
         BucketMetadataAuthority::Authoritative(metadata)
             if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
         {
@@ -1489,11 +1530,11 @@ pub(crate) async fn get_delete_config_from_disk_in(
                 }
             }
             let object_lock_config_updated_at = metadata.object_lock_config_updated_at;
-            Ok((
-                object_lock_config_state_from_authoritative_metadata(&metadata)?,
-                metadata.bucket_incarnation_id,
-                object_lock_config_updated_at,
-            ))
+            let state = object_lock_config_state_from_authoritative_metadata(metadata)?;
+            let incarnation_id = metadata.bucket_incarnation_id;
+            let quota_res = include_quota
+                .then(|| quota_config_and_incarnation_from_authority(bucket, authority).map_err(|err| err.to_string()));
+            Ok((state, incarnation_id, object_lock_config_updated_at, quota_res))
         }
         BucketMetadataAuthority::Authoritative(_) => {
             Err(Error::other(format!("bucket incarnation metadata is not authoritative: {bucket}")))
@@ -1564,6 +1605,23 @@ pub(crate) async fn get_quota_config_and_incarnation_from_disk_for_options_in(
         return Err(Error::other("quota snapshot metadata transaction fence was released"));
     }
 
+    #[cfg(test)]
+    let has_injected_error = sys.object_lock_disk_read_errors.read().await.contains(bucket);
+    #[cfg(not(test))]
+    let has_injected_error = false;
+    if !ctx.lock_manager().is_disabled()
+        && !has_injected_error
+        && let Some(fresh_quota) = snapshot.take_fresh_put_quota()
+    {
+        if snapshot_guard.is_lock_lost()
+            || snapshot_guard.is_released()
+            || !snapshot.is_valid_for_destructive_put(store.id, bucket, opts.expected_bucket_incarnation_id.unwrap_or_default())
+        {
+            return Err(Error::other("quota snapshot fence or bucket incarnation changed during the disk read"));
+        }
+        return fresh_quota.map_err(Error::other);
+    }
+
     // The snapshot carries the caller's lifecycle fence, excluding deletion.
     // Its transaction guard excludes config writes, so a same-name object
     // owner need not recursively acquire the bucket namespace here.
@@ -1750,9 +1808,22 @@ struct MetadataPublishGuard {
     _guard: tokio::sync::OwnedMutexGuard<MetadataPublishLockState>,
 }
 
+static LOCAL_META_MUTATION_GEN: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
+
+pub(crate) fn bump_local_meta_mutation_gen() {
+    LOCAL_META_MUTATION_GEN.fetch_add(1, std::sync::atomic::Ordering::Release);
+}
+
+pub(crate) fn current_local_meta_mutation_gen() -> u64 {
+    LOCAL_META_MUTATION_GEN.load(std::sync::atomic::Ordering::Acquire)
+}
+
+type LocalAuthorityCache = Arc<StdMutex<HashMap<String, (u64, Arc<BucketMetadata>)>>>;
+
 #[derive(Debug, Clone)]
 pub struct BucketMetadataSys {
     metadata_map: Arc<RwLock<HashMap<String, Arc<BucketMetadata>>>>,
+    local_authority_cache: LocalAuthorityCache,
     /// Serializes metadata-map commits and their derived cache updates for one
     /// bucket. Namespace locks, when present, are acquired before this lock.
     metadata_publish_locks: Arc<MetadataPublishLockRegistry>,
@@ -1796,6 +1867,7 @@ impl BucketMetadataSys {
     pub fn new(api: Arc<ECStore>) -> Self {
         Self {
             metadata_map: Arc::new(RwLock::new(HashMap::new())),
+            local_authority_cache: Arc::new(StdMutex::new(HashMap::new())),
             metadata_publish_locks: Arc::new(MetadataPublishLockRegistry {
                 locks: StdMutex::new(HashMap::new()),
             }),
@@ -1830,6 +1902,15 @@ impl BucketMetadataSys {
         self.api.upgrade()
     }
 
+    /// Whether the backing `ECStore` has been dropped.
+    ///
+    /// Used by `InstanceContext::clear_bucket_metadata_sys_if_stale` during
+    /// `ECStore::drop` so a shared bootstrap context does not retain a
+    /// `BucketMetadataSys` whose weak handle is dangling.
+    pub(crate) fn is_store_stale(&self) -> bool {
+        self.api.upgrade().is_none()
+    }
+
     fn metadata_publish_lock(&self, bucket: &str) -> Arc<Mutex<MetadataPublishLockState>> {
         Self::bucket_lock_in(&self.metadata_publish_locks, bucket)
     }
@@ -2160,6 +2241,7 @@ impl BucketMetadataSys {
     }
 
     pub async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
+        bump_local_meta_mutation_gen();
         if !is_meta_bucketname(&bucket) {
             let publish_lock = self.metadata_publish_lock(&bucket);
             let _publish_guard = publish_lock.lock().await;
@@ -2178,6 +2260,7 @@ impl BucketMetadataSys {
     ///
     /// Returns `true` if an entry was present. Reserved meta buckets are ignored.
     pub async fn remove(&self, bucket: &str) -> bool {
+        bump_local_meta_mutation_gen();
         if is_meta_bucketname(bucket) {
             return false;
         }
@@ -2198,6 +2281,7 @@ impl BucketMetadataSys {
     }
 
     async fn _reset(&mut self) {
+        bump_local_meta_mutation_gen();
         let mut map = self.metadata_map.write().await;
         map.clear();
         drop(map);
@@ -3001,30 +3085,77 @@ impl BucketMetadataSys {
             return Err(Error::other(format!("injected Object Lock metadata disk read failure: {bucket}")));
         }
 
-        match await_bucket_namespace_operation(Some(guard), bucket, "bucket metadata snapshot existence check", async {
-            self.object_store()
-                .get_bucket_info_from_sets_at_read_quorum(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
-                .await
-        })
-        .await
-        {
+        let store = self.object_store();
+        let can_use_local_authority_cache = !store.ctx.lock_manager().is_disabled() && store.all_disks_local_and_online();
+        let gen_before = current_local_meta_mutation_gen();
+        let cached_authority = can_use_local_authority_cache
+            .then(|| {
+                self.local_authority_cache
+                    .lock()
+                    .unwrap_or_else(std::sync::PoisonError::into_inner)
+                    .get(bucket)
+                    .filter(|(cached_gen, _)| *cached_gen == gen_before)
+                    .map(|(_, bm)| Arc::clone(bm))
+            })
+            .flatten();
+        if let Some(cached) = cached_authority {
+            await_bucket_namespace_operation(
+                Some(guard),
+                bucket,
+                "bucket metadata authoritative snapshot",
+                std::future::ready(Ok::<_, Error>(())),
+            )
+            .await?;
+            if current_local_meta_mutation_gen() == gen_before && store.all_disks_local_and_online() {
+                return Ok(BucketMetadataAuthority::Authoritative(cached));
+            }
+        }
+
+        let (bucket_info_res, metadata_res) = tokio::join!(
+            Box::pin(await_bucket_namespace_operation(
+                Some(guard),
+                bucket,
+                "bucket metadata snapshot existence check",
+                async {
+                    store
+                        .get_bucket_info_from_sets_at_read_quorum(
+                            bucket,
+                            &crate::storage_api_contracts::bucket::BucketOptions::default(),
+                        )
+                        .await
+                },
+            )),
+            Box::pin(await_bucket_namespace_operation(
+                Some(guard),
+                bucket,
+                "bucket metadata authoritative snapshot",
+                load_bucket_metadata_parse_with_presence(Arc::clone(&store), bucket, true),
+            )),
+        );
+        match bucket_info_res {
             Ok(_) => {}
             Err(Error::VolumeNotFound) => return Ok(BucketMetadataAuthority::MissingBucket),
             Err(err) => return Err(err),
         }
 
-        let (metadata, persisted) = await_bucket_namespace_operation(
-            Some(guard),
-            bucket,
-            "bucket metadata authoritative snapshot",
-            load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
-        )
-        .await?;
+        let (metadata, persisted) = metadata_res?;
         if persisted && metadata.lock_enabled && !metadata.bucket_creation_committed {
             return Err(Error::ErasureWriteQuorum);
         }
         if persisted {
-            Ok(BucketMetadataAuthority::Authoritative(Arc::new(metadata)))
+            let metadata = Arc::new(metadata);
+            if can_use_local_authority_cache
+                && metadata.bucket_incarnation_sidecar
+                && !metadata.bucket_incarnation_id.is_nil()
+                && current_local_meta_mutation_gen() == gen_before
+                && store.all_disks_local_and_online()
+            {
+                self.local_authority_cache
+                    .lock()
+                    .unwrap_or_else(std::sync::PoisonError::into_inner)
+                    .insert(bucket.to_string(), (gen_before, Arc::clone(&metadata)));
+            }
+            Ok(BucketMetadataAuthority::Authoritative(metadata))
         } else {
             Ok(BucketMetadataAuthority::Fabricated)
         }
@@ -3297,7 +3428,7 @@ mod tests {
         ),
         (
             BUCKET_TAGGING_CONFIG,
-            r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-🦀</Value></Tag></TagSet></Tagging>"#.as_bytes(),
+            r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-rust</Value></Tag></TagSet></Tagging>"#.as_bytes(),
         ),
         (BUCKET_REPLICATION_CONFIG, NEW_WRITER_REPLICATION_XML),
         (
@@ -4425,8 +4556,13 @@ mod tests {
             std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created");
         }
         let mut bm = BucketMetadata::new(bucket);
-        bm.update_config(config_file, raw.to_vec())
-            .expect("raw config should be stored");
+        if config_file == crate::bucket::metadata::BUCKET_TAGGING_CONFIG {
+            // Model legacy persisted corruption without passing the write-time validator.
+            bm.tagging_config_xml = raw.to_vec();
+        } else {
+            bm.update_config(config_file, raw.to_vec())
+                .expect("raw config should be stored");
+        }
         sys.persist_new_and_set(bm).await.expect("initial metadata should persist");
     }
 
@@ -5079,6 +5215,61 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn validated_config_update_rejects_a_different_bucket_incarnation() {
+        let (_dirs, store) = isolated_store_over_temp_disks().await;
+        init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let bucket = "catalog-cleanup-incarnation";
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create bucket");
+        let incarnation = store
+            .bucket_incarnation_id_from_disk(bucket)
+            .await
+            .expect("read bucket incarnation");
+        let called = std::sync::atomic::AtomicBool::new(false);
+        let result = store
+            .update_bucket_metadata_config_for_incarnation_validated(
+                bucket,
+                "quota.json",
+                br#"{"quota":1}"#.to_vec(),
+                Uuid::new_v4(),
+                || {
+                    called.store(true, std::sync::atomic::Ordering::SeqCst);
+                    Ok(())
+                },
+            )
+            .await;
+        assert!(matches!(result, Err(Error::BucketNotFound(name)) if name == bucket));
+        assert!(
+            !called.load(std::sync::atomic::Ordering::SeqCst),
+            "incarnation must be checked before the external fence"
+        );
+        let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
+            .await
+            .expect("read unchanged bucket metadata");
+        assert!(
+            metadata.quota_config_json.is_empty(),
+            "a stale cleanup must not modify the replacement bucket"
+        );
+        store
+            .update_bucket_metadata_config_for_incarnation_validated(
+                bucket,
+                "quota.json",
+                br#"{"quota":1}"#.to_vec(),
+                incarnation,
+                || Ok(()),
+            )
+            .await
+            .expect("the current incarnation permits the validated update");
+        let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
+            .await
+            .expect("read updated bucket metadata");
+        assert_eq!(metadata.quota_config_json, br#"{"quota":1}"#);
+    }
+
     #[tokio::test]
     #[serial]
     async fn quota_admission_rejects_stale_cached_no_quota() {
@@ -5552,7 +5743,8 @@ mod tests {
         // update only needs one interleaving to show up.
         const ROUNDS: usize = 8;
         for round in 0..ROUNDS {
-            let tagging = format!("<Tagging><Round>{round}</Round></Tagging>").into_bytes();
+            let tagging =
+                format!("<Tagging><TagSet><Tag><Key>round</Key><Value>{round}</Value></Tag></TagSet></Tagging>").into_bytes();
             let policy = format!(r#"{{"Version":"2012-10-17","Round":{round}}}"#).into_bytes();
 
             let start = Arc::new(tokio::sync::Barrier::new(2));
@@ -5625,7 +5817,7 @@ mod tests {
 
         let blocked = tokio::spawn({
             let node_b = node_b.clone();
-            async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging/>".to_vec()).await }
+            async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging><TagSet/></Tagging>".to_vec()).await }
         });
 
         // Long enough for the write to have finished had it not waited: the

crates/ecstore/src/bucket/quota/checker/tests/reporting_retirement.rs

@@ -119,6 +119,21 @@ async fn durable_quota_reporting_retires_degraded_floors_without_changing_ledger
             // Exercise the real storage entry point before testing the new
             // reporting semantics: the same ledger must admit and refuse
             // these writes even when a scanner floor claims a million bytes.
+            if protocol == 1 {
+                let mut bootstrap = PutObjReader::from_vec(vec![0x5a; 512]);
+                assert!(
+                    matches!(
+                        store
+                            .put_object(&bucket, "bootstrap", &mut bootstrap, &ObjectOptions::default())
+                            .await,
+                        Err(StorageError::SlowDown)
+                    ),
+                    "a missing authoritative ledger must not admit a write from scanner reporting"
+                );
+                crate::bucket::quota::reservation::retry_quota_reconciliation_for_test(store.clone(), &bucket)
+                    .await
+                    .expect("bootstrap authoritative usage through the production worker");
+            }
             let mut too_large = PutObjReader::from_vec(vec![0x5a; 8192]);
             assert!(
                 matches!(

crates/ecstore/src/bucket/quota/reservation.rs

@@ -19,7 +19,7 @@ use crate::data_usage::compute_bucket_usage;
 use crate::disk::RUSTFS_META_BUCKET;
 use crate::disk::{DiskAPI, error::DiskError};
 use crate::error::{Result, StorageError, is_err_object_not_found, is_err_version_not_found};
-use crate::object_api::{ObjectInfo, ObjectOptions, QuotaAdmission};
+use crate::object_api::{ObjectInfo, ObjectLockConfigSnapshot, ObjectOptions, QuotaAdmission};
 use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
 use crate::storage_api_contracts::namespace::NamespaceLocking;
 use crate::storage_api_contracts::{list::ListOperations as _, object::ObjectOperations};
@@ -28,7 +28,7 @@ use futures::{StreamExt, stream};
 use rustfs_lock::NamespaceLockGuard;
 use serde::{Deserialize, Serialize};
 use sha2::{Digest, Sha256};
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, HashSet};
 use std::sync::Arc;
 use std::time::Duration;
 use time::OffsetDateTime;
@@ -36,6 +36,8 @@ use tracing::warn;
 use uuid::Uuid;
 
 const QUOTA_LEDGER_FORMAT_VERSION: u8 = 1;
+const RECONCILE_RETRY_SECONDS: i64 = 30;
+const EVENT_QUOTA_RECONCILIATION: &str = "quota_reconciliation";
 const MAX_ORPHANS_REAPED_PER_WRITE: usize = 64;
 const MAX_ORPHAN_PROBES_PER_WRITE: usize = 128;
 const ORPHAN_PROBE_CONCURRENCY: usize = 32;
@@ -96,6 +98,10 @@ struct QuotaLedger {
     version: u8,
     bucket_incarnation: Uuid,
     quota_revision_unix_nanos: i128,
+    /// Changes on every durable write, including zero-growth reservations.
+    /// Missing on an older writer's output, which invalidates a concurrent scan.
+    #[serde(default)]
+    writer_revision: Uuid,
     accounted_usage: u64,
     reservations: BTreeMap<Uuid, PersistedReservation>,
     #[serde(default)]
@@ -110,6 +116,7 @@ impl QuotaLedger {
             version: QUOTA_LEDGER_FORMAT_VERSION,
             bucket_incarnation,
             quota_revision_unix_nanos: quota_revision.unix_timestamp_nanos(),
+            writer_revision: Uuid::nil(),
             accounted_usage,
             reservations: BTreeMap::new(),
             reconcile_required: false,
@@ -485,22 +492,23 @@ impl QuotaContext {
             let ledger_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_data.ledger_object).await?;
             let ledger_guard = Arc::new(ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?);
             fence_namespace_mutations(&store, RUSTFS_META_BUCKET, &ledger_data.ledger_object, None).await?;
-            let mut ledger = load_current_ledger_locked(
-                Arc::clone(&store),
-                &ledger_data.bucket,
-                &ledger_data.ledger_object,
-                bucket_incarnation,
-                quota_revision,
-            )
-            .await?;
-
+            let mut ledger =
+                load_current_ledger_locked(Arc::clone(&store), &ledger_data.ledger_object, bucket_incarnation, quota_revision)
+                    .await?;
+            let job = ReconcileJob::new(Arc::clone(&store), ledger_data.bucket.clone(), bucket_incarnation, quota_revision);
+            let Some(mut ledger) = ledger.take() else {
+                // Bootstrap has no authoritative usage yet. Release every
+                // request-owned fence by returning; the worker owns no request.
+                job.schedule();
+                return Err(StorageError::SlowDown);
+            };
             let growth = reservation.growth();
-            let mut current_usage = ledger.admitted_usage()?;
-            let mut expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?;
-            if growth > 0 && expected_usage > quota_limit && growth <= quota_limit && ledger.should_reconcile_after_denial() {
-                reconcile_exact(&store, &ledger_data.bucket, &mut ledger).await?;
-                current_usage = ledger.admitted_usage()?;
-                expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?;
+            let current_usage = ledger.admitted_usage()?;
+            let expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?;
+            if ledger.reconcile_required
+                || (growth > 0 && expected_usage > quota_limit && growth <= quota_limit && ledger.should_reconcile_after_denial())
+            {
+                job.schedule();
             }
             if growth > 0 && expected_usage > quota_limit {
                 return Err(StorageError::QuotaExceeded {
@@ -518,7 +526,7 @@ impl QuotaContext {
                 });
             }
             ledger.reserve(operation_id, reservation)?;
-            save_ledger_locked(Arc::clone(&store), &ledger_data.ledger_object, &ledger, &ledger_guard).await?;
+            save_ledger_locked(Arc::clone(&store), &ledger_data.ledger_object, &mut ledger, &ledger_guard).await?;
 
             Ok(QuotaReservation {
                 ledger: Some(ReservationLedgerData::Single(ledger_data)),
@@ -1082,7 +1090,7 @@ async fn mark_commit_started(data: &ReservationLedgerData) -> Result<()> {
         fence_namespace_mutations(&data.store, RUSTFS_META_BUCKET, &data.ledger_object, None).await?;
         let mut ledger = load_ledger_locked(Arc::clone(&data.store), &data.ledger_object).await?;
         ledger.mark_commit_started(data.operation_id, &data.reservation)?;
-        save_ledger_locked(Arc::clone(&data.store), &data.ledger_object, &ledger, &ledger_guard).await
+        save_ledger_locked(Arc::clone(&data.store), &data.ledger_object, &mut ledger, &ledger_guard).await
     })
     .await
     .map_err(|err| StorageError::other_with_context("quota commit marker task failed", err))?
@@ -1113,7 +1121,7 @@ async fn settle(data: &ReservationLedgerData, committed: bool) -> Result<()> {
         } else {
             ledger.abort(operation_id, &reservation)?;
         }
-        save_ledger_locked(Arc::clone(&store), &ledger_object, &ledger, &ledger_guard).await
+        save_ledger_locked(Arc::clone(&store), &ledger_object, &mut ledger, &ledger_guard).await
     })
     .await
     .map_err(|err| StorageError::other_with_context("quota ledger settlement task failed", err))?
@@ -1121,22 +1129,15 @@ async fn settle(data: &ReservationLedgerData, committed: bool) -> Result<()> {
 
 async fn load_current_ledger_locked(
     store: Arc<ECStore>,
-    bucket: &str,
     ledger_object: &str,
     bucket_incarnation: Uuid,
     quota_revision: OffsetDateTime,
-) -> Result<QuotaLedger> {
-    match load_ledger_locked(Arc::clone(&store), ledger_object).await {
-        Ok(ledger) if ledger.matches(bucket_incarnation, quota_revision) => Ok(ledger),
-        Ok(ledger) if ledger.reservations.is_empty() && !ledger.reconcile_required => {
-            let usage = exact_bucket_usage(&store, bucket).await?;
-            Ok(QuotaLedger::new(bucket_incarnation, quota_revision, usage))
-        }
+) -> Result<Option<QuotaLedger>> {
+    match load_ledger_locked(store, ledger_object).await {
+        Ok(ledger) if ledger.matches(bucket_incarnation, quota_revision) => Ok(Some(ledger)),
+        Ok(ledger) if ledger.reservations.is_empty() => Ok(None),
         Ok(_) => Err(StorageError::PartMissingOrCorrupt),
-        Err(StorageError::ConfigNotFound) => {
-            let usage = exact_bucket_usage(&store, bucket).await?;
-            Ok(QuotaLedger::new(bucket_incarnation, quota_revision, usage))
-        }
+        Err(StorageError::ConfigNotFound) => Ok(None),
         Err(err) => Err(err),
     }
 }
@@ -1245,15 +1246,11 @@ async fn reap_stale_reservations(store: Arc<ECStore>, bucket: &str, ledger_objec
     for (object, target) in orphan_commit_targets {
         fence_namespace_mutations(&store, bucket, &object, target).await?;
     }
-    let mut removed = remove_orphan_reservations(&mut ledger, &orphan_ids)?;
-    if ledger.reservations.is_empty() && ledger.reconcile_required {
-        reconcile_exact(&store, bucket, &mut ledger).await?;
-        removed = true;
-    }
+    let removed = remove_orphan_reservations(&mut ledger, &orphan_ids)?;
     if !removed && !cursor_changed {
         return Ok(());
     }
-    save_ledger_locked(store, ledger_object, &ledger, &ledger_guard).await
+    save_ledger_locked(store, ledger_object, &mut ledger, &ledger_guard).await
 }
 
 fn remove_orphan_reservations(ledger: &mut QuotaLedger, operation_ids: &[Uuid]) -> Result<bool> {
@@ -1273,13 +1270,217 @@ fn remove_orphan_reservations(ledger: &mut QuotaLedger, operation_ids: &[Uuid])
     Ok(removed)
 }
 
-async fn reconcile_exact(store: &Arc<ECStore>, bucket: &str, ledger: &mut QuotaLedger) -> Result<()> {
-    if !ledger.reservations.is_empty() {
-        return Err(StorageError::PartMissingOrCorrupt);
+#[derive(Debug, Serialize, Deserialize)]
+struct ReconcileControl {
+    bucket_incarnation: Uuid,
+    quota_revision_unix_nanos: i128,
+    next_attempt_at: i64,
+}
+
+#[derive(Clone)]
+struct ReconcileJob {
+    store: Arc<ECStore>,
+    bucket: String,
+    bucket_incarnation: Uuid,
+    quota_revision: OffsetDateTime,
+}
+
+type ReconcileKey = (Uuid, String, Uuid, i128);
+
+fn active_reconciliations() -> &'static std::sync::Mutex<HashSet<ReconcileKey>> {
+    static ACTIVE: std::sync::OnceLock<std::sync::Mutex<HashSet<ReconcileKey>>> = std::sync::OnceLock::new();
+    ACTIVE.get_or_init(Default::default)
+}
+
+struct ActiveReconciliation(ReconcileKey);
+
+impl Drop for ActiveReconciliation {
+    fn drop(&mut self) {
+        active_reconciliations()
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner)
+            .remove(&self.0);
+    }
+}
+
+impl ReconcileJob {
+    fn new(store: Arc<ECStore>, bucket: String, bucket_incarnation: Uuid, quota_revision: OffsetDateTime) -> Self {
+        Self {
+            store,
+            bucket,
+            bucket_incarnation,
+            quota_revision,
+        }
+    }
+
+    fn schedule(self) {
+        let key = (
+            self.store.id,
+            self.bucket.clone(),
+            self.bucket_incarnation,
+            self.quota_revision.unix_timestamp_nanos(),
+        );
+        if !active_reconciliations()
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner)
+            .insert(key.clone())
+        {
+            return;
+        }
+        let owner = ActiveReconciliation(key);
+        // The owner carries no object, metadata, lifecycle or reservation guard.
+        // Cancellation of the rejected request cannot cancel its reconciliation.
+        tokio::spawn(async move {
+            let _owner = owner;
+            if let Err(error) = self.run().await {
+                warn!(event = EVENT_QUOTA_RECONCILIATION, component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_QUOTA, state = "deferred", bucket = %self.bucket,
+                    error = %error, "Quota reconciliation deferred");
+            }
+        });
+    }
+
+    fn control_object(&self) -> String {
+        format!(
+            "{}.reconcile/{}/{}",
+            ledger_object(&self.bucket),
+            self.bucket_incarnation,
+            self.quota_revision.unix_timestamp_nanos()
+        )
+    }
+
+    async fn save_control(&self, object: &str, control: &ReconcileControl, guard: &Arc<NamespaceLockGuard>) -> Result<()> {
+        if guard.is_lock_lost() {
+            return Err(StorageError::SlowDown);
+        }
+        let mut opts = ObjectOptions {
+            max_parity: true,
+            no_lock: true,
+            ..Default::default()
+        };
+        let _ = opts.set_quota_admission(0, u64::MAX);
+        opts.add_owned_write_lock(Arc::clone(guard), RUSTFS_META_BUCKET, object);
+        opts.write_completion = crate::object_api::WriteCompletion::TailDrained;
+        save_config_with_opts(Arc::clone(&self.store), object, serde_json::to_vec(control)?, &opts).await
+    }
+
+    async fn run(&self) -> Result<()> {
+        let control_object = self.control_object();
+        let lock = self.store.new_ns_lock(RUSTFS_META_BUCKET, &control_object).await?;
+        let Ok(guard) = lock.get_write_lock_quiet(Duration::from_millis(50)).await else {
+            return Ok(());
+        };
+        let guard = Arc::new(guard);
+        let now = now_unix();
+        match read_config_no_lock(Arc::clone(&self.store), &control_object).await {
+            Ok(bytes) => {
+                let control: ReconcileControl = serde_json::from_slice(&bytes)?;
+                if control.bucket_incarnation != self.bucket_incarnation
+                    || control.quota_revision_unix_nanos != self.quota_revision.unix_timestamp_nanos()
+                {
+                    return Err(StorageError::PartMissingOrCorrupt);
+                }
+                if control.next_attempt_at > now {
+                    return Ok(());
+                }
+            }
+            Err(StorageError::ConfigNotFound) => {}
+            Err(error) => return Err(error),
+        }
+        fence_namespace_mutations(&self.store, RUSTFS_META_BUCKET, &control_object, None).await?;
+        let mut control = ReconcileControl {
+            bucket_incarnation: self.bucket_incarnation,
+            quota_revision_unix_nanos: self.quota_revision.unix_timestamp_nanos(),
+            next_attempt_at: now.saturating_add(RECONCILE_RETRY_SECONDS),
+        };
+        // Persist the throttle before work, including bootstrap and failed scans.
+        self.save_control(&control_object, &control, &guard).await?;
+        let result = self.scan_and_publish(&guard).await;
+        control.next_attempt_at = now_unix().saturating_add(RECONCILE_RETRY_SECONDS);
+        self.save_control(&control_object, &control, &guard).await?;
+        result
+    }
+
+    /// Take a short authoritative configuration snapshot without recursively
+    /// acquiring a reader behind a queued metadata/lifecycle writer.
+    async fn configuration_snapshot(&self) -> Result<Arc<ObjectLockConfigSnapshot>> {
+        let snapshot = self.store.object_lock_config_snapshot_for_single_put(&self.bucket).await?;
+        let guard = snapshot
+            .metadata_transaction_guard_for(self.store.id, &self.bucket, Some(self.bucket_incarnation))
+            .ok_or(StorageError::PreconditionFailed)?;
+        let opts = ObjectOptions {
+            expected_bucket_incarnation_id: Some(self.bucket_incarnation),
+            object_lock_config_snapshot: Some(Arc::clone(&snapshot)),
+            ..Default::default()
+        };
+        let (quota, incarnation, revision) =
+            metadata_sys::get_quota_config_and_incarnation_from_disk_for_options_in(&self.store.ctx, &self.bucket, &opts, &guard)
+                .await?;
+        if incarnation != self.bucket_incarnation
+            || revision != self.quota_revision
+            || !quota.as_ref().is_some_and(|quota| {
+                quota.uses_durable_reservations() && quota.reservation_protocol != Some(QUOTA_RESERVATION_PROTOCOL_V2)
+            })
+        {
+            return Err(StorageError::PreconditionFailed);
+        }
+        Ok(snapshot)
+    }
+
+    async fn scan_and_publish(&self, owner: &NamespaceLockGuard) -> Result<()> {
+        let ledger_object = ledger_object(&self.bucket);
+        let expected_revision = {
+            // Lock order: reconcile owner, lifecycle, metadata, ledger.
+            // Request paths never wait for the reconcile owner.
+            let _configuration = self.configuration_snapshot().await?;
+            let lock = self.store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_object).await?;
+            let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await?);
+            match load_ledger_locked(Arc::clone(&self.store), &ledger_object).await {
+                Ok(mut ledger) => {
+                    if !ledger.reservations.is_empty() {
+                        return Ok(());
+                    }
+                    // Seed a non-nil token even for a legacy ledger. An old
+                    // node drops this field on write and cannot pass the CAS.
+                    fence_namespace_mutations(&self.store, RUSTFS_META_BUCKET, &ledger_object, None).await?;
+                    save_ledger_locked(Arc::clone(&self.store), &ledger_object, &mut ledger, &guard).await?;
+                    Some(ledger.writer_revision)
+                }
+                Err(StorageError::ConfigNotFound) => None,
+                Err(error) => return Err(error),
+            }
+        };
+        #[cfg(test)]
+        tests::pause_reconciliation(&self.bucket, tests::ReconcilePauseStage::BeforeScan).await?;
+        // Only the independent reconcile owner is held during this whole-bucket
+        // walk. Concurrent reserve/settle/reap writes invalidate its result.
+        let usage = exact_bucket_usage(&self.store, &self.bucket).await?;
+        #[cfg(test)]
+        tests::pause_reconciliation(&self.bucket, tests::ReconcilePauseStage::BeforePublish).await?;
+        let configuration = self.configuration_snapshot().await?;
+        let lock = self.store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_object).await?;
+        let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await?);
+        let unchanged = match load_ledger_locked(Arc::clone(&self.store), &ledger_object).await {
+            Ok(ledger) => {
+                expected_revision == Some(ledger.writer_revision)
+                    && !ledger.writer_revision.is_nil()
+                    && ledger.reservations.is_empty()
+            }
+            Err(StorageError::ConfigNotFound) => expected_revision.is_none(),
+            Err(error) => return Err(error),
+        };
+        if !unchanged {
+            return Ok(());
+        }
+        if owner.is_lock_lost()
+            || !configuration.is_valid_for_destructive_put(self.store.id, &self.bucket, self.bucket_incarnation)
+        {
+            return Err(StorageError::SlowDown);
+        }
+        fence_namespace_mutations(&self.store, RUSTFS_META_BUCKET, &ledger_object, None).await?;
+        let mut ledger = QuotaLedger::new(self.bucket_incarnation, self.quota_revision, usage);
+        save_ledger_locked(Arc::clone(&self.store), &ledger_object, &mut ledger, &guard).await
     }
-    ledger.accounted_usage = exact_bucket_usage(store, bucket).await?;
-    ledger.reconcile_required = false;
-    Ok(())
 }
 
 async fn exact_bucket_usage(store: &Arc<ECStore>, bucket: &str) -> Result<u64> {
@@ -1935,7 +2136,7 @@ async fn load_ledger_locked(store: Arc<ECStore>, ledger_object: &str) -> Result<
 async fn save_ledger_locked(
     store: Arc<ECStore>,
     ledger_object: &str,
-    ledger: &QuotaLedger,
+    ledger: &mut QuotaLedger,
     ledger_guard: &Arc<NamespaceLockGuard>,
 ) -> Result<()> {
     if ledger_guard.is_lock_lost() {
@@ -1959,9 +2160,71 @@ async fn save_ledger_locked(
     let _ = opts.set_quota_admission(0, u64::MAX);
     opts.add_owned_write_lock(Arc::clone(ledger_guard), RUSTFS_META_BUCKET, ledger_object);
     opts.write_completion = crate::object_api::WriteCompletion::TailDrained;
+    ledger.writer_revision = Uuid::new_v4();
     save_config_with_opts(store, ledger_object, serde_json::to_vec(ledger)?, &opts).await
 }
 
+/// Advance the persisted reconciliation retry deadline for deterministic fixtures.
+/// The production reaper, ownership fence, scan and compare-and-swap still run.
+#[cfg(any(test, feature = "test-util"))]
+pub async fn retry_quota_reconciliation_for_test(store: Arc<ECStore>, bucket: &str) -> Result<()> {
+    let (_, incarnation, revision) = store.bucket_quota_and_incarnation_from_disk(bucket).await?;
+    let job = ReconcileJob::new(Arc::clone(&store), bucket.to_owned(), incarnation, revision);
+    let key = (store.id, bucket.to_owned(), incarnation, revision.unix_timestamp_nanos());
+    tokio::time::timeout(Duration::from_secs(10), async {
+        loop {
+            let active = active_reconciliations()
+                .lock()
+                .unwrap_or_else(std::sync::PoisonError::into_inner)
+                .contains(&key);
+            if !active {
+                break;
+            }
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .map_err(|_| StorageError::SlowDown)?;
+    reap_stale_reservations(Arc::clone(&store), bucket, &ledger_object(bucket)).await?;
+    let object = job.control_object();
+    {
+        let lock = store.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
+        let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await?);
+        match read_config_no_lock(Arc::clone(&store), &object).await {
+            Ok(bytes) => {
+                let mut control: ReconcileControl = serde_json::from_slice(&bytes)?;
+                control.next_attempt_at = 0;
+                fence_namespace_mutations(&store, RUSTFS_META_BUCKET, &object, None).await?;
+                job.save_control(&object, &control, &guard).await?;
+            }
+            Err(StorageError::ConfigNotFound) => {}
+            Err(error) => return Err(error),
+        }
+    }
+    job.run().await
+}
+
+/// Bootstrap a test bucket through the real background reconciliation path.
+/// Fixtures call this after enabling V1 quota, before issuing writes whose
+/// contract is independent of the initial retryable admission response.
+#[cfg(any(test, feature = "test-util"))]
+pub async fn initialize_quota_ledger_for_test(store: Arc<ECStore>, bucket: &str) -> Result<()> {
+    let (quota, incarnation, revision) = store.bucket_quota_and_incarnation_from_disk(bucket).await?;
+    if !quota
+        .as_ref()
+        .is_some_and(|quota| quota.uses_durable_reservations() && !quota.uses_sharded_reservations())
+    {
+        return Ok(());
+    }
+    ReconcileJob::new(Arc::clone(&store), bucket.to_owned(), incarnation, revision)
+        .run()
+        .await?;
+    match load_current_ledger_locked(store, &ledger_object(bucket), incarnation, revision).await? {
+        Some(_) => Ok(()),
+        None => Err(StorageError::SlowDown),
+    }
+}
+
 #[cfg(any(test, feature = "test-util"))]
 #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
 pub fn fail_next_quota_ledger_save_for_test() {
@@ -2003,6 +2266,434 @@ mod tests {
     use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
     use serial_test::serial;
 
+    #[derive(Clone, Copy, PartialEq, Eq)]
+    pub(super) enum ReconcilePauseStage {
+        BeforeScan,
+        BeforePublish,
+    }
+
+    struct ReconcilePause {
+        bucket: String,
+        stage: ReconcilePauseStage,
+        armed: std::sync::atomic::AtomicBool,
+        scans: std::sync::atomic::AtomicUsize,
+        fail_scan: std::sync::atomic::AtomicBool,
+        reached: tokio::sync::Notify,
+        release: tokio::sync::Notify,
+    }
+
+    static RECONCILE_PAUSE: std::sync::Mutex<Option<Arc<ReconcilePause>>> = std::sync::Mutex::new(None);
+
+    struct ReconcilePauseGuard(Arc<ReconcilePause>);
+
+    impl ReconcilePauseGuard {
+        fn install(bucket: &str, stage: ReconcilePauseStage) -> Self {
+            let pause = Arc::new(ReconcilePause {
+                bucket: bucket.to_owned(),
+                stage,
+                armed: true.into(),
+                scans: 0.into(),
+                fail_scan: false.into(),
+                reached: tokio::sync::Notify::new(),
+                release: tokio::sync::Notify::new(),
+            });
+            *RECONCILE_PAUSE.lock().expect("reconcile hooks") = Some(Arc::clone(&pause));
+            Self(pause)
+        }
+
+        async fn reached(&self) {
+            tokio::time::timeout(Duration::from_secs(10), self.0.reached.notified())
+                .await
+                .expect("reconciliation reached barrier");
+        }
+    }
+
+    impl Drop for ReconcilePauseGuard {
+        fn drop(&mut self) {
+            self.0.release.notify_one();
+            let mut hook = RECONCILE_PAUSE.lock().expect("reconcile hooks");
+            if hook.as_ref().is_some_and(|hook| Arc::ptr_eq(hook, &self.0)) {
+                *hook = None;
+            }
+        }
+    }
+
+    pub(super) async fn pause_reconciliation(bucket: &str, stage: ReconcilePauseStage) -> Result<()> {
+        use std::sync::atomic::Ordering;
+        let hook = RECONCILE_PAUSE
+            .lock()
+            .expect("reconcile hooks")
+            .as_ref()
+            .filter(|hook| hook.bucket == bucket)
+            .cloned();
+        if let Some(hook) = hook {
+            if stage == ReconcilePauseStage::BeforeScan {
+                hook.scans.fetch_add(1, Ordering::SeqCst);
+            }
+            if stage == hook.stage && hook.armed.swap(false, Ordering::SeqCst) {
+                hook.reached.notify_one();
+                hook.release.notified().await;
+            }
+            if stage == ReconcilePauseStage::BeforeScan && hook.fail_scan.load(Ordering::SeqCst) {
+                return Err(StorageError::other("injected quota usage failure"));
+            }
+        }
+        Ok(())
+    }
+
+    async fn reconciliation_fixture(limit: u64) -> (Vec<tempfile::TempDir>, ReconcileJob) {
+        use crate::object_api::PutObjReader;
+        use crate::storage_api_contracts::object::ObjectIO;
+        let (directories, store) = metadata_sys::test_support::isolated_store_over_temp_disks().await;
+        metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
+        crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test();
+        let bucket = format!("quota-reconcile-{}", Uuid::new_v4().simple());
+        store
+            .make_bucket(&bucket, &MakeBucketOptions::default())
+            .await
+            .expect("bucket");
+        let mut seed = PutObjReader::from_vec(vec![1; 10]);
+        store
+            .put_object(&bucket, "seed", &mut seed, &ObjectOptions::default())
+            .await
+            .expect("existing usage");
+        store
+            .update_bucket_metadata_config(
+                &bucket,
+                "quota.json",
+                serde_json::to_vec(&super::super::BucketQuota::new(Some(limit))).expect("quota JSON"),
+            )
+            .await
+            .expect("enable quota");
+        let (_, incarnation, revision) = store
+            .bucket_quota_and_incarnation_from_disk(&bucket)
+            .await
+            .expect("quota identity");
+        (directories, ReconcileJob::new(store, bucket, incarnation, revision))
+    }
+
+    async fn quota_reservation(job: &ReconcileJob, old: u64, new: u64) -> Result<QuotaReservation> {
+        begin(&job.store.ctx, &job.bucket, "candidate", &ObjectOptions::default(), 0, 0)
+            .await?
+            .reserve(old, new)
+            .await
+    }
+
+    async fn read_test_ledger(job: &ReconcileJob) -> QuotaLedger {
+        load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket))
+            .await
+            .expect("ledger")
+    }
+
+    async fn drain_reconciliation_scheduler(job: &ReconcileJob) {
+        let key = (
+            job.store.id,
+            job.bucket.clone(),
+            job.bucket_incarnation,
+            job.quota_revision.unix_timestamp_nanos(),
+        );
+        tokio::time::timeout(Duration::from_secs(10), async {
+            while active_reconciliations().lock().expect("active workers").contains(&key) {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("owned reconciliation finished");
+    }
+
+    async fn expire_reconciliation_backoff(job: &ReconcileJob) {
+        drain_reconciliation_scheduler(job).await;
+        let object = job.control_object();
+        let lock = job.store.new_ns_lock(RUSTFS_META_BUCKET, &object).await.expect("owner lock");
+        let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await.expect("owner guard"));
+        let mut control: ReconcileControl =
+            serde_json::from_slice(&read_config_no_lock(Arc::clone(&job.store), &object).await.expect("control"))
+                .expect("control JSON");
+        control.next_attempt_at = 0;
+        fence_namespace_mutations(&job.store, RUSTFS_META_BUCKET, &object, None)
+            .await
+            .expect("control fence");
+        job.save_control(&object, &control, &guard).await.expect("advance test clock");
+    }
+
+    async fn wait_for_reconcile_owner(job: &ReconcileJob) {
+        let object = job.control_object();
+        let lock = job.store.new_ns_lock(RUSTFS_META_BUCKET, &object).await.expect("owner lock");
+        let _guard = tokio::time::timeout(Duration::from_secs(10), lock.get_write_lock(get_lock_acquire_timeout()))
+            .await
+            .expect("worker finished")
+            .expect("owner guard");
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn quota_reconciliation_bootstrap_is_single_flight_without_request_guards() {
+        use std::sync::atomic::Ordering;
+        let (_directories, job) = reconciliation_fixture(10).await;
+        let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforeScan);
+        assert!(
+            matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::SlowDown)),
+            "unknown usage cannot admit a reservation"
+        );
+        pause.reached().await;
+        assert!(
+            matches!(
+                load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
+                Err(StorageError::ConfigNotFound)
+            ),
+            "bootstrap must not publish zero usage"
+        );
+        assert!(matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::SlowDown)));
+        // Bypass the local scheduler, exercising the cross-node ownership boundary.
+        tokio::time::timeout(Duration::from_secs(2), job.run())
+            .await
+            .expect("contender must not wait for a scan")
+            .expect("another owner is active");
+        let lifecycle = tokio::time::timeout(Duration::from_secs(2), job.store.acquire_bucket_lifecycle_write_lock(&job.bucket))
+            .await
+            .expect("scan does not retain lifecycle readers")
+            .expect("lifecycle writer");
+        let metadata = tokio::time::timeout(
+            Duration::from_secs(2),
+            metadata_sys::acquire_bucket_metadata_transaction_lock_in(&job.store.ctx, &job.bucket),
+        )
+        .await
+        .expect("scan does not retain metadata readers")
+        .expect("metadata writer");
+        drop(metadata);
+        drop(lifecycle);
+        pause.0.release.notify_one();
+        wait_for_reconcile_owner(&job).await;
+        let ledger = read_test_ledger(&job).await;
+        assert_eq!(ledger.accounted_usage, 10);
+        assert!(ledger.reservations.is_empty());
+        assert!(!ledger.writer_revision.is_nil());
+        for _ in 0..3 {
+            assert!(matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::QuotaExceeded { .. })));
+            job.run().await.expect("persisted cooldown");
+        }
+        drain_reconciliation_scheduler(&job).await;
+        assert_eq!(
+            pause.0.scans.load(Ordering::SeqCst),
+            1,
+            "denied requests cannot repeatedly scan a full bucket"
+        );
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn quota_reconciliation_rejects_concurrent_zero_growth_settle_reap_and_legacy_writes() {
+        let (_directories, job) = reconciliation_fixture(100).await;
+        job.run().await.expect("bootstrap");
+        for mutation in ["abort", "commit", "reap", "legacy"] {
+            expire_reconciliation_backoff(&job).await;
+            let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforePublish);
+            let worker = tokio::spawn({
+                let job = job.clone();
+                async move { job.run().await }
+            });
+            pause.reached().await;
+            let initial = read_test_ledger(&job).await.writer_revision;
+            if mutation == "legacy" {
+                let object = ledger_object(&job.bucket);
+                let lock = job.store.new_ns_lock(RUSTFS_META_BUCKET, &object).await.expect("ledger lock");
+                let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await.expect("ledger writer"));
+                let mut document = serde_json::to_value(read_test_ledger(&job).await).expect("ledger JSON");
+                document.as_object_mut().expect("ledger map").remove("writer_revision");
+                fence_namespace_mutations(&job.store, RUSTFS_META_BUCKET, &object, None)
+                    .await
+                    .expect("ledger fence");
+                let mut opts = ObjectOptions {
+                    max_parity: true,
+                    no_lock: true,
+                    ..Default::default()
+                };
+                let _ = opts.set_quota_admission(0, u64::MAX);
+                opts.add_owned_write_lock(Arc::clone(&guard), RUSTFS_META_BUCKET, &object);
+                opts.write_completion = crate::object_api::WriteCompletion::TailDrained;
+                save_config_with_opts(
+                    Arc::clone(&job.store),
+                    &object,
+                    serde_json::to_vec(&document).expect("old node JSON"),
+                    &opts,
+                )
+                .await
+                .expect("old node drops new field");
+            } else {
+                let mut reservation = quota_reservation(&job, 10, 10)
+                    .await
+                    .expect("zero-growth admission while scanning");
+                let reserved = read_test_ledger(&job).await.writer_revision;
+                assert_ne!(reserved, initial, "even zero-growth reserve must invalidate a scan");
+                reservation.mark_commit_started().await.expect("commit marker");
+                assert_ne!(read_test_ledger(&job).await.writer_revision, reserved);
+                match mutation {
+                    "abort" => reservation.abort().await,
+                    "commit" => reservation.commit().await,
+                    "reap" => {
+                        reservation.defer_after_fence();
+                        reap_stale_reservations(Arc::clone(&job.store), &job.bucket, &ledger_object(&job.bucket))
+                            .await
+                            .expect("reap orphan");
+                    }
+                    _ => unreachable!(),
+                }
+            }
+            let written = read_test_ledger(&job).await;
+            assert!(written.reservations.is_empty());
+            assert_ne!(written.writer_revision, initial);
+            pause.0.release.notify_one();
+            worker.await.expect("worker join").expect("stale scan discarded");
+            let current = read_test_ledger(&job).await;
+            assert_eq!(
+                current.writer_revision, written.writer_revision,
+                "{mutation}: stale usage must not replace a concurrent writer"
+            );
+            assert_eq!(current.accounted_usage, written.accounted_usage);
+        }
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn quota_reconciliation_failure_is_throttled_and_deletions_eventually_reclaim_usage() {
+        use std::sync::atomic::Ordering;
+        let (_directories, job) = reconciliation_fixture(10).await;
+        let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforeScan);
+        pause.0.fail_scan.store(true, Ordering::SeqCst);
+        pause.0.release.notify_one();
+        assert!(job.run().await.is_err());
+        assert!(matches!(
+            load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
+            Err(StorageError::ConfigNotFound)
+        ));
+        job.run().await.expect("failure cooldown");
+        assert_eq!(pause.0.scans.load(Ordering::SeqCst), 1);
+        pause.0.fail_scan.store(false, Ordering::SeqCst);
+        expire_reconciliation_backoff(&job).await;
+        job.run().await.expect("recover after failure");
+        assert_eq!(read_test_ledger(&job).await.accounted_usage, 10);
+        job.store
+            .delete_object(&job.bucket, "seed", ObjectOptions::default())
+            .await
+            .expect("delete stored usage");
+        assert!(
+            matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::QuotaExceeded { .. })),
+            "stale usage remains conservative during cooldown"
+        );
+        expire_reconciliation_backoff(&job).await;
+        job.run().await.expect("refresh deleted usage");
+        assert_eq!(read_test_ledger(&job).await.accounted_usage, 0);
+        quota_reservation(&job, 0, 1)
+            .await
+            .expect("delete must not leave bucket permanently full")
+            .abort()
+            .await;
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn quota_reconciliation_survives_waiter_cancellation_and_rechecks_configuration() {
+        let (_directories, job) = reconciliation_fixture(100).await;
+        let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforePublish);
+        let caller = tokio::spawn({
+            let job = job.clone();
+            async move {
+                job.schedule();
+                std::future::pending::<()>().await
+            }
+        });
+        pause.reached().await;
+        caller.abort();
+        assert!(caller.await.expect_err("cancel caller").is_cancelled());
+        job.store
+            .update_bucket_metadata_config(
+                &job.bucket,
+                "quota.json",
+                serde_json::to_vec(&super::super::BucketQuota::new(Some(200))).expect("updated quota"),
+            )
+            .await
+            .expect("quota changes while scanning");
+        pause.0.release.notify_one();
+        wait_for_reconcile_owner(&job).await;
+        assert!(
+            matches!(
+                load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
+                Err(StorageError::ConfigNotFound)
+            ),
+            "old config cannot publish bootstrap usage"
+        );
+        drop(pause);
+        let (_, incarnation, revision) = job
+            .store
+            .bucket_quota_and_incarnation_from_disk(&job.bucket)
+            .await
+            .expect("current configuration");
+        let current = ReconcileJob::new(Arc::clone(&job.store), job.bucket.clone(), incarnation, revision);
+        let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforeScan);
+        let caller = tokio::spawn({
+            let job = current.clone();
+            async move {
+                job.schedule();
+                std::future::pending::<()>().await
+            }
+        });
+        pause.reached().await;
+        caller.abort();
+        assert!(caller.await.expect_err("cancel new caller").is_cancelled());
+        pause.0.release.notify_one();
+        wait_for_reconcile_owner(&current).await;
+        assert!(
+            read_test_ledger(&current).await.matches(incarnation, revision),
+            "owned scan publishes after its waiter is canceled"
+        );
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn quota_reconciliation_cannot_publish_into_recreated_bucket() {
+        use crate::storage_api_contracts::bucket::DeleteBucketOptions;
+        let (_directories, job) = reconciliation_fixture(100).await;
+        let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforePublish);
+        let worker = tokio::spawn({
+            let job = job.clone();
+            async move { job.run().await }
+        });
+        pause.reached().await;
+        job.store
+            .delete_object(&job.bucket, "seed", ObjectOptions::default())
+            .await
+            .expect("empty old bucket");
+        job.store
+            .delete_bucket(&job.bucket, &DeleteBucketOptions::default())
+            .await
+            .expect("retire bucket while scan is paused");
+        job.store
+            .make_bucket(&job.bucket, &MakeBucketOptions::default())
+            .await
+            .expect("replacement bucket");
+        job.store
+            .update_bucket_metadata_config(
+                &job.bucket,
+                "quota.json",
+                serde_json::to_vec(&super::super::BucketQuota::new(Some(100))).expect("new quota"),
+            )
+            .await
+            .expect("replacement quota");
+        pause.0.release.notify_one();
+        assert!(worker.await.expect("old worker join").is_err(), "old incarnation must not publish");
+        assert!(matches!(
+            load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
+            Err(StorageError::ConfigNotFound)
+        ));
+        drop(pause);
+        initialize_quota_ledger_for_test(Arc::clone(&job.store), &job.bucket)
+            .await
+            .expect("replacement can bootstrap independently");
+        let replacement = read_test_ledger(&job).await;
+        assert_ne!(replacement.bucket_incarnation, job.bucket_incarnation);
+        assert_eq!(replacement.accounted_usage, 0);
+    }
+
     fn ledger(accounted_usage: u64) -> QuotaLedger {
         QuotaLedger::new(Uuid::new_v4(), OffsetDateTime::now_utc(), accounted_usage)
     }
@@ -2090,8 +2781,7 @@ mod tests {
             .expect("persist crash-left reservation fixture");
 
         assert!(matches!(
-            load_current_ledger_locked(Arc::clone(&store), &bucket, &ledger_path, new_incarnation, OffsetDateTime::now_utc())
-                .await,
+            load_current_ledger_locked(Arc::clone(&store), &ledger_path, new_incarnation, OffsetDateTime::now_utc()).await,
             Err(StorageError::PartMissingOrCorrupt)
         ));
         assert!(matches!(
@@ -2109,10 +2799,10 @@ mod tests {
             Err(StorageError::ConfigNotFound)
         ));
         let replacement =
-            load_current_ledger_locked(Arc::clone(&store), &bucket, &ledger_path, new_incarnation, OffsetDateTime::now_utc())
+            load_current_ledger_locked(Arc::clone(&store), &ledger_path, new_incarnation, OffsetDateTime::now_utc())
                 .await
                 .expect("new quota admission must not inherit a fresh stale reservation");
-        assert!(replacement.reservations.is_empty());
+        assert!(replacement.is_none(), "replacement must await authoritative bootstrap before admission");
     }
 
     #[test]

crates/ecstore/src/bucket/remote_s3_client.rs

@@ -40,7 +40,9 @@ use aws_smithy_runtime_api::client::http::{
     HttpConnector as SmithyHttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn,
 };
 use aws_smithy_runtime_api::client::interceptors::Intercept;
-use aws_smithy_runtime_api::client::interceptors::context::BeforeTransmitInterceptorContextMut;
+use aws_smithy_runtime_api::client::interceptors::context::{
+    BeforeDeserializationInterceptorContextMut, BeforeTransmitInterceptorContextMut,
+};
 use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse};
 use aws_smithy_runtime_api::client::result::ConnectorError;
 use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents;
@@ -155,6 +157,8 @@ pub struct RemoteS3EndpointSpec {
     pub ca_cert_pem: Option<String>,
     pub connect_timeout: Option<Duration>,
     pub read_timeout: Option<Duration>,
+    /// Entire SDK call, including non-streaming response bodies and retries.
+    pub operation_timeout: Option<Duration>,
     /// How many wire requests one logical call may cost. Every consumer
     /// declares it; see [`RemoteS3RetryPolicy`].
     pub retry: RemoteS3RetryPolicy,
@@ -252,6 +256,78 @@ pub(crate) fn remote_sdk_credentials(credentials: &RemoteCredentials, now: Syste
     Ok(builder.build())
 }
 
+/// Only installed on non-streaming upload operations. The budget begins at
+/// response headers, independently of the potentially long request upload.
+#[derive(Debug)]
+pub(crate) struct UploadResponseDeadline(pub Duration);
+
+impl Intercept for UploadResponseDeadline {
+    fn name(&self) -> &'static str {
+        "RustfsUploadResponseDeadline"
+    }
+
+    fn modify_before_deserialization(
+        &self,
+        context: &mut BeforeDeserializationInterceptorContextMut<'_>,
+        _runtime_components: &RuntimeComponents,
+        _cfg: &mut ConfigBag,
+    ) -> Result<(), BoxError> {
+        let response = context.response_mut();
+        let inner = std::mem::replace(response.body_mut(), SdkBody::taken());
+        *response.body_mut() = SdkBody::from_body_1_x(UploadResponseBody {
+            inner: Some(inner),
+            deadline: Box::pin(tokio::time::sleep(self.0)),
+        });
+        Ok(())
+    }
+}
+
+struct UploadResponseBody {
+    inner: Option<SdkBody>,
+    deadline: std::pin::Pin<Box<tokio::time::Sleep>>,
+}
+
+impl http_body::Body for UploadResponseBody {
+    type Data = bytes::Bytes;
+    type Error = BoxError;
+
+    fn poll_frame(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Option<Result<http_body::Frame<Self::Data>, Self::Error>>> {
+        use std::future::Future as _;
+        use std::task::Poll;
+        if self.inner.is_none() {
+            return Poll::Ready(None);
+        }
+        // The absolute response budget never renews on data, empty frames,
+        // or trailers. A retry gets its own bounded response window.
+        if self.deadline.as_mut().poll(cx).is_ready() {
+            self.inner.take();
+            return Poll::Ready(Some(Err(std::io::Error::new(
+                std::io::ErrorKind::TimedOut,
+                "replication response body deadline exceeded",
+            )
+            .into())));
+        }
+        let Some(inner) = self.inner.as_mut() else {
+            return Poll::Ready(None);
+        };
+        let result = std::pin::Pin::new(inner).poll_frame(cx);
+        if matches!(result, Poll::Ready(None) | Poll::Ready(Some(Err(_)))) {
+            self.inner.take();
+        }
+        result
+    }
+
+    fn is_end_stream(&self) -> bool {
+        self.inner.as_ref().is_none_or(http_body::Body::is_end_stream)
+    }
+    fn size_hint(&self) -> http_body::SizeHint {
+        self.inner.as_ref().map(http_body::Body::size_hint).unwrap_or_default()
+    }
+}
+
 /// Appends a caller-identifying token to the SDK `User-Agent`. Runs after
 /// signing: SigV4 excludes `user-agent` from the canonical request, so the
 /// signature stays valid.
@@ -309,7 +385,7 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
         config_builder = config_builder.http_client(http_client);
     }
 
-    if spec.connect_timeout.is_some() || spec.read_timeout.is_some() {
+    if spec.connect_timeout.is_some() || spec.read_timeout.is_some() || spec.operation_timeout.is_some() {
         let mut timeouts = TimeoutConfig::builder();
         if let Some(connect_timeout) = spec.connect_timeout {
             timeouts = timeouts.connect_timeout(connect_timeout);
@@ -317,6 +393,9 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
         if let Some(read_timeout) = spec.read_timeout {
             timeouts = timeouts.read_timeout(read_timeout);
         }
+        if let Some(operation_timeout) = spec.operation_timeout {
+            timeouts = timeouts.operation_timeout(operation_timeout);
+        }
         config_builder = config_builder.timeout_config(timeouts.build());
     }
 
@@ -674,6 +753,7 @@ mod tests {
             ca_cert_pem: None,
             connect_timeout: None,
             read_timeout: None,
+            operation_timeout: None,
             retry: RemoteS3RetryPolicy::Disabled,
             user_agent_suffix: "",
         }
@@ -1027,11 +1107,16 @@ mod tests {
         spec.user_agent_suffix = "RustFS-Test/0.0";
         spec.connect_timeout = Some(Duration::from_secs(5));
         spec.read_timeout = Some(Duration::from_secs(5));
+        spec.operation_timeout = Some(Duration::from_secs(12));
         let config = build_remote_s3_config(&spec)
             .await
             .expect("spec should build")
             .http_client(http_client)
             .build();
+        assert_eq!(
+            config.timeout_config().and_then(TimeoutConfig::operation_timeout),
+            Some(Duration::from_secs(12))
+        );
         S3Client::from_conf(config)
             .head_bucket()
             .bucket("bucket")

crates/ecstore/src/bucket/replication/replication_metadata_boundary.rs

@@ -33,6 +33,7 @@ pub(crate) struct ReplicationMetadataStore;
 impl ReplicationMetadataStore {
     pub(crate) const MRF_REPLICATION_FILE: &'static str = "config/replication/mrf.bin";
     pub(crate) const MRF_REPLICATION_RECOVERY_LOCK: &'static str = "config/replication/mrf.bin.recovery";
+    pub(crate) const MRF_REPLICATION_REPLAY_STAMP: &'static str = "config/replication/mrf.bin.replay";
     pub(crate) const FORCE_DELETE_REPLICATION_FILE: &'static str = "config/replication/force-delete.bin";
     pub(crate) const FORCE_DELETE_REPLICATION_TRANSACTION_LOCK: &'static str = "config/replication/force-delete.bin.transaction";
 
@@ -116,6 +117,10 @@ mod tests {
             ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
             "config/replication/mrf.bin.recovery"
         );
+        assert_eq!(
+            ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
+            "config/replication/mrf.bin.replay"
+        );
         assert_eq!(
             ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE,
             "config/replication/force-delete.bin"

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -16,8 +16,8 @@ use super::replication_config_store::ReplicationConfigStore;
 use super::replication_error_boundary::{Error as EcstoreError, is_err_object_not_found, is_err_version_not_found};
 use super::replication_filemeta_boundary::{
     MrfOpKind, MrfReplicateEntry, REPLICATE_HEAL_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicatedTargetInfo,
-    ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision, replicate_decision_for_admitted_targets,
-    replication_statuses_map, version_purge_statuses_map,
+    ReplicationGenerationSnapshot, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision,
+    replicate_decision_for_admitted_targets, replication_statuses_map, version_purge_statuses_map,
 };
 use super::replication_lock_boundary::ReplicationLockTiming;
 use super::replication_logging::{EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REPLICATION};
@@ -86,6 +86,10 @@ const MRF_CORRUPT_FILE_PREFIX: &str = "config/replication/mrf.corrupt";
 const MRF_PENDING_CAP: usize = 200_000;
 const MRF_RETRY_INITIAL_DELAY: Duration = Duration::from_millis(100);
 const MRF_RETRY_MAX_DELAY: Duration = Duration::from_secs(5);
+/// Interval between runtime replays of the MRF journal, cluster-wide. A pass
+/// re-attempts every retained entry, so a target that stays down costs one
+/// attempt per entry per interval.
+const MRF_REPLAY_INTERVAL: Duration = Duration::from_secs(300);
 
 #[derive(Debug, Default)]
 pub struct DurableMrfBacklog {
@@ -615,7 +619,7 @@ async fn read_mrf_entries<S: ReplicationObjectIO>(storage: Arc<S>) -> Result<Vec
 }
 
 /// Acknowledge only the generation read by the recovery leader. Entries appended
-/// after that generation are retained as a suffix and replayed on the next startup.
+/// after that generation are retained as a suffix and replayed on the next pass.
 /// Lock order: recovery leader lock -> MRF journal object lock.
 async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
     storage: Arc<S>,
@@ -667,7 +671,7 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
     Err(EcstoreError::PreconditionFailed)
 }
 
-/// Acquires the MRF recovery leader lock for the startup replay.
+/// Acquires the MRF recovery leader lock for one journal replay pass.
 /// Returns `None` (after logging) when the lock cannot be created or another
 /// node is already processing the backlog.
 async fn acquire_mrf_recovery_guard<S: ReplicationStorage>(storage: &Arc<S>) -> Option<rustfs_lock::NamespaceLockGuard> {
@@ -1018,7 +1022,7 @@ fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_t
 }
 
 /// Acknowledges the replayed MRF prefix and returns the retained backlog.
-/// On acknowledgement failure the backlog is preserved for the next startup and
+/// On acknowledgement failure the backlog is preserved for the next replay pass and
 /// re-read (falling back to the replayed snapshot) so the published summary stays accurate.
 async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
     storage: &Arc<S>,
@@ -1033,7 +1037,7 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
                 component = LOG_COMPONENT_ECSTORE,
                 subsystem = LOG_SUBSYSTEM_REPLICATION,
                 error = %error,
-                "Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
+                "Failed to acknowledge the MRF recovery prefix; preserving it for the next replay pass"
             );
             match read_mrf_entries(storage.clone()).await {
                 Ok(current) => current,
@@ -1051,6 +1055,120 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
     }
 }
 
+fn unix_millis_now() -> i64 {
+    i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000).unwrap_or(i64::MAX)
+}
+
+/// Whether a periodic pass may run, given the start time of the last pass on
+/// any node. A stamp more than one interval ahead of this node's clock is skew
+/// or a bad value, so it does not suppress replay.
+fn mrf_replay_due(last_started_ms: Option<i64>, now_ms: i64, interval: Duration) -> bool {
+    let Some(last_started_ms) = last_started_ms else {
+        return true;
+    };
+    let interval_ms = i64::try_from(interval.as_millis()).unwrap_or(i64::MAX);
+    now_ms.saturating_sub(last_started_ms) >= interval_ms || last_started_ms.saturating_sub(now_ms) >= interval_ms
+}
+
+async fn read_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>) -> Option<i64> {
+    let data = ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP)
+        .await
+        .ok()?;
+    std::str::from_utf8(&data).ok()?.trim().parse().ok()
+}
+
+async fn write_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>, started_ms: i64) {
+    if let Err(error) = ReplicationConfigStore::save(
+        storage.clone(),
+        ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
+        started_ms.to_string().into_bytes(),
+    )
+    .await
+    {
+        warn!(
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REPLICATION,
+            error = %error,
+            "Failed to record the MRF replay start time"
+        );
+    }
+}
+
+/// One MRF journal replay pass. The recovery leader reads the journal,
+/// re-drives every entry, and acknowledges the replayed prefix; entries that
+/// could not be admitted, and entries appended meanwhile, stay for the next pass.
+///
+/// Every node runs the periodic loop, and the leader lock only serializes
+/// passes. With `min_interval`, a pass is skipped when a pass on any node
+/// started within that interval, read from the shared replay stamp, so a
+/// cluster replays once per interval rather than once per node. The startup
+/// pass passes `None` and always runs. A pass that finds entries records its
+/// start time. Returns whether the journal was read.
+/// Lock order: recovery leader lock -> replay stamp object lock.
+async fn replay_mrf_journal<S: ReplicationStorage>(storage: &Arc<S>, min_interval: Option<Duration>) -> bool {
+    let Some(recovery_guard) = acquire_mrf_recovery_guard(storage).await else {
+        return false;
+    };
+    let started_ms = unix_millis_now();
+    if let Some(interval) = min_interval
+        && !mrf_replay_due(read_mrf_replay_stamp(storage).await, started_ms, interval)
+    {
+        return false;
+    }
+
+    let Some(entries) = load_mrf_recovery_entries(storage).await else {
+        return true;
+    };
+    if !entries.is_empty() {
+        write_mrf_replay_stamp(storage, started_ms).await;
+    }
+    set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
+
+    let total = entries.len();
+    let mut queued_count = 0usize;
+    let mut retry_entries = Vec::new();
+
+    for entry in entries.iter() {
+        let Some(admission) = replay_mrf_entry(entry, storage, &mut retry_entries).await else {
+            continue;
+        };
+
+        if admission == ReplicationQueueAdmission::Missed {
+            retry_entries.push(entry.clone());
+        } else if admission == ReplicationQueueAdmission::Queued {
+            queued_count += 1;
+        }
+    }
+
+    let retained = resolve_retained_mrf_entries(storage, &recovery_guard, &entries, &retry_entries).await;
+    let retained_count = retained.len();
+    set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
+
+    if queued_count > 0 {
+        info!(
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REPLICATION,
+            recovered = queued_count,
+            total,
+            retained = retained_count,
+            "Replayed MRF entries admitted for retry"
+        );
+    }
+    true
+}
+
+/// Re-runs [`replay_mrf_journal`] every `period`. The recovery leader lock keeps
+/// one pass at a time across the cluster, including the startup pass, and the
+/// shared replay stamp spaces passes from different nodes by `period`.
+fn spawn_mrf_replay_loop<S: ReplicationStorage>(storage: Arc<S>, period: Duration) -> JoinHandle<()> {
+    tokio::spawn(async move {
+        loop {
+            tokio::time::sleep(period).await;
+            replay_mrf_journal(&storage, Some(period)).await;
+        }
+    })
+}
+
 #[derive(Debug, thiserror::Error)]
 #[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
 struct ResyncActiveConflictError {
@@ -1171,6 +1289,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
         // Start background tasks
         pool.start_mrf_persister().await;
         pool.start_mrf_processor().await;
+        pool.start_mrf_replayer().await;
         pool.start_force_delete_processor().await;
 
         pool
@@ -1462,7 +1581,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 
     /// Queues a replica task
     pub async fn queue_replica_task(&self, ri: ReplicateObjectInfo) -> ReplicationQueueAdmission {
-        // A version that is already queued or being uploaded is not driven a
+        // A source generation already queued or being uploaded is not driven a
         // second time: the scanner heal pass sees it as PENDING until the
         // first upload lands and would otherwise re-queue it every cycle
         // (backlog#2362). The key is released when the worker finishes, or
@@ -1639,55 +1758,26 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 
     /// Starts the MRF processor — one-shot at startup.
     ///
-    /// Reads the on-disk MRF file, re-injects admitted entries as Heal operations, and
-    /// rewrites any entries that could not be admitted for a later startup retry.
+    /// Replays the on-disk MRF journal once; [`Self::start_mrf_replayer`] repeats
+    /// the pass while the server runs.
     async fn start_mrf_processor(&self) {
         let storage = self.storage.clone();
-
         let handle = tokio::spawn(async move {
-            let Some(recovery_guard) = acquire_mrf_recovery_guard(&storage).await else {
-                return;
-            };
-
-            let Some(entries) = load_mrf_recovery_entries(&storage).await else {
-                return;
-            };
-            set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
-
-            let total = entries.len();
-            let mut queued_count = 0usize;
-            let mut retry_entries = Vec::new();
-
-            for entry in entries.iter() {
-                let Some(admission) = replay_mrf_entry(entry, &storage, &mut retry_entries).await else {
-                    continue;
-                };
-
-                if admission == ReplicationQueueAdmission::Missed {
-                    retry_entries.push(entry.clone());
-                } else if admission == ReplicationQueueAdmission::Queued {
-                    queued_count += 1;
-                }
-            }
-
-            let retained = resolve_retained_mrf_entries(&storage, &recovery_guard, &entries, &retry_entries).await;
-            let retained_count = retained.len();
-            set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
-
-            if queued_count > 0 {
-                info!(
-                    component = LOG_COMPONENT_ECSTORE,
-                    subsystem = LOG_SUBSYSTEM_REPLICATION,
-                    recovered = queued_count,
-                    total,
-                    retained = retained_count,
-                    "Replayed MRF entries admitted for retry"
-                );
-            }
+            replay_mrf_journal(&storage, None).await;
         });
         self.task_handles.lock().await.push(handle);
     }
 
+    /// Starts the runtime MRF replayer — ongoing background task.
+    ///
+    /// Entries reach the journal while the server runs (worker-queue overflow,
+    /// delete-marker purge intents whose watch window drained with targets still
+    /// dirty). Replaying only at startup left them waiting for a restart.
+    async fn start_mrf_replayer(&self) {
+        let handle = spawn_mrf_replay_loop(self.storage.clone(), MRF_REPLAY_INTERVAL);
+        self.task_handles.lock().await.push(handle);
+    }
+
     async fn start_force_delete_processor(&self) {
         let storage = self.storage.clone();
         let handle =
@@ -1755,7 +1845,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
     /// writes them to the on-disk MRF file every flush interval (default 10s,
     /// overridable via `RUSTFS_REPL_MRF_FLUSH_INTERVAL_MS`) or when 1 000 new
     /// entries accumulate. Runtime writers append under the journal lock; only the
-    /// startup recovery leader may remove an acknowledged prefix.
+    /// recovery leader of a replay pass may remove an acknowledged prefix.
     async fn start_mrf_persister(&self) {
         let Some(mut rx) = self.mrf_save_rx.lock().await.take() else {
             return;
@@ -2314,28 +2404,47 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 }
 
 /// Object versions currently queued or being uploaded, keyed by bucket,
-/// object name and version. `queue_replica_task` admits a version only once
+/// object name, version and source generation. `queue_replica_task` admits a generation only once
 /// while it is in flight; the scanner heal pass and MRF replays that arrive
 /// in the meantime are `Skipped` instead of driving a second complete upload
 /// (backlog#2362). Entries are removed when the worker finishes the task or
 /// when no worker accepted it.
 #[derive(Debug, Default)]
 pub(crate) struct ReplicationInFlight {
-    keys: std::sync::Mutex<std::collections::HashSet<(String, String, Option<uuid::Uuid>)>>,
+    keys: std::sync::Mutex<std::collections::HashSet<ReplicationInFlightKey>>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+struct ReplicationInFlightKey {
+    bucket: String,
+    object: String,
+    version_id: Option<uuid::Uuid>,
+    generation: ReplicationGenerationSnapshot,
+}
+
+impl From<&ReplicateObjectInfo> for ReplicationInFlightKey {
+    fn from(info: &ReplicateObjectInfo) -> Self {
+        Self {
+            bucket: info.bucket.clone(),
+            object: info.name.clone(),
+            version_id: info.version_id,
+            generation: info.replication_generation.clone(),
+        }
+    }
 }
 
 impl ReplicationInFlight {
-    fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<(String, String, Option<uuid::Uuid>)>> {
+    fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<ReplicationInFlightKey>> {
         self.keys.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
     }
 
-    /// Claim `ri`; `false` when the same version is already in flight.
+    /// Claim `ri`; a metadata mutation is independent of its earlier generation.
     fn try_begin(&self, ri: &ReplicateObjectInfo) -> bool {
-        self.lock().insert((ri.bucket.clone(), ri.name.clone(), ri.version_id))
+        self.lock().insert(ReplicationInFlightKey::from(ri))
     }
 
     fn finish(&self, ri: &ReplicateObjectInfo) {
-        self.lock().remove(&(ri.bucket.clone(), ri.name.clone(), ri.version_id));
+        self.lock().remove(&ReplicationInFlightKey::from(ri));
     }
 
     #[cfg(test)]
@@ -2348,26 +2457,21 @@ impl ReplicationInFlight {
 /// including when replication panics.
 struct ReplicationInFlightGuard {
     in_flight: Arc<ReplicationInFlight>,
-    key: ReplicateObjectInfo,
+    key: ReplicationInFlightKey,
 }
 
 impl ReplicationInFlightGuard {
     fn new(in_flight: Arc<ReplicationInFlight>, ri: &ReplicateObjectInfo) -> Self {
         Self {
             in_flight,
-            key: ReplicateObjectInfo {
-                bucket: ri.bucket.clone(),
-                name: ri.name.clone(),
-                version_id: ri.version_id,
-                ..Default::default()
-            },
+            key: ReplicationInFlightKey::from(ri),
         }
     }
 }
 
 impl Drop for ReplicationInFlightGuard {
     fn drop(&mut self) {
-        self.in_flight.finish(&self.key);
+        self.in_flight.lock().remove(&self.key);
     }
 }
 
@@ -3421,6 +3525,8 @@ mod tests {
         allow_first_read: Notify,
         read_count: AtomicUsize,
         delete_storage_calls: AtomicUsize,
+        delete_error: StdMutex<EcstoreError>,
+        replay_stamp: StdMutex<Option<Vec<u8>>>,
         reported_size: StdMutex<Option<i64>>,
         stream_read_bytes: Arc<AtomicUsize>,
         write_count: AtomicUsize,
@@ -3488,6 +3594,24 @@ mod tests {
             _h: Self::HeaderMap,
             _opts: &Self::ObjectOptions,
         ) -> Result<Self::GetObjectReader, Self::Error> {
+            if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
+                let stamp = self
+                    .shared
+                    .replay_stamp
+                    .lock()
+                    .expect("test replay stamp lock should not be poisoned")
+                    .clone()
+                    .ok_or(EcstoreError::FileNotFound)?;
+                return Ok(Self::GetObjectReader {
+                    object_info: ObjectInfo {
+                        size: i64::try_from(stamp.len()).expect("test stamp length should fit i64"),
+                        ..Default::default()
+                    },
+                    stream: Box::new(Cursor::new(stamp)),
+                    buffered_body: None,
+                    body_source: Default::default(),
+                });
+            }
             if !object.ends_with("/.replication/resync.bin")
                 && !object.ends_with("config/replication/mrf.bin")
                 && !object.ends_with("config/replication/force-delete.bin")
@@ -3551,6 +3675,16 @@ mod tests {
                 let lock = self.new_ns_lock(bucket, object).await?;
                 Some(lock.get_write_lock(Duration::from_secs(10)).await?)
             };
+            if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
+                let mut stamp = Vec::new();
+                data.stream.read_to_end(&mut stamp).await.map_err(EcstoreError::from)?;
+                *self
+                    .shared
+                    .replay_stamp
+                    .lock()
+                    .expect("test replay stamp lock should not be poisoned") = Some(stamp);
+                return Ok(ObjectInfo::default());
+            }
             if opts.http_preconditions.is_some()
                 && let Some(replacement) = self
                     .shared
@@ -3672,7 +3806,12 @@ mod tests {
             _opts: Self::ObjectOptions,
         ) -> Result<Self::ObjectInfo, Self::Error> {
             self.shared.delete_storage_calls.fetch_add(1, Ordering::SeqCst);
-            Err(EcstoreError::NotImplemented)
+            Err(self
+                .shared
+                .delete_error
+                .lock()
+                .expect("delete error lock should not be poisoned")
+                .clone())
         }
 
         async fn delete_objects(
@@ -3929,6 +4068,48 @@ mod tests {
         assert_eq!(current_queue(&pool, "in-flight-bucket").await, (3, 3 * 4096));
     }
 
+    #[tokio::test]
+    async fn queue_replica_task_admits_metadata_changes_during_an_initial_upload() {
+        let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
+        let (tx, mut rx) = mpsc::channel(4);
+        pool.workers.write().await.push(tx);
+        let initial = ReplicateObjectInfo {
+            bucket: "pending-tags-bucket".to_string(),
+            name: "object".to_string(),
+            version_id: Some(uuid::Uuid::new_v4()),
+            size: 4096,
+            replication_generation: ReplicationGenerationSnapshot {
+                mutation_id: Some(uuid::Uuid::new_v4().to_string()),
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        let metadata = ReplicateObjectInfo {
+            op_type: ReplicationType::Metadata,
+            user_tags: String::new(),
+            replication_generation: ReplicationGenerationSnapshot {
+                mutation_id: Some(uuid::Uuid::new_v4().to_string()),
+                ..Default::default()
+            },
+            ..initial.clone()
+        };
+        assert_eq!(pool.queue_replica_task(initial.clone()).await, ReplicationQueueAdmission::Queued);
+        assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Queued);
+        assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
+        assert_eq!(pool.in_flight.len(), 2);
+        drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &initial));
+        assert_eq!(pool.in_flight.len(), 1);
+        assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
+        for expected in [&initial, &metadata] {
+            let ReplicationOperation::Object(queued) = rx.recv().await.expect("queued operation") else {
+                panic!("expected object replication");
+            };
+            assert_eq!(queued.replication_generation, expected.replication_generation);
+        }
+        drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &metadata));
+        assert_eq!(pool.in_flight.len(), 0);
+    }
+
     #[tokio::test]
     async fn queue_replica_task_releases_the_version_when_no_worker_accepts_it() {
         let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
@@ -4171,6 +4352,8 @@ mod tests {
             allow_first_read: Notify::new(),
             read_count: AtomicUsize::new(0),
             delete_storage_calls: AtomicUsize::new(0),
+            delete_error: StdMutex::new(EcstoreError::NotImplemented),
+            replay_stamp: StdMutex::new(None),
             reported_size: StdMutex::new(None),
             stream_read_bytes: Arc::new(AtomicUsize::new(0)),
             write_count: AtomicUsize::new(0),
@@ -4221,6 +4404,103 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn delete_worker_already_purged_source_completes_without_retry() {
+        let arn = "arn:test";
+        for delete_marker_version in [false, true] {
+            for version_id in [Uuid::new_v4(), Uuid::nil()] {
+                for error in [
+                    EcstoreError::VersionNotFound("source".to_string(), "purged".to_string(), version_id.to_string()),
+                    EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
+                    EcstoreError::FileVersionNotFound,
+                    EcstoreError::FileNotFound,
+                ] {
+                    let shared = empty_resync_shared_state();
+                    *shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
+                    let storage = Arc::new(LoadResyncNodeStore::new("purged-source", shared.clone()));
+                    let delete = DeletedObjectReplicationInfo {
+                        bucket: "source".to_string(),
+                        delete_object: ReplicationDeletedObject {
+                            object_name: "purged".to_string(),
+                            version_id: (!delete_marker_version).then_some(version_id),
+                            delete_marker_version_id: delete_marker_version.then_some(version_id),
+                            replication_state: Some(ReplicationState {
+                                replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
+                                purge_targets: HashMap::from([(arn.to_string(), VersionPurgeStatusType::Complete)]),
+                                ..Default::default()
+                            }),
+                            ..Default::default()
+                        },
+                        op_type: ReplicationType::Delete,
+                        ..Default::default()
+                    };
+
+                    assert!(
+                        replicate_delete_with_outcome(delete, storage).await,
+                        "a confirmed target purge must complete when the source is already absent (marker={delete_marker_version}, version={version_id})"
+                    );
+                    assert_eq!(
+                        shared.delete_storage_calls.load(Ordering::SeqCst),
+                        2 + usize::from(!delete_marker_version && !version_id.is_nil()),
+                        "the worker must lock and attempt writeback, reading a non-null data version first"
+                    );
+                }
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn delete_worker_missing_source_does_not_hide_unfinished_replication() {
+        let arn = "arn:test";
+        for (version_purge, target_status, error) in [
+            (
+                true,
+                VersionPurgeStatusType::Pending,
+                EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
+            ),
+            (true, VersionPurgeStatusType::Complete, EcstoreError::ErasureWriteQuorum),
+            (
+                false,
+                VersionPurgeStatusType::Empty,
+                EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
+            ),
+        ] {
+            let shared = empty_resync_shared_state();
+            *shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
+            let storage = Arc::new(LoadResyncNodeStore::new("unfinished-delete", shared));
+            let delete = DeletedObjectReplicationInfo {
+                bucket: "source".to_string(),
+                delete_object: ReplicationDeletedObject {
+                    object_name: "purged".to_string(),
+                    version_id: version_purge.then(Uuid::new_v4),
+                    delete_marker: !version_purge,
+                    replication_state: Some(ReplicationState {
+                        replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
+                        targets: if version_purge {
+                            HashMap::new()
+                        } else {
+                            HashMap::from([(arn.to_string(), ReplicationStatusType::Completed)])
+                        },
+                        purge_targets: if version_purge {
+                            HashMap::from([(arn.to_string(), target_status)])
+                        } else {
+                            HashMap::new()
+                        },
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                },
+                op_type: ReplicationType::Delete,
+                ..Default::default()
+            };
+
+            assert!(
+                !replicate_delete_with_outcome(delete, storage).await,
+                "unfinished target work, source I/O failures, and marker creation must remain retryable"
+            );
+        }
+    }
+
     #[tokio::test]
     async fn delete_worker_preserves_persisted_admission_without_live_config() {
         let arn = "arn:test";
@@ -5304,6 +5584,8 @@ mod tests {
                 allow_first_read: Notify::new(),
                 read_count: AtomicUsize::new(0),
                 delete_storage_calls: AtomicUsize::new(0),
+                delete_error: StdMutex::new(EcstoreError::NotImplemented),
+                replay_stamp: StdMutex::new(None),
                 reported_size: StdMutex::new(None),
                 stream_read_bytes: Arc::new(AtomicUsize::new(0)),
                 write_count: AtomicUsize::new(0),
@@ -6268,6 +6550,108 @@ mod tests {
         assert_eq!(retained.target_arns, entry.target_arns);
     }
 
+    #[tokio::test]
+    async fn mrf_replay_loop_drains_entries_persisted_after_startup() {
+        let shared = empty_resync_shared_state();
+        let storage = Arc::new(LoadResyncNodeStore::new("mrf-runtime-replay", shared.clone()));
+        // A force-delete intent from a superseded generation replays as
+        // Skipped, so a pass that reaches it acknowledges it off the journal.
+        let stale_intent = |object: &str| MrfReplicateEntry {
+            bucket: "mrf-runtime-replay".to_string(),
+            object: object.to_string(),
+            op: MrfOpKind::Delete,
+            force_delete_id: Some(uuid::Uuid::new_v4()),
+            force_delete_local_commit: false,
+            target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
+            ..Default::default()
+        };
+        let journal_drained = || async {
+            tokio::time::timeout(Duration::from_secs(5), async {
+                while !shared.data.lock().expect("test data lock should not be poisoned").is_empty() {
+                    tokio::time::sleep(Duration::from_millis(10)).await;
+                }
+            })
+            .await
+            .is_ok()
+        };
+
+        let handle = spawn_mrf_replay_loop(storage, Duration::from_millis(20));
+        for object in ["first", "second"] {
+            *shared.data.lock().expect("test data lock should not be poisoned") =
+                encode_mrf_file(&[stale_intent(object)]).expect("MRF entry should encode");
+            assert!(
+                journal_drained().await,
+                "an MRF entry persisted while the server runs must be replayed without a restart ({object})"
+            );
+        }
+        handle.abort();
+    }
+
+    #[tokio::test]
+    async fn mrf_periodic_replay_runs_once_per_interval_across_nodes() {
+        let shared = empty_resync_shared_state();
+        let node_a = Arc::new(LoadResyncNodeStore::new("mrf-node-a", shared.clone()));
+        let node_b = Arc::new(LoadResyncNodeStore::new("mrf-node-b", shared.clone()));
+        let interval = Duration::from_secs(1);
+        let persist_backlog = |object: &str| {
+            let entry = MrfReplicateEntry {
+                bucket: "mrf-node-replay".to_string(),
+                object: object.to_string(),
+                op: MrfOpKind::Delete,
+                force_delete_id: Some(uuid::Uuid::new_v4()),
+                force_delete_local_commit: false,
+                target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
+                ..Default::default()
+            };
+            *shared.data.lock().expect("test data lock should not be poisoned") =
+                encode_mrf_file(&[entry]).expect("MRF entry should encode");
+        };
+        let journal_is_empty = || shared.data.lock().expect("test data lock should not be poisoned").is_empty();
+
+        persist_backlog("first");
+        assert!(replay_mrf_journal(&node_a, Some(interval)).await, "node A's periodic pass should replay");
+        assert!(journal_is_empty(), "node A's pass should acknowledge the replayed entry");
+
+        // Node B's timer fires within the same interval with a backlog present.
+        persist_backlog("second");
+        assert!(
+            !replay_mrf_journal(&node_b, Some(interval)).await,
+            "a node must not replay within one interval of another node's pass"
+        );
+        assert!(!journal_is_empty(), "the backlog should wait for the next interval");
+
+        assert!(
+            replay_mrf_journal(&node_b, None).await,
+            "the startup pass should replay regardless of the shared stamp"
+        );
+        assert!(journal_is_empty(), "the startup pass should acknowledge the replayed entry");
+
+        persist_backlog("third");
+        tokio::time::sleep(interval).await;
+        assert!(
+            replay_mrf_journal(&node_a, Some(interval)).await,
+            "a periodic pass should replay once the interval has elapsed"
+        );
+        assert!(journal_is_empty(), "the due pass should acknowledge the replayed entry");
+    }
+
+    #[test]
+    fn mrf_replay_due_tolerates_missing_and_skewed_stamps() {
+        let interval = Duration::from_secs(300);
+        let now = 1_000_000_000;
+        assert!(mrf_replay_due(None, now, interval));
+        assert!(!mrf_replay_due(Some(now - 299_999), now, interval));
+        assert!(mrf_replay_due(Some(now - 300_000), now, interval));
+        assert!(
+            !mrf_replay_due(Some(now + 1_000), now, interval),
+            "small clock skew should still space passes"
+        );
+        assert!(
+            mrf_replay_due(Some(now + 300_000), now, interval),
+            "a stamp an interval ahead must not suppress replay"
+        );
+    }
+
     #[test]
     fn mrf_entry_delete_marker_roundtrip() {
         let dm_vid = Uuid::new_v4();

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -2809,17 +2809,22 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
             true
         }
         Err(e) => {
-            error!(
-                event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
-                component = LOG_COMPONENT_ECSTORE,
-                subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
-                bucket = %bucket,
-                arn = %dobj.target_arn,
-                object = %dobj.delete_object.object_name,
-                operation = "apply_replication_delete_state",
-                error = %e,
-                "Replication target operation failed"
-            );
+            // A batch purge or another worker may have removed this version
+            // already. Remote failures still control the final outcome below.
+            let source_already_purged = is_version_purge && (is_err_object_not_found(&e) || is_err_version_not_found(&e));
+            if !source_already_purged {
+                error!(
+                    event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
+                    bucket = %bucket,
+                    arn = %dobj.target_arn,
+                    object = %dobj.delete_object.object_name,
+                    operation = "apply_replication_delete_state",
+                    error = %e,
+                    "Replication target operation failed"
+                );
+            }
             send_event(EventArgs {
                 event_name,
                 bucket_name: bucket.clone(),
@@ -2832,7 +2837,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
                 },
                 ..Default::default()
             });
-            false
+            source_already_purged
         }
     };
 
@@ -3010,7 +3015,7 @@ const DELETE_MARKER_PURGE_WATCH_INTERVAL: TokioDuration = TokioDuration::from_se
 /// If the marker disappears (deleted before or while the replica landed),
 /// purge the replicated marker from the targets, retrying failed targets on
 /// later rounds. When the window drains with targets still dirty, persist the
-/// purge intent as a durable MRF entry so the next startup replays it through
+/// purge intent as a durable MRF entry so the next MRF replay pass re-drives it through
 /// `purge_stale_delete_marker_targets`.
 async fn watch_and_purge_source_delete_marker<S: ReplicationStorage>(
     bucket: String,
@@ -3101,7 +3106,7 @@ async fn enqueue_delete_marker_purge_mrf(dobj: &DeletedObjectReplicationInfo, fa
 /// Heal callers retry for the full watch window because the startup MRF
 /// processor runs before bucket metadata (and thus target clients) finishes
 /// initializing — the first pass can see `target_client_missing` and a later
-/// round resolves the client; the replay loop is serial and startup-only, so
+/// round resolves the client; replay passes are serial background work, so
 /// blocking it for up to the window per dirty entry is acceptable. Live
 /// callers run on replication workers where a down target would pin a worker
 /// for the whole window, so they attempt once and lean on the durable intent
@@ -3282,10 +3287,18 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
 
     let mut join_set = JoinSet::new();
     let mut all_succeeded = true;
+    let mut target_infos = Vec::with_capacity(tgt_arns.len());
 
     for arn in tgt_arns {
         let Some(tgt_client) = ReplicationTargetStore::remote_target_client(bucket, &arn).await else {
             all_succeeded = false;
+            target_infos.push(force_delete_target_info(
+                dobj.op_type,
+                &arn,
+                "",
+                false,
+                Err(TARGET_CLIENT_UNAVAILABLE_ERROR.to_string()),
+            ));
             debug!(
                 event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
                 component = LOG_COMPONENT_ECSTORE,
@@ -3311,6 +3324,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
 
         let bucket = bucket.clone();
         let object_name = object_name.clone();
+        let op_type = dobj.op_type;
 
         join_set.spawn(async move {
             if ReplicationTargetStore::target_is_offline(&tgt_client).await {
@@ -3335,9 +3349,16 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
                     user_agent: "Internal: [Replication]".to_string(),
                     ..Default::default()
                 });
-                return false;
+                return force_delete_target_info(
+                    op_type,
+                    &tgt_client.arn,
+                    &tgt_client.endpoint,
+                    tgt_client.secure,
+                    Err(FORCE_DELETE_TARGET_OFFLINE_ERROR.to_string()),
+                );
             }
 
+            let started = Instant::now();
             if let Err(e) = tgt_client
                 .remove_object(&tgt_client.bucket, &object_name, None, replication_force_delete_remove_options())
                 .await
@@ -3364,16 +3385,25 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
                     user_agent: "Internal: [Replication]".to_string(),
                     ..Default::default()
                 });
-                return false;
+                return force_delete_target_info(
+                    op_type,
+                    &tgt_client.arn,
+                    &tgt_client.endpoint,
+                    tgt_client.secure,
+                    Err(e.to_string()),
+                );
             }
 
-            true
+            force_delete_target_info(op_type, &tgt_client.arn, &tgt_client.endpoint, tgt_client.secure, Ok(started.elapsed()))
         });
     }
 
     while let Some(result) = join_set.join_next().await {
         match result {
-            Ok(success) => all_succeeded &= success,
+            Ok(info) => {
+                all_succeeded &= info.replication_status == ReplicationStatusType::Completed;
+                target_infos.push(info);
+            }
             Err(error) => {
                 all_succeeded = false;
                 error!(
@@ -3390,6 +3420,8 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
         }
     }
 
+    record_force_delete_target_stats(bucket, &target_infos).await;
+
     if all_succeeded
         && let Some(operation_id) = dobj.delete_object.force_delete_id
         && let Err(error) = super::replication_pool::complete_force_delete_intent(storage, operation_id).await
@@ -3409,6 +3441,53 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
     all_succeeded
 }
 
+const FORCE_DELETE_TARGET_OFFLINE_ERROR: &str = "replication target is offline";
+
+/// One target's outcome of a replicated force-delete in the shape the
+/// replication statistics consume. A force-delete carries no per-version
+/// replication state, so the previous status is always `Pending`: that is
+/// the transition `ReplicationStats::update` counts as a failure, whereas an
+/// `Empty` previous status would drop the outcome and leave `replication
+/// status` reporting zero failures while the peer still holds the objects.
+fn force_delete_target_info(
+    op_type: ReplicationType,
+    arn: &str,
+    endpoint: &str,
+    secure: bool,
+    outcome: std::result::Result<std::time::Duration, String>,
+) -> ReplicatedTargetInfo {
+    let mut rinfo = ReplicatedTargetInfo {
+        arn: arn.to_string(),
+        op_type,
+        endpoint: endpoint.to_string(),
+        secure,
+        prev_replication_status: ReplicationStatusType::Pending,
+        ..Default::default()
+    };
+    match outcome {
+        Ok(duration) => {
+            rinfo.replication_status = ReplicationStatusType::Completed;
+            rinfo.duration = duration;
+        }
+        Err(error) => {
+            rinfo.replication_status = ReplicationStatusType::Failed;
+            rinfo.error = Some(error);
+        }
+    }
+    rinfo
+}
+
+async fn record_force_delete_target_stats(bucket: &str, targets: &[ReplicatedTargetInfo]) {
+    let Some(stats) = runtime_sources::replication_stats() else {
+        return;
+    };
+    for target in targets {
+        stats
+            .update(bucket, target, target.replication_status.clone(), target.prev_replication_status.clone())
+            .await;
+    }
+}
+
 fn unavailable_delete_target_info(dobj: &DeletedObjectReplicationInfo, arn: &str) -> ReplicatedTargetInfo {
     let mut rinfo = dobj
         .delete_object
@@ -6530,6 +6609,7 @@ mod tests {
             health_check_duration: std::time::Duration::from_secs(5),
             replicate_sync: false,
             client: Arc::new(aws_sdk_s3::Client::from_conf(config)),
+            timeouts: Default::default(),
         })
     }
 
@@ -7444,6 +7524,51 @@ mod tests {
         server.join().expect("test HTTP server should finish");
     }
 
+    /// A force-delete the peer rejects must reach the replication failure
+    /// counters. A replicated force-delete used to only log the rejection, so
+    /// `replication status` reported zero failures while the sites diverged.
+    #[tokio::test]
+    async fn force_delete_target_outcomes_reach_replication_stats() {
+        let stats = super::super::replication_state::ReplicationStats::new();
+        let denied = force_delete_target_info(
+            ReplicationType::Delete,
+            "arn:rustfs:replication::denied:bucket",
+            "http://peer-a:9000",
+            false,
+            Err("remove_object failed: AccessDenied".to_string()),
+        );
+        assert_eq!(denied.replication_status, ReplicationStatusType::Failed);
+        assert_eq!(denied.prev_replication_status, ReplicationStatusType::Pending);
+        assert_eq!(denied.error.as_deref(), Some("remove_object failed: AccessDenied"));
+
+        let accepted = force_delete_target_info(
+            ReplicationType::Heal,
+            "arn:rustfs:replication::accepted:bucket",
+            "https://peer-b:9000",
+            true,
+            Ok(std::time::Duration::from_millis(3)),
+        );
+        assert_eq!(accepted.replication_status, ReplicationStatusType::Completed);
+        assert!(accepted.error.is_none());
+
+        for target in [&denied, &accepted] {
+            stats
+                .update(
+                    "bucket",
+                    target,
+                    target.replication_status.clone(),
+                    target.prev_replication_status.clone(),
+                )
+                .await;
+        }
+
+        let bucket_stats = stats.get("bucket").await;
+        let denied_stat = bucket_stats.stats.get(&denied.arn).expect("denied target: stats entry");
+        assert_eq!((denied_stat.failed.count, denied_stat.replicated_count), (1, 0));
+        let accepted_stat = bucket_stats.stats.get(&accepted.arn).expect("accepted target: stats entry");
+        assert_eq!((accepted_stat.failed.count, accepted_stat.replicated_count), (0, 1));
+    }
+
     /// AWS answers `HEAD ?versionId=<source marker id>` with a bodiless 400.
     /// The marker is verified through the latest version instead: a
     /// versionless 404 with `x-amz-delete-marker: true` proves it propagated.

crates/ecstore/src/bucket/utils.rs

@@ -259,6 +259,10 @@ pub fn check_object_name_for_length_and_slash(bucket: &str, object: &str) -> Res
 pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) -> Result<()> {
     check_put_object_args(bucket, object)?;
 
+    if !data_movement && !is_meta_bucketname(bucket) && has_xml_invalid_control_characters(object) {
+        return Err(StorageError::ObjectNameInvalid(bucket.to_owned(), object.to_owned()));
+    }
+
     if cfg!(target_os = "windows") && !data_movement && !is_meta_bucketname(bucket) && object.contains('\\') {
         return Err(StorageError::InvalidArgument(
             bucket.to_owned(),
@@ -270,6 +274,14 @@ pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) ->
     Ok(())
 }
 
+/// New public keys must remain representable in an unencoded XML listing.
+/// Existing keys still use the read/delete validators so they can be removed.
+pub fn has_xml_invalid_control_characters(object: &str) -> bool {
+    object
+        .chars()
+        .any(|character| character < ' ' && !matches!(character, '\t' | '\n' | '\r'))
+}
+
 pub fn check_copy_obj_args(bucket: &str, object: &str) -> Result<()> {
     check_bucket_and_object_names(bucket, object)
 }

crates/ecstore/src/cache_value/metacache_set.rs

@@ -224,6 +224,8 @@ pub struct ListPathRawOptions {
     pub recursive: bool,
     pub incl_deleted: bool,
     pub skip_hidden_prefix_check: bool,
+    pub hidden_entry_budget: usize,
+    pub pending_prefix: Option<String>,
     pub filter_prefix: Option<String>,
     pub forward_to: Option<String>,
     pub min_disks: usize,
@@ -260,6 +262,8 @@ impl Clone for ListPathRawOptions {
             recursive: self.recursive,
             incl_deleted: self.incl_deleted,
             skip_hidden_prefix_check: self.skip_hidden_prefix_check,
+            hidden_entry_budget: self.hidden_entry_budget,
+            pending_prefix: self.pending_prefix.clone(),
             filter_prefix: self.filter_prefix.clone(),
             forward_to: self.forward_to.clone(),
             min_disks: self.min_disks,
@@ -289,6 +293,8 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
         recursive: opts.recursive,
         incl_deleted: opts.incl_deleted,
         skip_hidden_prefix_check: opts.skip_hidden_prefix_check,
+        hidden_entry_budget: opts.hidden_entry_budget,
+        pending_prefix: opts.pending_prefix.clone(),
         report_notfound: opts.report_not_found,
         filter_prefix: opts.filter_prefix.clone(),
         forward_to: opts.forward_to.clone(),
@@ -301,13 +307,15 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
 }
 
 /// How a merged raw listing stream ended.
-#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
 pub struct ListPathRawOutcome {
     /// At least one producer stopped at its per-disk limit instead of at the
     /// end of its namespace, so the closed stream is not proof of exhaustion.
     /// Carried in-band by the metacache stream trailer, so remote producers
     /// report it as well.
     pub limit_reached: bool,
+    /// Inclusive sorted boundary proved by every reader before a bounded stop.
+    pub scan_cut: Option<String>,
 }
 
 pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> disk::error::Result<()> {
@@ -722,6 +730,8 @@ async fn list_path_raw_inner(
         // Sticky across rounds: once any reader's trailer says its producer
         // stopped at the per-disk limit, the merged stream may hide more keys.
         let mut limit_reached = false;
+        let mut scan_cut = None;
+        let mut last_scanned_key: Option<String> = None;
 
         loop {
             let mut current = MetaCacheEntry::default();
@@ -733,7 +743,7 @@ async fn list_path_raw_inner(
             // );
 
             if revjob_rx.is_cancelled() {
-                return Ok(ListPathRawOutcome { limit_reached });
+                return Ok(ListPathRawOutcome { limit_reached, scan_cut });
             }
 
             let mut top_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
@@ -747,6 +757,16 @@ async fn list_path_raw_inner(
             // Start every missing head read in the same round so one stalled
             // disk cannot multiply the wait budget by the erasure-set width.
             // Outcomes are still consumed below in stable disk-index order.
+            if !revjob_rx.is_cancelled() {
+                for (i, reader) in readers.iter_mut().enumerate() {
+                    if errs[i].is_none()
+                        && pending_entries[i].is_none()
+                        && let Some(Ok(Some(entry))) = reader.try_peek_buffered()
+                    {
+                        pending_entries[i] = Some(entry);
+                    }
+                }
+            }
             let concurrent_peeks = readers.iter_mut().enumerate().filter_map(|(i, reader)| {
                 if errs[i].is_some() || pending_entries[i].is_some() {
                     return None;
@@ -783,7 +803,12 @@ async fn list_path_raw_inner(
                                     continue;
                                 }
                                 // eof
-                                limit_reached |= r.limit_reached() == Some(true);
+                                if r.limit_reached() == Some(true) {
+                                    limit_reached = true;
+                                    scan_cut = Some(last_scanned_key.clone().ok_or_else(|| {
+                                        DiskError::other("bounded listing reader ended without a scan boundary")
+                                    })?);
+                                }
                                 at_eof += 1;
                                 // warn!("list_path_raw: peek eof, disk: {}", i);
                                 continue;
@@ -857,7 +882,7 @@ async fn list_path_raw_inner(
                             *r = MetacacheReader::new(detached_rd);
                             continue;
                         }
-                        PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached }),
+                        PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
                     }
                 };
 
@@ -878,7 +903,7 @@ async fn list_path_raw_inner(
                 }
                 if entry.name == current.name {
                     // If exact match, we agree.
-                    if current.matches_strict(&entry) {
+                    if current.matches_strict_borrowed(&entry) {
                         top_entries[i] = Some(entry);
                         agree += 1;
                     } else {
@@ -916,7 +941,7 @@ async fn list_path_raw_inner(
                 if let Some(finished_fn) = opts.finished.as_ref() {
                     tokio::select! {
                         biased;
-                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
+                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
                         _ = finished_fn(&errs) => {}
                     }
                 }
@@ -966,6 +991,14 @@ async fn list_path_raw_inner(
                 return Err(classify_listing_quorum_failure(&failures));
             }
 
+            // The next head from a bounded reader is unknown. The previous
+            // globally consumed group is a safe cut; consuming another reader's
+            // later head would turn this absence of evidence into a skipped key.
+            // Check quorum failures above before accepting the bounded result.
+            if limit_reached {
+                break;
+            }
+
             // Break if all at EOF or error.
             if at_eof + has_err == readers.len() {
                 if has_err > 0
@@ -973,7 +1006,7 @@ async fn list_path_raw_inner(
                 {
                     tokio::select! {
                         biased;
-                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
+                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
                         _ = finished_fn(&errs) => {}
                     }
                 }
@@ -984,16 +1017,19 @@ async fn list_path_raw_inner(
                 break;
             }
 
+            last_scanned_key = Some(current.name.clone());
             if agree == readers.len() && !opts.preserve_replica_metadata {
                 for r in readers.iter_mut() {
-                    let _ = r.skip(1).await;
+                    if !r.skip_current() {
+                        let _ = r.skip(1).await;
+                    }
                 }
 
                 if let Some(agreed_fn) = opts.agreed.as_ref() {
                     // warn!("list_path_raw: agreed_fn start, current: {:?}", &current.name);
                     tokio::select! {
                         biased;
-                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
+                        _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
                         result = agreed_fn(current) => result?
                     }
                     // warn!("list_path_raw: agreed_fn done");
@@ -1009,7 +1045,7 @@ async fn list_path_raw_inner(
             }
 
             for (i, r) in readers.iter_mut().enumerate() {
-                if top_entries[i].is_some() {
+                if top_entries[i].is_some() && !r.skip_current() {
                     let _ = r.skip(1).await;
                 }
             }
@@ -1017,12 +1053,12 @@ async fn list_path_raw_inner(
             if let Some(partial_fn) = opts.partial.as_ref() {
                 tokio::select! {
                     biased;
-                    _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
+                    _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
                     _ = partial_fn(MetaCacheEntries(top_entries), &errs) => {}
                 }
             }
         }
-        Ok(ListPathRawOutcome { limit_reached })
+        Ok(ListPathRawOutcome { limit_reached, scan_cut })
     });
 
     let merge_started = std::time::Instant::now();
@@ -1187,6 +1223,24 @@ mod tests {
         assert!(!options.skip_total_timeout);
     }
 
+    #[test]
+    fn cloned_reader_options_preserve_hidden_scan_budget_and_pending_prefix() {
+        let request = ListPathRawOptions {
+            bucket: "versioned".to_owned(),
+            hidden_entry_budget: 100,
+            pending_prefix: Some("pending/".to_owned()),
+            forward_to: Some("pending/0099".to_owned()),
+            ..Default::default()
+        };
+        // Primary and fallback readers both construct their walk from the
+        // per-job clone, rather than directly from the request options.
+        let job = request.clone();
+        let walk = walk_dir_options(&job);
+        assert_eq!(walk.hidden_entry_budget, 100);
+        assert_eq!(walk.pending_prefix.as_deref(), request.pending_prefix.as_deref());
+        assert_eq!(walk.forward_to.as_deref(), Some("pending/0099"));
+    }
+
     #[tokio::test]
     async fn list_path_raw_empty_disks_returns_read_quorum() {
         let err = list_path_raw(CancellationToken::new(), ListPathRawOptions::default())
@@ -1304,6 +1358,7 @@ mod tests {
             outcome.limit_reached,
             "a reader whose trailer reports a limit stop must mark the merged stream as possibly incomplete"
         );
+        assert_eq!(outcome.scan_cut.as_deref(), Some("bucket/object"));
     }
 
     #[tokio::test]

crates/ecstore/src/cluster/rpc/background_pipe.rs

@@ -0,0 +1,682 @@
+// Copyright 2026 RustFS Team
+// Licensed under the Apache License, Version 2.0.
+
+use crate::disk::background_memory::BackgroundBuffer;
+use std::io::{self, IoSlice};
+use std::pin::Pin;
+use std::sync::{Arc, Mutex};
+use std::task::{Context, Poll, Waker, ready};
+use tokio::io::{AsyncRead, AsyncWrite, DuplexStream, ReadBuf};
+
+pub struct BackgroundPipeReader {
+    inner: PipeEnd,
+}
+
+pub struct BackgroundPipeWriter {
+    inner: PipeEnd,
+}
+
+impl BackgroundPipeWriter {
+    /// Capture only the payload owner, without retaining node or physical units.
+    pub fn memory_budget(&self) -> Option<Arc<crate::disk::BackgroundMemoryBudget>> {
+        match &self.inner {
+            PipeEnd::Foreground(_) => None,
+            PipeEnd::Background(state) => state
+                .lock()
+                .unwrap_or_else(std::sync::PoisonError::into_inner)
+                .buffer
+                .memory_budget(),
+        }
+    }
+}
+
+enum PipeEnd {
+    Foreground(DuplexStream),
+    Background(Arc<Mutex<PipeState>>),
+}
+
+// The pipe lock may reserve memory. The memory budget never locks a pipe.
+struct PipeState {
+    buffer: BackgroundBuffer,
+    read_offset: usize,
+    readable: usize,
+    capacity: usize,
+    reader_closed: bool,
+    writer_closed: bool,
+    admission_failure: Option<io::ErrorKind>,
+    read_waker: Option<Waker>,
+    write_waker: Option<Waker>,
+}
+
+/// Capture only memory ownership while the request is admitted. Payload storage
+/// is reserved and initialized once, before the first nonempty background write.
+pub fn background_pipe(capacity: usize) -> (BackgroundPipeReader, BackgroundPipeWriter) {
+    let capacity = capacity.max(1);
+    let memory = crate::disk::current_resource_unit().and_then(|resources| resources.memory_budget());
+    match memory {
+        Some(memory) => {
+            let state = Arc::new(Mutex::new(PipeState {
+                buffer: BackgroundBuffer::new(Some(memory)),
+                read_offset: 0,
+                readable: 0,
+                capacity,
+                reader_closed: false,
+                writer_closed: false,
+                admission_failure: None,
+                read_waker: None,
+                write_waker: None,
+            }));
+            (
+                BackgroundPipeReader {
+                    inner: PipeEnd::Background(state.clone()),
+                },
+                BackgroundPipeWriter {
+                    inner: PipeEnd::Background(state),
+                },
+            )
+        }
+        None => {
+            let (reader, writer) = tokio::io::duplex(capacity);
+            (
+                BackgroundPipeReader {
+                    inner: PipeEnd::Foreground(reader),
+                },
+                BackgroundPipeWriter {
+                    inner: PipeEnd::Foreground(writer),
+                },
+            )
+        }
+    }
+}
+
+fn advance_ring(offset: usize, count: usize, capacity: usize) -> usize {
+    let to_end = capacity - offset;
+    if count < to_end { offset + count } else { count - to_end }
+}
+
+fn poll_background_read(state: &Mutex<PipeState>, cx: &mut Context<'_>, output: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
+    if output.remaining() == 0 {
+        return Poll::Ready(Ok(()));
+    }
+    let mut state = state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
+    if state.readable == 0 {
+        if let Some(kind) = state.admission_failure.take() {
+            drop(state);
+            return Poll::Ready(Err(io::Error::new(kind, "background pipe storage admission failed")));
+        }
+        if state.writer_closed {
+            return Poll::Ready(Ok(()));
+        }
+        let replaced = state.read_waker.replace(cx.waker().clone());
+        drop(state);
+        drop(replaced);
+        return Poll::Pending;
+    }
+    let length = state.readable.min(output.remaining());
+    let first = length.min(state.capacity - state.read_offset);
+    output.put_slice(&state.buffer.as_slice()[state.read_offset..state.read_offset + first]);
+    output.put_slice(&state.buffer.as_slice()[..length - first]);
+    state.read_offset = advance_ring(state.read_offset, length, state.capacity);
+    state.readable -= length;
+    let wake = state.write_waker.take();
+    drop(state);
+    if let Some(wake) = wake {
+        wake.wake();
+    }
+    Poll::Ready(Ok(()))
+}
+
+fn poll_background_write(state: &Mutex<PipeState>, cx: &mut Context<'_>, input: &[IoSlice<'_>]) -> Poll<io::Result<usize>> {
+    let mut state = state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
+    if state.reader_closed || state.writer_closed {
+        return Poll::Ready(Err(io::ErrorKind::BrokenPipe.into()));
+    }
+    if input.iter().all(|part| part.is_empty()) {
+        return Poll::Ready(Ok(0));
+    }
+    let available = state.capacity - state.readable;
+    if available == 0 {
+        let replaced = state.write_waker.replace(cx.waker().clone());
+        drop(state);
+        drop(replaced);
+        return Poll::Pending;
+    }
+    let capacity = state.capacity;
+    if let Err(error) = state.buffer.ensure_len(capacity) {
+        // Legacy walk completion may suppress ordinary producer errors. A new
+        // pipe admission failure must also reach the reader instead of EOF.
+        state.admission_failure = Some(error.kind());
+        state.writer_closed = true;
+        let wake = state.read_waker.take();
+        drop(state);
+        if let Some(wake) = wake {
+            wake.wake();
+        }
+        return Poll::Ready(Err(error));
+    }
+    let mut offset = advance_ring(state.read_offset, state.readable, capacity);
+    let mut written = 0;
+    for part in input {
+        let length = part.len().min(available - written);
+        let first = length.min(capacity - offset);
+        state.buffer.as_mut_slice()[offset..offset + first].copy_from_slice(&part[..first]);
+        state.buffer.as_mut_slice()[..length - first].copy_from_slice(&part[first..length]);
+        offset = advance_ring(offset, length, capacity);
+        written += length;
+        if written == available {
+            break;
+        }
+    }
+    state.readable += written;
+    let wake = state.read_waker.take();
+    drop(state);
+    if let Some(wake) = wake {
+        wake.wake();
+    }
+    Poll::Ready(Ok(written))
+}
+
+impl AsyncRead for BackgroundPipeReader {
+    fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, output: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
+        match &mut self.get_mut().inner {
+            PipeEnd::Foreground(reader) => Pin::new(reader).poll_read(cx, output),
+            PipeEnd::Background(state) => {
+                let coop = ready!(tokio::task::coop::poll_proceed(cx));
+                let result = poll_background_read(state, cx, output);
+                if result.is_ready() {
+                    coop.made_progress();
+                }
+                result
+            }
+        }
+    }
+}
+
+impl AsyncWrite for BackgroundPipeWriter {
+    fn poll_write(self: Pin<&mut Self>, cx: &mut Context<'_>, input: &[u8]) -> Poll<io::Result<usize>> {
+        match &mut self.get_mut().inner {
+            PipeEnd::Foreground(writer) => Pin::new(writer).poll_write(cx, input),
+            PipeEnd::Background(state) => {
+                let coop = ready!(tokio::task::coop::poll_proceed(cx));
+                let result = poll_background_write(state, cx, &[IoSlice::new(input)]);
+                if result.is_ready() {
+                    coop.made_progress();
+                }
+                result
+            }
+        }
+    }
+
+    fn poll_write_vectored(self: Pin<&mut Self>, cx: &mut Context<'_>, input: &[IoSlice<'_>]) -> Poll<io::Result<usize>> {
+        match &mut self.get_mut().inner {
+            PipeEnd::Foreground(writer) => Pin::new(writer).poll_write_vectored(cx, input),
+            PipeEnd::Background(state) => {
+                let coop = ready!(tokio::task::coop::poll_proceed(cx));
+                let result = poll_background_write(state, cx, input);
+                if result.is_ready() {
+                    coop.made_progress();
+                }
+                result
+            }
+        }
+    }
+
+    fn is_write_vectored(&self) -> bool {
+        match &self.inner {
+            PipeEnd::Foreground(writer) => writer.is_write_vectored(),
+            PipeEnd::Background(_) => true,
+        }
+    }
+
+    fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+        match &mut self.get_mut().inner {
+            PipeEnd::Foreground(writer) => Pin::new(writer).poll_flush(cx),
+            PipeEnd::Background(_) => Poll::Ready(Ok(())),
+        }
+    }
+
+    fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+        match &mut self.get_mut().inner {
+            PipeEnd::Foreground(writer) => Pin::new(writer).poll_shutdown(cx),
+            PipeEnd::Background(state) => {
+                let mut state = state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
+                state.writer_closed = true;
+                let wake = state.read_waker.take();
+                drop(state);
+                if let Some(wake) = wake {
+                    wake.wake();
+                }
+                Poll::Ready(Ok(()))
+            }
+        }
+    }
+}
+
+impl Drop for BackgroundPipeReader {
+    fn drop(&mut self) {
+        if let PipeEnd::Background(state) = &self.inner {
+            let mut state = state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
+            state.reader_closed = true;
+            let wake = state.write_waker.take();
+            let cancelled = state.read_waker.take();
+            drop(state);
+            drop(cancelled);
+            if let Some(wake) = wake {
+                wake.wake();
+            }
+        }
+    }
+}
+
+impl Drop for BackgroundPipeWriter {
+    fn drop(&mut self) {
+        if let PipeEnd::Background(state) = &self.inner {
+            let mut state = state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
+            state.writer_closed = true;
+            let wake = state.read_waker.take();
+            let cancelled = state.write_waker.take();
+            drop(state);
+            drop(cancelled);
+            if let Some(wake) = wake {
+                wake.wake();
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::disk::BackgroundMemoryBudget;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::time::Duration;
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
+
+    pub(super) fn budget(capacity: usize) -> Arc<BackgroundMemoryBudget> {
+        let budget = BackgroundMemoryBudget::new();
+        budget.set_sampled_limit(u64::try_from(capacity).expect("test capacity fits u64"), Duration::from_secs(30));
+        budget
+    }
+
+    pub(super) fn local_pipe(
+        memory: &Arc<BackgroundMemoryBudget>,
+        capacity: usize,
+    ) -> (BackgroundPipeReader, BackgroundPipeWriter) {
+        let state = Arc::new(Mutex::new(PipeState {
+            buffer: BackgroundBuffer::new(Some(memory.clone())),
+            read_offset: 0,
+            readable: 0,
+            capacity,
+            reader_closed: false,
+            writer_closed: false,
+            admission_failure: None,
+            read_waker: None,
+            write_waker: None,
+        }));
+        (
+            BackgroundPipeReader {
+                inner: PipeEnd::Background(state.clone()),
+            },
+            BackgroundPipeWriter {
+                inner: PipeEnd::Background(state),
+            },
+        )
+    }
+
+    pub(super) fn write_now(writer: &mut (impl AsyncWrite + Unpin), bytes: &[u8]) -> Poll<io::Result<usize>> {
+        Pin::new(writer).poll_write(&mut Context::from_waker(Waker::noop()), bytes)
+    }
+
+    pub(super) fn read_now(reader: &mut (impl AsyncRead + Unpin), bytes: &mut [u8]) -> Poll<io::Result<usize>> {
+        let mut output = ReadBuf::new(bytes);
+        match Pin::new(reader).poll_read(&mut Context::from_waker(Waker::noop()), &mut output) {
+            Poll::Ready(Ok(())) => Poll::Ready(Ok(output.filled().len())),
+            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
+            Poll::Pending => Poll::Pending,
+        }
+    }
+    #[test]
+    fn fixed_ring_allocates_lazily_once_and_keeps_last_owner_charged() {
+        for capacity in [1, 7, 64 * 1024, 1024 * 1024] {
+            let memory = budget(capacity);
+            let (mut reader, mut writer) = local_pipe(&memory, capacity);
+            assert_eq!(memory.snapshot().used, 0);
+            assert!(matches!(write_now(&mut writer, &[]), Poll::Ready(Ok(0))));
+            assert_eq!(memory.snapshot().used, 0);
+            assert!(matches!(write_now(&mut writer, b"x"), Poll::Ready(Ok(1))));
+            assert_eq!(memory.snapshot().used, capacity as u64);
+            memory.set_sampled_limit(0, Duration::ZERO);
+            let mut output = [0];
+            assert!(matches!(read_now(&mut reader, &mut output), Poll::Ready(Ok(1))));
+            assert_eq!(output, *b"x");
+            assert!(matches!(write_now(&mut writer, b"y"), Poll::Ready(Ok(1))));
+            drop(writer);
+            assert_eq!(memory.snapshot().used, capacity as u64, "reader still owns the real ring");
+            assert!(matches!(read_now(&mut reader, &mut output), Poll::Ready(Ok(1))));
+            assert_eq!(output, *b"y");
+            assert!(matches!(read_now(&mut reader, &mut output), Poll::Ready(Ok(0))));
+            assert_eq!(memory.snapshot().used, capacity as u64, "draining does not destroy allocation ownership");
+            drop(reader);
+            assert_eq!(memory.snapshot().used, 0);
+        }
+    }
+
+    #[test]
+    fn admission_error_reaches_writer_and_reader_once_without_payload() {
+        for sample in [None, Some((6, Duration::from_secs(30))), Some((7, Duration::ZERO))] {
+            let memory = BackgroundMemoryBudget::new();
+            if let Some((limit, validity)) = sample {
+                memory.set_sampled_limit(limit, validity);
+            }
+            let (mut reader, mut writer) = local_pipe(&memory, 7);
+            let error = match write_now(&mut writer, b"x") {
+                Poll::Ready(Err(e)) => e,
+                _ => panic!("reject before copying first payload"),
+            };
+            assert!(
+                crate::disk::background_memory::is_memory_pressure(&error),
+                "writer retains the original budget marker"
+            );
+            assert_eq!(memory.snapshot().used, 0);
+            let error = match read_now(&mut reader, &mut [0]) {
+                Poll::Ready(Err(e)) => e,
+                _ => panic!("reader must see a pipe admission failure"),
+            };
+            assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
+            assert!(matches!(read_now(&mut reader, &mut [0]), Poll::Ready(Ok(0))), "failure is delivered once");
+            assert!(matches!(write_now(&mut writer,b"x"),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::BrokenPipe));
+            drop((reader, writer));
+            assert_eq!(memory.snapshot().used, 0);
+        }
+    }
+
+    #[test]
+    fn allocation_overflow_is_a_reader_failure_without_a_leaked_reservation() {
+        let memory = BackgroundMemoryBudget::new();
+        memory.set_sampled_limit(u64::MAX, Duration::from_secs(30));
+        let (mut reader, mut writer) = local_pipe(&memory, usize::MAX);
+        assert!(matches!(write_now(&mut writer,b"x"),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::OutOfMemory));
+        assert_eq!(memory.snapshot().used, 0);
+        assert!(matches!(read_now(&mut reader,&mut[0]),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::OutOfMemory));
+    }
+
+    #[test]
+    fn scalar_and_vectored_wrap_match_a_reference_queue() {
+        use std::collections::VecDeque;
+        for capacity in [1, 3, 7, 31] {
+            let memory = budget(capacity);
+            let (mut reader, mut writer) = local_pipe(&memory, capacity);
+            let mut expected = VecDeque::new();
+            let mut next = 0u8;
+            for turn in 0..300 {
+                let input: Vec<_> = (0..(turn % 11 + 1))
+                    .map(|_| {
+                        let byte = next;
+                        next = next.wrapping_add(1);
+                        byte
+                    })
+                    .collect();
+                let available = capacity - expected.len();
+                let written = if turn % 2 == 0 {
+                    write_now(&mut writer, &input)
+                } else {
+                    let split = input.len() / 2;
+                    Pin::new(&mut writer).poll_write_vectored(
+                        &mut Context::from_waker(Waker::noop()),
+                        &[
+                            io::IoSlice::new(&[]),
+                            io::IoSlice::new(&input[..split]),
+                            io::IoSlice::new(&input[split..]),
+                        ],
+                    )
+                };
+                if available == 0 {
+                    assert!(written.is_pending());
+                } else {
+                    let accepted = input.len().min(available);
+                    assert!(matches!(written,Poll::Ready(Ok(n)) if n==accepted));
+                    expected.extend(&input[..accepted]);
+                }
+                let mut output = vec![0; turn % 5 + 1];
+                let count = output.len().min(expected.len());
+                assert!(matches!(read_now(&mut reader,&mut output),Poll::Ready(Ok(n)) if n==count));
+                for byte in &output[..count] {
+                    assert_eq!(Some(*byte), expected.pop_front());
+                }
+                assert_eq!(memory.snapshot().used, capacity as u64);
+            }
+            drop(writer);
+            let mut output = vec![0; capacity];
+            let count = expected.len();
+            assert!(matches!(read_now(&mut reader,&mut output),Poll::Ready(Ok(n)) if n==count));
+            assert_eq!(&output[..count], expected.into_iter().collect::<Vec<_>>().as_slice());
+            assert!(matches!(read_now(&mut reader, &mut output), Poll::Ready(Ok(0))));
+            drop(reader);
+            assert_eq!(memory.snapshot().used, 0);
+        }
+    }
+
+    #[tokio::test]
+    async fn reader_drop_wakes_blocked_writer_and_writer_drop_wakes_empty_reader() {
+        let memory = budget(7);
+        let (reader, mut writer) = local_pipe(&memory, 7);
+        writer.write_all(b"1234567").await.expect("fill the ring");
+        let task = tokio::spawn(async move { writer.write_all(b"blocked").await });
+        tokio::task::yield_now().await;
+        assert!(!task.is_finished());
+        drop(reader);
+        let error = tokio::time::timeout(Duration::from_secs(1), task)
+            .await
+            .expect("reader drop wakes writer")
+            .expect("writer task joins")
+            .expect_err("closed reader is BrokenPipe");
+        assert_eq!(error.kind(), io::ErrorKind::BrokenPipe);
+        assert_eq!(memory.snapshot().used, 0);
+        let (mut reader, writer) = local_pipe(&memory, 7);
+        let task = tokio::spawn(async move { reader.read(&mut [0]).await });
+        tokio::task::yield_now().await;
+        assert!(!task.is_finished());
+        drop(writer);
+        assert_eq!(
+            tokio::time::timeout(Duration::from_secs(1), task)
+                .await
+                .expect("writer drop wakes reader")
+                .expect("reader task joins")
+                .expect("clean EOF"),
+            0
+        );
+    }
+
+    #[tokio::test]
+    async fn read_wakes_a_full_writer_without_changing_byte_limit() {
+        let memory = budget(7);
+        let (mut reader, mut writer) = local_pipe(&memory, 7);
+        writer.write_all(b"1234567").await.expect("fill initial byte limit");
+        let task = tokio::spawn(async move {
+            writer.write_all(b"x").await.expect("resume after one byte is read");
+            writer
+        });
+        tokio::task::yield_now().await;
+        assert!(!task.is_finished());
+        let mut first = [0];
+        reader.read_exact(&mut first).await.expect("read one byte");
+        let writer = tokio::time::timeout(Duration::from_secs(1), task)
+            .await
+            .expect("read wakes writer")
+            .expect("writer joins");
+        assert_eq!(&first, b"1");
+        drop(writer);
+        let mut all = Vec::new();
+        reader.read_to_end(&mut all).await.expect("drain wrapped bytes");
+        assert_eq!(all, b"234567x");
+        drop(reader);
+        assert_eq!(memory.snapshot().used, 0);
+    }
+
+    #[tokio::test]
+    async fn shutdown_drains_then_eof_and_cancelled_writes_accept_nothing() {
+        let memory = budget(7);
+        let (mut reader, mut writer) = local_pipe(&memory, 7);
+        writer
+            .write_all(b"1234567")
+            .await
+            .expect("fill before cancelling a blocked write");
+        {
+            let blocked = writer.write(b"lost");
+            tokio::pin!(blocked);
+            assert!(futures::poll!(&mut blocked).is_pending());
+        }
+        writer.shutdown().await.expect("shutdown does not discard buffered bytes");
+        assert!(matches!(write_now(&mut writer,b"x"),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::BrokenPipe));
+        let mut out = Vec::new();
+        reader.read_to_end(&mut out).await.expect("drain after shutdown");
+        assert_eq!(out, b"1234567");
+        drop(reader);
+        assert_eq!(memory.snapshot().used, 7, "writer still holds the allocation");
+        drop(writer);
+        assert_eq!(memory.snapshot().used, 0);
+    }
+
+    #[test]
+    fn foreground_keeps_native_duplex_semantics() {
+        let (reader, mut writer) = background_pipe(7);
+        assert!(matches!(write_now(&mut writer, b"12345678"), Poll::Ready(Ok(7))));
+        assert!(write_now(&mut writer, b"x").is_pending());
+        drop(reader);
+        assert!(matches!(write_now(&mut writer,b"x"),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::BrokenPipe));
+        let (mut reader, writer) = background_pipe(7);
+        drop(writer);
+        assert!(matches!(read_now(&mut reader, &mut [0]), Poll::Ready(Ok(0))));
+    }
+
+    #[test]
+    fn cancelled_reader_leaves_the_real_ring_charged_until_writer_drop() {
+        let memory = budget(7);
+        let (reader, mut writer) = local_pipe(&memory, 7);
+        assert!(matches!(write_now(&mut writer, b"x"), Poll::Ready(Ok(1))));
+        drop(reader);
+        assert_eq!(memory.snapshot().used, 7, "the surviving writer still owns the ring allocation");
+        assert!(matches!(write_now(&mut writer,b"y"),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::BrokenPipe));
+        drop(writer);
+        assert_eq!(memory.snapshot().used, 0);
+    }
+
+    #[test]
+    fn runtime_shutdown_releases_queued_writer_but_not_the_surviving_reader() {
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .expect("build isolated runtime");
+        let memory = budget(7);
+        let (reader, mut writer) = local_pipe(&memory, 7);
+        runtime.block_on(async {
+            writer.write_all(b"x").await.expect("allocate ring before runtime shutdown");
+            tokio::spawn(async move {
+                let _writer = writer;
+                std::future::pending::<()>().await;
+            });
+            tokio::task::yield_now().await;
+        });
+        drop(runtime);
+        assert_eq!(memory.snapshot().used, 7, "reader holds the actual allocation after runtime cancellation");
+        drop(reader);
+        assert_eq!(memory.snapshot().used, 0);
+    }
+
+    #[tokio::test]
+    async fn ready_operations_keep_tokio_cooperative_yields() {
+        let memory = budget(7);
+        let (_reader, mut writer) = local_pipe(&memory, 7);
+        let observed = Arc::new(AtomicUsize::new(0));
+        let flag = observed.clone();
+        let task = tokio::spawn(async move {
+            flag.store(1, Ordering::SeqCst);
+        });
+        for _ in 0..1000 {
+            assert_eq!(writer.write(&[]).await.expect("empty operation remains successful"), 0);
+        }
+        assert_eq!(
+            observed.load(Ordering::SeqCst),
+            1,
+            "ready pipe operations must not starve other ready tasks"
+        );
+        task.await.expect("observer task joins");
+    }
+}
+
+#[cfg(test)]
+mod waker_tests {
+    use super::*;
+    use std::sync::{
+        Weak,
+        atomic::{AtomicUsize, Ordering},
+    };
+    use std::task::Wake;
+
+    struct LockCheck {
+        state: Weak<Mutex<PipeState>>,
+        wakes: Arc<AtomicUsize>,
+        drops: Arc<AtomicUsize>,
+    }
+    impl LockCheck {
+        fn unlocked(&self) {
+            if let Some(state) = self.state.upgrade() {
+                assert!(state.try_lock().is_ok(), "waker code must run outside the pipe lock");
+            }
+        }
+    }
+    impl Wake for LockCheck {
+        fn wake(self: Arc<Self>) {
+            self.unlocked();
+            self.wakes.fetch_add(1, Ordering::SeqCst);
+        }
+    }
+    impl Drop for LockCheck {
+        fn drop(&mut self) {
+            self.unlocked();
+            self.drops.fetch_add(1, Ordering::SeqCst);
+        }
+    }
+
+    #[test]
+    fn replaced_and_woken_reader_and_writer_wakers_run_outside_the_lock() {
+        for waiting_reader in [true, false] {
+            let memory = super::tests::budget(7);
+            let (mut reader, mut writer) = super::tests::local_pipe(&memory, 7);
+            let PipeEnd::Background(state) = &reader.inner else { panic!("background fixture") };
+            let weak = Arc::downgrade(state);
+            let wakes = Arc::new(AtomicUsize::new(0));
+            let drops = Arc::new(AtomicUsize::new(0));
+            if !waiting_reader {
+                assert!(matches!(super::tests::write_now(&mut writer, b"1234567"), Poll::Ready(Ok(7))));
+            }
+            for _ in 0..2 {
+                let waker = Waker::from(Arc::new(LockCheck {
+                    state: weak.clone(),
+                    wakes: wakes.clone(),
+                    drops: drops.clone(),
+                }));
+                let mut cx = Context::from_waker(&waker);
+                if waiting_reader {
+                    let mut bytes = [0];
+                    assert!(
+                        Pin::new(&mut reader)
+                            .poll_read(&mut cx, &mut ReadBuf::new(&mut bytes))
+                            .is_pending()
+                    );
+                } else {
+                    assert!(Pin::new(&mut writer).poll_write(&mut cx, b"x").is_pending());
+                }
+                drop(waker);
+            }
+            assert_eq!(drops.load(Ordering::SeqCst), 1, "the replaced waker has been dropped");
+            if waiting_reader {
+                assert!(matches!(super::tests::write_now(&mut writer, b"x"), Poll::Ready(Ok(1))));
+            } else {
+                assert!(matches!(super::tests::read_now(&mut reader, &mut [0]), Poll::Ready(Ok(1))));
+            }
+            assert_eq!(wakes.load(Ordering::SeqCst), 1);
+            assert_eq!(drops.load(Ordering::SeqCst), 2);
+        }
+    }
+}

crates/ecstore/src/cluster/rpc/background_reader_stream.rs

@@ -0,0 +1,586 @@
+// Copyright 2026 RustFS Team
+// Licensed under the Apache License, Version 2.0.
+
+use crate::disk::background_memory::{BackgroundMemoryBudget, Reservation};
+use bytes::{Bytes, BytesMut};
+use futures::Stream;
+use std::io;
+use std::pin::Pin;
+use std::sync::Arc;
+use std::task::{Context, Poll};
+use tokio::io::{AsyncRead, ReadBuf};
+use tokio_util::io::ReaderStream;
+
+struct ReservedReadBytes {
+    // Free each payload view before releasing its shared allocation receipt.
+    bytes: Bytes,
+    _reservation: Arc<Reservation>,
+}
+
+impl AsRef<[u8]> for ReservedReadBytes {
+    fn as_ref(&self) -> &[u8] {
+        &self.bytes
+    }
+}
+
+struct ReaderBuffer {
+    // These initialized bytes are spare storage, not unconsumed source data.
+    bytes: BytesMut,
+    reservation: Option<Arc<Reservation>>,
+    budget: Arc<BackgroundMemoryBudget>,
+    capacity: usize,
+}
+
+impl ReaderBuffer {
+    fn new(budget: Arc<BackgroundMemoryBudget>, capacity: usize) -> Self {
+        Self {
+            bytes: BytesMut::new(),
+            reservation: None,
+            budget,
+            capacity: capacity.max(1),
+        }
+    }
+
+    fn clear(&mut self) {
+        self.bytes = BytesMut::new();
+        self.reservation = None;
+    }
+
+    fn ensure_readable(&mut self) -> io::Result<()> {
+        if !self.bytes.is_empty() {
+            return Ok(());
+        }
+        if self.reservation.is_some() {
+            if self.bytes.try_reclaim(self.capacity) {
+                self.bytes.resize(self.capacity, 0);
+                return Ok(());
+            }
+            // A live output owns this allocation. Drop our empty view before
+            // its receipt, then reserve independent replacement storage.
+            self.clear();
+        }
+        let reservation = self.budget.reserve(self.capacity)?;
+        let mut bytes = Vec::new();
+        bytes
+            .try_reserve_exact(self.capacity)
+            .map_err(|_| io::ErrorKind::OutOfMemory)?;
+        bytes.resize(self.capacity, 0);
+        // Fresh ordinary Bytes is unique and spans the complete allocation.
+        // Unlike from_owner output, this conversion cannot require a copy.
+        let bytes = Bytes::from(bytes.into_boxed_slice())
+            .try_into_mut()
+            .map_err(|_| io::Error::other("fresh reader storage was not unique"))?;
+        self.bytes = bytes;
+        self.reservation = Some(Arc::new(reservation));
+        Ok(())
+    }
+
+    fn take_bytes(&mut self, length: usize) -> io::Result<Bytes> {
+        let reservation = self
+            .reservation
+            .as_ref()
+            .cloned()
+            .ok_or_else(|| io::Error::other("reader storage has no memory reservation"))?;
+        Ok(Bytes::from_owner(ReservedReadBytes {
+            bytes: self.bytes.split_to(length).freeze(),
+            _reservation: reservation,
+        }))
+    }
+}
+
+/// RPC response chunks retain their background memory reservation until the
+/// last clone or slice is dropped. Foreground readers use the existing stream.
+pub struct BackgroundReaderStream<R> {
+    inner: ReaderState<R>,
+}
+
+enum ReaderState<R> {
+    Foreground(ReaderStream<R>),
+    Background { reader: Option<R>, buffer: ReaderBuffer },
+}
+
+impl<R: AsyncRead> BackgroundReaderStream<R> {
+    /// Capture memory ownership inside the admitted scope, before moving the
+    /// response body out of it. This stream retains no node or device capacity.
+    pub fn with_capacity(reader: R, capacity: usize) -> Self {
+        let memory = crate::disk::current_resource_unit().and_then(|resources| resources.memory_budget());
+        let inner = match memory {
+            Some(memory) => ReaderState::Background {
+                reader: Some(reader),
+                buffer: ReaderBuffer::new(memory, capacity),
+            },
+            None => ReaderState::Foreground(ReaderStream::with_capacity(reader, capacity)),
+        };
+        Self { inner }
+    }
+}
+
+impl<R: AsyncRead + Unpin> Stream for BackgroundReaderStream<R> {
+    type Item = io::Result<Bytes>;
+
+    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        let this = self.get_mut();
+        match &mut this.inner {
+            ReaderState::Foreground(reader) => Pin::new(reader).poll_next(cx),
+            ReaderState::Background { reader, buffer } => {
+                let Some(inner) = reader.as_mut() else {
+                    return Poll::Ready(None);
+                };
+                if let Err(error) = buffer.ensure_readable() {
+                    *reader = None;
+                    return Poll::Ready(Some(Err(error)));
+                }
+                let mut output = ReadBuf::new(&mut buffer.bytes);
+                match std::task::ready!(Pin::new(inner).poll_read(cx, &mut output)) {
+                    Ok(()) => {
+                        let length = output.filled().len();
+                        if length == 0 {
+                            buffer.clear();
+                            *reader = None;
+                            Poll::Ready(None)
+                        } else {
+                            let bytes = buffer.take_bytes(length);
+                            if bytes.is_err() {
+                                buffer.clear();
+                                *reader = None;
+                            }
+                            Poll::Ready(Some(bytes))
+                        }
+                    }
+                    Err(error) => {
+                        buffer.clear();
+                        *reader = None;
+                        Poll::Ready(Some(Err(error)))
+                    }
+                }
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
+    use futures::StreamExt;
+    use std::sync::Arc;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::time::Duration;
+    use tokio::io::AsyncWriteExt;
+
+    struct ObservedReader {
+        polls: Arc<AtomicUsize>,
+        fail: bool,
+    }
+
+    impl AsyncRead for ObservedReader {
+        fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, output: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
+            self.polls.fetch_add(1, Ordering::SeqCst);
+            if self.fail {
+                Poll::Ready(Err(io::Error::from(io::ErrorKind::PermissionDenied)))
+            } else {
+                output.put_slice(b"data");
+                Poll::Ready(Ok(()))
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn background_frames_keep_capacity_until_the_last_slice() {
+        let memory = BackgroundMemoryBudget::new();
+        memory.set_sampled_limit(16, Duration::from_secs(30));
+        let (resources, gates) = test_resource_units(2);
+        let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
+            BackgroundReaderStream::with_capacity(io::Cursor::new(b"data"), 16)
+        })
+        .await;
+        assert!(
+            gates.iter().all(|gate| gate.snapshot().active == 0),
+            "the stream captures only memory ownership"
+        );
+        assert_eq!(memory.snapshot().used, 0, "construction does not allocate payload storage");
+        let bytes = stream.next().await.expect("first frame").expect("read frame");
+        assert_eq!(bytes.as_ref(), b"data");
+        assert_eq!(memory.snapshot().used, 16, "charge full capacity rather than visible length");
+        let slice = bytes.slice(1..3);
+        memory.set_sampled_limit(0, Duration::ZERO);
+        drop(bytes);
+        drop(stream);
+        assert_eq!(slice.as_ref(), b"at");
+        assert_eq!(memory.snapshot().used, 16);
+        drop(slice);
+        assert_eq!(memory.snapshot().used, 0);
+    }
+
+    #[tokio::test]
+    async fn background_denial_happens_before_reading_and_never_falls_back() {
+        for sample in [None, Some((7, Duration::from_secs(30))), Some((8, Duration::ZERO))] {
+            let memory = BackgroundMemoryBudget::new();
+            if let Some((limit, validity)) = sample {
+                memory.set_sampled_limit(limit, validity);
+            }
+            let polls = Arc::new(AtomicUsize::new(0));
+            let (resources, _) = test_resource_units(1);
+            let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
+                BackgroundReaderStream::with_capacity(
+                    ObservedReader {
+                        polls: polls.clone(),
+                        fail: false,
+                    },
+                    8,
+                )
+            })
+            .await;
+            let error = stream
+                .next()
+                .await
+                .expect("allocation failure")
+                .expect_err("background allocation denied");
+            assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
+            assert_eq!(polls.load(Ordering::SeqCst), 0, "denial cannot consume source bytes");
+            assert!(stream.next().await.is_none());
+            assert_eq!(memory.snapshot().used, 0);
+        }
+    }
+
+    #[tokio::test]
+    async fn background_pending_read_keeps_storage_through_shrink_or_cancellation() {
+        for cancelled in [false, true] {
+            let memory = BackgroundMemoryBudget::new();
+            memory.set_sampled_limit(16, Duration::from_secs(30));
+            let (resources, _) = test_resource_units(1);
+            let (mut sender, reader) = tokio::io::duplex(4);
+            let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
+                BackgroundReaderStream::with_capacity(reader, 16)
+            })
+            .await;
+            assert!(futures::poll!(stream.next()).is_pending());
+            memory.set_sampled_limit(0, Duration::ZERO);
+            assert_eq!(memory.snapshot().used, 16);
+            let bytes = if cancelled {
+                None
+            } else {
+                sender.write_all(b"data").await.expect("release pending read");
+                Some(
+                    stream
+                        .next()
+                        .await
+                        .expect("pending frame")
+                        .expect("existing allocation survives shrink"),
+                )
+            };
+            drop(stream);
+            if let Some(bytes) = bytes {
+                assert_eq!(bytes.as_ref(), b"data");
+                assert_eq!(memory.snapshot().used, 16);
+                drop(bytes);
+            }
+            assert_eq!(memory.snapshot().used, 0);
+        }
+    }
+
+    #[tokio::test]
+    async fn background_read_error_and_eof_release_storage() {
+        for fail in [false, true] {
+            let memory = BackgroundMemoryBudget::new();
+            memory.set_sampled_limit(8, Duration::from_secs(30));
+            let (resources, _) = test_resource_units(1);
+            let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
+                let reader: Box<dyn AsyncRead + Unpin> = if fail {
+                    Box::new(ObservedReader {
+                        polls: Arc::new(AtomicUsize::new(0)),
+                        fail: true,
+                    })
+                } else {
+                    Box::new(tokio::io::empty())
+                };
+                BackgroundReaderStream::with_capacity(reader, 8)
+            })
+            .await;
+            if fail {
+                let error = stream.next().await.expect("read failure").expect_err("preserve reader error");
+                assert_eq!(error.kind(), io::ErrorKind::PermissionDenied);
+            }
+            assert!(stream.next().await.is_none());
+            assert_eq!(memory.snapshot().used, 0);
+        }
+    }
+
+    #[tokio::test]
+    async fn retained_frame_blocks_new_storage_until_released() {
+        let memory = BackgroundMemoryBudget::new();
+        memory.set_sampled_limit(8, Duration::from_secs(30));
+        let (resources, _) = test_resource_units(1);
+        let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
+            BackgroundReaderStream::with_capacity(io::Cursor::new(b"firstonefollowup"), 8)
+        })
+        .await;
+        let first = stream.next().await.expect("first frame").expect("fits exact budget");
+        assert_eq!(first.as_ref(), b"firstone");
+        let error = stream
+            .next()
+            .await
+            .expect("next allocation")
+            .expect_err("held frame exhausts budget");
+        assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
+        assert_eq!(memory.snapshot().used, 8);
+        drop(stream);
+        drop(first);
+        assert_eq!(memory.snapshot().used, 0);
+    }
+
+    #[tokio::test]
+    async fn consumed_frames_keep_reusable_capacity_until_eof() {
+        let memory = BackgroundMemoryBudget::new();
+        memory.set_sampled_limit(8, Duration::from_secs(30));
+        let (resources, _) = test_resource_units(1);
+        let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
+            BackgroundReaderStream::with_capacity(io::Cursor::new(b"firstonefollowup"), 8)
+        })
+        .await;
+        for expected in [b"firstone", b"followup"] {
+            let bytes = stream.next().await.expect("next frame").expect("released previous capacity");
+            assert_eq!(bytes.as_ref(), expected);
+            assert_eq!(memory.snapshot().used, 8);
+            drop(bytes);
+            assert_eq!(memory.snapshot().used, 8, "the reusable allocation remains owned by the stream");
+        }
+        assert!(stream.next().await.is_none());
+        assert_eq!(memory.snapshot().used, 0);
+    }
+
+    #[tokio::test]
+    async fn foreground_stream_and_unpolled_cancellation_preserve_existing_behavior() {
+        let mut stream = BackgroundReaderStream::with_capacity(io::Cursor::new(b"foreground"), 4);
+        assert!(matches!(stream.inner, ReaderState::Foreground(_)));
+        let mut bytes = Vec::new();
+        while let Some(chunk) = stream.next().await {
+            bytes.extend_from_slice(&chunk.expect("foreground chunk"));
+        }
+        assert_eq!(bytes, b"foreground");
+
+        let memory = BackgroundMemoryBudget::new();
+        let (resources, _) = test_resource_units(1);
+        let stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
+            BackgroundReaderStream::with_capacity(tokio::io::empty(), 8)
+        })
+        .await;
+        drop(stream);
+        assert_eq!(memory.snapshot().used, 0);
+    }
+}
+
+#[cfg(test)]
+mod reuse_tests {
+    use super::*;
+    use bytes::Bytes;
+    use futures::StreamExt;
+    use std::sync::Arc;
+    use std::time::Duration;
+    use std::{
+        io,
+        pin::Pin,
+        task::{Context, Poll},
+    };
+    use tokio::io::{AsyncRead, ReadBuf};
+
+    static DATA: [u8; 1024] = [0x37; 1024];
+    struct ShortReader {
+        offset: usize,
+        length: usize,
+        chunk: usize,
+        fail_at: Option<usize>,
+    }
+    impl AsyncRead for ShortReader {
+        fn poll_read(mut self: Pin<&mut Self>, _: &mut Context<'_>, out: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
+            if self.fail_at == Some(self.offset) {
+                return Poll::Ready(Err(io::ErrorKind::PermissionDenied.into()));
+            }
+            let n = (self.length - self.offset).min(self.chunk).min(out.remaining());
+            out.put_slice(&DATA[self.offset..self.offset + n]);
+            self.offset += n;
+            Poll::Ready(Ok(()))
+        }
+    }
+    fn budget(limit: u64) -> Arc<crate::disk::BackgroundMemoryBudget> {
+        let b = crate::disk::BackgroundMemoryBudget::new();
+        b.set_sampled_limit(limit, Duration::from_secs(30));
+        b
+    }
+    async fn stream<R: AsyncRead>(
+        budget: &Arc<crate::disk::BackgroundMemoryBudget>,
+        reader: R,
+        capacity: usize,
+    ) -> BackgroundReaderStream<R> {
+        let (resources, _) = crate::disk::test_resource_units(1);
+        crate::disk::with_optional_resource_unit(Some(resources.with_memory_budget(budget.clone())), async {
+            BackgroundReaderStream::with_capacity(reader, capacity)
+        })
+        .await
+    }
+    #[tokio::test]
+    async fn complete_consumed_frames_reclaim_the_same_storage_after_expiry() {
+        let b = budget(64);
+        let mut s = stream(&b, io::Cursor::new(&DATA[..]), 64).await;
+        let mut pointer = None;
+        for _ in 0..16 {
+            let frame = s.next().await.expect("next full frame").expect("reclaimed backing");
+            assert_eq!(frame.len(), 64);
+            assert_eq!(frame.as_ref(), &DATA[..64]);
+            if let Some(p) = pointer {
+                assert_eq!(frame.as_ptr(), p);
+            } else {
+                pointer = Some(frame.as_ptr());
+            }
+            b.set_sampled_limit(0, Duration::ZERO);
+            drop(frame);
+            assert_eq!(b.snapshot().used, 64);
+        }
+        assert!(s.next().await.is_none());
+        assert_eq!(b.snapshot().used, 0);
+    }
+    #[tokio::test]
+    async fn short_live_frames_use_disjoint_tail_then_reserve_separate_storage() {
+        let b = budget(32);
+        let mut s = stream(
+            &b,
+            ShortReader {
+                offset: 0,
+                length: 32,
+                chunk: 4,
+                fail_at: None,
+            },
+            16,
+        )
+        .await;
+        let mut frames: Vec<Bytes> = Vec::new();
+        for i in 0..4 {
+            let frame = s.next().await.expect("short frame").expect("remaining initialized tail");
+            if i != 0 {
+                assert_eq!(frame.as_ptr(), frames[0].as_ptr().wrapping_add(i * 4));
+            }
+            assert_eq!(frame.as_ref(), &DATA[..4]);
+            frames.push(frame);
+            assert_eq!(b.snapshot().used, 16);
+        }
+        let next = s.next().await.expect("independent frame").expect("second backing fits");
+        assert_ne!(next.as_ptr(), frames[0].as_ptr());
+        assert_eq!(b.snapshot().used, 32);
+        drop(frames);
+        assert_eq!(b.snapshot().used, 16);
+        drop(s);
+        assert_eq!(b.snapshot().used, 16);
+        drop(next);
+        assert_eq!(b.snapshot().used, 0);
+    }
+    #[tokio::test]
+    async fn final_slice_prevents_reclaim_and_keeps_old_plus_new_charged() {
+        let b = budget(32);
+        let mut s = stream(&b, io::Cursor::new(&DATA[..32]), 16).await;
+        let first = s.next().await.expect("old frame").expect("old backing");
+        let last = first.slice(15..);
+        drop(first);
+        let next = s.next().await.expect("new frame").expect("second backing fits");
+        assert_eq!(last.as_ref(), &DATA[..1]);
+        assert_ne!(next.as_ptr(), last.as_ptr());
+        assert_eq!(b.snapshot().used, 32);
+        drop(s);
+        assert_eq!(b.snapshot().used, 32);
+        drop(next);
+        assert_eq!(b.snapshot().used, 16);
+        drop(last);
+        assert_eq!(b.snapshot().used, 0);
+    }
+    #[tokio::test]
+    async fn expiry_uses_tail_but_rejects_new_storage_before_consuming_source() {
+        let b = budget(16);
+        let mut s = stream(
+            &b,
+            ShortReader {
+                offset: 0,
+                length: 32,
+                chunk: 4,
+                fail_at: None,
+            },
+            16,
+        )
+        .await;
+        let first = s.next().await.expect("first frame").expect("initial backing");
+        b.set_sampled_limit(0, Duration::ZERO);
+        for _ in 0..3 {
+            drop(s.next().await.expect("tail frame").expect("reuse charged storage"));
+        }
+        let error = s
+            .next()
+            .await
+            .expect("new backing denied")
+            .expect_err("live frame prevents reclaim");
+        assert!(crate::disk::background_memory::is_memory_pressure(&error));
+        assert!(s.next().await.is_none());
+        assert_eq!(b.snapshot().used, 16);
+        drop(first);
+        assert_eq!(b.snapshot().used, 0);
+    }
+    #[tokio::test]
+    async fn eof_and_error_release_spare_but_keep_live_output_charged() {
+        for fail_at in [None, Some(4)] {
+            let b = budget(16);
+            let mut s = stream(
+                &b,
+                ShortReader {
+                    offset: 0,
+                    length: 4,
+                    chunk: 4,
+                    fail_at,
+                },
+                16,
+            )
+            .await;
+            let frame = s.next().await.expect("first short frame").expect("payload");
+            match fail_at {
+                None => assert!(s.next().await.is_none()),
+                Some(_) => assert_eq!(
+                    s.next()
+                        .await
+                        .expect("read error")
+                        .expect_err("preserve source failure")
+                        .kind(),
+                    io::ErrorKind::PermissionDenied
+                ),
+            }
+            assert!(s.next().await.is_none());
+            assert_eq!(b.snapshot().used, 16);
+            assert_eq!(frame.as_ref(), &DATA[..4]);
+            drop(frame);
+            assert_eq!(b.snapshot().used, 0);
+        }
+    }
+    #[test]
+    fn runtime_shutdown_drops_a_pending_stream_without_refunding_early() {
+        for abort in [false, true] {
+            let b = budget(16);
+            let rt = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .expect("runtime");
+            let (_tx, reader) = tokio::io::duplex(4);
+            let (began, wait) = tokio::sync::oneshot::channel();
+            let memory = b.clone();
+            let handle = rt.spawn(async move {
+                let mut s = stream(&memory, reader, 16).await;
+                assert!(futures::poll!(s.next()).is_pending());
+                assert_eq!(memory.snapshot().used, 16);
+                began.send(()).expect("notify pending allocation");
+                let _ = s.next().await;
+            });
+            rt.block_on(wait).expect("stream pending");
+            assert_eq!(b.snapshot().used, 16);
+            if abort {
+                handle.abort();
+                let _ = rt.block_on(handle);
+            }
+            drop(rt);
+            assert_eq!(b.snapshot().used, 0);
+        }
+    }
+}

crates/ecstore/src/cluster/rpc/client.rs

@@ -17,10 +17,10 @@ use crate::cluster::rpc::http_auth::RPC_REPLAY_SCOPE_VERSION_HEADER;
 use crate::cluster::rpc::http_auth::{
     AuthenticatedPeerReplayCapabilities, RPC_AUTH_VERSION_HEADER, RPC_AUTH_VERSION_V2, RPC_BOOT_EPOCH_CHALLENGE_HEADER,
     RPC_CONTENT_SHA256_HEADER, RPC_REPLAY_CACHE_CAPABILITY_HEADER, RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER,
-    RollingMutationBodyDigest, TIMESTAMP_HEADER, internode_rpc_body_digest_strict,
-    verify_tonic_peer_replay_capabilities_response,
+    RollingMutationBodyDigest, TIMESTAMP_HEADER, gen_tonic_replay_scope_headers_with_admission, internode_rpc_body_digest_strict,
+    rpc_background_admission, verify_tonic_peer_replay_capabilities_response,
 };
-use crate::cluster::rpc::{gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience};
+use crate::cluster::rpc::{gen_tonic_signature_headers, normalize_tonic_rpc_audience};
 #[cfg(test)]
 use crate::cluster::rpc::{tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers};
 use crate::disk::error::{DiskError, Error as DiskErrorType, RpcStatusError};
@@ -415,7 +415,16 @@ where
                     .get(RPC_CONTENT_SHA256_HEADER)
                     .and_then(|value| value.to_str().ok()),
             ) {
-                match gen_tonic_replay_scope_headers(audience, request.uri().path(), timestamp, content_sha256, boot_epoch) {
+                match rpc_background_admission(request.headers()).and_then(|background| {
+                    gen_tonic_replay_scope_headers_with_admission(
+                        audience,
+                        request.uri().path(),
+                        timestamp,
+                        content_sha256,
+                        boot_epoch,
+                        background,
+                    )
+                }) {
                     Ok(headers) => request.headers_mut().extend(headers),
                     Err(error) => debug!(error = %error, "could not attach replay-scoped RPC signature"),
                 }

crates/ecstore/src/cluster/rpc/http_auth.rs

@@ -32,7 +32,6 @@ use crate::storage_api_contracts::internode::{
     PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_CAPABILITY_VERSION,
 };
 
-use hmac::{Hmac, KeyInit, Mac};
 use http::uri::Authority;
 use http::{HeaderMap, HeaderValue, Method, Uri};
 #[cfg(test)]
@@ -47,6 +46,7 @@ use rustfs_io_metrics::internode_metrics::{
     global_internode_metrics,
 };
 use rustfs_object_data_cache::{MemoryBasis, resolve_effective_memory};
+use rustfs_utils::crypto::{HmacSha256, KeyInit, Mac};
 use rustfs_utils::get_env_bool;
 use sha2::Digest as _;
 use sha2::Sha256;
@@ -58,9 +58,10 @@ use time::OffsetDateTime;
 use tracing::{error, info, warn};
 use uuid::Uuid;
 
-type HmacSha256 = Hmac<Sha256>;
-
 const SIGNATURE_HEADER: &str = "x-rustfs-signature";
+pub const RPC_BACKGROUND_ADMISSION_HEADER: &str = "x-rustfs-rpc-background-admission";
+const RPC_BACKGROUND_ADMISSION_V1: &str = "v1";
+const RPC_BACKGROUND_ADMISSION_DOMAIN: &[u8] = b"rustfs-rpc-background-admission-v1\0";
 pub(crate) const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
 pub(crate) const RPC_AUTH_VERSION_HEADER: &str = "x-rustfs-rpc-auth-version";
 const RPC_SIGNATURE_V2_HEADER: &str = "x-rustfs-rpc-signature-v2";
@@ -77,6 +78,46 @@ pub(crate) const RPC_REPLAY_CACHE_CAPABILITY_HEADER: &str = "x-rustfs-rpc-replay
 pub(crate) const RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER: &str = "x-rustfs-rpc-replay-cache-capability-proof";
 const RPC_REPLAY_SCOPE_VERSION_V3: &str = "3";
 const RPC_RESPONSE_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-response-proof-v1\0";
+
+tokio::task_local! {
+    static RPC_BACKGROUND_REQUEST: bool;
+}
+
+pub(crate) fn current_rpc_background_request() -> bool {
+    RPC_BACKGROUND_REQUEST
+        .try_with(|background| *background)
+        .unwrap_or_else(|_| crate::disk::has_resource_io_scope())
+}
+
+/// Preserve the authenticated wire class while polling locally admitted work.
+/// This scope carries only the class, never node, disk, or memory ownership.
+pub async fn with_rpc_background_request<F: std::future::Future>(background: bool, operation: F) -> F::Output {
+    RPC_BACKGROUND_REQUEST.scope(background, operation).await
+}
+
+/// Parse the request class only; callers must verify the signature before using it.
+pub fn rpc_background_admission(headers: &HeaderMap) -> std::io::Result<bool> {
+    let mut values = headers.get_all(RPC_BACKGROUND_ADMISSION_HEADER).iter();
+    match (values.next(), values.next()) {
+        (None, None) => Ok(false),
+        (Some(value), None) if value == RPC_BACKGROUND_ADMISSION_V1 => Ok(true),
+        _ => Err(std::io::Error::other("Invalid RPC background admission version")),
+    }
+}
+
+fn bind_background_admission(mac: &mut HmacSha256, background: bool) {
+    if background {
+        // Every auth lane binds the class, so stripping the marker or falling
+        // back to an older signature cannot turn admitted work into foreground I/O.
+        mac.update(RPC_BACKGROUND_ADMISSION_DOMAIN);
+    }
+}
+
+fn insert_background_admission(headers: &mut HeaderMap, background: bool) {
+    if background {
+        headers.insert(RPC_BACKGROUND_ADMISSION_HEADER, HeaderValue::from_static(RPC_BACKGROUND_ADMISSION_V1));
+    }
+}
 const RPC_REPLAY_SCOPE_DOMAIN: &[u8] = b"rustfs-rpc-replay-scope-v3\0";
 const RPC_BOOT_EPOCH_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-boot-epoch-proof-v1\0";
 const RPC_REPLAY_CACHE_CAPABILITY_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-replay-cache-capability-proof-v1\0";
@@ -519,21 +560,23 @@ fn redacted_rpc_path(url: &str) -> String {
 }
 
 /// Generate HMAC-SHA256 signature for the given data
-fn generate_signature(secret: &str, url: &str, method: &Method, timestamp: i64) -> String {
+fn generate_signature(secret: &str, url: &str, method: &Method, timestamp: i64, background: bool) -> String {
     let data = signature_payload(url, method, timestamp);
     let mut mac = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).expect("HMAC can take key of any size");
+    bind_background_admission(&mut mac, background);
     mac.update(data.as_bytes());
     let result = mac.finalize();
     base64_simd::STANDARD.encode_to_string(result.into_bytes())
 }
 
-fn verify_signature(secret: &str, url: &str, method: &Method, timestamp: i64, signature: &str) -> bool {
+fn verify_signature(secret: &str, url: &str, method: &Method, timestamp: i64, signature: &str, background: bool) -> bool {
     let Ok(signature) = base64_simd::STANDARD.decode_to_vec(signature) else {
         return false;
     };
 
     let data = signature_payload(url, method, timestamp);
     let mut mac = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).expect("HMAC can take key of any size");
+    bind_background_admission(&mut mac, background);
     mac.update(data.as_bytes());
     mac.verify_slice(&signature).is_ok()
 }
@@ -771,6 +814,7 @@ pub fn verify_ns_scanner_capability_with_checkpoint_coverage(
 
 #[derive(Clone, Copy)]
 struct SignatureV2Scope<'a> {
+    background: bool,
     audience: &'a str,
     service: &'a str,
     rpc_method: &'a str,
@@ -780,6 +824,7 @@ struct SignatureV2Scope<'a> {
 }
 
 fn update_signature_v2(mac: &mut HmacSha256, scope: SignatureV2Scope<'_>) {
+    bind_background_admission(mac, scope.background);
     for part in [
         b"rustfs-rpc-auth-v2|".as_slice(),
         scope.audience.as_bytes(),
@@ -818,6 +863,7 @@ fn verify_signature_v2(secret: &str, scope: SignatureV2Scope<'_>, signature: &st
 
 #[derive(Clone, Copy)]
 struct ReplayScope<'a> {
+    background: bool,
     audience: &'a str,
     path: &'a str,
     timestamp: &'a str,
@@ -827,6 +873,7 @@ struct ReplayScope<'a> {
 }
 
 fn update_replay_scope(mac: &mut HmacSha256, scope: ReplayScope<'_>) {
+    bind_background_admission(mac, scope.background);
     mac.update(RPC_REPLAY_SCOPE_DOMAIN);
     for part in [
         scope.audience.as_bytes(),
@@ -970,6 +1017,24 @@ pub fn gen_tonic_replay_scope_headers(
     timestamp: &str,
     content_sha256: &str,
     boot_epoch: Uuid,
+) -> std::io::Result<HeaderMap> {
+    gen_tonic_replay_scope_headers_with_admission(
+        audience,
+        path,
+        timestamp,
+        content_sha256,
+        boot_epoch,
+        current_rpc_background_request(),
+    )
+}
+
+pub(crate) fn gen_tonic_replay_scope_headers_with_admission(
+    audience: &str,
+    path: &str,
+    timestamp: &str,
+    content_sha256: &str,
+    boot_epoch: Uuid,
+    background: bool,
 ) -> std::io::Result<HeaderMap> {
     if audience.is_empty() || !path.starts_with('/') || !valid_content_sha256(content_sha256) || boot_epoch.is_nil() {
         return Err(std::io::Error::other("Invalid replay-scoped RPC signing scope"));
@@ -983,6 +1048,7 @@ pub fn gen_tonic_replay_scope_headers(
     let signature = generate_replay_scope_signature(
         &get_shared_secret()?,
         ReplayScope {
+            background,
             audience,
             path,
             timestamp,
@@ -992,6 +1058,7 @@ pub fn gen_tonic_replay_scope_headers(
         },
     )?;
     let mut headers = HeaderMap::new();
+    insert_background_admission(&mut headers, background);
     headers.insert(RPC_REPLAY_SCOPE_VERSION_HEADER, HeaderValue::from_static(RPC_REPLAY_SCOPE_VERSION_V3));
     headers.insert(
         RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
@@ -1226,10 +1293,12 @@ pub fn build_auth_headers(url: &str, method: &Method, headers: &mut HeaderMap) -
 pub fn gen_signature_headers(url: &str, method: &Method) -> std::io::Result<HeaderMap> {
     let secret = get_shared_secret()?;
     let timestamp = OffsetDateTime::now_utc().unix_timestamp();
+    let background = current_rpc_background_request();
 
-    let signature = generate_signature(&secret, url, method, timestamp);
+    let signature = generate_signature(&secret, url, method, timestamp, background);
 
     let mut headers = HeaderMap::new();
+    insert_background_admission(&mut headers, background);
     headers.insert(SIGNATURE_HEADER, HeaderValue::from_str(&signature).expect("Invalid header value"));
     headers.insert(
         TIMESTAMP_HEADER,
@@ -1264,10 +1333,12 @@ pub fn gen_tonic_signature_headers(
     let timestamp_header = timestamp.to_string();
     let body_nonce = (content_sha256 != UNSIGNED_PAYLOAD).then(|| Uuid::new_v4().to_string());
     let nonce = body_nonce.as_deref().unwrap_or(UNSIGNED_PAYLOAD_NONCE);
-    let legacy_signature = generate_signature(&secret, TONIC_RPC_PREFIX, &Method::GET, timestamp);
+    let background = current_rpc_background_request();
+    let legacy_signature = generate_signature(&secret, TONIC_RPC_PREFIX, &Method::GET, timestamp, background);
     let signature_v2 = generate_signature_v2(
         &secret,
         SignatureV2Scope {
+            background,
             audience,
             service,
             rpc_method,
@@ -1278,6 +1349,7 @@ pub fn gen_tonic_signature_headers(
     )?;
 
     let mut headers = HeaderMap::new();
+    insert_background_admission(&mut headers, background);
     headers.insert(SIGNATURE_HEADER, header_value(&legacy_signature, SIGNATURE_HEADER)?);
     headers.insert(TIMESTAMP_HEADER, header_value(&timestamp_header, TIMESTAMP_HEADER)?);
     headers.insert(RPC_AUTH_VERSION_HEADER, HeaderValue::from_static(RPC_AUTH_VERSION_V2));
@@ -1501,6 +1573,7 @@ fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &Hea
     if !verify_replay_scope_signature(
         &secret,
         ReplayScope {
+            background: rpc_background_admission(headers)?,
             audience,
             path,
             timestamp,
@@ -1515,7 +1588,13 @@ fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &Hea
     if boot_epoch != tonic_rpc_boot_epoch() {
         return Err(std::io::Error::other("RPC boot epoch is stale"));
     }
-    check_and_record_tonic_nonce(nonce, signed_at, path)
+    check_and_record_tonic_nonce(nonce, signed_at, path)?;
+    // A body-bound request also carries a usable v2 signature. Consume both
+    // nonces so stripping the replay-scope headers cannot authorize it again.
+    if content_sha256 != UNSIGNED_PAYLOAD {
+        verify_tonic_rpc_signature_with_strictness(audience, path, headers, true)?;
+    }
+    Ok(())
 }
 
 /// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
@@ -1625,6 +1704,9 @@ fn verify_tonic_rpc_signature_with_strictness(
     strict: bool,
 ) -> std::io::Result<()> {
     if !has_v2_auth_headers(headers) {
+        if rpc_background_admission(headers)? {
+            return Err(std::io::Error::other("RPC background admission requires v2 authentication"));
+        }
         // RUSTFS_COMPAT_TODO(heal-rpc-auth-v2): accept old peers during rolling upgrades. Remove after the minimum
         // supported RustFS peer version sends v2 authentication on every internode gRPC request.
         if strict {
@@ -1696,6 +1778,7 @@ fn verify_tonic_rpc_signature_with_strictness(
     if !verify_signature_v2(
         &secret,
         SignatureV2Scope {
+            background: rpc_background_admission(headers)?,
             audience,
             service,
             rpc_method,
@@ -1736,7 +1819,7 @@ pub fn verify_rpc_signature(url: &str, method: &Method, headers: &HeaderMap) ->
     // Verify signature with constant-time HMAC comparison.
     let secret = get_shared_secret()?;
 
-    if !verify_signature(&secret, url, method, timestamp, signature) {
+    if !verify_signature(&secret, url, method, timestamp, signature, rpc_background_admission(headers)?) {
         let rpc_path = redacted_rpc_path(url);
         error!(
             rpc_path = %rpc_path,
@@ -1828,6 +1911,98 @@ mod tests {
         runtime_sources::ensure_test_rpc_secret();
     }
 
+    #[tokio::test]
+    async fn background_admission_is_bound_in_http_and_both_tonic_signature_lanes() {
+        ensure_test_rpc_secret();
+        let secret = get_shared_secret().expect("RPC secret");
+        let path = "/node_service.NodeService/ReadAll";
+        for background in [false, true] {
+            let (http, tonic) = with_rpc_background_request(background, async {
+                (
+                    gen_signature_headers("/rustfs/rpc/read?disk=a", &Method::GET).expect("HTTP signature"),
+                    gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "ReadAll", None)
+                        .expect("gRPC signature"),
+                )
+            })
+            .await;
+            assert_eq!(rpc_background_admission(&http).expect("HTTP class"), background);
+            assert_eq!(rpc_background_admission(&tonic).expect("gRPC class"), background);
+            verify_rpc_signature("/rustfs/rpc/read?disk=a", &Method::GET, &http).expect("HTTP authentication");
+            verify_tonic_rpc_signature_with_strictness("node-a:9000", path, &tonic, true).expect("v2 authentication");
+
+            let timestamp = tonic[TIMESTAMP_HEADER].to_str().expect("timestamp");
+            let content = tonic[RPC_CONTENT_SHA256_HEADER].to_str().expect("content digest");
+            assert_eq!(
+                verify_signature_v2(
+                    &secret,
+                    SignatureV2Scope {
+                        background: false,
+                        audience: "node-a:9000",
+                        service: "node_service.NodeService",
+                        rpc_method: "ReadAll",
+                        timestamp,
+                        nonce: tonic[RPC_NONCE_HEADER].to_str().expect("nonce"),
+                        content_sha256: content,
+                    },
+                    tonic[RPC_SIGNATURE_V2_HEADER].to_str().expect("signature"),
+                ),
+                !background,
+                "an old peer cannot authenticate a background request as foreground"
+            );
+
+            let mut replay = tonic.clone();
+            // The channel can run outside the caller's task-local scope.
+            replay.extend(
+                gen_tonic_replay_scope_headers_with_admission(
+                    "node-a:9000",
+                    path,
+                    timestamp,
+                    content,
+                    tonic_rpc_boot_epoch(),
+                    background,
+                )
+                .expect("v3 signature"),
+            );
+            for mutation in ["strip-or-add", "unknown", "duplicate"] {
+                let alter = |headers: &HeaderMap| {
+                    let mut changed = headers.clone();
+                    match mutation {
+                        "strip-or-add" if background => {
+                            changed.remove(RPC_BACKGROUND_ADMISSION_HEADER);
+                        }
+                        "strip-or-add" => insert_background_admission(&mut changed, true),
+                        "unknown" => {
+                            changed.insert(RPC_BACKGROUND_ADMISSION_HEADER, HeaderValue::from_static("v2"));
+                        }
+                        _ => {
+                            insert_background_admission(&mut changed, true);
+                            changed.append(RPC_BACKGROUND_ADMISSION_HEADER, HeaderValue::from_static("v1"));
+                        }
+                    }
+                    changed
+                };
+                assert!(verify_rpc_signature("/rustfs/rpc/read?disk=a", &Method::GET, &alter(&http)).is_err());
+                assert!(verify_tonic_rpc_signature_with_strictness("node-a:9000", path, &alter(&tonic), true).is_err());
+                assert!(verify_tonic_replay_scope_signature("node-a:9000", path, &alter(&replay)).is_err());
+            }
+            verify_tonic_replay_scope_signature("node-a:9000", path, &replay).expect("v3 authentication");
+            assert!(
+                verify_tonic_replay_scope_signature("node-a:9000", path, &replay).is_err(),
+                "replay rejected"
+            );
+
+            if background {
+                let mut downgraded = HeaderMap::new();
+                for name in [SIGNATURE_HEADER, TIMESTAMP_HEADER, RPC_BACKGROUND_ADMISSION_HEADER] {
+                    downgraded.insert(name, tonic[name].clone());
+                }
+                assert!(verify_tonic_rpc_signature_with_strictness("node-a:9000", path, &downgraded, false).is_err());
+                downgraded.remove(RPC_BACKGROUND_ADMISSION_HEADER);
+                assert!(verify_tonic_rpc_signature_with_strictness("node-a:9000", path, &downgraded, false).is_err());
+            }
+        }
+    }
+
     #[test]
     fn namespace_scanner_capability_proof_binds_challenge_and_server_epoch() {
         let secret = "test-scanner-capability-secret";
@@ -1965,8 +2140,8 @@ mod tests {
         let method = Method::GET;
         let timestamp = 1640995200; // Fixed timestamp
 
-        let signature1 = generate_signature(secret, url, &method, timestamp);
-        let signature2 = generate_signature(secret, url, &method, timestamp);
+        let signature1 = generate_signature(secret, url, &method, timestamp, false);
+        let signature2 = generate_signature(secret, url, &method, timestamp, false);
 
         assert_eq!(signature1, signature2, "Same inputs should produce same signature");
         assert!(!signature1.is_empty(), "Signature should not be empty");
@@ -1979,10 +2154,10 @@ mod tests {
         let method = Method::GET;
         let timestamp = 1640995200;
 
-        let signature1 = generate_signature(secret, url, &method, timestamp);
-        let signature2 = generate_signature(secret, "http://different.com/api/test2", &method, timestamp);
-        let signature3 = generate_signature(secret, url, &Method::POST, timestamp);
-        let signature4 = generate_signature(secret, url, &method, timestamp + 1);
+        let signature1 = generate_signature(secret, url, &method, timestamp, false);
+        let signature2 = generate_signature(secret, "http://different.com/api/test2", &method, timestamp, false);
+        let signature3 = generate_signature(secret, url, &Method::POST, timestamp, false);
+        let signature4 = generate_signature(secret, url, &method, timestamp + 1, false);
 
         assert_ne!(signature1, signature2, "Different URLs should produce different signatures");
         assert_ne!(signature1, signature3, "Different methods should produce different signatures");
@@ -2088,14 +2263,14 @@ mod tests {
         let url = "http://example.com/api/test";
         let method = Method::GET;
         let timestamp = 1640995200;
-        let signature = generate_signature(secret, url, &method, timestamp);
+        let signature = generate_signature(secret, url, &method, timestamp, false);
         let mut tampered = base64_simd::STANDARD.decode_to_vec(&signature).unwrap();
         tampered[0] ^= 1;
         let tampered_signature = base64_simd::STANDARD.encode_to_string(tampered);
 
-        assert!(verify_signature(secret, url, &method, timestamp, &signature));
-        assert!(!verify_signature(secret, url, &method, timestamp, &tampered_signature));
-        assert!(!verify_signature(secret, url, &method, timestamp, "invalid-signature"));
+        assert!(verify_signature(secret, url, &method, timestamp, &signature, false));
+        assert!(!verify_signature(secret, url, &method, timestamp, &tampered_signature, false));
+        assert!(!verify_signature(secret, url, &method, timestamp, "invalid-signature", false));
     }
 
     #[test]
@@ -2112,11 +2287,11 @@ mod tests {
         );
         let method = Method::GET;
         let timestamp = 1_640_995_200;
-        let signature = generate_signature(secret, signed_url, &method, timestamp);
+        let signature = generate_signature(secret, signed_url, &method, timestamp, false);
 
-        assert!(verify_signature(secret, signed_url, &method, timestamp, &signature));
-        assert!(!verify_signature(secret, downgraded_url, &method, timestamp, &signature));
-        assert!(!verify_signature(secret, tampered_body_digest, &method, timestamp, &signature));
+        assert!(verify_signature(secret, signed_url, &method, timestamp, &signature, false));
+        assert!(!verify_signature(secret, downgraded_url, &method, timestamp, &signature, false));
+        assert!(!verify_signature(secret, tampered_body_digest, &method, timestamp, &signature, false));
     }
 
     #[test]
@@ -2126,7 +2301,7 @@ mod tests {
         let method = Method::GET;
         let timestamp = OffsetDateTime::now_utc().unix_timestamp();
         let secret = get_shared_secret().expect("test RPC secret should resolve");
-        let expected_signature = generate_signature(&secret, url, &method, timestamp);
+        let expected_signature = generate_signature(&secret, url, &method, timestamp, false);
         let invalid_signature = "invalid-signature";
         let logs = CapturedLogs::default();
         let subscriber = tracing_subscriber::fmt()
@@ -2164,7 +2339,7 @@ mod tests {
         // Set expired timestamp (older than SIGNATURE_VALID_DURATION)
         let expired_timestamp = OffsetDateTime::now_utc().unix_timestamp() - SIGNATURE_VALID_DURATION - 10;
         let secret = get_shared_secret().expect("test RPC secret should resolve");
-        let signature = generate_signature(&secret, url, &method, expired_timestamp);
+        let signature = generate_signature(&secret, url, &method, expired_timestamp, false);
 
         headers.insert(SIGNATURE_HEADER, HeaderValue::from_str(&signature).unwrap());
         headers.insert(TIMESTAMP_HEADER, HeaderValue::from_str(&expired_timestamp.to_string()).unwrap());
@@ -2186,7 +2361,7 @@ mod tests {
 
         let future_timestamp = OffsetDateTime::now_utc().unix_timestamp() + SIGNATURE_VALID_DURATION + 10;
         let secret = get_shared_secret().expect("test RPC secret should resolve");
-        let signature = generate_signature(&secret, url, &method, future_timestamp);
+        let signature = generate_signature(&secret, url, &method, future_timestamp, false);
 
         headers.insert(SIGNATURE_HEADER, HeaderValue::from_str(&signature).unwrap());
         headers.insert(TIMESTAMP_HEADER, HeaderValue::from_str(&future_timestamp.to_string()).unwrap());
@@ -2300,7 +2475,7 @@ mod tests {
         // Test timestamp just within valid duration
         let valid_timestamp = current_time - SIGNATURE_VALID_DURATION + 1;
 
-        let signature = generate_signature(&secret, url, &method, valid_timestamp);
+        let signature = generate_signature(&secret, url, &method, valid_timestamp, false);
 
         headers.insert(SIGNATURE_HEADER, HeaderValue::from_str(&signature).unwrap());
         headers.insert(TIMESTAMP_HEADER, HeaderValue::from_str(&valid_timestamp.to_string()).unwrap());
@@ -2311,7 +2486,7 @@ mod tests {
         // Test timestamp just outside valid duration
         let mut headers = HeaderMap::new();
         let invalid_timestamp = current_time - SIGNATURE_VALID_DURATION - 15;
-        let signature = generate_signature(&secret, url, &method, invalid_timestamp);
+        let signature = generate_signature(&secret, url, &method, invalid_timestamp, false);
 
         headers.insert(SIGNATURE_HEADER, HeaderValue::from_str(&signature).unwrap());
         headers.insert(TIMESTAMP_HEADER, HeaderValue::from_str(&invalid_timestamp.to_string()).unwrap());
@@ -2410,6 +2585,43 @@ mod tests {
         assert_eq!(path_error.to_string(), "Invalid RPC replay scope signature");
     }
 
+    #[test]
+    fn replay_scope_consumes_the_body_bound_v2_nonce() {
+        ensure_test_rpc_secret();
+        let audience = "node-a:9000";
+        let path = "/node_service.NodeService/RecoverDeleteTransaction";
+        let digest = "a".repeat(64);
+        for v3_first in [true, false] {
+            let mut headers =
+                gen_tonic_signature_headers(audience, "node_service.NodeService", "RecoverDeleteTransaction", Some(&digest))
+                    .expect("body-bound v2 headers should build");
+            let timestamp = headers[TIMESTAMP_HEADER].to_str().expect("timestamp").to_string();
+            headers.extend(
+                gen_tonic_replay_scope_headers(audience, path, &timestamp, &digest, tonic_rpc_boot_epoch())
+                    .expect("replay-scope headers should build"),
+            );
+            let mut downgraded = headers.clone();
+            for header in [
+                RPC_REPLAY_SCOPE_VERSION_HEADER,
+                RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
+                RPC_REPLAY_SCOPE_NONCE_HEADER,
+                RPC_BOOT_EPOCH_HEADER,
+            ] {
+                downgraded.remove(header);
+            }
+            let (first, replay) = if v3_first {
+                (&headers, &downgraded)
+            } else {
+                (&downgraded, &headers)
+            };
+            verify_tonic_rpc_signature_with_policy(audience, path, first, true, false, false)
+                .expect("the first delivery must be accepted");
+            let error = verify_tonic_rpc_signature_with_policy(audience, path, replay, true, false, false)
+                .expect_err("a mutation cannot replay through the other signature generation");
+            assert_eq!(error.to_string(), "RPC request replay detected");
+        }
+    }
+
     #[test]
     fn replay_scope_rejects_partial_metadata_and_stale_epoch_without_fallback() {
         ensure_test_rpc_secret();

crates/ecstore/src/cluster/rpc/internode_data_transport.rs

@@ -33,7 +33,9 @@ use rustfs_config::{
     DEFAULT_INTERNODE_DATA_TRANSPORT, ENV_RUSTFS_INTERNODE_DATA_TRANSPORT, INTERNODE_DATA_TRANSPORT_TCP,
     KNOWN_INTERNODE_DATA_TRANSPORT_BACKENDS,
 };
-use rustfs_rio::{ChunkReaderBox, HttpChunkReader, HttpReader, HttpWriter};
+use rustfs_rio::{
+    ChunkReaderBox, HttpChunkReader, HttpReader, HttpWriter, HttpWriterMemoryReservation, HttpWriterMemoryReserver,
+};
 use sha2::{Digest, Sha256};
 use std::collections::HashMap;
 use std::future::Future;
@@ -66,6 +68,19 @@ const PUT_FILE_CAPABILITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
 const CONTENT_TYPE_JSON: &str = "application/json";
 const CONTENT_TYPE_MSGPACK: &str = "application/msgpack";
 
+fn http_writer_memory_reserver() -> Option<HttpWriterMemoryReserver> {
+    crate::disk::current_resource_unit()
+        .and_then(|resources| resources.memory_budget())
+        .map(|budget| {
+            // Payloads can outlive the operation; never retain its node or disk units.
+            Arc::new(move |capacity| {
+                budget
+                    .reserve(capacity)
+                    .map(|reservation| Box::new(reservation) as HttpWriterMemoryReservation)
+            }) as HttpWriterMemoryReserver
+        })
+}
+
 fn unsupported_transport_message(transport: &str) -> String {
     format!(
         "invalid {ENV_RUSTFS_INTERNODE_DATA_TRANSPORT}={transport:?}; supported values: {}",
@@ -346,13 +361,17 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
     }
 
     async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter> {
+        let memory_reserver = http_writer_memory_reserver();
         let server_epoch = self.put_file_auth_capability(&request.endpoint).await?;
         let auth_scope = server_epoch.map(|server_epoch| (Uuid::new_v4(), server_epoch));
         let url = build_put_file_stream_url(&request, auth_scope);
         let endpoint = request.endpoint;
         let mut headers = json_headers();
         build_auth_headers(&url, &Method::PUT, &mut headers)?;
-        let writer = HttpWriter::new(url.clone(), Method::PUT, headers).await?;
+        let writer = match memory_reserver {
+            Some(reserve) => HttpWriter::new_with_memory_reserver(url.clone(), Method::PUT, headers, reserve).await?,
+            None => HttpWriter::new(url.clone(), Method::PUT, headers).await?,
+        };
         match auth_scope {
             Some((nonce, server_epoch)) => Ok(Box::new(PutFileAuthWriter::new(writer, url, nonce, endpoint, server_epoch))),
             None => Ok(Box::new(writer)),
@@ -948,6 +967,193 @@ mod tests {
     use tokio::io::AsyncWriteExt;
     use tokio::sync::{Barrier, Notify};
 
+    fn memory_test_write_request(authenticated: bool) -> WriteStreamRequest {
+        crate::runtime::sources::ensure_test_rpc_secret();
+        let endpoint = format!("http://writer-memory-{}.invalid", Uuid::new_v4());
+        put_file_capability_cache_entry(&endpoint).write().cached = Some(if authenticated {
+            PutFileCapabilityState::V1 {
+                server_epoch: Uuid::new_v4(),
+                revalidate_after: Instant::now() + PUT_FILE_V1_CAPABILITY_TTL,
+            }
+        } else {
+            PutFileCapabilityState::LegacyUntil(Instant::now() + PUT_FILE_LEGACY_CAPABILITY_TTL)
+        });
+        WriteStreamRequest {
+            endpoint,
+            disk: "disk-a".to_owned(),
+            volume: "bucket".to_owned(),
+            path: "part.1".to_owned(),
+            append: false,
+            size: 4,
+        }
+    }
+
+    #[tokio::test]
+    async fn http_write_memory_reserver_outlives_only_the_memory_scope() {
+        use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
+
+        assert!(http_writer_memory_reserver().is_none(), "foreground must use the existing writer");
+        let memory = BackgroundMemoryBudget::new();
+        memory.set_sampled_limit(2 * 1024 * 1024, Duration::from_secs(30));
+        let (resources, gates) = test_resource_units(2);
+        let request = memory_test_write_request(true);
+        let endpoint = request.endpoint.clone();
+        let writer = with_optional_resource_unit(
+            Some(resources.with_memory_budget(memory.clone())),
+            TcpHttpInternodeDataTransport.open_write(request),
+        )
+        .await;
+        PUT_FILE_CAPABILITY_CACHE.write().remove(&endpoint);
+        let mut writer = writer.expect("admitted transport writer");
+        assert!(
+            gates.iter().all(|gate| gate.snapshot().active == 0),
+            "the callback must not retain disk units"
+        );
+        assert_eq!(memory.snapshot().used, 1024 * 1024);
+        writer.write_all(b"data").await.expect("buffered bytes");
+        writer.flush().await.expect("transfer charged allocation to queue");
+        memory.set_sampled_limit(2 * 1024 * 1024, Duration::ZERO);
+        writer
+            .write_all(&vec![0x5a; 1024 * 1024 - 4])
+            .await
+            .expect("reuse the already charged tail after observation expiry");
+        assert_eq!(memory.snapshot().used, 1024 * 1024, "reuse must not allocate separate storage");
+        let error = writer
+            .write(b"rejected")
+            .await
+            .expect_err("expired observation must reject new storage");
+        assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
+        assert!(
+            crate::disk::background_memory::is_memory_pressure(&error),
+            "preserve the typed memory-pressure marker"
+        );
+        assert_eq!(memory.snapshot().used, 1024 * 1024, "queued storage remains charged after expiry");
+        drop(writer);
+        tokio::task::yield_now().await;
+        assert_eq!(memory.snapshot().used, 0);
+    }
+
+    #[tokio::test]
+    async fn http_write_memory_denial_preserves_the_budget_error() {
+        use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
+
+        for (authenticated, sample) in [false, true].into_iter().flat_map(|authenticated| {
+            [
+                None,
+                Some((1024 * 1024 - 1, Duration::from_secs(30))),
+                Some((1024 * 1024, Duration::ZERO)),
+            ]
+            .into_iter()
+            .map(move |sample| (authenticated, sample))
+        }) {
+            let memory = BackgroundMemoryBudget::new();
+            if let Some((limit, valid_for)) = sample {
+                memory.set_sampled_limit(limit, valid_for);
+            }
+            let (resources, _) = test_resource_units(1);
+            let request = memory_test_write_request(authenticated);
+            let endpoint = request.endpoint.clone();
+            let result = with_optional_resource_unit(
+                Some(resources.with_memory_budget(memory.clone())),
+                TcpHttpInternodeDataTransport.open_write(request),
+            )
+            .await;
+            PUT_FILE_CAPABILITY_CACHE.write().remove(&endpoint);
+            let error = match result {
+                Err(Error::Io(error)) => error,
+                Err(error) => panic!("unexpected transport error: {error}"),
+                Ok(_) => panic!("unavailable memory must reject construction"),
+            };
+            assert!(crate::disk::background_memory::is_memory_pressure(&error));
+            assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
+            assert_eq!(memory.snapshot().used, 0);
+        }
+    }
+
+    #[tokio::test]
+    async fn admitted_remote_reads_bind_receiver_admission_to_the_http_signature() {
+        use crate::disk::with_background_resource_unit;
+        use rustfs_concurrency::adaptive::AdaptiveAdmission;
+        use std::num::NonZeroU16;
+
+        crate::runtime::sources::ensure_test_rpc_secret();
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
+            .await
+            .expect("remote admission listener");
+        let address = listener.local_addr().expect("remote admission address");
+        let server = tokio::spawn(async move {
+            let (mut socket, _) = listener.accept().await.expect("remote read connection");
+            let mut bytes = Vec::new();
+            while !bytes.windows(4).any(|window| window == b"\r\n\r\n") {
+                let mut chunk = [0; 1024];
+                let read = socket.read(&mut chunk).await.expect("remote read headers");
+                assert!(read > 0 && bytes.len() < 16 * 1024, "bounded complete request headers");
+                bytes.extend_from_slice(&chunk[..read]);
+            }
+            let request = std::str::from_utf8(&bytes).expect("HTTP request");
+            let mut lines = request.split("\r\n");
+            let target = lines
+                .next()
+                .expect("request line")
+                .split_whitespace()
+                .nth(1)
+                .expect("request target")
+                .to_owned();
+            let mut headers = HeaderMap::new();
+            for line in lines.take_while(|line| !line.is_empty()) {
+                let (name, value) = line.split_once(':').expect("header field");
+                headers.append(
+                    http::header::HeaderName::from_bytes(name.as_bytes()).expect("header name"),
+                    HeaderValue::from_str(value.trim()).expect("header value"),
+                );
+            }
+            socket
+                .write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 5\r\nconnection: close\r\n\r\nhello")
+                .await
+                .expect("remote read response");
+            (target, headers)
+        });
+        let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
+        let resource = gate.try_acquire().expect("sender device admission");
+        let mut reader = tokio::time::timeout(
+            Duration::from_secs(5),
+            with_background_resource_unit(
+                resource,
+                TcpHttpInternodeDataTransport.open_read(ReadStreamRequest {
+                    endpoint: format!("http://{address}"),
+                    disk: "disk-a".to_owned(),
+                    volume: "bucket".to_owned(),
+                    path: "part.1".to_owned(),
+                    offset: 0,
+                    length: 5,
+                    stall_timeout: Some(Duration::from_secs(5)),
+                }),
+            ),
+        )
+        .await
+        .expect("bounded remote request")
+        .expect("remote reader");
+        let mut body = Vec::new();
+        reader.read_to_end(&mut body).await.expect("complete remote body");
+        assert_eq!(body, b"hello");
+        let (target, headers) = server.await.expect("remote request capture");
+        assert_eq!(gate.snapshot().active, 0, "remote body does not retain the sender's device");
+        assert_eq!(
+            headers
+                .get("x-rustfs-rpc-background-admission")
+                .and_then(|value| value.to_str().ok()),
+            Some("v1"),
+            "resource-controlled remote work must request receiver-side admission"
+        );
+        crate::cluster::rpc::verify_rpc_signature(&target, &Method::GET, &headers).expect("bound request signature");
+        let mut stripped = headers;
+        stripped.remove("x-rustfs-rpc-background-admission");
+        assert!(
+            crate::cluster::rpc::verify_rpc_signature(&target, &Method::GET, &stripped).is_err(),
+            "removing background admission must invalidate the signature"
+        );
+    }
+
     async fn wait_for_capability_flight_waiters(entry: &PutFileCapabilityCacheEntry, waiters: usize) {
         tokio::time::timeout(Duration::from_secs(5), async {
             loop {

crates/ecstore/src/cluster/rpc/mod.rs

@@ -13,6 +13,8 @@
 // limitations under the License.
 
 pub(crate) mod background_monitor;
+pub(crate) mod background_pipe;
+pub(crate) mod background_reader_stream;
 pub(crate) mod client;
 pub(crate) mod context_propagation;
 pub(crate) mod http_auth;
@@ -33,16 +35,17 @@ pub use client::{
 // Re-exported through `api::rpc`; not every item is consumed inside this crate.
 #[allow(unused_imports)]
 pub use http_auth::{
-    TONIC_RPC_PREFIX, build_auth_headers, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, gen_signature_headers,
-    gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
-    set_tonic_mutation_body_digest, set_tonic_rolling_canonical_body_digest, set_tonic_rolling_mutation_body_digest,
-    sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
-    sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
-    tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability,
-    verify_ns_scanner_capability_with_checkpoint_coverage, verify_ns_scanner_capability_with_tier_registry_generation,
-    verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response,
-    verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned,
-    verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
+    RPC_BACKGROUND_ADMISSION_HEADER, TONIC_RPC_PREFIX, build_auth_headers, build_put_file_auth_trailer,
+    check_and_record_signed_rpc_nonce, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
+    normalize_tonic_rpc_audience, rpc_background_admission, set_tonic_canonical_body_digest, set_tonic_mutation_body_digest,
+    set_tonic_rolling_canonical_body_digest, set_tonic_rolling_mutation_body_digest, sign_ns_scanner_capability,
+    sign_ns_scanner_capability_with_checkpoint_coverage, sign_ns_scanner_capability_with_tier_registry_generation,
+    sign_put_file_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
+    tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
+    verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer, verify_put_file_capability,
+    verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
+    verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
+    verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
 };
 #[cfg(test)]
 pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -2014,9 +2014,21 @@ impl PeerRestClient {
             }
             result
         })
-        .await
-        .unwrap_or_else(|_| Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))));
-        self.finalize_result(result).await
+        .await;
+        self.settle_bucket_metadata_reload(bucket, result).await
+    }
+
+    async fn settle_bucket_metadata_reload(
+        &self,
+        bucket: &str,
+        result: std::result::Result<Result<()>, tokio::time::error::Elapsed>,
+    ) -> Result<()> {
+        match result {
+            Ok(result) => self.finalize_result(result).await,
+            // A local operation deadline does not establish that the peer is
+            // unreachable. In particular, it must not gate IAM revocations.
+            Err(_) => Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))),
+        }
     }
 
     async fn load_bucket_metadata_once(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
@@ -4753,6 +4765,31 @@ mod tests {
         assert!(client.offline.load(Ordering::Acquire));
     }
 
+    #[tokio::test]
+    async fn bucket_metadata_deadline_does_not_gate_healthy_peer_notifications() {
+        let client = test_peer_client();
+        let elapsed = tokio::time::timeout(std::time::Duration::ZERO, std::future::pending::<Result<()>>()).await;
+        let error = client
+            .settle_bucket_metadata_reload("slow-policy", elapsed)
+            .await
+            .expect_err("local budget expires");
+        assert!(error.to_string().contains("timed out"));
+        assert!(
+            !client.offline.load(Ordering::Acquire),
+            "local work must not suppress subsequent IAM RPCs"
+        );
+        client
+            .settle_bucket_metadata_reload("healthy", Ok(Ok(())))
+            .await
+            .expect("peer reply");
+        let error = client
+            .settle_bucket_metadata_reload("unreachable", Ok(Err(Error::from(tonic::Status::unavailable("connection refused")))))
+            .await
+            .expect_err("real transport failure");
+        assert!(error.to_string().contains("connection refused"));
+        assert!(client.offline.load(Ordering::Acquire), "transport failures must still enter recovery");
+    }
+
     #[tokio::test]
     async fn peer_rest_client_finalize_result_keeps_online_for_business_errors() {
         let client = test_peer_client();

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -17,6 +17,7 @@ use crate::cluster::rpc::client::{
     node_service_delete_recovery_client, node_service_time_out_client, node_service_time_out_client_for_class,
     node_service_time_out_client_no_auth,
 };
+use crate::cluster::rpc::http_auth::{current_rpc_background_request, with_rpc_background_request};
 use crate::cluster::rpc::internode_data_transport::{
     InternodeDataTransport, NsScannerCapabilityRequest, NsScannerStreamRequest, ReadStreamRequest, WalkDirStreamRequest,
     WriteStreamRequest,
@@ -570,6 +571,7 @@ struct RetryingRemoteReader {
     reader: Option<FileReader>,
     transport: Arc<dyn InternodeDataTransport>,
     request: ReadStreamRequest,
+    background: bool,
     emitted: usize,
     retried: bool,
     initial_read_timeout: Option<Duration>,
@@ -590,6 +592,7 @@ impl RetryingRemoteReader {
             reader: Some(reader),
             transport,
             request,
+            background: current_rpc_background_request(),
             emitted: 0,
             retried: false,
             initial_read_timeout,
@@ -607,9 +610,10 @@ impl RetryingRemoteReader {
         let request = resumed_read_request(&self.request, self.emitted)?;
         let recovery_timeout = self.recovery_timeout;
         let transport = Arc::clone(&self.transport);
-        self.resume = Some(AbortOnDropTask::new(tokio::spawn(async move {
-            with_remote_read_recovery_timeout(recovery_timeout, transport.open_read_fresh(request)).await
-        })));
+        self.resume = Some(AbortOnDropTask::new(tokio::spawn(with_rpc_background_request(
+            self.background,
+            async move { with_remote_read_recovery_timeout(recovery_timeout, transport.open_read_fresh(request)).await },
+        ))));
         Ok(())
     }
 
@@ -717,6 +721,7 @@ struct RetryingRemoteChunkReader {
     reader: Option<rustfs_rio::ChunkReaderBox>,
     transport: Arc<dyn InternodeDataTransport>,
     request: ReadStreamRequest,
+    background: bool,
     emitted: usize,
     retried: bool,
     initial_read_timeout: Option<Duration>,
@@ -737,6 +742,7 @@ impl RetryingRemoteChunkReader {
             reader: Some(reader),
             transport,
             request,
+            background: current_rpc_background_request(),
             emitted: 0,
             retried: false,
             initial_read_timeout,
@@ -754,9 +760,10 @@ impl RetryingRemoteChunkReader {
         let request = resumed_read_request(&self.request, self.emitted)?;
         let recovery_timeout = self.recovery_timeout;
         let transport = Arc::clone(&self.transport);
-        self.resume = Some(AbortOnDropTask::new(tokio::spawn(async move {
-            with_remote_read_recovery_timeout(recovery_timeout, transport.open_read_chunks_fresh(request)).await
-        })));
+        self.resume = Some(AbortOnDropTask::new(tokio::spawn(with_rpc_background_request(
+            self.background,
+            async move { with_remote_read_recovery_timeout(recovery_timeout, transport.open_read_chunks_fresh(request)).await },
+        ))));
         Ok(())
     }
 
@@ -1075,7 +1082,10 @@ impl RemoteDisk {
         if payload.len() > operation.max_wire_payload_bytes() {
             return Err(Error::other("delete recovery operation exceeds payload limit"));
         }
-        let settle_timeout = if matches!(&operation, DeleteRecoveryRpcOperation::Settle { .. }) {
+        let settle_timeout = if matches!(
+            &operation,
+            DeleteRecoveryRpcOperation::Settle { .. } | DeleteRecoveryRpcOperation::SettleBounded(_)
+        ) {
             Some(crate::disk::get_delete_recovery_timeout()?)
         } else {
             None
@@ -1630,6 +1640,7 @@ impl RemoteDisk {
         let initial_probe_ok = Self::perform_connectivity_check(&addr).await.is_ok();
         if initial_probe_ok {
             health.record_operation_success(&endpoint, "connectivity_probe_success");
+            rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
         } else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
             warn!(
                 event = EVENT_REMOTE_DISK_HEALTH,
@@ -1701,6 +1712,7 @@ impl RemoteDisk {
                     // Perform basic connectivity check
                     if Self::perform_connectivity_check(&addr).await.is_ok() {
                         health.record_operation_success(&endpoint, "connectivity_probe_success");
+                        rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
                     } else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
                         warn!(
                             event = EVENT_REMOTE_DISK_HEALTH,
@@ -1771,6 +1783,7 @@ impl RemoteDisk {
                 }
                 _ = interval.tick() => {
                     if Self::perform_recovery_probe(&addr, &endpoint).await.is_ok() {
+                        rustfs_io_metrics::internode_metrics::record_peer_reachable(&addr);
                         let became_online = health.mark_recovery_success(&endpoint, "disk_info_probe_success");
                         debug!(
                             event = EVENT_REMOTE_DISK_HEALTH,
@@ -2002,6 +2015,7 @@ impl RemoteDisk {
             let operation_result = operation().await;
             if operation_result.is_ok() {
                 self.health.record_operation_success(&self.endpoint, "operation_success");
+                rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
             }
             self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
                 .await;
@@ -2016,6 +2030,7 @@ impl RemoteDisk {
                 // Log success; the waiting guard balances every exit path.
                 if operation_result.is_ok() {
                     self.health.record_operation_success(&self.endpoint, "operation_success");
+                    rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
                 }
                 self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
                     .await;
@@ -2704,6 +2719,54 @@ impl RemoteDisk {
 
 #[async_trait::async_trait]
 impl DiskAPI for RemoteDisk {
+    async fn list_multipart_index_roots(
+        &self,
+        bucket: &str,
+        marker: Option<&str>,
+    ) -> Result<crate::disk::multipart_index::MultipartIndexPage> {
+        use crate::disk::multipart_index::{MULTIPART_INDEX_RPC_MAX_BYTES, MultipartIndexRpcReply, MultipartIndexRpcRequest};
+        let expected_disk = self
+            .id
+            .lock()
+            .await
+            .filter(|id| !id.is_nil())
+            .ok_or(DiskError::DiskNotFound)?;
+        let query = MultipartIndexRpcRequest::new(bucket, marker);
+        let endpoint = self.endpoint.to_string();
+        let canonical = query.canonical_request(&endpoint)?;
+        let opts = serde_json::to_string(&query)?;
+        self.execute_with_timeout(
+            || async {
+                let mut client = self.get_client().await?;
+                let mut request = Request::new(DiskInfoRequest {
+                    disk: endpoint.clone(),
+                    opts: opts.clone(),
+                });
+                crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
+                let response = client.disk_info(request).await?.into_inner();
+                if !response.success {
+                    return Err(response.error.unwrap_or_default().into());
+                }
+                if response.disk_info.len() > MULTIPART_INDEX_RPC_MAX_BYTES {
+                    return Err(DiskError::FileCorrupt);
+                }
+                // An old server ignores the additive query and returns ordinary DiskInfo.
+                // It cannot satisfy the dedicated reply schema or the fresh challenge.
+                let reply: MultipartIndexRpcReply =
+                    serde_json::from_str(&response.disk_info).map_err(|_| DiskError::MethodNotAllowed)?;
+                reply.validate(&query, expected_disk)?;
+                crate::cluster::rpc::verify_tonic_rpc_response_proof(
+                    &query.canonical_response(&endpoint, &reply.page)?,
+                    &reply.proof,
+                )
+                .map_err(Error::other)?;
+                Ok(reply.page)
+            },
+            get_drive_disk_info_timeout(),
+        )
+        .await
+    }
+
     async fn supports_guarded_delete_versions(&self) -> Result<()> {
         let response = self.guarded_delete_versions_rpc("", Vec::new(), true).await?;
         if !response.errors.is_empty() || !response.item_errors.is_empty() {
@@ -4287,6 +4350,35 @@ impl DiskAPI for RemoteDisk {
         }
     }
 
+    async fn discover_delete_transactions_bounded(
+        &self,
+        volume: &str,
+        object: &str,
+        max_entries: u32,
+    ) -> Result<crate::disk::BoundedDeleteRecoveryDiscovery> {
+        // Older peers reject this variant. Do not retry through the unbounded protocol.
+        match self
+            .recover_delete_transaction(
+                volume,
+                DeleteRecoveryRpcOperation::DiscoverBounded {
+                    object: object.to_owned(),
+                    max_entries,
+                },
+            )
+            .await
+        {
+            Ok(DeleteRecoveryRpcReply::DiscoveredBounded(page)) => Ok(page),
+            Ok(_) => Err(Error::FileCorrupt),
+            Err(error)
+                if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
+                    || matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
+                    .is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
+            {
+                Err(Error::DeleteRecoveryWorkLimit)
+            }
+            Err(error) => Err(error),
+        }
+    }
     async fn discover_delete_transactions(
         &self,
         volume: &str,
@@ -4322,6 +4414,58 @@ impl DiskAPI for RemoteDisk {
             _ => Err(Error::other("unexpected delete discovery reply")),
         }
     }
+    async fn settle_delete_transaction_bounded(
+        &self,
+        volume: &str,
+        request: crate::disk::BoundedDeleteRecoverySettlement,
+        _namespace_owner: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<crate::disk::BoundedDeleteRecoverySettlementReply> {
+        match self
+            .recover_delete_transaction(volume, DeleteRecoveryRpcOperation::SettleBounded(request))
+            .await
+        {
+            Ok(DeleteRecoveryRpcReply::SettledBounded(reply)) => Ok(reply),
+            Ok(_) => Err(Error::FileCorrupt),
+            Err(error)
+                if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
+                    || matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
+                    .is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
+            {
+                Err(Error::DeleteRecoveryWorkLimit)
+            }
+            Err(error) => Err(error),
+        }
+    }
+    async fn inspect_delete_transaction_bounded(
+        &self,
+        volume: &str,
+        object: &str,
+        transaction: Uuid,
+        max_entries: u32,
+    ) -> Result<crate::disk::BoundedDeleteRecoveryInspection> {
+        match self
+            .recover_delete_transaction(
+                volume,
+                DeleteRecoveryRpcOperation::InspectBounded {
+                    object: object.to_owned(),
+                    transaction,
+                    max_entries,
+                },
+            )
+            .await
+        {
+            Ok(DeleteRecoveryRpcReply::InspectedBounded(page)) => Ok(page),
+            Ok(_) => Err(Error::FileCorrupt),
+            Err(error)
+                if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
+                    || matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
+                    .is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
+            {
+                Err(Error::DeleteRecoveryWorkLimit)
+            }
+            Err(error) => Err(error),
+        }
+    }
     async fn inspect_delete_transaction(
         &self,
         volume: &str,
@@ -4842,6 +4986,113 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn bounded_delete_discovery_lost_remote_reply_leaves_no_cursor() {
+        runtime_sources::ensure_test_rpc_secret();
+        let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
+            .await
+            .expect("test peer must bind");
+        let directory = tempfile::tempdir().expect("temporary disk");
+        let endpoint = Endpoint::try_from(directory.path().to_str().expect("disk path")).expect("endpoint");
+        let disk = Arc::new(
+            crate::disk::local::LocalDisk::new(&endpoint, false)
+                .await
+                .expect("local disk"),
+        );
+        disk.make_volume("bucket").await.expect("bucket");
+        let prefix = directory.path().join("bucket/prefix");
+        std::fs::create_dir(&prefix).expect("prefix");
+        for index in 0..64 {
+            std::fs::create_dir(prefix.join(format!("child-{index}"))).expect("legitimate child");
+        }
+        let fixture = Arc::new(TestBoundedDiscovery {
+            disk: Arc::clone(&disk),
+            response_gate: TestFormatReadGate::default(),
+        });
+        *peer.peer.bounded_discovery.lock().expect("bounded fixture") = Some(Arc::clone(&fixture));
+        let remote = remote_disk_for_addr(&peer.addr).await;
+        let task = tokio::spawn(async move { remote.discover_delete_transactions_bounded("bucket", "prefix", 8).await });
+        tokio::time::timeout(Duration::from_secs(5), fixture.response_gate.entered.notified())
+            .await
+            .expect("disk completed bounded scan");
+        task.abort();
+        assert!(task.await.expect_err("caller cancelled").is_cancelled());
+        assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
+        fixture.response_gate.release.cancel();
+        peer.stop().await;
+        assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
+        assert!(prefix.join("child-63").is_dir());
+    }
+
+    #[tokio::test]
+    async fn multipart_index_old_disk_info_handler_is_not_a_ready_empty_index() {
+        runtime_sources::ensure_test_rpc_secret();
+        let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
+            .await
+            .expect("old peer must bind");
+        let remote = remote_disk_for_addr(&peer.addr).await;
+        remote
+            .set_disk_id(Some(Uuid::new_v4()))
+            .await
+            .expect("expected disk identity");
+        assert!(matches!(
+            remote.list_multipart_index_roots("bucket", None).await,
+            Err(DiskError::MethodNotAllowed)
+        ));
+        assert_eq!(
+            peer.peer.disk_info_calls.load(Ordering::Acquire),
+            1,
+            "exercise the actual legacy JSON handler"
+        );
+        peer.stop().await;
+    }
+
+    #[tokio::test]
+    async fn bounded_delete_discovery_old_peer_rejects_without_paginated_fallback() {
+        runtime_sources::ensure_test_rpc_secret();
+        let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
+            .await
+            .expect("old peer must bind");
+        let remote = remote_disk_for_addr(&peer.addr).await;
+        assert_eq!(
+            remote
+                .discover_delete_transactions_bounded("bucket", "prefix", 4096)
+                .await
+                .expect_err("old peer cannot prove a bounded inventory"),
+            DiskError::DeleteRecoveryWorkLimit
+        );
+        let transaction = Uuid::new_v4();
+        assert_eq!(
+            remote
+                .inspect_delete_transaction_bounded("bucket", "prefix", transaction, 65536)
+                .await
+                .expect_err("old peer cannot inspect with a budget"),
+            DiskError::DeleteRecoveryWorkLimit
+        );
+        assert_eq!(
+            remote
+                .settle_delete_transaction_bounded(
+                    "bucket",
+                    crate::disk::BoundedDeleteRecoverySettlement {
+                        object: "prefix".to_owned(),
+                        transaction,
+                        expected_fingerprint: [0; 32],
+                        action: DeleteRecoveryAction::PublishCommit,
+                        max_entries: 65536,
+                    },
+                    None
+                )
+                .await
+                .expect_err("old peer cannot settle with a budget"),
+            DiskError::DeleteRecoveryWorkLimit
+        );
+        assert_eq!(
+            *peer.peer.unsupported_methods.lock().expect("observed methods"),
+            vec!["/node_service.NodeService/RecoverDeleteTransaction"; 3]
+        );
+        peer.stop().await;
+    }
+
     #[tokio::test]
     async fn multipart_source_recovery_old_peer_rejects_without_generic_fallback() {
         runtime_sources::ensure_test_rpc_secret();
@@ -5734,9 +5985,16 @@ mod tests {
         format_data: Bytes,
         read_all_data: Bytes,
         next_format_read_gate: Arc<StdMutex<Option<Arc<TestFormatReadGate>>>>,
+        bounded_discovery: Arc<StdMutex<Option<Arc<TestBoundedDiscovery>>>>,
         unsupported_methods: Arc<StdMutex<Vec<String>>>,
     }
 
+    #[derive(Debug)]
+    struct TestBoundedDiscovery {
+        disk: Arc<crate::disk::local::LocalDisk>,
+        response_gate: TestFormatReadGate,
+    }
+
     #[derive(Debug, Default)]
     struct TestFormatReadGate {
         entered: tokio::sync::Notify,
@@ -5753,6 +6011,7 @@ mod tests {
                 format_data,
                 read_all_data,
                 next_format_read_gate: Arc::default(),
+                bounded_discovery: Arc::default(),
                 unsupported_methods: Arc::default(),
             }
         }
@@ -5852,6 +6111,56 @@ mod tests {
                         Ok(grpc.unary(method, request).await)
                     })
                 }
+                "/node_service.NodeService/RecoverDeleteTransaction"
+                    if self.peer.bounded_discovery.lock().expect("bounded fixture").is_some() =>
+                {
+                    #[derive(Clone)]
+                    struct BoundedSvc(Arc<TestBoundedDiscovery>);
+                    impl tonic::server::UnaryService<RecoverDeleteTransactionRequest> for BoundedSvc {
+                        type Response = rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse;
+                        type Future = Pin<Box<dyn Future<Output = std::result::Result<Response<Self::Response>, Status>> + Send>>;
+                        fn call(&mut self, request: Request<RecoverDeleteTransactionRequest>) -> Self::Future {
+                            let fixture = Arc::clone(&self.0);
+                            Box::pin(async move {
+                                let request = request.into_inner();
+                                let operation: DeleteRecoveryRpcOperation =
+                                    rustfs_protos::decode_delete_recovery_payload(&request.operation)
+                                        .map_err(|_| Status::invalid_argument("invalid bounded request"))?;
+                                let DeleteRecoveryRpcOperation::DiscoverBounded { object, max_entries } = operation else {
+                                    return Err(Status::invalid_argument("unexpected fallback"));
+                                };
+                                let page = fixture
+                                    .disk
+                                    .discover_delete_transactions_bounded(&request.volume, &object, max_entries)
+                                    .await
+                                    .map_err(|_| Status::internal("bounded discovery failed"))?;
+                                // Lose the response only after the real disk operation has completed.
+                                fixture.response_gate.entered.notify_one();
+                                fixture.response_gate.release.cancelled().await;
+                                let result = rmp_serde::to_vec_named(&DeleteRecoveryRpcReply::DiscoveredBounded(page))
+                                    .map_err(|_| Status::internal("encode bounded reply"))?;
+                                Ok(Response::new(rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse {
+                                    success: true,
+                                    protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
+                                    result: result.into(),
+                                    error: None,
+                                }))
+                            })
+                        }
+                    }
+                    let fixture = self
+                        .peer
+                        .bounded_discovery
+                        .lock()
+                        .expect("bounded fixture")
+                        .clone()
+                        .expect("configured fixture");
+                    Box::pin(async move {
+                        let codec = tonic_prost::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec);
+                        Ok(grpc.unary(BoundedSvc(fixture), request).await)
+                    })
+                }
                 "/node_service.NodeService/ReadAll" => {
                     #[derive(Clone)]
                     struct ReadAllSvc(AuthenticatedReadPeer);
@@ -7769,6 +8078,98 @@ mod tests {
         .await;
     }
 
+    #[tokio::test]
+    async fn remote_read_retries_preserve_signed_admission_after_sender_scope_ends() {
+        use crate::cluster::rpc::{rpc_background_admission, verify_rpc_signature};
+        use crate::disk::with_background_resource_unit;
+        use rustfs_concurrency::adaptive::AdaptiveAdmission;
+        use std::num::NonZeroU16;
+
+        crate::runtime::sources::ensure_test_rpc_secret();
+        for background in [false, true] {
+            for chunked in [false, true] {
+                let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("retry listener");
+                let address = listener.local_addr().expect("retry address");
+                let server = tokio::spawn(async move {
+                    let (mut socket, _) = listener.accept().await.expect("fresh connection");
+                    let mut bytes = Vec::new();
+                    while !bytes.windows(4).any(|window| window == b"\r\n\r\n") {
+                        let mut chunk = [0; 1024];
+                        let read = socket.read(&mut chunk).await.expect("retry request");
+                        assert!(read > 0 && bytes.len() < 16 * 1024, "bounded complete headers");
+                        bytes.extend_from_slice(&chunk[..read]);
+                    }
+                    let request = std::str::from_utf8(&bytes).expect("HTTP request");
+                    let mut lines = request.split("\r\n");
+                    let target = lines
+                        .next()
+                        .expect("request line")
+                        .split_whitespace()
+                        .nth(1)
+                        .expect("target")
+                        .to_owned();
+                    let mut headers = http::HeaderMap::new();
+                    for line in lines.take_while(|line| !line.is_empty()) {
+                        let (name, value) = line.split_once(':').expect("header");
+                        headers.append(
+                            http::header::HeaderName::from_bytes(name.as_bytes()).expect("header name"),
+                            http::HeaderValue::from_str(value.trim()).expect("header value"),
+                        );
+                    }
+                    socket
+                        .write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 6\r\nconnection: close\r\n\r\n456789")
+                        .await
+                        .expect("retry response");
+                    (target, headers)
+                });
+                let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
+                let request = ReadStreamRequest {
+                    endpoint: format!("http://{address}"),
+                    ..resume_request(10)
+                };
+                let create = async {
+                    let transport = Arc::new(TcpHttpInternodeDataTransport);
+                    if chunked {
+                        Box::new(RetryingRemoteChunkReader::new_with_timeouts(
+                            resume_step_chunk_reader(ResumeReadStep::PartialThenReset(b"0123".to_vec())),
+                            transport,
+                            request,
+                            None,
+                            None,
+                        )) as FileReader
+                    } else {
+                        Box::new(RetryingRemoteReader::new_with_timeouts(
+                            resume_step_reader(ResumeReadStep::PartialThenReset(b"0123".to_vec())),
+                            transport,
+                            request,
+                            None,
+                            None,
+                        )) as FileReader
+                    }
+                };
+                let mut reader = if background {
+                    with_background_resource_unit(gate.try_acquire().expect("sender admission"), create).await
+                } else {
+                    create.await
+                };
+                assert_eq!(gate.snapshot().active, 0, "retry reader cannot retain sender disk capacity");
+                let mut output = Vec::new();
+                tokio::time::timeout(
+                    Duration::from_secs(5),
+                    with_rpc_background_request(!background, reader.read_to_end(&mut output)),
+                )
+                .await
+                .expect("bounded retry")
+                .expect("resumed body");
+                assert_eq!(output, b"0123456789", "chunked={chunked}, background={background}");
+                let (target, headers) = server.await.expect("captured retry");
+                assert!(target.contains("offset=11") && target.contains("length=6"), "{target}");
+                assert_eq!(rpc_background_admission(&headers).expect("admission marker"), background);
+                verify_rpc_signature(&target, &http::Method::GET, &headers).expect("signed retry class");
+            }
+        }
+    }
+
     #[tokio::test]
     async fn remote_reader_resumes_from_emitted_bytes_without_duplicates() {
         let transport = Arc::new(ResumeTransport::with_read_steps(vec![ResumeReadStep::Data(b"456789".to_vec())]));

crates/ecstore/src/cluster/rpc/remote_locker.rs

@@ -445,6 +445,10 @@ impl RemoteClient {
                 }
             }
         });
+        // A successful lock RPC over a cached or fresh channel proves the peer is
+        // reachable; reconcile the internode peer-health gauge so it does not stay
+        // stale when no fresh dial fires (issue #9119).
+        rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
     }
 
     fn admit_late_release(&self) -> bool {
@@ -672,6 +676,7 @@ impl RemoteClient {
                     with_lock_peer_health(&addr, |health| {
                         health.last_success = Some(Instant::now());
                     });
+                    rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
                     rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "success");
                     if let Some(late) = late {
                         late(response).await;

crates/ecstore/src/config/com.rs

@@ -3506,6 +3506,7 @@ mod tests {
                 replication_decision: String::new(),
                 checksum: None,
                 multipart_completion_replayed: false,
+                backfilled_old_current_size: None,
             }
         }
     }

crates/ecstore/src/core/pool_meta_native_integration_tests.rs

@@ -18,6 +18,7 @@ use crate::storage_api_contracts::namespace::NamespaceLocking as _;
 use codec::{fixture_inline_pool_meta_replicas as replicas, fixture_split_inline_pool_meta_replicas as split};
 use std::path::PathBuf;
 use tokio::fs;
+use uuid::Uuid;
 
 struct PhysicalFixture {
     _directories: Vec<tempfile::TempDir>,
@@ -628,3 +629,559 @@ async fn pool_meta_native_physical_three_plus_one_checkpoint_converges_all_shard
         );
     }
 }
+
+async fn owner_repair_no_backups_fixture() -> PhysicalFixture {
+    let fixture = fixture(0).await;
+    for pool in disks(&fixture.store).await {
+        for disk in pool {
+            let object = object_path(&disk);
+            let mut entries = fs::read_dir(&object).await.expect("fixture metadata directory");
+            while let Some(entry) = entries.next_entry().await.expect("fixture directory entry") {
+                if entry.file_type().await.expect("fixture entry type").is_dir() {
+                    fs::remove_dir_all(entry.path())
+                        .await
+                        .expect("remove task-owned fixture rollback directory");
+                }
+            }
+        }
+    }
+    fixture
+}
+
+async fn owner_repair_identity_heads(store: &ECStore) -> Vec<Vec<(Uuid, Bytes)>> {
+    let mut all = Vec::new();
+    for pool in &store.pools {
+        let set = pool.get_disks_by_key(POOL_META_IDENTITY_NAME);
+        let members = set
+            .disks
+            .read()
+            .await
+            .iter()
+            .cloned()
+            .map(|disk| disk.expect("identity fixture disk must be online"))
+            .collect::<Vec<_>>();
+        let mut heads = Vec::new();
+        for disk in members {
+            match disk
+                .pool_meta_native_recovery(Request::InspectIdentity {})
+                .await
+                .expect("physical identity inspection")
+            {
+                Response::IdentityInspected { disk_id, raw } => heads.push((disk_id, raw)),
+                _ => panic!("identity inspection must return its physical head"),
+            }
+        }
+        all.push(heads);
+    }
+    all
+}
+
+async fn legacy_native_owner_fixture(version: u16, stale: bool) -> (PhysicalFixture, Vec<u8>) {
+    let fixture = owner_repair_no_backups_fixture().await;
+    let mut meta = match decode_pool_meta_replica(fixture.canonical.clone()) {
+        PoolMetaReplica::Valid {
+            meta, committed: true, ..
+        } => meta,
+        _ => panic!("fixture must provide valid committed pool metadata"),
+    };
+    meta.version = version;
+    let raw = meta
+        .encode_config_data_for_v2_gate(version == POOL_META_VERSION)
+        .expect("legacy fixture should encode in its original version");
+    assert_eq!(u16::from_le_bytes([raw[2], raw[3]]), version);
+    let all = disks(&fixture.store).await;
+    let now = OffsetDateTime::now_utc();
+    for pool in &all {
+        let heads = replicas(&raw, now).await;
+        for (index, disk) in pool.iter().enumerate() {
+            fs::write(object_path(disk).join("xl.meta"), &heads[index])
+                .await
+                .expect("install exact legacy head");
+        }
+    }
+    if stale {
+        let old = replicas(&raw, now - Duration::seconds(10)).await;
+        fs::write(object_path(&all[0][0]).join("xl.meta"), &old[0])
+            .await
+            .expect("install stale same-raw legacy head");
+    }
+    (fixture, raw)
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_v1_with_all_upgrade_gates_disabled_keeps_exact_raw() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            let (fixture, raw) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+            let guard = owner(&fixture.store).await;
+            let mut state = fixture.store.pool_meta_save_gate.lock().await;
+            let result = tokio::time::timeout(
+                std::time::Duration::from_secs(30),
+                fixture.store.heal_pool_meta_replica_under_fence(0, guard, &mut state),
+            )
+            .await
+            .expect("legacy native heal must terminate");
+            assert!(result.is_ok(), "fully online legacy native heal should succeed: {result:?}");
+            drop(state);
+            let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            for read in &reads {
+                assert!(
+                    matches!(&read.replica, PoolMetaReplica::Valid { raw: bytes, revision, .. }
+                    if bytes == &raw && revision.version == POOL_META_V1_VERSION),
+                    "native heal must not upgrade a V1 raw payload when both upgrade gates are disabled"
+                );
+            }
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_legacy_source_with_trailing_bytes_without_writes() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            for version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
+                let (fixture, raw) = legacy_native_owner_fixture(version, false).await;
+                let mut source_raw = raw.clone();
+                // A second MessagePack value is readable by legacy decoding but
+                // must not be copied as part of an unverified repair payload.
+                source_raw.push(0xc0);
+                assert!(
+                    matches!(decode_pool_meta_replica(source_raw.clone()), PoolMetaReplica::Valid { raw, .. } if raw == source_raw),
+                    "ordinary legacy reads must remain compatible with trailing bytes"
+                );
+                let all_disks = disks(&fixture.store).await;
+                let source_heads = replicas(&source_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+                for (index, disk) in all_disks[0].iter().enumerate() {
+                    fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
+                        .await
+                        .expect("install readable legacy source with trailing bytes");
+                }
+                let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
+                    .await
+                    .expect("legacy semantic selection must remain readable");
+                assert_eq!(selection.legacy_raw.as_deref(), Some(source_raw.as_slice()));
+                let before = snapshots(&fixture.store).await;
+                let identities = owner_repair_identity_heads(&fixture.store).await;
+                let guard = owner(&fixture.store).await;
+                let mut state = fixture.store.pool_meta_save_gate.lock().await;
+                let result = tokio::time::timeout(
+                    std::time::Duration::from_secs(30),
+                    fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
+                )
+                .await
+                .expect("legacy native heal must terminate");
+                drop(state);
+                assert!(result.is_err(), "version {version}: unparsed trailing source bytes must not be republished");
+                assert_eq!(snapshots(&fixture.store).await, before, "version {version}: no pool.bin head or backup may change");
+                assert_eq!(
+                    owner_repair_identity_heads(&fixture.store).await,
+                    identities,
+                    "version {version}: no identity shard may change"
+                );
+                let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+                assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &source_raw));
+                assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw: target, .. } if target == &raw));
+            }
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_mixed_v1_v2_preserves_observed_format_floor() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            for first_version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
+                for target_pool in [0, 1] {
+                    let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+                    let mut meta = match decode_pool_meta_replica(raw_v1.clone()) {
+                        PoolMetaReplica::Valid { meta, committed: true, .. } => meta,
+                        _ => panic!("initial V1 metadata must decode"),
+                    };
+                    meta.version = POOL_META_VERSION;
+                    let raw_v2 = meta.encode_config_data_for_v2_gate(true).expect("equivalent V2 payload must encode");
+                    assert_eq!(u16::from_le_bytes([raw_v1[2], raw_v1[3]]), POOL_META_V1_VERSION);
+                    assert_eq!(u16::from_le_bytes([raw_v2[2], raw_v2[3]]), POOL_META_VERSION);
+                    let semantic_v1 = match decode_pool_meta_replica(raw_v1.clone()) {
+                        PoolMetaReplica::Valid { canonical, .. } => canonical,
+                        _ => panic!("V1 payload must remain valid"),
+                    };
+                    let semantic_v2 = match decode_pool_meta_replica(raw_v2.clone()) {
+                        PoolMetaReplica::Valid { canonical, .. } => canonical,
+                        _ => panic!("V2 payload must remain valid"),
+                    };
+                    assert_eq!(semantic_v1, semantic_v2, "the two formats must carry equivalent metadata");
+                    let other_version = if first_version == POOL_META_V1_VERSION { POOL_META_VERSION } else { POOL_META_V1_VERSION };
+                    // Pool two follows pool zero; pools zero and one cover both
+                    // orderings and both target formats in separate runs.
+                    let versions = [first_version, other_version, first_version];
+                    let physical = disks(&fixture.store).await;
+                    let timestamp = OffsetDateTime::now_utc();
+                    for (pool_index, pool) in physical.iter().enumerate() {
+                        let raw = if versions[pool_index] == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
+                        let heads = replicas(raw, timestamp).await;
+                        for (index, disk) in pool.iter().enumerate() {
+                            fs::write(object_path(disk).join("xl.meta"), &heads[index]).await.expect("install mixed legacy pool head");
+                        }
+                    }
+                    let before = snapshots(&fixture.store).await;
+                    let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
+                        .await.expect("equivalent legacy pools should be selectable");
+                    assert_eq!(selection.revision.version, POOL_META_VERSION, "the observed V2 format floor must be retained");
+                    assert_eq!(selection.canonical.as_deref(), Some(semantic_v2.as_slice()));
+                    let guard = owner(&fixture.store).await;
+                    let mut state = fixture.store.pool_meta_save_gate.lock().await;
+                    tokio::time::timeout(std::time::Duration::from_secs(30),
+                        fixture.store.heal_pool_meta_replica_under_fence(target_pool, guard, &mut state))
+                        .await.expect("mixed legacy heal must terminate")
+                        .expect("equivalent legacy metadata should heal without downgrading the format floor");
+                    drop(state);
+                    let after = snapshots(&fixture.store).await;
+                    let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+                    for (pool_index, read) in reads.iter().enumerate() {
+                        let expected_version = if pool_index == target_pool { POOL_META_VERSION } else { versions[pool_index] };
+                        let expected_raw = if expected_version == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
+                        assert!(matches!(&read.replica,
+                            PoolMetaReplica::Valid { raw, canonical, revision, committed: true, .. }
+                                if raw == expected_raw && canonical == &semantic_v2 && revision.version == expected_version
+                        ), "pool {pool_index}, first={first_version}, target={target_pool}: preserve equivalent payload and expected legacy version");
+                        for (index, snapshot) in after[pool_index].iter().enumerate() {
+                            codec::validate_inline_pool_meta_shard(snapshot.current.as_ref().expect("physical head"), index, expected_raw)
+                                .await.expect("every physical shard must match its selected legacy payload");
+                        }
+                        if pool_index != target_pool {
+                            assert_eq!(after[pool_index], before[pool_index], "healing one pool must not modify other pools");
+                        }
+                    }
+                }
+            }
+        },
+    ).await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_v1_and_v3_prepare_in_either_pool_order() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            for prepare_pool in [0, 1] {
+                let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+                let (legacy_meta, legacy_canonical) = match decode_pool_meta_replica(raw_v1.clone()) {
+                    PoolMetaReplica::Valid {
+                        meta,
+                        canonical,
+                        revision,
+                        committed: true,
+                        ..
+                    } if revision.version == POOL_META_V1_VERSION => (meta, canonical),
+                    _ => panic!("baseline V1 metadata must be committed and readable"),
+                };
+                let previous = PoolMetaCommittedCandidate {
+                    canonical: legacy_canonical,
+                    meta: legacy_meta.clone(),
+                    revision: PoolMetaRevision::legacy(POOL_META_V1_VERSION),
+                };
+                let mut next = legacy_meta;
+                next.version = POOL_META_GENERATION_VERSION;
+                next.pools[0].last_update += Duration::seconds(1);
+                let revision = PoolMetaRevision {
+                    version: POOL_META_GENERATION_VERSION,
+                    cluster_id: fixture.revision.cluster_id,
+                    epoch: fixture.revision.epoch,
+                    generation: 1,
+                    transaction_id: Some(Uuid::new_v4()),
+                };
+                let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&previous))
+                    .expect("V3 PREPARE with V1 predecessor must encode");
+                assert!(matches!(decode_pool_meta_replica(pending.clone()),
+                    PoolMetaReplica::Valid { committed: false, revision: actual, .. } if actual == revision));
+                let heads = replicas(&pending, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+                let physical = disks(&fixture.store).await;
+                for (index, disk) in physical[prepare_pool].iter().enumerate() {
+                    fs::write(object_path(disk).join("xl.meta"), &heads[index])
+                        .await
+                        .expect("install task-owned V3 PREPARE evidence");
+                }
+                let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+                assert!(matches!(&reads[prepare_pool].replica,
+                    PoolMetaReplica::Valid { raw, committed: false, revision: actual, .. }
+                        if raw == &pending && *actual == revision));
+                let other_pool = 1 - prepare_pool;
+                assert!(matches!(&reads[other_pool].replica,
+                    PoolMetaReplica::Valid { raw, committed: true, revision: actual, .. }
+                        if raw == &raw_v1 && actual.version == POOL_META_V1_VERSION));
+                let before = snapshots(&fixture.store).await;
+                let identities = owner_repair_identity_heads(&fixture.store).await;
+                let guard = owner(&fixture.store).await;
+                let mut state = fixture.store.pool_meta_save_gate.lock().await;
+                let result = tokio::time::timeout(
+                    std::time::Duration::from_secs(30),
+                    fixture
+                        .store
+                        .heal_pool_meta_replica_under_fence(prepare_pool, guard, &mut state),
+                )
+                .await
+                .expect("mixed V1/V3 PREPARE heal must terminate");
+                drop(state);
+                assert!(
+                    result.is_err(),
+                    "pool {prepare_pool}: native heal must not overwrite a V3 PREPARE with legacy data"
+                );
+                assert_eq!(
+                    snapshots(&fixture.store).await,
+                    before,
+                    "pool {prepare_pool}: rejection must retain all physical head and backup evidence"
+                );
+                assert_eq!(
+                    owner_repair_identity_heads(&fixture.store).await,
+                    identities,
+                    "pool {prepare_pool}: rejection must not mutate identity evidence"
+                );
+            }
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_v1_legacy_dont_save_true_without_writes() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+            let meta = match decode_pool_meta_replica(safe_v1.clone()) {
+                PoolMetaReplica::Valid {
+                    meta, committed: true, ..
+                } => meta,
+                _ => panic!("baseline V1 metadata must decode"),
+            };
+            let statuses = meta
+                .pools
+                .iter()
+                .map(|pool| {
+                    assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
+                    LegacyPoolStatus {
+                        id: pool.id,
+                        cmd_line: pool.cmd_line.clone(),
+                        last_update: pool.last_update,
+                        decommission: None,
+                    }
+                })
+                .collect();
+            let legacy = LegacyPoolMeta {
+                version: POOL_META_V1_VERSION,
+                pools: statuses,
+                dont_save: true,
+            };
+            let mut dangerous_raw = Vec::new();
+            dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
+            dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
+            dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
+            let old_reader: LegacyPoolMeta =
+                rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode source");
+            assert!(old_reader.dont_save, "the old reader would treat this source as permanently do-not-save");
+            assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
+            let source = match decode_pool_meta_replica(dangerous_raw.clone()) {
+                PoolMetaReplica::Valid {
+                    canonical,
+                    revision,
+                    committed: true,
+                    ..
+                } => {
+                    assert_eq!(revision.version, POOL_META_V1_VERSION);
+                    canonical
+                }
+                _ => panic!("current reader must recognize the legacy source"),
+            };
+            assert!(matches!(decode_pool_meta_replica(safe_v1.clone()),
+                PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
+                    if canonical == source && revision.version == POOL_META_V1_VERSION));
+            let physical = disks(&fixture.store).await;
+            let source_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+            for (index, disk) in physical[0].iter().enumerate() {
+                fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
+                    .await
+                    .expect("install legacy source on pool zero");
+            }
+            let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
+            assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
+            let before = snapshots(&fixture.store).await;
+            let identities = owner_repair_identity_heads(&fixture.store).await;
+            let guard = owner(&fixture.store).await;
+            let mut state = fixture.store.pool_meta_save_gate.lock().await;
+            let result = tokio::time::timeout(
+                std::time::Duration::from_secs(30),
+                fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
+            )
+            .await
+            .expect("unsafe legacy heal must terminate");
+            drop(state);
+            let error = result.expect_err("native heal must reject a source whose old-reader dont_save is true");
+            assert!(
+                error.to_string().contains("dont_save"),
+                "rejection must identify the unsafe legacy field: {error}"
+            );
+            assert_eq!(
+                snapshots(&fixture.store).await,
+                before,
+                "rejection must preserve every original head and backup"
+            );
+            assert_eq!(
+                owner_repair_identity_heads(&fixture.store).await,
+                identities,
+                "rejection must not write identity shards"
+            );
+            let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw));
+            assert!(
+                matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1),
+                "the dangerous source bytes must not be copied into the target pool"
+            );
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial_test::serial(pool_meta_version_env)]
+async fn pool_meta_native_heal_rejects_target_v1_dont_save_true_without_writes() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
+            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
+        ],
+        async {
+            let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
+            let meta = match decode_pool_meta_replica(safe_v1.clone()) {
+                PoolMetaReplica::Valid {
+                    meta, committed: true, ..
+                } => meta,
+                _ => panic!("baseline V1 metadata must decode"),
+            };
+            let statuses = meta
+                .pools
+                .iter()
+                .map(|pool| {
+                    assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
+                    LegacyPoolStatus {
+                        id: pool.id,
+                        cmd_line: pool.cmd_line.clone(),
+                        last_update: pool.last_update,
+                        decommission: None,
+                    }
+                })
+                .collect();
+            let legacy = LegacyPoolMeta {
+                version: POOL_META_V1_VERSION,
+                pools: statuses,
+                dont_save: true,
+            };
+            let mut dangerous_raw = Vec::new();
+            dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
+            dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
+            dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
+            let old_reader: LegacyPoolMeta =
+                rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode target");
+            assert!(old_reader.dont_save, "the old reader interprets the target as do-not-save");
+            assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
+            let source_canonical = match decode_pool_meta_replica(safe_v1.clone()) {
+                PoolMetaReplica::Valid {
+                    canonical,
+                    revision,
+                    committed: true,
+                    ..
+                } => {
+                    assert_eq!(revision.version, POOL_META_V1_VERSION);
+                    canonical
+                }
+                _ => panic!("safe pool-zero source must decode"),
+            };
+            assert!(
+                matches!(decode_pool_meta_replica(dangerous_raw.clone()),
+                PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
+                    if canonical == source_canonical && revision.version == POOL_META_V1_VERSION),
+                "the dangerous target must remain semantically equivalent to the safe source"
+            );
+            let physical = disks(&fixture.store).await;
+            let target_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
+            for (index, disk) in physical[1].iter().enumerate() {
+                fs::write(object_path(disk).join("xl.meta"), &target_heads[index])
+                    .await
+                    .expect("install legacy dangerous target head");
+            }
+            let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
+            assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
+            let before = snapshots(&fixture.store).await;
+            let identities = owner_repair_identity_heads(&fixture.store).await;
+            let guard = owner(&fixture.store).await;
+            let mut state = fixture.store.pool_meta_save_gate.lock().await;
+            let result = tokio::time::timeout(
+                std::time::Duration::from_secs(30),
+                fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
+            )
+            .await
+            .expect("unsafe legacy target heal must terminate");
+            drop(state);
+            let error = result.expect_err("native heal must reject a target whose old-reader dont_save is true");
+            assert!(
+                error.to_string().contains("dont_save"),
+                "rejection must identify the unsafe legacy field: {error}"
+            );
+            assert_eq!(
+                snapshots(&fixture.store).await,
+                before,
+                "rejection must preserve all physical heads and backups"
+            );
+            assert_eq!(
+                owner_repair_identity_heads(&fixture.store).await,
+                identities,
+                "rejection must not write identity shards"
+            );
+            let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
+            assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1));
+            assert!(
+                matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw),
+                "unsafe target evidence must not be overwritten by the safe source"
+            );
+        },
+    )
+    .await;
+}

crates/ecstore/src/core/pool_meta_recovery_codec.rs

@@ -391,7 +391,7 @@ pub(super) async fn fixture_inline_pool_meta_replicas(payload: &[u8], modified:
         writer.shutdown().await.expect("fixture bitrot writer should finish");
         let mut info = base.clone();
         info.erasure.index = *coding_index;
-        info.data = Some(Bytes::from(writer.into_inline_data().expect("fixture writer should retain inline bytes")));
+        info.data = Some(writer.into_inline_data().expect("fixture writer should retain inline bytes"));
         let mut meta = FileMeta::new();
         meta.add_version(info).expect("test inline FileInfo should serialize");
         raw.push(Bytes::from(meta.marshal_msg().expect("test xl.meta should serialize")));

crates/ecstore/src/core/pools.rs

@@ -94,7 +94,6 @@ use serde::{Deserialize, Serialize};
 use sha2::{Digest, Sha256};
 use std::collections::{HashMap, HashSet};
 use std::fmt::Display;
-#[cfg(test)]
 use std::io::Cursor;
 use std::io::Write;
 use std::path::PathBuf;
@@ -3720,33 +3719,45 @@ fn is_decommission_copy_quorum_deferral(err: &Error) -> bool {
     data_movement::data_movement_stage_source_as::<DecommissionCopyQuorumFailure>(err).is_some()
 }
 
-/// The storage error of a version copy that kept timing out on a namespace
-/// lock through its in-place retries, kept typed so the entry failure policy
-/// defers it.
+/// A copy whose object namespace lock remained unavailable through its
+/// in-place retries, retained as a typed deferral.
 #[derive(Debug, thiserror::Error)]
 #[error("{0}")]
-struct DecommissionCopyLockTimeout(#[source] Error);
+struct DecommissionCopyLockUnavailable(#[source] Error);
 
-fn is_decommission_copy_lock_timeout_error(err: &Error) -> bool {
-    let err = data_movement::data_movement_stage_source(err).unwrap_or(err);
-    matches!(err, Error::Lock(rustfs_lock::LockError::Timeout { .. }))
+fn is_decommission_copy_lock_unavailable_error(err: &Error, bucket: &str, object: &str) -> bool {
+    let source = data_movement::data_movement_stage_source(err).unwrap_or(err);
+    match source {
+        Error::Lock(rustfs_lock::LockError::Timeout { .. }) => true,
+        Error::NamespaceLockQuorumUnavailable {
+            mode: "read",
+            bucket: lock_bucket,
+            object: lock_object,
+            ..
+        } => {
+            data_movement::data_movement_error_is_stage(err, "prepare_put_object")
+                && lock_bucket == bucket
+                && lock_object == object
+        }
+        _ => false,
+    }
 }
 
-/// A version copy that timed out acquiring a namespace lock leaves the source
+/// A version copy that could not acquire its namespace lock leaves the source
 /// in place; the next attempt copies the version again or finds an equivalent
 /// committed target. The lock is held by another mutation of the key or, when
 /// this node restarted mid-copy, by its previous incarnation until that lock's
 /// lease expires on the peers. Either clears by waiting, so the entry is
 /// deferred instead of being counted as a permanent per-object failure.
-fn decommission_copy_lock_timeout_deferral(stage: &str, bucket: &str, object: &str, err: Error) -> Error {
+fn decommission_copy_lock_unavailable_deferral(stage: &str, bucket: &str, object: &str, err: Error) -> Error {
     data_movement::data_movement_context_error(
-        format!("decommission {stage} timed out on a namespace lock for {bucket}/{object}: {err}"),
-        DecommissionCopyLockTimeout(err),
+        format!("decommission {stage} could not acquire a namespace lock for {bucket}/{object}: {err}"),
+        DecommissionCopyLockUnavailable(err),
     )
 }
 
-fn is_decommission_copy_lock_timeout_deferral(err: &Error) -> bool {
-    data_movement::data_movement_stage_source_as::<DecommissionCopyLockTimeout>(err).is_some()
+fn is_decommission_copy_lock_unavailable_deferral(err: &Error) -> bool {
+    data_movement::data_movement_stage_source_as::<DecommissionCopyLockUnavailable>(err).is_some()
 }
 
 /// The deferral of a version copy whose in-place retries ended on `error`, if
@@ -3756,8 +3767,8 @@ fn take_decommission_copy_deferral(stage: &str, bucket: &str, object: &str, erro
         return Some(decommission_copy_quorum_deferral(stage, bucket, object, err));
     }
     error
-        .take_if(|err| is_decommission_copy_lock_timeout_error(err))
-        .map(|err| decommission_copy_lock_timeout_deferral(stage, bucket, object, err))
+        .take_if(|err| is_decommission_copy_lock_unavailable_error(err, bucket, object))
+        .map(|err| decommission_copy_lock_unavailable_deferral(stage, bucket, object, err))
 }
 
 fn resolve_decommission_entry_reload_result(result: Result<()>, bucket: &str, object_name: &str) -> Result<()> {
@@ -5822,13 +5833,13 @@ enum DecommissionEntryOutcome {
 /// entry is done and must not be counted as a failure or escalated. A quorum
 /// failure of the source cleanup is deferred too: the copied source is still
 /// in place and the next attempt cleans it up again. So is a version copy that
-/// lost drive quorum or timed out on a namespace lock through its in-place
+/// lost drive quorum or could not acquire an object namespace lock through its in-place
 /// retries: the source still holds the version and the next attempt copies it
 /// again. Every other failure stays terminal.
 fn resolve_decommission_entry_failure(err: Error) -> Result<DecommissionEntryOutcome> {
     if is_decommission_source_cleanup_retryable_error(&err)
         || is_decommission_copy_quorum_deferral(&err)
-        || is_decommission_copy_lock_timeout_deferral(&err)
+        || is_decommission_copy_lock_unavailable_deferral(&err)
     {
         return Ok(DecommissionEntryOutcome::Deferred(err));
     }
@@ -6827,11 +6838,47 @@ impl Drop for PoolMetaTransactionArm {
     }
 }
 
+fn ensure_legacy_pool_meta_republish_safe(data: &[u8]) -> Result<()> {
+    if data.len() >= 4
+        && LittleEndian::read_u16(&data[2..4]) == POOL_META_V1_VERSION
+        // The selection already validates the payload. The runtime-only
+        // flag can only appear in the legacy three-field sequence or map.
+        && rmp::decode::read_array_len(&mut &data[4..])
+            .or_else(|_| rmp::decode::read_map_len(&mut &data[4..]))
+            .ok()
+            == Some(3)
+        && let Ok(legacy) = rmp_serde::from_slice::<LegacyPoolMeta>(&data[4..])
+        && legacy.dont_save
+    {
+        return Err(Error::other("pool metadata legacy source contains dont_save=true"));
+    }
+    Ok(())
+}
+
+// Ordinary legacy reads retain their compatibility behavior, but bytes that
+// will be copied to another pool must not contain an unparsed extension.
+fn ensure_legacy_pool_meta_payload_complete(data: &[u8]) -> Result<()> {
+    let payload = data
+        .get(4..)
+        .ok_or_else(|| Error::other("pool metadata legacy source is too short"))?;
+    let mut decoder = rmp_serde::Deserializer::new(Cursor::new(payload));
+    serde::de::IgnoredAny::deserialize(&mut decoder).map_err(|_| Error::other("pool metadata legacy source is invalid"))?;
+    let payload_len = u64::try_from(payload.len()).map_err(|_| Error::other("pool metadata legacy source is too large"))?;
+    if decoder.get_ref().position() != payload_len {
+        return Err(Error::other("pool metadata legacy source has trailing bytes"));
+    }
+    Ok(())
+}
+
 #[derive(Debug)]
 struct PoolMetaSelection {
     meta: PoolMeta,
     revision: PoolMetaRevision,
     canonical: Option<Vec<u8>>,
+    // Legacy semantic comparison canonicalizes to V2; a heal must instead
+    // preserve verified raw bytes at the highest observed format floor.
+    legacy_raw: Option<Vec<u8>>,
+    legacy_has_unsafe_flag: bool,
     replica_state: PoolMetaReplicaState,
     cas_tokens: Vec<PoolMetaCasToken>,
     absent: bool,
@@ -7222,6 +7269,8 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
             meta: PoolMeta::default(),
             revision: PoolMetaRevision::legacy(0),
             canonical: None,
+            legacy_raw: None,
+            legacy_has_unsafe_flag: false,
             replica_state: PoolMetaReplicaState {
                 needs_repair: false,
                 repair_write_safe: true,
@@ -7276,6 +7325,8 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
             meta: selected.meta.clone(),
             revision: selected.revision,
             canonical: Some(selected.canonical.clone()),
+            legacy_raw: None,
+            legacy_has_unsafe_flag: false,
             replica_state: PoolMetaReplicaState {
                 needs_repair,
                 repair_write_safe,
@@ -7289,6 +7340,10 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
 
     // Legacy V1/V2 has no durable generation. Pool zero remains the commit
     // record; divergent backups without it are ambiguous and fail closed.
+    let legacy_has_unsafe_flag = committed
+        .iter()
+        .any(|(_, raw, _)| ensure_legacy_pool_meta_republish_safe(raw).is_err());
+    let mut floor_raw = None;
     let mut selected: Option<(usize, Vec<u8>, PoolMetaCommittedCandidate)> = None;
     for (idx, raw, candidate) in committed {
         if let Some((selected_idx, selected_raw, selected_candidate)) = selected.as_ref() {
@@ -7302,19 +7357,36 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
                 }
             } else {
                 needs_repair |= selected_raw != &raw;
+                if floor_raw.is_none()
+                    && candidate.revision.version == observed_version
+                    && raw
+                        .get(2..4)
+                        .is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
+                {
+                    floor_raw = Some(raw);
+                }
             }
         } else {
             selected = Some((idx, raw, candidate));
         }
     }
-    let (_, _, mut selected) =
+    let (_, selected_raw, mut selected) =
         selected.ok_or_else(|| Error::other("pool metadata recovery required: no valid legacy replica is available"))?;
+    if selected.revision.version == observed_version
+        && selected_raw
+            .get(2..4)
+            .is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
+    {
+        floor_raw = Some(selected_raw);
+    }
     selected.meta.version = observed_version;
     selected.revision.version = observed_version;
     Ok(PoolMetaSelection {
         meta: selected.meta,
         revision: selected.revision,
         canonical: Some(selected.canonical),
+        legacy_raw: floor_raw,
+        legacy_has_unsafe_flag,
         replica_state: PoolMetaReplicaState {
             needs_repair,
             repair_write_safe,
@@ -14534,6 +14606,23 @@ impl ECStore {
             .canonical
             .as_ref()
             .ok_or_else(|| Error::other("pool metadata heal has no committed source"))?;
+        if selection.generation_protocol_observed && !selection.revision.is_generation_protocol() {
+            return Err(Error::other("pool metadata heal requires recovery of the observed generation protocol"));
+        }
+        if selection.legacy_has_unsafe_flag {
+            return Err(Error::other("pool metadata legacy replica contains dont_save=true"));
+        }
+        let repair_data = if selection.revision.is_generation_protocol() {
+            canonical.clone()
+        } else {
+            let raw = selection
+                .legacy_raw
+                .clone()
+                .ok_or_else(|| Error::other("pool metadata heal cannot prove the legacy format floor"))?;
+            ensure_legacy_pool_meta_payload_complete(&raw)?;
+            raw
+        };
+        ensure_legacy_pool_meta_republish_safe(&repair_data)?;
         let pool = self
             .pools
             .get(pool_index)
@@ -14544,10 +14633,10 @@ impl ECStore {
             .get(pool_index)
             .ok_or_else(|| Error::other("pool metadata heal has no target revision"))?;
         let mut arm = state.arm_transaction();
-        save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, canonical.clone(), token, &fence, "native_heal", &mut arm)
+        save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, repair_data.clone(), token, &fence, "native_heal", &mut arm)
             .await?;
         let (confirmed, _) = read_config_no_lock_preserve_empty_with_metadata(pool, POOL_META_NAME).await?;
-        if confirmed != *canonical {
+        if confirmed != repair_data {
             return Err(Error::other("pool metadata native heal readback diverged"));
         }
         fence.ensure_held()?;
@@ -28353,7 +28442,7 @@ mod tests {
             "one version must commit before the next version times out"
         );
         assert!(
-            matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_timeout_deferral(err)),
+            matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
             "{outcome:?}"
         );
         assert_eq!(
@@ -28387,6 +28476,88 @@ mod tests {
         assert_eq!(actual, b"written after versioning was enabled");
     }
 
+    /// A target lookup that loses its read-lock quorum must retain the source
+    /// and resume an entry after an earlier version has already committed.
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn decommission_defers_target_lookup_read_lock_quorum_and_resumes() {
+        use tokio::io::AsyncReadExt as _;
+
+        let fixture = NullVersionDecommission::new(NullVersionTop::NewerVersion).await;
+        let object = fixture.object;
+        let attempted = Arc::new(AtomicUsize::new(0));
+        let fault = DecommissionTestFaultGuard::install_with_error(
+            Arc::new({
+                let attempted = Arc::clone(&attempted);
+                move |stage, _, faulted, _, success| {
+                    stage == "before_target_commit"
+                        && faulted == object
+                        && success
+                        && attempted.fetch_add(1, Ordering::SeqCst) > 0
+                }
+            }),
+            Some(Arc::new(|_, bucket, object| {
+                data_movement::data_movement_stage_error_for_test(
+                    "decommission_object",
+                    "prepare_put_object",
+                    bucket,
+                    object,
+                    Error::NamespaceLockQuorumUnavailable {
+                        mode: "read",
+                        bucket: bucket.to_owned(),
+                        object: object.to_owned(),
+                        required: 1,
+                        achieved: 0,
+                    },
+                )
+            })),
+        );
+        let (outcome, _) = fixture.run_entry().await;
+        assert!(
+            matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
+            "{outcome:?}"
+        );
+        fixture.assert_source_retained().await;
+        assert_eq!(
+            fixture
+                .target_versions()
+                .await
+                .expect("first version committed")
+                .versions
+                .len(),
+            1
+        );
+        assert_eq!(
+            fixture.store.pool_meta.read().await.pools[0]
+                .decommission
+                .as_ref()
+                .expect("active decommission")
+                .items_decommission_failed,
+            0
+        );
+
+        drop(fault);
+        let (outcome, _) = fixture.run_entry().await;
+        assert!(matches!(outcome, DecommissionEntryOutcome::Complete), "{outcome:?}");
+        fixture.assert_moved().await;
+        let mut reader = fixture.store.pools[1]
+            .get_object_reader(
+                fixture.bucket,
+                fixture.object,
+                None,
+                HeaderMap::new(),
+                &ObjectOptions {
+                    version_id: Some(fixture.seeded.versions[0].version_id.expect("newer version").to_string()),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("the newer version must remain readable");
+        let mut actual = Vec::new();
+        reader.stream.read_to_end(&mut actual).await.expect("read the newer version");
+        assert_eq!(actual, b"written after versioning was enabled");
+    }
+
     /// A namespace lock still held on the target key, as by a copy whose node
     /// was killed mid-write, makes target selection fail with a typed
     /// object-lock timeout. A real held lock, not an injected error, must defer
@@ -28429,7 +28600,7 @@ mod tests {
 
             let (outcome, _) = fixture.run_entry().await;
             assert!(
-                matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_timeout_deferral(err)),
+                matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
                 "{outcome:?}"
             );
             assert_eq!(
@@ -28838,17 +29009,17 @@ mod tests {
             timeout(),
         );
         for err in [timeout(), prepare] {
-            assert!(is_decommission_copy_lock_timeout_error(&err), "{err}");
+            assert!(is_decommission_copy_lock_unavailable_error(&err, "bucket-a", "object-a"), "{err}");
             assert!(!is_decommission_copy_quorum_error(&err), "{err}");
             let mut error = Some(err);
             let deferred = take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error)
                 .expect("a lock timeout copy must be deferred");
             assert!(error.is_none());
-            assert!(is_decommission_copy_lock_timeout_deferral(&deferred));
+            assert!(is_decommission_copy_lock_unavailable_deferral(&deferred));
             assert!(!is_decommission_copy_quorum_deferral(&deferred));
-            let failure = data_movement::data_movement_stage_source_as::<DecommissionCopyLockTimeout>(&deferred)
+            let failure = data_movement::data_movement_stage_source_as::<DecommissionCopyLockUnavailable>(&deferred)
                 .expect("the deferred error must keep its lock-timeout origin");
-            assert!(is_decommission_copy_lock_timeout_error(&failure.0));
+            assert!(is_decommission_copy_lock_unavailable_error(&failure.0, "bucket-a", "object-a"));
             assert!(matches!(
                 resolve_decommission_entry_failure(deferred),
                 Ok(DecommissionEntryOutcome::Deferred(_))
@@ -28873,6 +29044,43 @@ mod tests {
         }
     }
 
+    #[test]
+    fn decommission_copy_defers_only_the_matching_object_read_lock_quorum() {
+        let quorum = |mode, bucket: &str, object: &str| Error::NamespaceLockQuorumUnavailable {
+            mode,
+            bucket: bucket.to_owned(),
+            object: object.to_owned(),
+            required: 1,
+            achieved: 0,
+        };
+        let wrap = |stage, err| {
+            data_movement::data_movement_stage_error_for_test("decommission_object", stage, "bucket-a", "object-a", err)
+        };
+        let mut error = Some(wrap("prepare_put_object", quorum("read", "bucket-a", "object-a")));
+        let deferred = take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error)
+            .expect("matching read-lock quorum must defer");
+        assert!(error.is_none());
+        assert!(matches!(
+            resolve_decommission_entry_failure(deferred),
+            Ok(DecommissionEntryOutcome::Deferred(_))
+        ));
+        for err in [
+            quorum("read", "bucket-a", "object-a"),
+            wrap("source_recheck", quorum("read", "bucket-a", "object-a")),
+            wrap("prepare_put_object", quorum("write", "bucket-a", "object-a")),
+            wrap(
+                "prepare_put_object",
+                quorum("exact_delete_capacity_reconciliation", "bucket-a", "object-a"),
+            ),
+            wrap("prepare_put_object", quorum("read", "other-bucket", "object-a")),
+            wrap("prepare_put_object", quorum("read", "bucket-a", "other-object")),
+        ] {
+            let mut error = Some(err);
+            assert!(take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error).is_none());
+            assert!(error.is_some(), "unrelated lock failures must stay terminal");
+        }
+    }
+
     #[test]
     fn decommission_target_capacity_error_accepts_wrapped_capacity_errors() {
         let disk_full = Error::other(format!("decommission_object: put_object failed for bucket/object: {}", Error::DiskFull));
@@ -29351,6 +29559,53 @@ mod tests {
         );
     }
 
+    #[test]
+    fn legacy_republish_guard_checks_only_three_field_sequence_or_map() {
+        let frame = |payload: Vec<u8>| {
+            let mut data = Vec::new();
+            data.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
+            data.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
+            data.extend_from_slice(&payload);
+            data
+        };
+        let two_fields = PersistedPoolMetaV1 {
+            version: POOL_META_V1_VERSION,
+            pools: Vec::new(),
+        };
+        for payload in [rmp_serde::to_vec(&two_fields), rmp_serde::to_vec_named(&two_fields)] {
+            let data = frame(payload.expect("two-field V1 payload must serialize"));
+            assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
+            assert!(ensure_legacy_pool_meta_republish_safe(&data).is_ok());
+            assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
+            let mut trailing = data;
+            trailing.push(0xc0);
+            assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
+        }
+        for dont_save in [false, true] {
+            let legacy = LegacyPoolMeta {
+                version: POOL_META_V1_VERSION,
+                pools: Vec::new(),
+                dont_save,
+            };
+            for payload in [rmp_serde::to_vec(&legacy), rmp_serde::to_vec_named(&legacy)] {
+                let data = frame(payload.expect("three-field V1 payload must serialize"));
+                assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
+                assert_eq!(ensure_legacy_pool_meta_republish_safe(&data).is_err(), dont_save);
+                assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
+                let mut trailing = data;
+                trailing.push(0xc0);
+                assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
+            }
+        }
+        // This guard is only called for already-decoded replicas; malformed
+        // containers must fail selection rather than becoming repair sources.
+        let truncated = frame(vec![0x93, 0x01]);
+        assert!(matches!(
+            decode_pool_meta_replica(truncated),
+            PoolMetaReplica::Corrupt(_) | PoolMetaReplica::Incompatible(_)
+        ));
+    }
+
     #[test]
     fn pool_meta_decode_supports_legacy_payload() {
         let start_time = OffsetDateTime::now_utc();

crates/ecstore/src/core/sets.rs

@@ -317,8 +317,9 @@ impl Sets {
         &self.ctx
     }
 
-    #[cfg(test)]
-    pub(crate) async fn stop_endpoint_monitor_for_test(&self) {
+    /// Stop endpoint reconnection and wait for any in-flight pass before a test owns disk probes.
+    #[cfg(any(test, feature = "test-util"))]
+    pub async fn stop_endpoint_monitor_for_test(&self) {
         if let Some(exit_signal) = &self.exit_signal {
             let _ = exit_signal.send(());
             // Wait for an in-flight reconnect pass as well as its receiver to exit.

crates/ecstore/src/crash_inject.rs

@@ -84,7 +84,7 @@ pub(crate) enum CrashPoint {
 }
 
 #[cfg(test)]
-pub(crate) use armed::{arm, disarm, should_crash_at};
+pub(crate) use armed::{arm, disarm, is_armed, should_crash_at};
 
 /// Production no-op: the compiler inlines this to `false`, so armed crash points
 /// exist only in `#[cfg(test)]` builds and never add a branch to real commits.
@@ -94,6 +94,12 @@ pub(crate) fn should_crash_at(_point: CrashPoint, _key: &str) -> bool {
     false
 }
 
+#[cfg(not(test))]
+#[inline(always)]
+pub(crate) fn is_armed(_point: CrashPoint, _key: &str) -> bool {
+    false
+}
+
 #[cfg(test)]
 mod armed {
     use super::CrashPoint;
@@ -132,6 +138,10 @@ mod armed {
         guard().retain(|(p, k)| !(*p == point && k == key));
     }
 
+    pub(crate) fn is_armed(point: CrashPoint, key: &str) -> bool {
+        guard().iter().any(|(p, k)| *p == point && k == key)
+    }
+
     /// Returns `true` (consuming the arm) iff a crash is armed for exactly this
     /// `point` and `key`.
     pub(crate) fn should_crash_at(point: CrashPoint, key: &str) -> bool {

crates/ecstore/src/data_movement/mod.rs

@@ -605,6 +605,7 @@ fn resolve_data_movement_abort_result(
 #[derive(Debug)]
 struct DataMovementStageError {
     rendered: String,
+    stage: Option<String>,
     source: Box<dyn std::error::Error + Send + Sync>,
 }
 
@@ -626,6 +627,7 @@ where
 {
     Error::other(DataMovementStageError {
         rendered,
+        stage: None,
         source: Box::new(err),
     })
 }
@@ -634,7 +636,21 @@ fn data_movement_stage_error<E>(op_label: &str, stage: &str, bucket: &str, objec
 where
     E: std::error::Error + Send + Sync + 'static,
 {
-    data_movement_context_error(format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"), err)
+    Error::other(DataMovementStageError {
+        rendered: format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"),
+        stage: Some(stage.to_owned()),
+        source: Box::new(err),
+    })
+}
+
+pub(crate) fn data_movement_error_is_stage(err: &Error, stage: &str) -> bool {
+    let Error::Io(io_err) = err else {
+        return false;
+    };
+    io_err
+        .get_ref()
+        .and_then(|source| source.downcast_ref::<DataMovementStageError>())
+        .is_some_and(|source| source.stage.as_deref() == Some(stage))
 }
 
 #[cfg(test)]

crates/ecstore/src/disk/cleanup_runtime.rs

@@ -624,58 +624,58 @@ mod tests {
 
     #[tokio::test]
     async fn cleanup_dispatch_retains_resource_units_without_inheriting_disk_policy() {
-        use crate::disk::io_admission::{current_disk_io_context, with_background_disk_io, with_background_resource_unit};
-        use rustfs_concurrency::adaptive::AdaptiveAdmission;
-        use std::num::NonZeroU16;
+        use crate::disk::io_admission::{current_disk_io_context, test_resource_units, with_background_disk_io};
         use tokio_util::sync::CancellationToken;
 
-        for mode in 0..4 {
-            let runtime = Arc::new(CleanupRuntime::new(config(1, 1, 1)).expect("cleanup runtime"));
-            let worker_runtime = runtime.clone();
-            let gate = AdaptiveAdmission::new(NonZeroU16::new(1).expect("one unit"), 1);
-            let (started_tx, started_rx) = tokio::sync::oneshot::channel();
-            let (release_tx, release_rx) = tokio::sync::oneshot::channel();
-            let work = async move {
-                started_tx
-                    .send((current_resource_unit().is_some(), current_disk_io_context().is_none()))
-                    .expect("report worker context");
-                release_rx.await.expect("release owned worker");
-                Ok::<_, super::super::error::DiskError>(())
-            };
-            let task = tokio::spawn(with_background_disk_io(
-                CancellationToken::new(),
-                with_background_resource_unit(gate.acquire().await, async move {
-                    match mode {
-                        0 => worker_runtime
-                            .disk_operation(work, DiskAdmission::Request)
-                            .await
-                            .expect("disk operation"),
-                        1 => worker_runtime.gc_operation(move |_| work).await.expect("GC operation"),
-                        2 => {
-                            let reservation = worker_runtime.reserve().expect("cleanup reservation");
-                            run(Some(reservation), async { work.await.expect("cleanup operation") }).await;
+        for physical_count in [1, 2] {
+            for mode in 0..4 {
+                let runtime = Arc::new(CleanupRuntime::new(config(1, 1, 1)).expect("cleanup runtime"));
+                let worker_runtime = runtime.clone();
+                let (unit, gates) = test_resource_units(physical_count);
+                let (started_tx, started_rx) = tokio::sync::oneshot::channel();
+                let (release_tx, release_rx) = tokio::sync::oneshot::channel();
+                let work = async move {
+                    started_tx
+                        .send((current_resource_unit().is_some(), current_disk_io_context().is_none()))
+                        .expect("report worker context");
+                    release_rx.await.expect("release owned worker");
+                    Ok::<_, super::super::error::DiskError>(())
+                };
+                let task = tokio::spawn(with_background_disk_io(
+                    CancellationToken::new(),
+                    with_optional_resource_unit(Some(unit), async move {
+                        match mode {
+                            0 => worker_runtime
+                                .disk_operation(work, DiskAdmission::Request)
+                                .await
+                                .expect("disk operation"),
+                            1 => worker_runtime.gc_operation(move |_| work).await.expect("GC operation"),
+                            2 => {
+                                let reservation = worker_runtime.reserve().expect("cleanup reservation");
+                                run(Some(reservation), async { work.await.expect("cleanup operation") }).await;
+                            }
+                            _ => spawn(None, work)
+                                .await
+                                .expect("unisolated worker")
+                                .expect("unisolated operation"),
                         }
-                        _ => spawn(None, work)
-                            .await
-                            .expect("unisolated worker")
-                            .expect("unisolated operation"),
+                    }),
+                ));
+                let (inherited, original_policy) = deadline(started_rx).await.expect("owned work starts");
+                task.abort();
+                assert!(task.await.expect_err("waiter cancelled").is_cancelled());
+                let held: Vec<_> = gates.iter().map(|gate| gate.snapshot().active).collect();
+                release_tx.send(()).expect("owned work survives cancellation");
+                deadline(async {
+                    while gates.iter().any(|gate| gate.snapshot().active != 0) {
+                        tokio::task::yield_now().await;
                     }
-                }),
-            ));
-            let (inherited, original_policy) = deadline(started_rx).await.expect("owned work starts");
-            task.abort();
-            assert!(task.await.expect_err("waiter cancelled").is_cancelled());
-            let held = gate.snapshot().active;
-            release_tx.send(()).expect("owned work survives cancellation");
-            deadline(async {
-                while gate.snapshot().active != 0 {
-                    tokio::task::yield_now().await;
-                }
-            })
-            .await;
-            assert!(inherited, "mode {mode} lost resource ownership");
-            assert!(original_policy, "mode {mode} changed the detached worker's disk policy");
-            assert_eq!(held, 1, "mode {mode} released a live worker's unit");
+                })
+                .await;
+                assert!(inherited, "mode {mode} lost resource ownership");
+                assert!(original_policy, "mode {mode} changed the detached worker's disk policy");
+                assert_eq!(held, vec![1; physical_count], "mode {mode} released a live worker's unit");
+            }
         }
     }
     fn config(pending: usize, workers: usize, blocking_threads: usize) -> Config {