fix(heal): detect stale delete-marker metadata (#8102)

A flat (recursive) listing resumed inside "s/" re-emitted everything in
its sibling "s-x/". scan_dir dropped the entries before forward_to by
comparing directory names without their trailing slash, where "s" sorts
before "s-x", while the keys they stand for sort the other way round:
'-' (0x2d) is below '/' (0x2f), so all of "s-x/..." precedes "s/...".
The drain stopped at "s" and kept "s-x".

The next page then started with keys at or before the marker; the
listing layer filtered all of them out, found no more candidates and
answered IsTruncated=false. A bucket of 104,137 objects with backup
directories named "<id>" and "<id>-rollbacks" listed as 5,000; two such
directories of 1,200 keys each listed as 2,000.

Compare every remaining entry as the key prefix it stands for, slash
included, and keep it only when it sorts at or after forward_to or
contains it. The remainder of forward_to is taken before `current` is
trimmed, so this also holds below the bucket root; retain also drops
entries when all of them precede forward_to, which the drain never did.

.agents/skills/security-advisory-lessons/references/advisory-patterns.md

@@ -101,6 +101,7 @@ Update this file only when an advisory adds or changes a reusable lesson, affect
 ### SSE and on-disk storage invariants
 
 - `GHSA-xrrf-67jm-3c2r`: SSE metadata reported encryption while reader composition bypassed `EncryptReader` and stored plaintext. Lesson: test actual bytes on disk and wrapper order, not only API metadata.
+- `GHSA-wqmc-vjgv-jrpw`: SSE-C persisted and listed the plaintext MD5 as the ETag, allowing list-only users to confirm or brute-force low-entropy content and correlate equal plaintext across keys. Lesson: encrypted-object ETags must not expose deterministic plaintext fingerprints; protect them with per-object key material and keep listing, conditional, copy, and multipart semantics consistent.
 
 ### Object Lock and retention invariants
 
@@ -126,7 +127,7 @@ rg -n "normalize_extract_entry_key|Snowball|auto-extract|PathBuf::join|canonical
 rg -n "DEFAULT_SECRET|DEFAULT_ACCESS|TEST_PRIVATE_KEY|rustfs rpc|RUSTFS_RPC_SECRET" rustfs crates
 rg -n "TONIC_RPC_PREFIX|verify_rpc_signature|check_auth|NodeServiceServer|x-rustfs-signature" rustfs crates
 rg -n "debug!|trace!|info!|error!|\\?resp|\\?merged_config|session_token|secret_key" rustfs crates
-rg -n "HashReader|EncryptReader|SSE|server-side encryption|Access-Control-Allow-Credentials|Origin" rustfs crates
+rg -n "HashReader|EncryptReader|try_resolve_etag|ETag|SSE|server-side encryption|Access-Control-Allow-Credentials|Origin" rustfs crates
 rg -n "ObjectLock|object_lock|retention|COMPLIANCE|GOVERNANCE|delete_prefix|lifecycle|scanner" rustfs crates
 rg -n "deny_unknown_fields|serde.default|as u32|as usize|as i32" rustfs crates
 ```
@@ -145,5 +146,5 @@ rg -n "deny_unknown_fields|serde.default|as u32|as usize|as i32" rustfs crates
 - IAM export fixes: assert exported archives omit plaintext user and service-account secrets unless the format deliberately encrypts or seals them.
 - RPC auth fixes: include captured metadata replay across two concrete methods, stale timestamps, wrong path, wrong method surrogate, wrong secret, and valid same-method calls.
 - Browser/CORS fixes: assert no credentials on reflected/default origins, correct behavior for explicit allowlists, and no same-origin script execution for previewed object content.
-- SSE fixes: inspect stored bytes and verify API metadata, read-back behavior, and on-disk ciphertext together.
+- SSE fixes: inspect stored bytes and verify API metadata, read-back behavior, and on-disk ciphertext together; for ETags, upload equal plaintext under different keys and assert list-only callers cannot derive or correlate the plaintext digest, including multipart objects.
 - Object Lock fixes: include unreadable metadata, fabricated metadata defaults, unparsable config, confirmed absent config, COMPLIANCE/GOVERNANCE retention, lifecycle expiry, scanner sweeps, and force-delete paths.

.config/make/tests.mak

@@ -29,6 +29,7 @@ script-tests: ## Run shell script tests
 	./scripts/test_entrypoint_credentials.sh
 	./scripts/test_internode_grpc_ab_bench.sh
 	./scripts/test_object_batch_bench_enhanced.sh
+	./scripts/test_package_service_scripts.sh
 	./scripts/test_hotpath_warp_ab_gate.sh
 	./scripts/test_hotpath_warp_abba.sh
 	./scripts/test_scanner_validation_harness.sh

.github/actions/quick-checks/action.yml

@@ -19,7 +19,7 @@ runs:
   using: composite
   steps:
     - name: Install quality tools
-      uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
+      uses: taiki-e/install-action@7623a79cdfecb99d681017af368ca353d9f49bb5 # v2
       with:
         tool: |
           ripgrep@15.2.0

.github/actions/setup/action.yml

@@ -109,7 +109,7 @@ runs:
       uses: taiki-e/install-action@96c7780c1d8a2b8723e12031def873a434d39d8d # nextest
 
     - name: Setup Rust cache
-      uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
+      uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
       with:
         # false is rust-cache's own default. With true, cleanup.ts returns
         # *before* pruning ~/.cargo/registry/src, and config.ts archives the

.github/workflows/architecture-migration-rules.yml

@@ -48,12 +48,12 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
       - name: Install ripgrep
-        uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
+        uses: taiki-e/install-action@7623a79cdfecb99d681017af368ca353d9f49bb5 # v2
         with:
           tool: ripgrep@15.2.0
 

.github/workflows/audit.yml

@@ -24,8 +24,11 @@ on:
       - '.github/actions/**'
       - '.github/workflows/**'
       - 'scripts/release/create_or_update_release.sh'
+      - 'scripts/release/package_service_scripts.sh'
       - 'scripts/release/package_versions.sh'
+      - 'scripts/test_package_service_scripts.sh'
       - 'scripts/test_package_versions.sh'
+      - 'scripts/test_docker_workflow.py'
       - 'scripts/security/check_performance_ab_workflow.sh'
       - 'scripts/security/check_preview_release_workflow.sh'
       - 'scripts/security/check_tier_artifact_workflow.sh'
@@ -40,8 +43,11 @@ on:
       - '.github/actions/**'
       - '.github/workflows/**'
       - 'scripts/release/create_or_update_release.sh'
+      - 'scripts/release/package_service_scripts.sh'
       - 'scripts/release/package_versions.sh'
+      - 'scripts/test_package_service_scripts.sh'
       - 'scripts/test_package_versions.sh'
+      - 'scripts/test_docker_workflow.py'
       - 'scripts/security/check_performance_ab_workflow.sh'
       - 'scripts/security/check_preview_release_workflow.sh'
       - 'scripts/security/check_tier_artifact_workflow.sh'
@@ -87,7 +93,7 @@ jobs:
     timeout-minutes: 20
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -106,7 +112,7 @@ jobs:
       # default is now "false", but state it explicitly — see
       # scripts/security/check_cache_save_if.sh.
       - name: Setup Rust cache
-        uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
+        uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
         with:
           # Same reasoning as the setup composite: true archives every
           # dependency's unpacked source tree.
@@ -116,7 +122,7 @@ jobs:
           save-if: ${{ github.ref == 'refs/heads/main' }}
 
       - name: Install cargo-deny
-        uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
+        uses: taiki-e/install-action@7623a79cdfecb99d681017af368ca353d9f49bb5 # v2
         with:
           tool: cargo-deny
 
@@ -130,7 +136,7 @@ jobs:
     timeout-minutes: 5
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -158,6 +164,9 @@ jobs:
       - name: Check package version contract
         run: ./scripts/test_package_versions.sh
 
+      - name: Check package service lifecycle contract
+        run: ./scripts/test_package_service_scripts.sh
+
   dependency-review:
     name: Dependency Review
     runs-on: ubuntu-latest
@@ -168,7 +177,7 @@ jobs:
       pull-requests: write
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -201,7 +210,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/build.yml

@@ -98,7 +98,7 @@ jobs:
       is_prerelease: ${{ steps.check.outputs.is_prerelease }}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -257,7 +257,7 @@ jobs:
       matrix: ${{ fromJson(needs.prepare-platform-matrix.outputs.matrix) }}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
           fetch-depth: 0
@@ -702,7 +702,7 @@ jobs:
           exit 1
 
       - name: Upload to GitHub artifacts
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: ${{ steps.package.outputs.package_name }}
           path: "rustfs-*.zip"
@@ -818,9 +818,7 @@ jobs:
 
           echo ""
           echo "🐳 Docker Images:"
-          if [[ "$BUILD_TYPE" == "preview" ]]; then
-            echo "⏭️  Preview tags do not publish Docker images"
-          elif [[ "$INPUT_BUILD_DOCKER" == "false" ]]; then
+          if [[ "$INPUT_BUILD_DOCKER" == "false" ]]; then
             echo "⏭️  Docker image build was skipped (binary only build)"
           elif [[ "$BUILD_STATUS" == "success" ]]; then
             echo "🔄 Docker images will be built and pushed automatically via workflow_run event"
@@ -842,7 +840,7 @@ jobs:
       release_url: ${{ steps.create.outputs.release_url }}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
           fetch-depth: 0
@@ -901,12 +899,12 @@ jobs:
       actions: read
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
       - name: Download all build artifacts
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           path: ./artifacts
           pattern: rustfs-*
@@ -1140,7 +1138,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/cache-warm.yml

@@ -109,7 +109,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -141,7 +141,7 @@ jobs:
 
       - name: Upload cargo timings report
         if: inputs.emit_timings
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: cargo-timings-ci-dev
           path: target/cargo-timings/
@@ -199,7 +199,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -230,7 +230,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -260,7 +260,7 @@ jobs:
     timeout-minutes: 60
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 

.github/workflows/ci.yml

@@ -77,7 +77,7 @@ jobs:
     outputs:
       mode: ${{ steps.scope.outputs.mode }}
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           fetch-depth: 2
           persist-credentials: false
@@ -101,11 +101,11 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Typos check with custom config file
-        uses: crate-ci/typos@37bb98842b0d8c4ffebdb75301a13db0267cef89 # master
+        uses: crate-ci/typos@512fc24f32f44ab01972217aaaf3dc86ec234d53 # v1.50.2
 
   # Fail early with compile-free checks for every pull request.
   quick-checks:
@@ -115,7 +115,7 @@ jobs:
     timeout-minutes: 10
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -132,7 +132,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           # Checkout otherwise writes the token into .git/config, where a PR's
           # own build.rs or proc-macro could read it back out.
@@ -261,7 +261,7 @@ jobs:
 
       - name: Upload test reports and diagnostics
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: junit-test-and-lint-${{ github.run_number }}
           path: |
@@ -305,7 +305,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -344,7 +344,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -391,7 +391,7 @@ jobs:
 
       - name: Upload ILM test diagnostics
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: ilm-integration-${{ github.run_number }}-${{ github.run_attempt }}
           path: |
@@ -408,7 +408,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -449,7 +449,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -521,7 +521,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -561,7 +561,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -621,7 +621,7 @@ jobs:
           PYBUILD
 
       - name: Upload debug binary
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: rustfs-debug-binary
           path: |
@@ -646,7 +646,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -662,7 +662,7 @@ jobs:
         run: python3 scripts/e2e_binary.py build --bins --features rio-v2,e2e-test-hooks
 
       - name: Upload debug binary
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: rustfs-debug-binary-rio-v2
           path: |
@@ -690,7 +690,7 @@ jobs:
     timeout-minutes: 30
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -754,7 +754,7 @@ jobs:
     timeout-minutes: 30
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -770,7 +770,7 @@ jobs:
           install-build-packaging-tools: 'false'
 
       - name: Set up Python
-        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: "3.12"
 
@@ -785,7 +785,7 @@ jobs:
       # Download after the cache restore so the freshly built binary from the
       # build job always wins over anything restored into target/debug.
       - name: Download debug binary
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: rustfs-debug-binary
           path: target/debug
@@ -821,7 +821,7 @@ jobs:
 
       - name: Upload e2e smoke diagnostics
         if: failure()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: e2e-smoke-diagnostics-${{ github.run_number }}
           path: |
@@ -831,7 +831,7 @@ jobs:
 
       - name: Upload e2e smoke JUnit report
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: e2e-smoke-junit-${{ github.run_number }}
           path: target/nextest/e2e-smoke/junit.xml
@@ -856,7 +856,7 @@ jobs:
 
       - name: Upload test logs
         if: failure()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: e2e-test-logs-${{ github.run_number }}
           path: ${{ runner.temp }}/rustfs-e2e-*/rustfs.log
@@ -879,7 +879,7 @@ jobs:
     timeout-minutes: 55
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -903,7 +903,7 @@ jobs:
           fi
 
       - name: Set up Python
-        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: "3.12"
 
@@ -945,7 +945,7 @@ jobs:
       # Download after the cache restore so the freshly built binary from the
       # build job always wins over anything restored into target/debug.
       - name: Download debug binary
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: rustfs-debug-binary
           path: target/debug
@@ -1005,7 +1005,7 @@ jobs:
 
       - name: Upload junit
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: e2e-full-junit-${{ github.run_number }}
           path: |
@@ -1015,7 +1015,7 @@ jobs:
 
       - name: Upload startup CAS evidence
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: fresh-startup-cas-evidence-${{ github.run_number }}
           path: |
@@ -1034,7 +1034,7 @@ jobs:
     timeout-minutes: 30
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -1044,7 +1044,7 @@ jobs:
           rm -f /tmp/rustfs.log
 
       - name: Download debug binary
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: rustfs-debug-binary-rio-v2
           path: target/debug
@@ -1077,7 +1077,7 @@ jobs:
 
       - name: Upload test logs
         if: failure()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: e2e-test-logs-rio-v2-${{ github.run_number }}
           path: /tmp/rustfs.log
@@ -1090,12 +1090,12 @@ jobs:
     timeout-minutes: 60
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
       - name: Download debug binary
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: rustfs-debug-binary
           path: target/debug
@@ -1120,7 +1120,7 @@ jobs:
 
       - name: Upload s3 test artifacts
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: s3tests-implemented-${{ github.run_number }}
           path: artifacts/s3tests-single/**
@@ -1164,12 +1164,12 @@ jobs:
     timeout-minutes: 30
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
       - name: Download debug binary
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: rustfs-debug-binary
           path: target/debug
@@ -1202,7 +1202,7 @@ jobs:
 
       - name: Upload s3 test artifacts
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: s3tests-lifecycle-behavior-${{ github.run_number }}
           path: artifacts/s3tests-single/**
@@ -1233,7 +1233,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Require the expected result of every CI lane
@@ -1272,7 +1272,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/cla.yml

@@ -66,7 +66,7 @@ jobs:
     steps:
       - name: Report CLA result for merge queue
         if: github.event_name == 'merge_group'
-        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         with:
           script: |
             await github.rest.checks.create({

.github/workflows/connect-diagnostic-schedule-acceptance.yml

@@ -0,0 +1,198 @@
+# Copyright 2026 RustFS Team
+# SPDX-License-Identifier: Apache-2.0
+
+name: Connect diagnostic scheduler receipt acceptance
+
+on:
+  workflow_dispatch:
+    inputs:
+      build_run_id:
+        description: Main-branch Build and Release run ID
+        required: true
+        type: string
+      artifact_id:
+        description: Linux x86_64 GNU artifact from that run
+        required: true
+        type: string
+      source_sha:
+        description: Exact RustFS source commit
+        required: true
+        type: string
+      artifact_digest:
+        description: GitHub artifact digest including sha256 prefix
+        required: true
+        type: string
+      binary_sha256:
+        description: Independently verified rustfs binary SHA-256
+        required: true
+        type: string
+      connect_sha:
+        description: Exact Connect scheduler acceptance harness commit
+        required: true
+        type: string
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  diagnostic-schedule:
+    name: Verify production scheduler receipts over mTLS
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - name: Checkout exact RustFS source
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          path: rustfs-source
+          persist-credentials: false
+          ref: ${{ inputs.source_sha }}
+
+      - name: Checkout exact Connect harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/connect
+          path: connect-harness
+          persist-credentials: false
+          ref: ${{ inputs.connect_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 22.22.2
+          cache: npm
+          cache-dependency-path: connect-harness/web/package-lock.json
+
+      - name: Verify official source and artifact identity
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          BINARY_SHA256: ${{ inputs.binary_sha256 }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+        run: |
+          set -euo pipefail
+          [[ "$BUILD_RUN_ID" =~ ^[0-9]+$ && "$ARTIFACT_ID" =~ ^[0-9]+$ ]]
+          [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ && "$CONNECT_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ "$ARTIFACT_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
+          [[ "$BINARY_SHA256" =~ ^[0-9a-f]{64}$ ]]
+          [[ $(git -C rustfs-source rev-parse HEAD) == "$SOURCE_SHA" ]]
+          [[ $(git -C connect-harness rev-parse HEAD) == "$CONNECT_SHA" ]]
+          [[ $(git -C rustfs-source remote get-url origin) == https://github.com/rustfs/rustfs ]]
+          [[ $(git -C connect-harness remote get-url origin) == https://github.com/rustfs/connect ]]
+
+          run=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}")
+          jq -e --arg source "$SOURCE_SHA" --arg repository "$GITHUB_REPOSITORY" '
+            .head_sha == $source and .head_branch == "main"
+            and .head_repository.full_name == $repository
+            and .name == "Build and Release" and .path == ".github/workflows/build.yml"
+            and .status == "completed" and .conclusion == "success"
+          ' <<<"$run" >/dev/null
+          jobs=$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
+          jq -e '
+            [.[].jobs[] | select(.name | test("^Build RustFS \\(linux-x86_64-gnu, [a-z0-9-]+, x86_64-unknown-linux-gnu, false, linux, pyroscope\\)$"))] as $matches
+            | ($matches | length) == 1 and $matches[0].conclusion == "success"
+          ' <<<"$jobs" >/dev/null
+          artifact=$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}")
+          jq -e --argjson run "$BUILD_RUN_ID" --arg source "$SOURCE_SHA" --arg digest "$ARTIFACT_DIGEST" '
+            .workflow_run.id == $run and .workflow_run.head_sha == $source
+            and .name == ("rustfs-linux-x86_64-gnu-dev-" + $source[0:7])
+            and .digest == $digest and .expired == false
+            and (.expires_at | fromdateiso8601) > now
+            and .size_in_bytes > 0 and .size_in_bytes <= 2147483648
+          ' <<<"$artifact" >/dev/null
+
+          gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}/zip" > official-artifact.zip
+          printf '%s  official-artifact.zip\n' "${ARTIFACT_DIGEST#sha256:}" | sha256sum --check --strict
+
+      - name: Extract the verified official binary
+        shell: bash
+        env:
+          SOURCE_SHA: ${{ inputs.source_sha }}
+          BINARY_SHA256: ${{ inputs.binary_sha256 }}
+        run: |
+          set -euo pipefail
+          python3 - <<'PY'
+          import io
+          import os
+          import pathlib
+          import stat
+          import zipfile
+
+          def regular(entry):
+              mode = entry.external_attr >> 16
+              return not entry.is_dir() and (stat.S_IFMT(mode) in (0, stat.S_IFREG))
+
+          package = f"rustfs-linux-x86_64-gnu-dev-{os.environ['SOURCE_SHA'][:7]}.zip"
+          with zipfile.ZipFile('official-artifact.zip') as outer:
+              entries = outer.infolist()
+              assert 1 <= len(entries) <= 16
+              assert len({item.filename for item in entries}) == len(entries)
+              assert all(regular(item) and pathlib.PurePosixPath(item.filename).name == item.filename for item in entries)
+              assert sum(item.file_size for item in entries) <= 2147483648
+              assert outer.testzip() is None
+              with zipfile.ZipFile(io.BytesIO(outer.read(package))) as inner:
+                  entries = inner.infolist()
+                  assert {item.filename for item in entries} == {'rustfs', 'rustfs-cli'} and len(entries) == 2
+                  assert all(regular(item) and 0 < item.file_size <= 1073741824 for item in entries)
+                  assert inner.testzip() is None
+                  pathlib.Path('binary').mkdir()
+                  with inner.open('rustfs') as source, open('binary/rustfs', 'xb') as destination:
+                      import shutil
+                      shutil.copyfileobj(source, destination)
+          PY
+          chmod 0755 binary/rustfs
+          printf '%s  binary/rustfs\n' "$BINARY_SHA256" | sha256sum --check --strict
+          file binary/rustfs | grep -Eq 'ELF 64-bit LSB.*x86-64'
+
+      - name: Run the real scheduler and receipt delivery scenario
+        shell: bash
+        env:
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          git -C connect-harness diff --exit-code
+          printf '%s\n' "$SOURCE_SHA" >connect-harness/tests/e2e/connected/rustfs-ref
+          [[ $(git -C connect-harness diff --name-only) == tests/e2e/connected/rustfs-ref ]]
+          npm --prefix connect-harness/web ci
+          connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
+          make -C connect-harness e2e-connected-dispatch-check
+          (cd connect-harness && ./tests/connectivity/diagnostic-schedule-evidence.test.sh)
+          RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
+            RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
+            CONNECT_E2E_DIAGNOSTIC_SCHEDULE_EVIDENCE="$GITHUB_WORKSPACE/diagnostic-schedule-evidence.json" \
+            make -C connect-harness e2e-connected E2E_SCENARIO=diagnostic-schedule
+
+      - name: Bind and validate sanitized evidence
+        shell: bash
+        env:
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          BINARY_SHA256: ${{ inputs.binary_sha256 }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          jq -e --arg connect "$CONNECT_SHA" --arg source "$SOURCE_SHA" --arg binary "$BINARY_SHA256" '
+            .provenance.connectSha == $connect and .provenance.sourceSha == $source
+            and .provenance.binarySha256 == $binary
+          ' diagnostic-schedule-evidence.json >/dev/null
+          jq --arg run "$GITHUB_RUN_ID" --arg build "$BUILD_RUN_ID" \
+            --arg artifact "$ARTIFACT_ID" --arg digest "$ARTIFACT_DIGEST" '
+            .provenance += {workflowRunId:$run,buildRunId:$build,artifactId:$artifact,artifactDigest:$digest}
+          ' diagnostic-schedule-evidence.json > diagnostic-schedule-evidence.bound.json
+          mv diagnostic-schedule-evidence.bound.json diagnostic-schedule-evidence.json
+          node connect-harness/tests/connectivity/verify-diagnostic-schedule-evidence.mjs diagnostic-schedule-evidence.json --bound
+
+      - name: Upload verified receipt evidence
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: connect-diagnostic-schedule-evidence-${{ github.run_id }}
+          path: diagnostic-schedule-evidence.json
+          retention-days: 14
+          if-no-files-found: error

.github/workflows/connect-performance-drive-acceptance.yml

@@ -53,19 +53,19 @@ jobs:
     timeout-minutes: 45
     steps:
       - name: Checkout acceptance harness
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
       - name: Checkout exact RustFS source
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           path: rustfs-source
           persist-credentials: false
           ref: ${{ inputs.source_sha }}
 
       - name: Checkout exact Connect harness
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/connect
           path: connect-harness
@@ -122,7 +122,7 @@ jobs:
           [[ $(jq -r '.expired' <<<"$artifact") == false ]]
 
       - name: Download exact build artifact
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           artifact-ids: ${{ inputs.artifact_id }}
           path: artifact
@@ -183,7 +183,7 @@ jobs:
 
       - name: Upload acceptance evidence
         if: ${{ always() && hashFiles('performance-drive-service-job-evidence.json') != '' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: connect-performance-drive-evidence-${{ github.run_id }}
           path: |

.github/workflows/connect-performance-network-acceptance.yml

@@ -53,19 +53,19 @@ jobs:
     timeout-minutes: 45
     steps:
       - name: Checkout acceptance harness
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
       - name: Checkout exact RustFS source
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           path: rustfs-source
           persist-credentials: false
           ref: ${{ inputs.source_sha }}
 
       - name: Checkout exact Connect harness
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/connect
           path: connect-harness
@@ -122,7 +122,7 @@ jobs:
           [[ $(jq -r '.expired' <<<"$artifact") == false ]]
 
       - name: Download exact build artifact
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           artifact-ids: ${{ inputs.artifact_id }}
           path: artifact
@@ -183,7 +183,7 @@ jobs:
 
       - name: Upload acceptance evidence
         if: ${{ always() && hashFiles('performance-network-service-job-evidence.json') != '' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: connect-performance-network-evidence-${{ github.run_id }}
           path: |

.github/workflows/connect-profile-cpu-acceptance.yml

@@ -53,19 +53,19 @@ jobs:
     timeout-minutes: 45
     steps:
       - name: Checkout acceptance harness
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
       - name: Checkout exact RustFS source
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           path: rustfs-source
           persist-credentials: false
           ref: ${{ inputs.source_sha }}
 
       - name: Checkout exact Connect harness
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/connect
           path: connect-harness
@@ -122,7 +122,7 @@ jobs:
           [[ $(jq -r '.expired' <<<"$artifact") == false ]]
 
       - name: Download exact build artifact
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           artifact-ids: ${{ inputs.artifact_id }}
           path: artifact
@@ -183,7 +183,7 @@ jobs:
 
       - name: Upload acceptance evidence
         if: ${{ always() && hashFiles('profile-service-job-evidence.json') != '' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: connect-profile-cpu-evidence-${{ github.run_id }}
           path: |

.github/workflows/connect-profile-memory-acceptance.yml

@@ -0,0 +1,201 @@
+# Copyright 2026 RustFS Team
+# SPDX-License-Identifier: Apache-2.0
+
+name: Connect service memory profile acceptance
+
+on:
+  workflow_dispatch:
+    inputs:
+      build_run_id:
+        description: Successful main-branch Build and Release run ID
+        required: true
+        type: string
+      artifact_id:
+        description: Linux x86_64 GNU artifact from that run
+        required: true
+        type: string
+      source_sha:
+        description: Exact RustFS source commit
+        required: true
+        type: string
+      artifact_digest:
+        description: GitHub artifact digest including sha256 prefix
+        required: true
+        type: string
+      binary_sha256:
+        description: Independently verified rustfs binary SHA-256
+        required: true
+        type: string
+      connect_sha:
+        description: Exact Connect memory acceptance harness commit
+        required: true
+        type: string
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  profile-memory:
+    name: Verify signed service memory profile over mTLS
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - name: Checkout exact RustFS source
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          path: rustfs-source
+          persist-credentials: false
+          ref: ${{ inputs.source_sha }}
+
+      - name: Checkout exact Connect harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/connect
+          path: connect-harness
+          persist-credentials: false
+          ref: ${{ inputs.connect_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 22.22.2
+          cache: npm
+          cache-dependency-path: connect-harness/web/package-lock.json
+
+      - name: Verify official source and artifact identity
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          BINARY_SHA256: ${{ inputs.binary_sha256 }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+        run: |
+          set -euo pipefail
+          [[ "$GITHUB_REPOSITORY" == rustfs/rustfs ]]
+          [[ "$BUILD_RUN_ID" =~ ^[1-9][0-9]*$ && "$ARTIFACT_ID" =~ ^[1-9][0-9]*$ ]]
+          [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ && "$CONNECT_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ "$ARTIFACT_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
+          [[ "$BINARY_SHA256" =~ ^[0-9a-f]{64}$ ]]
+          [[ $(git -C rustfs-source rev-parse HEAD) == "$SOURCE_SHA" ]]
+          [[ $(git -C connect-harness rev-parse HEAD) == "$CONNECT_SHA" ]]
+          [[ $(git -C rustfs-source remote get-url origin) == https://github.com/rustfs/rustfs ]]
+          [[ $(git -C connect-harness remote get-url origin) == https://github.com/rustfs/connect ]]
+          run=$(gh api "repos/rustfs/rustfs/actions/runs/${BUILD_RUN_ID}")
+          jq -e --arg source "$SOURCE_SHA" '
+            .head_sha == $source and .head_branch == "main"
+            and .head_repository.full_name == "rustfs/rustfs"
+            and .name == "Build and Release" and .path == ".github/workflows/build.yml"
+            and .status == "completed" and .conclusion == "success"
+          ' <<<"$run" >/dev/null
+          jobs=$(gh api --paginate --slurp "repos/rustfs/rustfs/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
+          jq -e '
+            [.[].jobs[] | select(.name | test("^Build RustFS \\(linux-x86_64-gnu, [a-z0-9-]+, x86_64-unknown-linux-gnu, false, linux, pyroscope\\)$"))] as $matches
+            | ($matches | length) == 1 and $matches[0].conclusion == "success"
+          ' <<<"$jobs" >/dev/null
+          artifact=$(gh api "repos/rustfs/rustfs/actions/artifacts/${ARTIFACT_ID}")
+          jq -e --argjson run "$BUILD_RUN_ID" --arg source "$SOURCE_SHA" --arg digest "$ARTIFACT_DIGEST" '
+            .workflow_run.id == $run and .workflow_run.head_sha == $source
+            and .name == ("rustfs-linux-x86_64-gnu-dev-" + $source[0:7])
+            and .digest == $digest and .expired == false
+            and (.expires_at | fromdateiso8601) > now
+            and .size_in_bytes > 0 and .size_in_bytes <= 2147483648
+          ' <<<"$artifact" >/dev/null
+          jq -r '.expires_at' <<<"$artifact" > artifact-expires-at
+          gh api "repos/rustfs/rustfs/actions/artifacts/${ARTIFACT_ID}/zip" > official-artifact.zip
+          [[ $(stat --format=%s official-artifact.zip) == $(jq -r '.size_in_bytes' <<<"$artifact") ]]
+          printf '%s  official-artifact.zip\n' "${ARTIFACT_DIGEST#sha256:}" | sha256sum --check --strict
+
+      - name: Extract only the verified official service binary
+        shell: bash
+        env:
+          SOURCE_SHA: ${{ inputs.source_sha }}
+          BINARY_SHA256: ${{ inputs.binary_sha256 }}
+        run: |
+          set -euo pipefail
+          python3 - <<'PY'
+          import io
+          import os
+          import pathlib
+          import shutil
+          import stat
+          import zipfile
+
+          def regular(entry):
+              mode = entry.external_attr >> 16
+              return not entry.is_dir() and stat.S_IFMT(mode) in (0, stat.S_IFREG)
+
+          package = f"rustfs-linux-x86_64-gnu-dev-{os.environ['SOURCE_SHA'][:7]}.zip"
+          with zipfile.ZipFile('official-artifact.zip') as outer:
+              entries = outer.infolist()
+              assert 1 <= len(entries) <= 16
+              assert len({entry.filename for entry in entries}) == len(entries)
+              assert all(regular(entry) and pathlib.PurePosixPath(entry.filename).name == entry.filename for entry in entries)
+              assert sum(entry.file_size for entry in entries) <= 2147483648
+              assert outer.testzip() is None
+              with zipfile.ZipFile(io.BytesIO(outer.read(package))) as inner:
+                  entries = inner.infolist()
+                  assert {entry.filename for entry in entries} == {'rustfs', 'rustfs-cli'} and len(entries) == 2
+                  assert all(regular(entry) and 0 < entry.file_size <= 1073741824 for entry in entries)
+                  assert inner.testzip() is None
+                  pathlib.Path('binary').mkdir()
+                  with inner.open('rustfs') as source, open('binary/rustfs', 'xb') as destination:
+                      shutil.copyfileobj(source, destination)
+          PY
+          chmod 0755 binary/rustfs
+          printf '%s  binary/rustfs\n' "$BINARY_SHA256" | sha256sum --check --strict
+          file binary/rustfs | grep -Eq 'ELF 64-bit LSB.*x86-64'
+
+      - name: Run the real service job under S3 workload
+        shell: bash
+        env:
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          git -C connect-harness diff --exit-code
+          printf '%s\n' "$SOURCE_SHA" >connect-harness/tests/e2e/connected/rustfs-ref
+          changed=$(git -C connect-harness diff --name-only)
+          [[ -z "$changed" || "$changed" == tests/e2e/connected/rustfs-ref ]]
+          npm --prefix connect-harness/web ci
+          connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
+          make -C connect-harness e2e-connected-dispatch-check
+          (cd connect-harness && ./tests/connectivity/profile-memory-evidence.test.sh)
+          RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
+            RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
+            CONNECT_E2E_PROFILE_MEMORY_EVIDENCE="$GITHUB_WORKSPACE/profile-memory-evidence.json" \
+            make -C connect-harness e2e-connected E2E_SCENARIO=profile-memory
+
+      - name: Bind and validate sanitized evidence
+        shell: bash
+        env:
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          BINARY_SHA256: ${{ inputs.binary_sha256 }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          jq -e --arg connect "$CONNECT_SHA" --arg source "$SOURCE_SHA" --arg binary "$BINARY_SHA256" '
+            .provenance.connectSha == $connect and .provenance.sourceSha == $source
+            and .provenance.binarySha256 == $binary
+          ' profile-memory-evidence.json >/dev/null
+          jq --arg run "$GITHUB_RUN_ID" --arg build "$BUILD_RUN_ID" \
+            --arg artifact "$ARTIFACT_ID" --arg digest "$ARTIFACT_DIGEST" \
+            --arg expiry "$(cat artifact-expires-at)" '
+            .provenance += {workflowRunId:$run,buildRunId:$build,artifactId:$artifact,artifactDigest:$digest,artifactExpiresAt:$expiry}
+          ' profile-memory-evidence.json > profile-memory-evidence.bound.json
+          mv profile-memory-evidence.bound.json profile-memory-evidence.json
+          node connect-harness/tests/connectivity/profile-memory-evidence.mjs profile-memory-evidence.json --bound
+
+      - name: Upload only verified sanitized evidence
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: connect-profile-memory-evidence-${{ github.run_id }}
+          path: profile-memory-evidence.json
+          retention-days: 14
+          if-no-files-found: error

.github/workflows/connect-profile-threads-acceptance.yml

@@ -53,19 +53,19 @@ jobs:
     timeout-minutes: 45
     steps:
       - name: Checkout acceptance harness
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
       - name: Checkout exact RustFS source
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           path: rustfs-source
           persist-credentials: false
           ref: ${{ inputs.source_sha }}
 
       - name: Checkout exact Connect harness
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/connect
           path: connect-harness
@@ -122,7 +122,7 @@ jobs:
           [[ $(jq -r '.expired' <<<"$artifact") == false ]]
 
       - name: Download exact build artifact
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           artifact-ids: ${{ inputs.artifact_id }}
           path: artifact
@@ -183,7 +183,7 @@ jobs:
 
       - name: Upload acceptance evidence
         if: ${{ always() && hashFiles('profile-threads-service-job-evidence.json') != '' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: connect-profile-threads-evidence-${{ github.run_id }}
           path: |

.github/workflows/connect-top-api-acceptance.yml

@@ -53,19 +53,19 @@ jobs:
     timeout-minutes: 45
     steps:
       - name: Checkout acceptance harness
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
       - name: Checkout exact RustFS source
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           path: rustfs-source
           persist-credentials: false
           ref: ${{ inputs.source_sha }}
 
       - name: Checkout exact Connect harness
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/connect
           path: connect-harness
@@ -122,7 +122,7 @@ jobs:
           [[ $(jq -r '.expired' <<<"$artifact") == false ]]
 
       - name: Download exact build artifact
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           artifact-ids: ${{ inputs.artifact_id }}
           path: artifact
@@ -183,7 +183,7 @@ jobs:
 
       - name: Upload acceptance evidence
         if: ${{ always() && hashFiles('top-api-service-job-evidence.json') != '' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: connect-top-api-evidence-${{ github.run_id }}
           path: |

.github/workflows/connect-top-disk-acceptance.yml

@@ -49,7 +49,7 @@ jobs:
     timeout-minutes: 15
     steps:
       - name: Checkout acceptance harness
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -83,7 +83,7 @@ jobs:
           [[ $(jq -r '.expired' <<<"$artifact") == false ]]
 
       - name: Download exact build artifact
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           artifact-ids: ${{ inputs.artifact_id }}
           path: artifact
@@ -132,7 +132,7 @@ jobs:
           mv top-disk-runtime-evidence.bound.json top-disk-runtime-evidence.json
 
       - name: Upload acceptance evidence
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: connect-top-disk-evidence-${{ github.run_id }}
           path: |

.github/workflows/connect-top-locks-acceptance.yml

@@ -53,19 +53,19 @@ jobs:
     timeout-minutes: 45
     steps:
       - name: Checkout acceptance harness
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
       - name: Checkout exact RustFS source
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           path: rustfs-source
           persist-credentials: false
           ref: ${{ inputs.source_sha }}
 
       - name: Checkout exact Connect harness
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/connect
           path: connect-harness
@@ -122,7 +122,7 @@ jobs:
           [[ $(jq -r '.expired' <<<"$artifact") == false ]]
 
       - name: Download exact build artifact
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           artifact-ids: ${{ inputs.artifact_id }}
           path: artifact
@@ -183,7 +183,7 @@ jobs:
 
       - name: Upload acceptance evidence
         if: ${{ always() && hashFiles('top-locks-service-job-evidence.json') != '' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: connect-top-locks-evidence-${{ github.run_id }}
           path: |

.github/workflows/connect-top-rpc-acceptance.yml

@@ -53,19 +53,19 @@ jobs:
     timeout-minutes: 45
     steps:
       - name: Checkout acceptance harness
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
       - name: Checkout exact RustFS source
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           path: rustfs-source
           persist-credentials: false
           ref: ${{ inputs.source_sha }}
 
       - name: Checkout exact Connect harness
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/connect
           path: connect-harness
@@ -122,7 +122,7 @@ jobs:
           [[ $(jq -r '.expired' <<<"$artifact") == false ]]
 
       - name: Download exact build artifact
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           artifact-ids: ${{ inputs.artifact_id }}
           path: artifact
@@ -183,7 +183,7 @@ jobs:
 
       - name: Upload acceptance evidence
         if: ${{ always() && hashFiles('top-rpc-service-job-evidence.json') != '' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: connect-top-rpc-evidence-${{ github.run_id }}
           path: |

.github/workflows/coverage.yml

@@ -76,7 +76,7 @@ jobs:
       NEXTEST_PROFILE: ci
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -89,7 +89,7 @@ jobs:
           install-build-packaging-tools: 'false'
 
       - name: Install cargo-llvm-cov
-        uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
+        uses: taiki-e/install-action@7623a79cdfecb99d681017af368ca353d9f49bb5 # v2
         with:
           tool: cargo-llvm-cov
 
@@ -112,7 +112,7 @@ jobs:
 
       - name: Upload coverage artifact
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: coverage-lcov-${{ github.run_number }}
           path: |
@@ -134,7 +134,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/docker.yml

@@ -47,7 +47,7 @@ on:
         default: true
         type: boolean
       version:
-        description: "Version to build (latest for stable release, or specific version like v1.0.0, v1.0.0-alpha1)"
+        description: "Version to build (latest, v1.0.0, v1.0.0-alpha1, or v1.0.1-preview.1)"
         required: false
         default: "latest"
         type: string
@@ -82,8 +82,7 @@ jobs:
       github.event_name == 'workflow_dispatch' ||
       (github.event.workflow_run.conclusion == 'success' &&
        github.event.workflow_run.event == 'push' &&
-       github.event.workflow_run.head_branch != 'main' &&
-       !contains(github.event.workflow_run.head_branch, '-preview'))
+       github.event.workflow_run.head_branch != 'main')
     runs-on: ubuntu-latest
     timeout-minutes: 30
     outputs:
@@ -97,7 +96,7 @@ jobs:
       source_ref: ${{ steps.check.outputs.source_ref }}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
           # For workflow_run events, checkout the specific commit that triggered the workflow
@@ -197,7 +196,14 @@ jobs:
                   version="${version#v}"
                 fi
 
-                if [[ "$version" == *"alpha"* ]] || [[ "$version" == *"beta"* ]] || [[ "$version" == *"rc"* ]]; then
+                if [[ "$version" =~ -preview\.[0-9]+$ ]]; then
+                  build_type="preview"
+                  is_prerelease=true
+                  echo "🔍 Building Docker image for preview: $version (version tags only)"
+                elif [[ "$version" == *"-preview"* ]]; then
+                  echo "❌ Invalid preview tag: $version (expected suffix: -preview.<number>)" >&2
+                  exit 1
+                elif [[ "$version" == *"alpha"* ]] || [[ "$version" == *"beta"* ]] || [[ "$version" == *"rc"* ]]; then
                   build_type="prerelease"
                   is_prerelease=true
                   # Pre-GA policy: prereleases update latest until the first stable tag exists.
@@ -249,11 +255,14 @@ jobs:
                 echo "🚀 Building with latest stable release version"
                 ;;
               *-preview*)
+                if [[ ! "$input_version" =~ ^v?[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?-preview\.[0-9]+$ ]]; then
+                  echo "❌ Invalid preview version: $input_version" >&2
+                  exit 1
+                fi
                 build_type="preview"
                 is_prerelease=true
-                should_build=false
-                should_push=false
-                echo "⏭️  Preview tags do not publish Docker images"
+                version="${input_version#v}"
+                echo "🔍 Building with preview version: $version (version tags only)"
                 ;;
               # Prerelease versions (must match first, more specific)
               v*alpha*|v*beta*|v*rc*|*alpha*|*beta*|*rc*)
@@ -343,7 +352,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
           ref: ${{ needs.build-check.outputs.source_ref }}
@@ -535,13 +544,61 @@ jobs:
           category: container-image-${{ matrix.variant }}
 
       - name: Upload container scan report
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: container-image-scan-${{ matrix.variant }}
           path: trivy-${{ matrix.variant }}.sarif
           if-no-files-found: error
           retention-days: 30
 
+  # Docker Hub's overview is a separate `full_description` field that
+  # `docker push` never touches, so it drifted from README.md and kept
+  # linking to pages that no longer exist (#7995). Republish it from the
+  # commit the images were built from, only when images were pushed.
+  sync-dockerhub-description:
+    name: Sync Docker Hub Description
+    needs: [ build-check, build-docker ]
+    if: needs.build-check.outputs.should_build == 'true' && needs.build-check.outputs.should_push == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+          ref: ${{ needs.build-check.outputs.source_ref }}
+          sparse-checkout: README.md
+          sparse-checkout-cone-mode: false
+
+      - name: Sync Docker Hub description
+        uses: peter-evans/dockerhub-description@1b9a80c056b620d92cedb9d9b5a223409c68ddfa # v5.0.0
+        with:
+          username: ${{ env.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+          repository: ${{ env.REGISTRY_DOCKERHUB }}
+          readme-filepath: ./README.md
+          # README.md links to docs/ and CONTRIBUTING.md by relative path;
+          # rewrite those to github.com URLs so they resolve on Docker Hub.
+          enable-url-completion: true
+
+      # Docker Hub caps `full_description` at 25,000 bytes and the action
+      # truncates to fit with only a warning, which would publish a README
+      # cut off mid-sentence. README.md is already near the cap, so read the
+      # published overview back and fail if it hit the limit.
+      - name: Check published description was not truncated
+        env:
+          REPOSITORY: ${{ env.REGISTRY_DOCKERHUB }}
+        run: |
+          limit=25000
+          published=$(curl -sSf "https://hub.docker.com/v2/repositories/${REPOSITORY}/" \
+            | jq -j '.full_description' | wc -c | tr -d ' ')
+          echo "Published overview: ${published} bytes (Docker Hub limit: ${limit})"
+          # Truncation lands within one UTF-8 character of the cap.
+          if [ "$published" -ge $((limit - 4)) ]; then
+            echo "::error file=README.md::Docker Hub overview was truncated at ${limit} bytes; trim README.md before the next release."
+            exit 1
+          fi
+
   # Docker build summary
   docker-summary:
     name: Docker Build Summary
@@ -563,6 +620,10 @@ jobs:
           echo ""
 
           case "$BUILD_TYPE" in
+            "preview")
+              echo "🔍 Preview Docker image has been built with ${VERSION} tags"
+              echo "⏭️  Preview images do not update latest or prerelease channels"
+              ;;
             "release")
               echo "🚀 Release Docker image has been built with ${VERSION} tags"
               echo "✅ This image is ready for production use"

.github/workflows/e2e-distributed.yml

@@ -93,7 +93,7 @@ jobs:
       UPGRADE_SOURCE_SHA256: 7c789386bf85278f865b8e0d359bf4edb84d5aa408cc3fa54a18c25ca74cd6e7
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -174,7 +174,7 @@ jobs:
 
       - name: Upload distributed e2e diagnostics
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: e2e-distributed-${{ github.run_number }}
           path: |
@@ -206,7 +206,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/e2e-replication-nightly.yml

@@ -63,7 +63,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -77,7 +77,7 @@ jobs:
 
       # The STS dual-node test requires awscurl and fails if it is unavailable.
       - name: Set up Python
-        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: "3.12"
 
@@ -108,7 +108,7 @@ jobs:
 
       - name: Upload nextest junit report
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: e2e-replication-nightly-junit-${{ github.run_number }}
           path: |
@@ -126,7 +126,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -157,7 +157,7 @@ jobs:
 
       - name: Upload cluster fault diagnostics
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: e2e-cluster-nightly-${{ github.run_number }}
           path: |
@@ -176,7 +176,7 @@ jobs:
       RUSTFS_BUILD_FEATURES: ftps,webdav,sftp
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -213,7 +213,7 @@ jobs:
 
       - name: Upload protocol diagnostics
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: e2e-protocol-nightly-${{ github.run_number }}
           path: |
@@ -236,7 +236,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/e2e-s3tests.yml

@@ -159,7 +159,7 @@ jobs:
       TEST_MODE: ${{ matrix.test-mode }}
       S3_SHARD_INDEX: ${{ matrix.shard-index }}
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -167,7 +167,7 @@ jobs:
       # ship a working pip (ci-1: a bare python3 without pip is what broke the
       # scheduled sweep). Keeps the workflow correct across runner drift.
       - name: Set up Python
-        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: "3.12"
 
@@ -264,7 +264,7 @@ jobs:
               hostname: rustfs4
               volumes: [rustfs4-data:/data]
             lb:
-              image: haproxy:2.9
+              image: haproxy:3.4.4
               hostname: lb
               networks: [rustfs-net]
               ports:
@@ -308,6 +308,11 @@ jobs:
 
           docker compose -f compose.yml up -d
 
+          # Validate the pinned HAProxy image parses the exact config before
+          # the S3 compatibility sweep starts.
+          docker compose -f compose.yml exec -T lb \
+            haproxy -c -f /usr/local/etc/haproxy/haproxy.cfg
+
       - name: Wait for RustFS ready
         run: |
           for _ in {1..120}; do
@@ -357,7 +362,7 @@ jobs:
 
       - name: Upload artifacts
         if: always() && env.ACT != 'true'
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: s3tests-${{ env.TEST_MODE }}-shard-${{ matrix.shard-index }}
           path: artifacts/**
@@ -369,12 +374,12 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
       - name: Set up Python
-        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: "3.12"
 
@@ -435,7 +440,7 @@ jobs:
 
       - name: Upload canary artifacts
         if: always() && env.ACT != 'true'
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: s3tests-upstream-head
           path: artifacts/s3tests-upstream-head/**
@@ -454,7 +459,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/e2e-upgrade.yml

@@ -96,7 +96,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -137,7 +137,7 @@ jobs:
 
       - name: Upload scanner/heal G09 evidence
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: ${{ matrix.artifact }}-g09-evidence-${{ github.run_number }}
           path: ${{ runner.temp }}/rustfs-upgrade-g09-evidence/${{ matrix.artifact }}
@@ -146,7 +146,7 @@ jobs:
 
       - name: Upload server logs
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: ${{ matrix.artifact }}-server-logs-${{ github.run_number }}
           path: ${{ runner.temp }}/rustfs-upgrade-logs

.github/workflows/functional-chain-health.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 15
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Collect and publish verified chain health
@@ -24,7 +24,7 @@ jobs:
         run: python3 scripts/functional_chain_health.py --source-ref "${NIGHTLY_SOURCE_REF}" --publish --output "${RUNNER_TEMP}/chain-health.json"
       - name: Retain health observation
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-chain-health-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-health.json

.github/workflows/fuzz.yml

@@ -81,7 +81,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -93,7 +93,7 @@ jobs:
           cache-save-if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'schedule' }}
 
       - name: Install cargo-fuzz
-        uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
+        uses: taiki-e/install-action@7623a79cdfecb99d681017af368ca353d9f49bb5 # v2
         with:
           tool: cargo-fuzz
 
@@ -113,7 +113,7 @@ jobs:
           done
 
       - name: Upload prebuilt fuzz binaries
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: fuzz-prebuilt-binaries-${{ github.run_number }}
           path: fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/
@@ -143,12 +143,12 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
       - name: Download prebuilt fuzz binaries
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: fuzz-prebuilt-binaries-${{ github.run_number }}
           path: fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release
@@ -167,7 +167,7 @@ jobs:
 
       - name: Upload fuzz smoke artifacts
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: fuzz-smoke-${{ matrix.target }}-${{ github.run_number }}
           path: |
@@ -197,12 +197,12 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
       - name: Download prebuilt fuzz binaries
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: fuzz-prebuilt-binaries-${{ github.run_number }}
           path: fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release
@@ -221,7 +221,7 @@ jobs:
 
       - name: Upload nightly fuzz artifacts
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: fuzz-nightly-${{ matrix.target }}-${{ github.run_number }}
           path: |
@@ -246,7 +246,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/helm-package.yml

@@ -49,7 +49,7 @@ jobs:
 
     steps:
       - name: Checkout helm chart repo
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -107,7 +107,7 @@ jobs:
             --version "${{ steps.version.outputs.chart_version }}"
 
       - name: Upload helm package as artifact
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: helm-package
           path: helm/rustfs/*.tgz
@@ -121,7 +121,7 @@ jobs:
 
     steps:
       - name: Checkout helm package repo
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           # persist-credentials-exempt: this checkout's token IS the push credential —
           # the job git-pushes to rustfs/helm below. Clearing it breaks chart publishing.
@@ -129,7 +129,7 @@ jobs:
           token: ${{ secrets.RUSTFS_HELM_PACKAGE }}
 
       - name: Download helm package
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: helm-package
           path: ./

.github/workflows/minio-interop.yml

@@ -83,7 +83,7 @@ jobs:
       INTEROP_REQUIRED_TESTS: '["reads_minio_generated_sse_s3_multipart_fixture", "reads_minio_generated_sse_kms_multipart_fixture", "rejects_minio_generated_sse_s3_fixture_with_wrong_kms_key", "rejects_minio_generated_sse_s3_fixture_with_truncated_ciphertext"]'
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -137,7 +137,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/mint.yml

@@ -117,7 +117,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 120
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -246,7 +246,7 @@ jobs:
 
       - name: Upload artifacts
         if: always() && env.ACT != 'true'
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: mint
           path: artifacts/**
@@ -264,7 +264,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/nightly-gnu.yml

@@ -51,7 +51,7 @@ jobs:
       source_ref: ${{ steps.source.outputs.ref }}
     steps:
       - name: Checkout selected source
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
           ref: ${{ env.NIGHTLY_BUILD_REF }}
@@ -71,7 +71,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
           ref: ${{ needs.resolve-source.outputs.source_sha }}
@@ -117,10 +117,15 @@ jobs:
 
           cat > "${PKG_DIR}/etc/default/rustfs" << 'ENVEOF'
           # RustFS Environment Configuration
-          # See https://rustfs.com/docs/ for more information
+          # See https://docs.rustfs.com for more information
+          # RUSTFS_ACCESS_KEY=<changeme>
+          # RUSTFS_SECRET_KEY=<changeme>
           # RUSTFS_VOLUMES=""
-          # RUSTFS_ROOT_USER=""
-          # RUSTFS_ROOT_PASSWORD=""
+          # RUSTFS_ADDRESS=":9000"
+          # RUSTFS_CONSOLE_ADDRESS=":9001"
+          # RUSTFS_CONSOLE_ENABLE=true
+          # RUSTFS_OBS_LOGGER_LEVEL=error
+          # RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"
           ENVEOF
 
           # dpkg versions must start with a digit and cannot contain hyphens;
@@ -146,37 +151,13 @@ jobs:
           /etc/default/rustfs
           CONFFILES
 
-          cat > "${PKG_DIR}/DEBIAN/postinst" << 'POSTINST'
-          #!/bin/bash
-          set -e
-          if ! getent passwd rustfs > /dev/null 2>&1; then
-              useradd -r -s /bin/false -d /opt/rustfs rustfs
-          fi
-          mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs
-          chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs
-          if [ -d /run/systemd/system ]; then
-              systemctl daemon-reload
-          fi
-          echo "RustFS installed. Configure /etc/default/rustfs then: systemctl start rustfs"
-          POSTINST
+          ./scripts/release/package_service_scripts.sh after-install > "${PKG_DIR}/DEBIAN/postinst"
           chmod 755 "${PKG_DIR}/DEBIAN/postinst"
 
-          cat > "${PKG_DIR}/DEBIAN/prerm" << 'PRERM'
-          #!/bin/bash
-          set -e
-          if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then
-              systemctl stop rustfs
-          fi
-          PRERM
+          ./scripts/release/package_service_scripts.sh before-remove > "${PKG_DIR}/DEBIAN/prerm"
           chmod 755 "${PKG_DIR}/DEBIAN/prerm"
 
-          cat > "${PKG_DIR}/DEBIAN/postrm" << 'POSTRM'
-          #!/bin/bash
-          set -e
-          if [ -d /run/systemd/system ]; then
-              systemctl daemon-reload
-          fi
-          POSTRM
+          ./scripts/release/package_service_scripts.sh after-remove > "${PKG_DIR}/DEBIAN/postrm"
           chmod 755 "${PKG_DIR}/DEBIAN/postrm"
 
           cp LICENSE "${PKG_DIR}/usr/share/doc/rustfs/"
@@ -214,10 +195,15 @@ jobs:
           mkdir -p ./tmp-pkg/etc/default
           cat > ./tmp-pkg/etc/default/rustfs << 'ENVEOF'
           # RustFS Environment Configuration
-          # See https://rustfs.com/docs/ for more information
+          # See https://docs.rustfs.com for more information
+          # RUSTFS_ACCESS_KEY=<changeme>
+          # RUSTFS_SECRET_KEY=<changeme>
           # RUSTFS_VOLUMES=""
-          # RUSTFS_ROOT_USER=""
-          # RUSTFS_ROOT_PASSWORD=""
+          # RUSTFS_ADDRESS=":9000"
+          # RUSTFS_CONSOLE_ADDRESS=":9001"
+          # RUSTFS_CONSOLE_ENABLE=true
+          # RUSTFS_OBS_LOGGER_LEVEL=error
+          # RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"
           ENVEOF
 
           fpm -s dir -t rpm \
@@ -231,35 +217,13 @@ jobs:
             --description "High-performance distributed object storage" \
             --url "https://rustfs.com" \
             --license "Apache-2.0" \
-            --after-install <(cat << 'POSTINST'
-          #!/bin/bash
-          set -e
-          if ! getent passwd rustfs > /dev/null 2>&1; then
-              useradd -r -s /bin/false -d /opt/rustfs rustfs
-          fi
-          mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs
-          chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs
-          if [ -d /run/systemd/system ]; then
-              systemctl daemon-reload
-          fi
-          POSTINST
-            ) \
-            --before-remove <(cat << 'PRERM'
-          #!/bin/bash
-          set -e
-          if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then
-              systemctl stop rustfs
-          fi
-          PRERM
-            ) \
-            --after-remove <(cat << 'POSTRM'
-          #!/bin/bash
-          set -e
-          if [ -d /run/systemd/system ]; then
-              systemctl daemon-reload
-          fi
-          POSTRM
-            ) \
+            --before-install <(./scripts/release/package_service_scripts.sh rpm-before-install) \
+            --before-upgrade <(./scripts/release/package_service_scripts.sh rpm-before-upgrade) \
+            --after-install <(./scripts/release/package_service_scripts.sh after-install) \
+            --after-upgrade <(./scripts/release/package_service_scripts.sh after-install) \
+            --before-remove <(./scripts/release/package_service_scripts.sh before-remove) \
+            --after-remove <(./scripts/release/package_service_scripts.sh after-remove) \
+            --rpm-posttrans <(./scripts/release/package_service_scripts.sh rpm-posttrans) \
             --config-files /etc/default/rustfs \
             "rustfs-nightly-${DEB_DATE}/usr/bin/rustfs=/usr/bin/rustfs" \
             ./tmp-pkg/etc/default/rustfs=/etc/default/rustfs \
@@ -273,12 +237,12 @@ jobs:
           echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
 
       - name: Upload DEB artifact
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: ${{ steps.deb.outputs.deb_file }}
           path: ${{ steps.deb.outputs.deb_file }}
       - name: Upload RPM artifact
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: ${{ steps.rpm.outputs.rpm_file }}
           path: ${{ steps.rpm.outputs.rpm_file }}
@@ -364,7 +328,7 @@ jobs:
 
       - name: Upload nightly candidate manifest
         if: ${{ steps.publish.outputs.candidate_file != '' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: nightly-candidate-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ steps.publish.outputs.candidate_file }}
@@ -419,7 +383,7 @@ jobs:
       NO_PROXY: 127.0.0.1,localhost
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
           ref: ${{ needs.resolve-source.outputs.source_sha }}
@@ -509,7 +473,7 @@ jobs:
       NO_PROXY: 127.0.0.1,localhost
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
           ref: ${{ needs.resolve-source.outputs.source_sha }}
@@ -537,7 +501,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/nix-flake-update.yml

@@ -39,24 +39,24 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         # persist-credentials-exempt: update-flake-lock pushes the branch and opens
         # the PR. It passes FLAKE_UPDATE_TOKEN to create-pull-request itself rather
         # than reusing .git/config, but that is unverified — exempt until a
         # workflow_dispatch run confirms it (rustfs/backlog#1602).
 
       - name: Install Nix
-        uses: DeterminateSystems/determinate-nix-action@629b284231c2a82554b724e357e47fc6020833c8 # v3
+        uses: DeterminateSystems/determinate-nix-action@8d87e8d5e5b8a8309d4281094560f127d9a265f1 # v3.22.5
 
       - name: Cache Nix
-        uses: DeterminateSystems/flakehub-cache-action@1f9a51a2959d3e26c7838c6f3bf9f48acae525ea # v3.20.0
+        uses: DeterminateSystems/flakehub-cache-action@83282a8aef353db1d659d3bacf4a28b6683de0b3 # v3.22.5
 
       - name: Check Nix flake inputs
-        uses: DeterminateSystems/flake-checker-action@3164002371bc90729c68af0e24d5aacf20d7c9f6 # v12
+        uses: DeterminateSystems/flake-checker-action@786422608c7bded2bbc9741ad9f91356842bf520 # v14
 
       - name: Update flake.lock
         id: update
-        uses: DeterminateSystems/update-flake-lock@fd9359ac79d0e912f1b4b947a48470b3e2799b56 # main
+        uses: DeterminateSystems/update-flake-lock@da03c0f078bc4b2c37ee4f7e072d34bf8f188bb3 # v29
         with:
           git-author-name: houseme
           git-author-email: housemecn@gmail.com

.github/workflows/nix.yml

@@ -72,12 +72,12 @@ jobs:
       NIX_CURL_FLAGS: -A cargo/stable
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
       - name: Install Nix
-        uses: DeterminateSystems/determinate-nix-action@4eea0b33e3d1f02ecfe37cf16e7204c424009606 # v3.21.0
+        uses: DeterminateSystems/determinate-nix-action@8d87e8d5e5b8a8309d4281094560f127d9a265f1 # v3.22.5
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           extra-conf: |
@@ -86,10 +86,10 @@ jobs:
             max-jobs = 1
 
       - name: Cache Nix
-        uses: DeterminateSystems/flakehub-cache-action@c01e819d047464c3edf6ba778f075952af5a3aa7 # v3.21.0
+        uses: DeterminateSystems/flakehub-cache-action@83282a8aef353db1d659d3bacf4a28b6683de0b3 # v3.22.5
 
       - name: Check Nix Flake Inputs
-        uses: DeterminateSystems/flake-checker-action@3164002371bc90729c68af0e24d5aacf20d7c9f6 # v12
+        uses: DeterminateSystems/flake-checker-action@786422608c7bded2bbc9741ad9f91356842bf520 # v14
         with:
           fail-mode: true
           ignore-missing-flake-lock: false

.github/workflows/oidc-keycloak.yml

@@ -57,7 +57,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -82,7 +82,7 @@ jobs:
 
       - name: Upload service logs
         if: failure()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: oidc-keycloak-live-${{ github.run_number }}
           path: ${{ runner.temp }}/rustfs-keycloak-live-*/**/*.log
@@ -101,7 +101,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/on-demand-migration-interop.yml

@@ -93,7 +93,7 @@ jobs:
       NEXTEST_LISTING: ${{ github.workspace }}/artifacts/odm-interop/minio/selection.json
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -166,7 +166,7 @@ jobs:
 
       - name: Upload the MinIO interop report
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: odm-interop-minio-${{ github.run_number }}-${{ github.run_attempt }}
           path: |
@@ -237,7 +237,7 @@ jobs:
 
       - name: Checkout repository
         if: steps.credentials.outputs.present == 'true'
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -286,7 +286,7 @@ jobs:
 
       - name: Upload the ${{ matrix.provider }} interop report
         if: always() && steps.credentials.outputs.present == 'true'
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: odm-interop-${{ matrix.provider }}-${{ github.run_number }}-${{ github.run_attempt }}
           path: |
@@ -306,7 +306,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/package.yml

@@ -25,9 +25,10 @@
 #
 # Flow:
 # 1. Resolve and validate the selected Build workflow run and source identity
-# 2. Download Linux binaries (x86_64-gnu, aarch64-gnu) from build artifacts
-# 3. Build DEB packages for amd64 and arm64
-# 4. Build RPM packages for x86_64 and aarch64
+# 2. Download Linux binaries (x86_64-gnu, aarch64-gnu, x86_64-musl,
+#    aarch64-musl) from build artifacts
+# 3. Build DEB packages for amd64 and arm64, gnu and musl variants
+# 4. Build RPM packages for x86_64 and aarch64, gnu and musl variants
 # 5. Upload all packages to Cloudflare R2 and the GitHub release
 
 name: Package DEB/RPM
@@ -224,9 +225,9 @@ jobs:
           echo "  Build run ID: $BUILD_RUN_ID"
           echo "  Build run number: $RUN_NUMBER"
 
-  # Build DEB and RPM packages for each architecture
+  # Build DEB and RPM packages for each architecture and libc variant
   package:
-    name: Package (${{ matrix.arch }})
+    name: Package (${{ matrix.arch }}-${{ matrix.libc }})
     needs: resolve
     runs-on: ubuntu-latest
     timeout-minutes: 30
@@ -236,16 +237,38 @@ jobs:
       matrix:
         include:
           - arch: x86_64
+            libc: gnu
             deb_arch: amd64
             rpm_arch: x86_64
             artifact_name: "rustfs-linux-x86_64-gnu"
+            # gnu binaries dynamically link glibc; musl binaries are static
+            # and run without a libc dependency.
+            deb_depends: "libc6 (>= 2.31)"
+            rpm_depends: "glibc >= 2.31"
           - arch: aarch64
+            libc: gnu
             deb_arch: arm64
             rpm_arch: aarch64
             artifact_name: "rustfs-linux-aarch64-gnu"
+            deb_depends: "libc6 (>= 2.31)"
+            rpm_depends: "glibc >= 2.31"
+          - arch: x86_64
+            libc: musl
+            deb_arch: amd64
+            rpm_arch: x86_64
+            artifact_name: "rustfs-linux-x86_64-musl"
+            deb_depends: ""
+            rpm_depends: ""
+          - arch: aarch64
+            libc: musl
+            deb_arch: arm64
+            rpm_arch: aarch64
+            artifact_name: "rustfs-linux-aarch64-musl"
+            deb_depends: ""
+            rpm_depends: ""
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -256,17 +279,17 @@ jobs:
           BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
           SOURCE_VERSION: ${{ needs.resolve.outputs.version }}
           DEV_SEQUENCE: ${{ needs.resolve.outputs.dev_sequence }}
-          DEB_ARCH: ${{ matrix.deb_arch }}
-          RPM_ARCH: ${{ matrix.rpm_arch }}
+          ARCH: ${{ matrix.rpm_arch }}
+          LIBC: ${{ matrix.libc }}
         run: |
           set -euo pipefail
 
           normalized=$(./scripts/release/package_versions.sh \
-            "$BUILD_TYPE" "$SOURCE_VERSION" "$DEV_SEQUENCE" "$DEB_ARCH" "$RPM_ARCH")
+            "$BUILD_TYPE" "$SOURCE_VERSION" "$DEV_SEQUENCE" "$ARCH" "$LIBC")
           printf '%s\n' "$normalized" >> "$GITHUB_OUTPUT"
 
       - name: Download binary artifact from build run
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           pattern: ${{ matrix.artifact_name }}*
           path: ./binary-artifact
@@ -308,6 +331,8 @@ jobs:
           DEB_VERSION: ${{ steps.versions.outputs.deb_version }}
           DEB_ARCH: ${{ matrix.deb_arch }}
           DEB_FILE: ${{ steps.versions.outputs.deb_file }}
+          DEB_DEPENDS: ${{ matrix.deb_depends }}
+          LIBC: ${{ matrix.libc }}
         run: |
           set -euo pipefail
 
@@ -328,10 +353,15 @@ jobs:
 
           cat > "${PKG_DIR}/etc/default/rustfs" << 'ENVEOF'
           # RustFS Environment Configuration
-          # See https://rustfs.com/docs/ for more information
+          # See https://docs.rustfs.com for more information
+          # RUSTFS_ACCESS_KEY=<changeme>
+          # RUSTFS_SECRET_KEY=<changeme>
           # RUSTFS_VOLUMES=""
-          # RUSTFS_ROOT_USER=""
-          # RUSTFS_ROOT_PASSWORD=""
+          # RUSTFS_ADDRESS=":9000"
+          # RUSTFS_CONSOLE_ADDRESS=":9001"
+          # RUSTFS_CONSOLE_ENABLE=true
+          # RUSTFS_OBS_LOGGER_LEVEL=error
+          # RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"
           ENVEOF
 
           cat > "${PKG_DIR}/DEBIAN/control" << EOF
@@ -340,9 +370,17 @@ jobs:
           Section: utils
           Priority: optional
           Architecture: ${DEB_ARCH}
-          Depends: libc6 (>= 2.31)
+          EOF
+
+          # gnu binaries dynamically link glibc and must declare it; musl
+          # binaries are statically linked and need no libc dependency.
+          if [[ -n "$DEB_DEPENDS" ]]; then
+            echo "Depends: ${DEB_DEPENDS}" >> "${PKG_DIR}/DEBIAN/control"
+          fi
+
+          cat >> "${PKG_DIR}/DEBIAN/control" << EOF
           Maintainer: RustFS Team <support@rustfs.com>
-          Description: High-performance distributed object storage
+          Description: High-performance distributed object storage (${LIBC} build)
            RustFS is a high-performance distributed object storage software
            built using Rust. It is compatible with MinIO and S3 API.
           Homepage: https://rustfs.com
@@ -354,37 +392,13 @@ jobs:
           /etc/default/rustfs
           CONFFILES
 
-          cat > "${PKG_DIR}/DEBIAN/postinst" << 'POSTINST'
-          #!/bin/bash
-          set -e
-          if ! getent passwd rustfs > /dev/null 2>&1; then
-              useradd -r -s /bin/false -d /opt/rustfs rustfs
-          fi
-          mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs
-          chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs
-          if [ -d /run/systemd/system ]; then
-              systemctl daemon-reload
-          fi
-          echo "RustFS installed. Configure /etc/default/rustfs then: systemctl start rustfs"
-          POSTINST
+          ./scripts/release/package_service_scripts.sh after-install > "${PKG_DIR}/DEBIAN/postinst"
           chmod 755 "${PKG_DIR}/DEBIAN/postinst"
 
-          cat > "${PKG_DIR}/DEBIAN/prerm" << 'PRERM'
-          #!/bin/bash
-          set -e
-          if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then
-              systemctl stop rustfs
-          fi
-          PRERM
+          ./scripts/release/package_service_scripts.sh before-remove > "${PKG_DIR}/DEBIAN/prerm"
           chmod 755 "${PKG_DIR}/DEBIAN/prerm"
 
-          cat > "${PKG_DIR}/DEBIAN/postrm" << 'POSTRM'
-          #!/bin/bash
-          set -e
-          if [ -d /run/systemd/system ]; then
-              systemctl daemon-reload
-          fi
-          POSTRM
+          ./scripts/release/package_service_scripts.sh after-remove > "${PKG_DIR}/DEBIAN/postrm"
           chmod 755 "${PKG_DIR}/DEBIAN/postrm"
 
           cp LICENSE "${PKG_DIR}/usr/share/doc/rustfs/"
@@ -408,24 +422,38 @@ jobs:
           RPM_RELEASE: ${{ steps.versions.outputs.rpm_release }}
           RPM_ARCH: ${{ matrix.rpm_arch }}
           RPM_FILE: ${{ steps.versions.outputs.rpm_file }}
+          RPM_DEPENDS: ${{ matrix.rpm_depends }}
+          LIBC: ${{ matrix.libc }}
         run: |
           set -euo pipefail
 
-          echo "Building RPM for ${RPM_ARCH}"
+          echo "Building RPM for ${RPM_ARCH} (${LIBC})"
 
           sudo apt-get update && sudo apt-get install -y ruby ruby-dev build-essential rpm
           sudo gem install fpm
           ./scripts/test_package_versions.sh --require-package-managers
 
+          # gnu binaries dynamically link glibc and must declare it; musl
+          # binaries are statically linked and need no libc dependency.
+          depends_args=()
+          if [[ -n "$RPM_DEPENDS" ]]; then
+            depends_args+=(--depends "$RPM_DEPENDS")
+          fi
+
           # Create config file for fpm (DEB build creates it in its package dir structure,
           # but fpm needs the file to exist before packaging)
           mkdir -p ./tmp-pkg/etc/default
           cat > ./tmp-pkg/etc/default/rustfs << 'ENVEOF'
           # RustFS Environment Configuration
-          # See https://rustfs.com/docs/ for more information
+          # See https://docs.rustfs.com for more information
+          # RUSTFS_ACCESS_KEY=<changeme>
+          # RUSTFS_SECRET_KEY=<changeme>
           # RUSTFS_VOLUMES=""
-          # RUSTFS_ROOT_USER=""
-          # RUSTFS_ROOT_PASSWORD=""
+          # RUSTFS_ADDRESS=":9000"
+          # RUSTFS_CONSOLE_ADDRESS=":9001"
+          # RUSTFS_CONSOLE_ENABLE=true
+          # RUSTFS_OBS_LOGGER_LEVEL=error
+          # RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"
           ENVEOF
 
           fpm -s dir -t rpm \
@@ -434,40 +462,18 @@ jobs:
             --iteration "$RPM_RELEASE" \
             --architecture "$RPM_ARCH" \
             --package "$RPM_FILE" \
-            --depends "glibc >= 2.31" \
+            "${depends_args[@]}" \
+            --description "High-performance distributed object storage (${LIBC} build)" \
             --maintainer "RustFS Team <support@rustfs.com>" \
-            --description "High-performance distributed object storage" \
             --url "https://rustfs.com" \
             --license "Apache-2.0" \
-            --after-install <(cat <<'POSTINST'
-          #!/bin/bash
-          set -e
-          if ! getent passwd rustfs > /dev/null 2>&1; then
-              useradd -r -s /bin/false -d /opt/rustfs rustfs
-          fi
-          mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs
-          chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs
-          if [ -d /run/systemd/system ]; then
-              systemctl daemon-reload
-          fi
-          POSTINST
-            ) \
-            --before-remove <(cat <<'PRERM'
-          #!/bin/bash
-          set -e
-          if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then
-              systemctl stop rustfs
-          fi
-          PRERM
-            ) \
-            --after-remove <(cat <<'POSTRM'
-          #!/bin/bash
-          set -e
-          if [ -d /run/systemd/system ]; then
-              systemctl daemon-reload
-          fi
-          POSTRM
-            ) \
+            --before-install <(./scripts/release/package_service_scripts.sh rpm-before-install) \
+            --before-upgrade <(./scripts/release/package_service_scripts.sh rpm-before-upgrade) \
+            --after-install <(./scripts/release/package_service_scripts.sh after-install) \
+            --after-upgrade <(./scripts/release/package_service_scripts.sh after-install) \
+            --before-remove <(./scripts/release/package_service_scripts.sh before-remove) \
+            --after-remove <(./scripts/release/package_service_scripts.sh after-remove) \
+            --rpm-posttrans <(./scripts/release/package_service_scripts.sh rpm-posttrans) \
             --config-files /etc/default/rustfs \
             ./bin/rustfs=/usr/bin/rustfs \
             ./tmp-pkg/etc/default/rustfs=/etc/default/rustfs \
@@ -489,9 +495,9 @@ jobs:
           echo "✅ RPM built: $RPM_FILE"
 
       - name: Upload packages to artifacts
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: packages-${{ matrix.arch }}
+          name: packages-${{ matrix.arch }}-${{ matrix.libc }}
           path: |
             *.deb
             *.rpm
@@ -587,10 +593,11 @@ jobs:
             for f in "$DEB_FILE" "$RPM_FILE"; do
               if [[ -n "$f" && -f "$f" ]]; then
                 base="$(basename "$f")"
-                # GitHub stores release asset names with '~' normalized to '.'
-                # (e.g. rustfs_1.0.0~rc.2_amd64.deb is stored as
-                # rustfs_1.0.0.rc.2_amd64.deb), so checksum entries must
-                # reference the name as stored on the release.
+                # GitHub stores release asset names with '~' normalized to '.'.
+                # Current package file names embed the raw tag and contain no
+                # '~', but older releases may still ship such assets, so the
+                # checksum entries must reference the name as stored on the
+                # release.
                 github_base="${base//\~/.}"
                 # Remove any stale entry (both naming variants), then append
                 grep -Fv -- "$base" "$checksum_file" > "${checksum_file}.tmp" || true

.github/workflows/performance-ab.yml

@@ -57,7 +57,7 @@ jobs:
     timeout-minutes: 180
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
           fetch-depth: 0 # baseline may be an earlier successful scheduled head
@@ -94,7 +94,7 @@ jobs:
       - name: Find last successful scheduled baseline
         id: scheduled_baseline
         if: github.event_name == 'schedule'
-        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         with:
           result-encoding: string
           script: |
@@ -271,7 +271,7 @@ jobs:
 
       - name: Upload A/B results
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: hotpath-warp-ab-${{ github.run_number }}
           # Includes per-cell median_summary.csv / baseline_compare.csv, both gates,
@@ -372,7 +372,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/runner-hygiene.yml

@@ -47,7 +47,7 @@ jobs:
     timeout-minutes: 15
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -72,7 +72,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/rustfs-fault-tolerance-matrix.yml

@@ -0,0 +1,221 @@
+# RustFS Fault-Tolerance (degradation) Matrix Sweep
+#
+# Exhaustive topology x EC outage sweep, complementary to the scenario suite
+# (rustfs-fault-tolerance-test.yml). For each topology (nodes x drives per
+# node) and each EC level P in 1..N*drives/2 the runner installs that exact
+# cluster shape and drives progressive node outages plus recovery:
+#
+#   M-<topo>EC<P>-full   full-capacity reads/writes must succeed
+#   M-<topo>EC<P>-n<k>   with k nodes online: reads succeed iff the online
+#                        drives cover the read quorum (N-P), writes iff they
+#                        cover the write quorum (N-P+1); probes below quorum
+#                        assert the expected outage
+#   M-<topo>EC<P>-rec    after restoring every node, reads/writes succeed
+#
+# 10 topologies x EC levels = 38 combinations, 320 cases. The full sweep runs
+# 3-5 hours and is dispatched on demand (e.g. after a nightly chain goes
+# green), not nightly. Case ids and expectations come from product source
+# (default_parity_count, erasure set sizing) via rustfs/auto-testing.
+
+name: RustFS Fault-Tolerance Matrix
+
+on:
+  workflow_dispatch:
+    inputs:
+      package_url:
+        description: 'Direct .deb URL. Required unless the nightly default is wanted.'
+        required: false
+        type: string
+      auto_testing_ref:
+        description: 'auto-testing branch, tag, or SHA (defaults to main)'
+        required: false
+        default: main
+        type: string
+      matrix_topo:
+        description: 'Topology filter like 4x2; empty sweeps all ten topologies'
+        required: false
+        default: ''
+        type: string
+      matrix_ec:
+        description: 'EC/parity filter; empty sweeps every EC level'
+        required: false
+        default: ''
+        type: string
+      cleanup_before:
+        description: 'Reset the nodes before the test (DESTROYS existing data/config)'
+        type: boolean
+        default: true
+      cleanup_after:
+        description: 'Reset the nodes after the test (DESTROYS test data/config)'
+        type: boolean
+        default: true
+
+permissions:
+  contents: read
+
+# The suite stops services and revokes drive access on the shared fleet; only one
+# functional suite may touch the environment at a time.
+concurrency:
+  group: rustfs-shared-functional-tests-v2
+  cancel-in-progress: false
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
+  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
+  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
+  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
+  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
+
+jobs:
+  fault-tolerance-matrix:
+    runs-on: smoke-testing
+    # The full 38-combination sweep takes 3-5h; 360 is the runner job ceiling.
+    timeout-minutes: 360
+    steps:
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-fault-tolerance-matrix-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch" "${FUNCTIONAL_ARTIFACTS_DIR}/evidence"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'RESULTS_FILE=%s/results.json\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'EVIDENCE_DIR=%s/evidence\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
+      - name: Checkout auto-testing scripts
+        timeout-minutes: 5
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ inputs.auto_testing_ref }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
+
+      - name: Record auto-testing revision
+        run: |
+          set -euo pipefail
+          AUTO_TESTING_SHA="$(git -C auto-testing rev-parse HEAD)"
+          printf 'AUTO_TESTING_SHA=%s\n' "${AUTO_TESTING_SHA}" >> "${GITHUB_ENV}"
+          echo "auto-testing revision: ${AUTO_TESTING_SHA}"
+
+      - name: Show environment
+        run: |
+          uname -a
+          jq --version
+          aws --version
+          echo "auto-testing: ${AUTO_TESTING_SHA}"
+          df -h /data | tail -1
+
+      - name: Cleanup environment (before)
+        timeout-minutes: 5
+        if: ${{ inputs.cleanup_before != 'false' }}
+        run: |
+          ./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}"
+
+      - name: Run matrix sweep (topology x EC outage combinations)
+        timeout-minutes: 350
+        id: test
+        # No continue-on-error: an UNEXPECTED verdict (a probe that should
+        # fail by quorum math but passes, or vice versa) is the finding this
+        # sweep exists to surface, and the red run is the signal until matrix
+        # runs get backlog integration like the scenario suite has.
+        env:
+          MATRIX_EC: ${{ inputs.matrix_ec }}
+        run: |
+          set -euo pipefail
+          ARGS=(--matrix -y --package-url "${{ inputs.package_url || env.RUSTFS_NIGHTLY_PACKAGE_URL }}" --log-file "${LOG_FILE}" --results-file "${RESULTS_FILE}")
+          if [ -n "${{ inputs.matrix_topo }}" ]; then
+            # Trailing value form: consumed as `--matrix-topo <topo> [ec]`.
+            ARGS+=(--matrix-topo "${{ inputs.matrix_topo }}")
+            if [ -n "${MATRIX_EC}" ]; then
+              ARGS+=("${MATRIX_EC}")
+            fi
+          fi
+          ./auto-testing/rustfs-fault-tolerance-test.sh "${ARGS[@]}"
+
+      - name: Generate report
+        timeout-minutes: 2
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        run: |
+          set -euo pipefail
+          {
+            echo "# RustFS fault-tolerance matrix report"
+            echo ""
+            echo "- Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+            echo "- Package: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
+            echo "- auto-testing: \`${AUTO_TESTING_SHA}\`"
+            echo "- Topology filter: ${{ inputs.matrix_topo || 'all (10 topologies)' }}"
+            echo "- EC filter: ${{ inputs.matrix_ec || 'all levels' }}"
+            echo ""
+            echo "## Per-probe results"
+            echo ""
+            echo '```'
+            grep -E '^FT-(CASE|COMPLETE|SUMMARY|REPORT)' "${LOG_FILE}" || echo "(no FT-CASE lines found)"
+            echo '```'
+            echo ""
+            echo "## Health snapshots"
+            echo ""
+            for f in "${FUNCTIONAL_ARTIFACTS_DIR}"/evidence/*.code; do
+              [ -e "${f}" ] || continue
+              printf '%s -> %s\n' "$(basename "${f}" .code)" "$(cat "${f}")"
+            done
+          } > "${REPORT_FILE}"
+          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+
+          # An unfiltered sweep must land exactly 320 cases (38 topology x EC
+          # combinations at 2*(nodes+1) cases each); filtered sweeps are held
+          # to internal consistency. Missing/harness-error rows always fail.
+          EXPECTED_TOTAL=""
+          if [ -z "${{ inputs.matrix_topo }}" ] && [ -z "${MATRIX_EC}" ]; then
+            EXPECTED_TOTAL=320
+          fi
+          jq -e --arg testing_sha "${AUTO_TESTING_SHA}" --argjson expected_total "${EXPECTED_TOTAL:-null}" '
+            .schema_version == 1 and
+            .auto_testing_sha == $testing_sha and
+            .complete == true and
+            .counts.expected == ($expected_total // .counts.expected) and
+            .counts.expected == .counts.seen and
+            .counts.harness_errors == 0 and
+            (.expected_cases | length) == .counts.expected and
+            (.expected_cases | unique | length) == .counts.expected and
+            (.seen_cases | length) == .counts.seen and
+            (.missing_cases | length) == 0 and
+            (.harness_errors | length) == 0 and
+            (.cases | length) == .counts.seen and
+            ([.cases[].case_id] | unique | length) == .counts.seen and
+            ([.cases[].case_id] | sort) == (.expected_cases | sort) and
+            all(.cases[]; (.case_id | startswith("M-")) and (.verdict == "pass" or .verdict == "known-divergence" or .verdict == "UNEXPECTED")) and
+            .counts.unexpected == ([.cases[] | select(.verdict == "UNEXPECTED")] | length) and
+            .counts.known_divergence == ([.cases[] | select(.verdict == "known-divergence")] | length)
+          ' "${RESULTS_FILE}" >/dev/null
+
+      - name: Upload test logs & evidence
+        timeout-minutes: 2
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: rustfs-fault-tolerance-matrix-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results.json
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/evidence/
+          if-no-files-found: warn
+
+      - name: Cleanup environment (after)
+        timeout-minutes: 5
+        if: ${{ always() && inputs.cleanup_after != 'false' }}
+        run: |
+          ./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}" || true

.github/workflows/rustfs-fault-tolerance-test.yml

@@ -4,13 +4,14 @@
 # behavior under drive and node loss against the erasure-coding contract and
 # snapshots health-endpoint responses at every tier.
 #
-#   A  single-node 4 drives (SNMD): hide 1/2/3 drives, restore
+#   A  single-node 4 drives (SNMD): revoke access to 1/2/3 drives, restore
 #   B  multi-node 4x1 (one drive per node): stop 1/2/3 nodes, restore
 #   C  multi-node 4x4 (16 drives, EC:4): stop 1 node (read-quorum boundary),
 #      stop 2 nodes, restore
 #   C2 multi-node 4x4 with EC:8: 2 nodes down puts 8 drives online -- reads
 #      satisfy the EC read quorum while the lock majority is broken (the
 #      reported divergence window: reads 503 with lock_quorum_unavailable)
+#   D  multi-node 4x1: stop 1/2 nodes, then restore
 #
 # Expectations come from product source (default_parity_count, erasure set
 # sizing). By default a "reads refused although the read quorum is met"
@@ -32,6 +33,11 @@ on:
         description: 'Direct .deb URL. Required unless the nightly default is wanted.'
         required: false
         type: string
+      auto_testing_ref:
+        description: 'auto-testing branch, tag, or SHA (defaults to main)'
+        required: false
+        default: main
+        type: string
       strict:
         description: 'Fail the suite when reads are refused despite a met read quorum'
         type: boolean
@@ -52,7 +58,7 @@ on:
 permissions:
   contents: read
 
-# The suite stops services and hides drive dirs on the shared fleet; only one
+# The suite stops services and revokes drive access on the shared fleet; only one
 # functional suite may touch the environment at a time.
 concurrency:
   group: rustfs-shared-functional-tests-v2
@@ -77,7 +83,7 @@ jobs:
     if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -92,6 +98,7 @@ jobs:
             printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
             printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
             printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'RESULTS_FILE=%s/results.json\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
             printf 'EVIDENCE_DIR=%s/evidence\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
             printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
           } >> "${GITHUB_ENV}"
@@ -106,19 +113,27 @@ jobs:
 
       - name: Checkout auto-testing scripts
         timeout-minutes: 5
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/auto-testing
-          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          ref: ${{ steps.chain.outputs.testing_sha || inputs.auto_testing_ref || 'main' }}
           token: ${{ secrets.PF_TESTING_GH_TOKEN }}
           path: auto-testing
           persist-credentials: false
 
+      - name: Record auto-testing revision
+        run: |
+          set -euo pipefail
+          AUTO_TESTING_SHA="$(git -C auto-testing rev-parse HEAD)"
+          printf 'AUTO_TESTING_SHA=%s\n' "${AUTO_TESTING_SHA}" >> "${GITHUB_ENV}"
+          echo "auto-testing revision: ${AUTO_TESTING_SHA}"
+
       - name: Show environment
         run: |
           uname -a
           jq --version
           aws --version
+          echo "auto-testing: ${AUTO_TESTING_SHA}"
           df -h /data | tail -1
 
       - name: Cleanup environment (before)
@@ -134,14 +149,14 @@ jobs:
           CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
         run: python3 scripts/prepare_functional_package.py prepare
 
-      - name: Run fault-tolerance scenarios (A, B, C, C2)
+      - name: Run fault-tolerance scenarios (A, B, C, C2, D, E)
         timeout-minutes: 45
         id: test
         # Standalone case failures are reported by the backlog manager.
         # Chain evidence separately requires complete passing cases.
         continue-on-error: true
         run: |
-          ARGS=(--all -y --package-url "${{ steps.chain_package.outputs.package_url || inputs.package_url || env.RUSTFS_NIGHTLY_PACKAGE_URL }}" --log-file "${LOG_FILE}")
+          ARGS=(--all -y --package-url "${{ steps.chain_package.outputs.package_url || inputs.package_url || env.RUSTFS_NIGHTLY_PACKAGE_URL }}" --log-file "${LOG_FILE}" --results-file "${RESULTS_FILE}")
           if [ "${{ inputs.strict }}" = "true" ]; then
             ARGS+=(--strict)
           fi
@@ -158,12 +173,13 @@ jobs:
             echo ""
             echo "- Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
             echo "- Package: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
+            echo "- auto-testing: \`${AUTO_TESTING_SHA}\`"
             echo "- Strict mode: ${{ inputs.strict || 'false' }}"
             echo ""
             echo "## Per-probe results"
             echo ""
             echo '```'
-            grep -E '^FT-(CASE|SUMMARY|REPORT)' "${LOG_FILE}" || echo "(no FT-CASE lines found)"
+            grep -E '^FT-(CASE|COMPLETE|SUMMARY|REPORT)' "${LOG_FILE}" || echo "(no FT-CASE lines found)"
             echo '```'
             echo ""
             echo "## Health snapshots"
@@ -174,19 +190,30 @@ jobs:
             done
           } > "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          # Product gate: failing FT cases keep the run green — the report above and
-          # the backlog issue manager carry the signal. Only harness/environment
-          # breakdowns (no case verdicts at all) turn the workflow red.
-          FT_CASES="$(grep -cE '^FT-CASE:' "${LOG_FILE}" 2>/dev/null || true)"
-          FT_FAILED="$(grep -cE '^FT-CASE: .* verdict=UNEXPECTED' "${LOG_FILE}" 2>/dev/null || true)"
-          FT_PASSED=$(( ${FT_CASES:-0} - ${FT_FAILED:-0} ))
-          HARNESS_OK=0
-          if [ "${{ steps.test.outcome }}" = "success" ]; then
-            HARNESS_OK=1
-          elif [ "${{ steps.test.outcome }}" = "failure" ] && [ "${FT_CASES}" -gt 0 ] && [ "${FT_PASSED}" -ge 1 ]; then
-            HARNESS_OK=1
-          fi
-          [ "${HARNESS_OK}" = "1" ]
+          printf 'complete=false\n' >> "${GITHUB_OUTPUT}"
+          jq -e --arg testing_sha "${AUTO_TESTING_SHA}" '
+            .schema_version == 1 and
+            .auto_testing_sha == $testing_sha and
+            .selected_scenarios == ["A", "B", "C", "C2", "D", "E"] and
+            .complete == true and
+            .counts.expected == 44 and
+            .counts.seen == 44 and
+            .counts.harness_errors == 0 and
+            (.expected_cases | length) == 44 and
+            (.expected_cases | unique | length) == 44 and
+            (.seen_cases | length) == 44 and
+            (.seen_cases | unique | length) == 44 and
+            (.seen_cases | sort) == (.expected_cases | sort) and
+            (.missing_cases | length) == 0 and
+            (.harness_errors | length) == 0 and
+            (.cases | length) == 44 and
+            ([.cases[].case_id] | sort) == (.expected_cases | sort) and
+            ([.cases[].case_id] | unique | length) == 44 and
+            all(.cases[]; .verdict == "pass" or .verdict == "known-divergence" or .verdict == "UNEXPECTED") and
+            .counts.unexpected == ([.cases[] | select(.verdict == "UNEXPECTED")] | length) and
+            .counts.known_divergence == ([.cases[] | select(.verdict == "known-divergence")] | length)
+          ' "${RESULTS_FILE}" >/dev/null
+          printf 'complete=true\n' >> "${GITHUB_OUTPUT}"
 
       - name: Manage backlog issues (dedup / label / auto-close)
         timeout-minutes: 2
@@ -196,7 +223,7 @@ jobs:
         # green run. Never acts on cancelled runs. Logic lives in
         # auto-testing/scripts/issue_manager.py, which parses the
         # FT-CASE verdict lines from the suite log.
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: ${{ always() && steps.evidence.outcome == 'success' && steps.chain_report.outputs.complete == 'true' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
@@ -267,11 +294,12 @@ jobs:
       - name: Upload test logs & evidence
         timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: rustfs-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
             ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results.json
             ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
             ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/evidence/
           if-no-files-found: warn
@@ -358,7 +386,7 @@ jobs:
 
       - name: Upload chain evidence
         if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-chain-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}/fault-tolerance.json

.github/workflows/rustfs-functional-chain.yml

@@ -48,19 +48,22 @@ jobs:
       manifest: ${{ steps.candidate.outputs.manifest }}
       performance_ready: ${{ steps.perf_probe.outputs.performance_ready }}
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Resolve published candidate
         id: candidate
         env:
-          GH_TOKEN: ${{ github.token }}
+          # The staleness probe in resolve_functional_candidate.py reads the
+          # private rustfs/auto-testing repo; github.token cannot, and a probe
+          # failure silently keeps a stale testing_sha pinned for the chain.
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
           BUILD_RUN_ID: ${{ inputs.build_run_id }}
           BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
           CHAIN_OUTPUT: ${{ runner.temp }}/chain-candidate.json
         run: python3 scripts/resolve_functional_candidate.py
       - name: Retain candidate identity
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-candidate-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-candidate.json
@@ -188,13 +191,13 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Download suite evidence
         id: download
         continue-on-error: true
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           pattern: functional-chain-*-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-evidence
@@ -210,7 +213,7 @@ jobs:
           --output "${RUNNER_TEMP}/chain-report.json"
       - name: Retain chain report regardless of test verdict
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-chain-report-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
@@ -228,7 +231,7 @@ jobs:
           --directory "${RUNNER_TEMP}/chain-evidence"
           --output "${RUNNER_TEMP}/chain-complete.json"
       - name: Upload complete-chain evidence
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-chain-complete-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-complete.json

.github/workflows/rustfs-heal-test.yml

@@ -66,7 +66,7 @@ jobs:
     if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout chain tooling
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -93,7 +93,7 @@ jobs:
         run: python3 scripts/functional_chain_evidence.py consume
 
       - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/auto-testing
           ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
@@ -366,7 +366,7 @@ jobs:
       - name: Upload test logs
         timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: rustfs-heal-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
@@ -467,7 +467,7 @@ jobs:
 
       - name: Upload chain evidence
         if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-chain-heal-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-heal-${{ github.run_id }}-${{ github.run_attempt }}/heal.json

.github/workflows/rustfs-kms-test.yml

@@ -59,7 +59,7 @@ jobs:
     if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -85,7 +85,7 @@ jobs:
         run: python3 scripts/functional_chain_evidence.py consume
 
       - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/auto-testing
           ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
@@ -315,7 +315,7 @@ jobs:
       - name: Upload report and logs
         timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: rustfs-kms-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
@@ -414,7 +414,7 @@ jobs:
 
       - name: Upload chain evidence
         if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-chain-kms-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-kms-${{ github.run_id }}-${{ github.run_attempt }}/kms.json

.github/workflows/rustfs-performance-test.yml

@@ -61,10 +61,13 @@ on:
 permissions:
   contents: read
 
-# The default performance nodes overlap the other suites' remote VMs, even
-# though the runner differs. Hold the shared lock through cleanup as well.
+# Workflow-level so the lock is held through cleanup as well.
 concurrency:
-  group: rustfs-shared-functional-tests-v2
+  # Own group: the performance suite runs on the pf-testing fleet - a fully
+  # separate environment from the smoke-testing fleet the functional suites
+  # use. No shared node, no shared runner: it must never queue behind them
+  # (or block them).
+  group: rustfs-performance-suite
   cancel-in-progress: false
 
 defaults:
@@ -90,7 +93,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Check performance runner before entering its queue
@@ -113,7 +116,7 @@ jobs:
     if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout chain tooling
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -141,7 +144,7 @@ jobs:
         run: python3 scripts/functional_chain_evidence.py consume
 
       - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/auto-testing
           ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
@@ -304,7 +307,7 @@ jobs:
 
       - name: Upload test logs & results
         if: ${{ always() && steps.evidence.outcome == 'success' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: rustfs-perf-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
@@ -346,7 +349,7 @@ jobs:
 
       - name: Upload chain evidence
         if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-chain-performance-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-performance-${{ github.run_id }}-${{ github.run_attempt }}/performance.json

.github/workflows/rustfs-pool-expand-test.yml

@@ -92,7 +92,7 @@ jobs:
       RUSTFS_POOL_NODE_ENDPOINTS: ${{ secrets.RUSTFS_POOL_NODE_ENDPOINTS || vars.RUSTFS_POOL_NODE_ENDPOINTS || 'http://rustfs-node1:9000 http://rustfs-node2:9000 http://rustfs-node3:9000 http://rustfs-node4:9000' }}
     steps:
       - name: Checkout chain tooling
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -104,7 +104,7 @@ jobs:
         run: python3 scripts/functional_chain_evidence.py consume
 
       - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/auto-testing
           ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
@@ -656,7 +656,7 @@ jobs:
       - name: Upload test logs
         timeout-minutes: 2
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: rustfs-pool-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/rustfs-pool-${{ github.run_id }}-${{ github.run_attempt }}
@@ -764,7 +764,7 @@ jobs:
 
       - name: Upload chain evidence
         if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-chain-pool-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-pool-${{ github.run_id }}-${{ github.run_attempt }}/pool.json

.github/workflows/rustfs-replication-test.yml

@@ -71,7 +71,7 @@ jobs:
     if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -97,7 +97,7 @@ jobs:
         run: python3 scripts/functional_chain_evidence.py consume
 
       - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/auto-testing
           ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
@@ -323,7 +323,7 @@ jobs:
       - name: Upload report and logs
         timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: rustfs-replication-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
@@ -419,7 +419,7 @@ jobs:
 
       - name: Upload chain evidence
         if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-chain-replication-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-replication-${{ github.run_id }}-${{ github.run_attempt }}/replication.json

.github/workflows/rustfs-s3-compat-test.yml

@@ -47,7 +47,7 @@ jobs:
     if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -73,7 +73,7 @@ jobs:
         run: python3 scripts/functional_chain_evidence.py consume
 
       - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/auto-testing
           ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
@@ -292,7 +292,7 @@ jobs:
       - name: Upload report and logs
         timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: rustfs-s3-compat-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
@@ -391,7 +391,7 @@ jobs:
 
       - name: Upload chain evidence
         if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-chain-s3-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-s3-${{ github.run_id }}-${{ github.run_attempt }}/s3.json

.github/workflows/rustfs-security-test.yml

@@ -87,7 +87,7 @@ jobs:
       # the workspace root would wipe the auto-testing clone above (that is
       # exactly how run 33934141181 lost rustfs-security-test.sh).
       - name: Checkout repository (for the OIDC live gate script)
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
           path: rustfs-repo
@@ -109,7 +109,7 @@ jobs:
         run: python3 scripts/functional_chain_evidence.py consume
 
       - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/auto-testing
           ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
@@ -325,7 +325,7 @@ jobs:
       - name: Upload report and logs
         timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: rustfs-security-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
@@ -395,7 +395,7 @@ jobs:
 
       - name: Upload chain evidence
         if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-chain-security-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-security-${{ github.run_id }}-${{ github.run_attempt }}/security.json

.github/workflows/rustfs-storage-test.yml

@@ -56,7 +56,7 @@ jobs:
     if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -82,7 +82,7 @@ jobs:
         run: python3 scripts/functional_chain_evidence.py consume
 
       - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/auto-testing
           ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
@@ -307,7 +307,7 @@ jobs:
       - name: Upload report and logs
         timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: rustfs-storage-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
@@ -406,7 +406,7 @@ jobs:
 
       - name: Upload chain evidence
         if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-chain-storage-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-storage-${{ github.run_id }}-${{ github.run_attempt }}/storage.json

.github/workflows/rustfs-table-test.yml

@@ -46,7 +46,7 @@ jobs:
     timeout-minutes: 60
     steps:
       - name: Checkout repository (for report parser)
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -73,7 +73,7 @@ jobs:
 
       - name: Checkout auto-testing scripts
         timeout-minutes: 5
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/auto-testing
           ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
@@ -133,7 +133,7 @@ jobs:
           if [ -n "${PACKAGE_URL}" ]; then
             ARGS+=(--package-url "${PACKAGE_URL}")
           elif [ -n "${RUSTFS_VERSION}" ] && [ "${RUSTFS_VERSION}" != "null" ]; then
-            ARGS+=(--package-url "https://github.com/rustfs/rustfs/releases/download/${RUSTFS_VERSION}/rustfs_${RUSTFS_VERSION}_amd64.deb")
+            ARGS+=(--package-url "https://github.com/rustfs/rustfs/releases/download/${RUSTFS_VERSION}/rustfs-linux-x86_64-gnu-v${RUSTFS_VERSION}.deb")
           else
             ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
           fi
@@ -281,7 +281,7 @@ jobs:
       - name: Upload report and logs
         timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: rustfs-table-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
@@ -353,7 +353,7 @@ jobs:
 
       - name: Upload chain evidence
         if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-chain-table-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-table-${{ github.run_id }}-${{ github.run_attempt }}/table.json

.github/workflows/rustfs-tier-test.yml

@@ -71,7 +71,7 @@ jobs:
     if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout chain tooling
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -95,7 +95,7 @@ jobs:
         run: python3 scripts/functional_chain_evidence.py consume
 
       - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/auto-testing
           ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
@@ -460,7 +460,7 @@ jobs:
       - name: Upload report and logs
         timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: rustfs-tier-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ env.TIER_ARTIFACTS_DIR }}/
@@ -734,7 +734,7 @@ jobs:
 
       - name: Upload chain evidence
         if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-chain-tier-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-tier-${{ github.run_id }}-${{ github.run_attempt }}/tier.json

.github/workflows/rustfs-upgrade-test.yml

@@ -89,7 +89,7 @@ jobs:
     if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -115,7 +115,7 @@ jobs:
         run: python3 scripts/functional_chain_evidence.py consume
 
       - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/auto-testing
           ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
@@ -207,7 +207,7 @@ jobs:
             local version="$1" tag asset url
             [ -n "${version}" ] && [ "${version}" != "null" ] || return 0
             tag="${version#v}"
-            asset="rustfs_${tag//-/.}_amd64.deb"
+            asset="rustfs-linux-x86_64-gnu-v${tag}.deb"
             url="https://github.com/rustfs/rustfs/releases/download/${tag}/${asset}"
             if ! gh api "repos/rustfs/rustfs/releases/tags/${tag}" --jq '.assets[].name' 2>/dev/null | grep -qxF "${asset}"; then
               echo "ERROR: release ${tag} has no downloadable asset ${asset}:" >&2
@@ -400,7 +400,7 @@ jobs:
       - name: Upload report and logs
         timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: rustfs-upgrade-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
@@ -503,7 +503,7 @@ jobs:
 
       - name: Upload chain evidence
         if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-chain-upgrade-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-upgrade-${{ github.run_id }}-${{ github.run_attempt }}/upgrade.json

.github/workflows/schedule-failure-alert-drill.yml

@@ -64,7 +64,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/scheduled-validation-freshness.yml

@@ -36,7 +36,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Check latest scheduled runs

.github/workflows/scheduled-validation-watchdog.yml

@@ -49,7 +49,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update incomplete-run issue

.github/workflows/stale.yml

@@ -29,7 +29,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 30
     steps:
-      - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9
+      - uses: actions/stale@4391f3da665fdf50b6810c1a66712fb9ba21aa93 # v11.0.0
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           stale-issue-message: 'This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs.'

.github/workflows/targets-integration.yml

@@ -52,7 +52,7 @@ jobs:
       RUSTFS_TEST_NATS_URL: nats://127.0.0.1:4222
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -166,7 +166,7 @@ jobs:
 
       - name: Upload target integration diagnostics
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: targets-integration-${{ github.run_number }}-${{ github.run_attempt }}
           path: artifacts/targets-live
@@ -183,7 +183,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/windows-filesystem.yml

@@ -56,7 +56,7 @@ jobs:
     timeout-minutes: 60
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 

Cargo.lock

@@ -685,9 +685,9 @@ dependencies = [
 
 [[package]]
 name = "async-compression"
-version = "0.4.47"
+version = "0.4.48"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef217a77a86a6e3dab9a5b3c81dc445b603fe743a90c1cb10a2f2144628d8cfa"
+checksum = "fb61aea1a7def73ee7c350a184f0e70b32c182344e2e75bf70c9b621b83417fd"
 dependencies = [
  "compression-codecs",
  "compression-core",
@@ -812,7 +812,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 3.0.5",
+ "syn 3.0.6",
 ]
 
 [[package]]
@@ -961,9 +961,9 @@ dependencies = [
 
 [[package]]
 name = "aws-runtime"
-version = "1.9.2"
+version = "1.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef47857a1d4488b528f4a5d5715fa7c3300820897824152234d3fa22b1426657"
+checksum = "25b43ad47adc2517efe3d706559d94b97e50e80e0321b3cadbc9f77cee88adcd"
 dependencies = [
  "aws-credential-types",
  "aws-sigv4",
@@ -989,9 +989,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-kms"
-version = "1.118.0"
+version = "1.120.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c243864bc3754be9f0001414e0162fdf1370fa62c70b0cfbe1da6a6221d553c7"
+checksum = "d6b0fe38fee2ba5b6cd24d32d08365b314ae1adea123649e061a7eb6300b6f5b"
 dependencies = [
  "arc-swap",
  "aws-credential-types",
@@ -1015,9 +1015,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-s3"
-version = "1.146.1"
+version = "1.148.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2cd651b4400d4011b8927b83a9552bf90ff11e6e5da0b9f0a7583247aceec971"
+checksum = "8a61c1db3987ab6c8740fb87f248864687d50549401115244a84f02c3047b2d5"
 dependencies = [
  "arc-swap",
  "aws-credential-types",
@@ -1052,9 +1052,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-sso"
-version = "1.109.0"
+version = "1.111.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3cfe74df5d9ad2fedd691973ad3521ebf4f27a3c68c792556686aedb5519bab"
+checksum = "65700d75f6da78c89a61a49f1539979400a4609d3329376b1dc2133d7a1b2b90"
 dependencies = [
  "arc-swap",
  "aws-credential-types",
@@ -1078,9 +1078,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-ssooidc"
-version = "1.111.0"
+version = "1.113.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81b0ec31ed6191bd11350aae4b2004198f2db21350cb0a20c57e0a92e55dd161"
+checksum = "eb1147c254af7b6cee3ace90ae7da6f21366ce9a9b899035fa8a0fdf586eb75a"
 dependencies = [
  "arc-swap",
  "aws-credential-types",
@@ -1104,9 +1104,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-sts"
-version = "1.114.0"
+version = "1.116.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef45745026107ec30c4ef86bd8ae4b002e7e5f6a86e4225240bdf6b06a0b944a"
+checksum = "035c7392e751d20bf1060d625d023587eb14099646110a9cff7c697bfe4b253a"
 dependencies = [
  "arc-swap",
  "aws-credential-types",
@@ -1131,9 +1131,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sigv4"
-version = "1.5.1"
+version = "1.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "723c2234ad7511ceef63eab016b7ba6ff7c55590fefb96fa8467af014a07309f"
+checksum = "31d955e76ff96acd555bf06fa0fa6d5bf9335fa84ae7c64481b20ae61d231f70"
 dependencies = [
  "aws-credential-types",
  "aws-smithy-eventstream",
@@ -1737,7 +1737,7 @@ dependencies = [
  "prettyplease 0.3.0",
  "proc-macro2",
  "quote",
- "syn 3.0.5",
+ "syn 3.0.6",
 ]
 
 [[package]]
@@ -1984,9 +1984,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.4.6"
+version = "1.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a3eb0f42d6c360dc3f8a821f6bf2fdea7f72bfd36b3076eb0e6d1e9e0752fff4"
+checksum = "54413ede23c2daf518f35156dfde027feb2374004d63bd497f983c8db9c0e313"
 dependencies = [
  "find-msvc-tools",
  "jobserver",
@@ -1996,9 +1996,9 @@ dependencies = [
 
 [[package]]
 name = "cfg-if"
-version = "1.0.4"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
+checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
 
 [[package]]
 name = "cfg_aliases"
@@ -2141,7 +2141,7 @@ dependencies = [
  "heck 0.5.0",
  "proc-macro2",
  "quote",
- "syn 3.0.5",
+ "syn 3.0.6",
 ]
 
 [[package]]
@@ -2222,9 +2222,9 @@ dependencies = [
 
 [[package]]
 name = "compression-codecs"
-version = "0.4.42"
+version = "0.4.43"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "257c7085cbb71be72d8fb97edff08b03d86d5d9f2222b9cc34a6bec87093bc10"
+checksum = "bef16c47ba2797aa6a909cc37d39911f3a6743811fe7408ac0b0cc0276b656e9"
 dependencies = [
  "brotli 8.0.4",
  "bzip2",
@@ -2798,7 +2798,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "strsim",
- "syn 3.0.5",
+ "syn 3.0.6",
 ]
 
 [[package]]
@@ -2842,7 +2842,7 @@ checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785"
 dependencies = [
  "darling_core 0.24.1",
  "quote",
- "syn 3.0.5",
+ "syn 3.0.6",
 ]
 
 [[package]]
@@ -3343,7 +3343,7 @@ checksum = "3a711bd5a06ec85d9683e142d28a98ad0679a7220669f5c1b8589e57e9b29e46"
 dependencies = [
  "datafusion-doc",
  "quote",
- "syn 3.0.5",
+ "syn 3.0.6",
 ]
 
 [[package]]
@@ -3715,7 +3715,7 @@ checksum = "2e2b94854e8576378ccda7c8de8a66ed8b4e8acbd2c50ec3418ea6c8aaf4b567"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 3.0.5",
+ "syn 3.0.6",
 ]
 
 [[package]]
@@ -3792,9 +3792,9 @@ dependencies = [
 
 [[package]]
 name = "dial9"
-version = "0.5.0"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49b11bce55173f77d9003cce8f129b4af889a4639005c53c3498cef933bc895c"
+checksum = "bab33c9f10e909fb3f1f6af2a9d976398ea793b90d1d286b4e23647a7d131d53"
 dependencies = [
  "bon",
  "dial9-core",
@@ -3809,9 +3809,9 @@ dependencies = [
 
 [[package]]
 name = "dial9-core"
-version = "0.5.0"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e8cbbc8955394be626249a3b52ddd6bf664373661eaeae70d87eb12bf6f20b6"
+checksum = "ffd7231006b074b02765ae929bded4757a414224b06c7db324d3e6e902f182ee"
 dependencies = [
  "arc-swap",
  "bon",
@@ -3842,9 +3842,9 @@ dependencies = [
 
 [[package]]
 name = "dial9-metrique"
-version = "0.5.0"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d42fe0e3e14e8780c4ecf9ee14c183dbcb9b1ae3612cc9a689e7dc83b13adef1"
+checksum = "f987e56dbafcab5998acc8e77eb91aaf66fa5441a691b38e99c8058c70725f58"
 dependencies = [
  "dial9-core",
  "dial9-trace-format",
@@ -3856,9 +3856,9 @@ dependencies = [
 
 [[package]]
 name = "dial9-perf-self-profile"
-version = "0.5.0"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8f65948455c504bf08576c7b5cfe93bfcaa486d661dc4ee85c2a6309ab89629"
+checksum = "d81f74ada03919614ca5b1ba767271d00980376a5a0e0fc43f8c265bf90413d4"
 dependencies = [
  "blazesym",
  "bon",
@@ -3875,9 +3875,9 @@ dependencies = [
 
 [[package]]
 name = "dial9-tokio-telemetry"
-version = "0.5.0"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af1244091367c805b5d98a6a590f8ab992efda06e6d2560a6967ef898ffd30a2"
+checksum = "934c027f628c59ae4c9664ca4dce758843c3da027dee5052d358f86a5d292ae6"
 dependencies = [
  "bon",
  "bytes",
@@ -3991,7 +3991,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 3.0.5",
+ "syn 3.0.6",
 ]
 
 [[package]]
@@ -4410,9 +4410,9 @@ dependencies = [
 
 [[package]]
 name = "find-msvc-tools"
-version = "0.1.12"
+version = "0.1.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d"
+checksum = "ef25905e51abafe4dcea6c15fec58c57b601cdbd0ee53d22ea1d3016c587d39b"
 
 [[package]]
 name = "findshlibs"
@@ -4597,7 +4597,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 3.0.5",
+ "syn 3.0.6",
 ]
 
 [[package]]
@@ -5306,9 +5306,9 @@ dependencies = [
 
 [[package]]
 name = "hotpath"
-version = "0.25.1"
+version = "0.26.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "366c5db5a8e4643bdde121a1d6f35cfa035e7850b8b8ea94c6960a139d44e93a"
+checksum = "f37234f85a8d1bda6b84d486245dd0847c5b33cca10d5f3691375e66704de0f8"
 dependencies = [
  "arc-swap",
  "async-channel",
@@ -5318,7 +5318,7 @@ dependencies = [
  "flate2",
  "futures-channel",
  "futures-core",
- "futures-util",
+ "futures-sink",
  "hdrhistogram",
  "hotpath-macros",
  "http 1.5.0",
@@ -5339,9 +5339,9 @@ dependencies = [
 
 [[package]]
 name = "hotpath-macros"
-version = "0.25.1"
+version = "0.26.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "846bde0d9600d98434e1aac376977d7718bfe3d2f5312a041b7c59a6a466c51a"
+checksum = "2ac39b5517f540cab511bdbd5a4dc8cfc62858d6cd26a095463c1d2acfb3f290"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -6751,7 +6751,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rustversion",
- "syn 3.0.5",
+ "syn 3.0.6",
 ]
 
 [[package]]
@@ -8220,7 +8220,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0"
 dependencies = [
  "proc-macro2",
- "syn 3.0.5",
+ "syn 3.0.6",
 ]
 
 [[package]]
@@ -9054,7 +9054,7 @@ checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 3.0.5",
+ "syn 3.0.6",
 ]
 
 [[package]]
@@ -10408,6 +10408,8 @@ dependencies = [
  "opentelemetry_sdk",
  "percent-encoding",
  "pyroscope",
+ "rcgen",
+ "reqwest",
  "rustfs-audit",
  "rustfs-common",
  "rustfs-config",
@@ -10421,6 +10423,8 @@ dependencies = [
  "rustfs-storage-api",
  "rustfs-utils",
  "rustix",
+ "rustls",
+ "rustls-pki-types",
  "serde",
  "serde_json",
  "sysinfo",
@@ -10428,6 +10432,7 @@ dependencies = [
  "tempfile",
  "thiserror 2.0.20",
  "tokio",
+ "tokio-rustls",
  "tokio-util",
  "tracing",
  "tracing-appender",
@@ -11111,9 +11116,9 @@ dependencies = [
 
 [[package]]
 name = "rustix"
-version = "1.1.4"
+version = "1.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
+checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d"
 dependencies = [
  "bitflags 2.13.2",
  "errno",
@@ -11259,8 +11264,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
 [[package]]
 name = "s3s"
 version = "0.16.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d29a4db6a51f1cd3269a36354ce4aa7db2be8816cee42a642b9c85cd3769577"
+source = "git+https://github.com/s3s-project/s3s.git?rev=50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6#50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6"
 dependencies = [
  "arc-swap",
  "arrayvec",
@@ -11278,7 +11282,6 @@ dependencies = [
  "http 1.5.0",
  "http-body 1.1.0",
  "http-body-util",
- "httparse",
  "hyper",
  "itoa",
  "jiff",
@@ -11291,6 +11294,7 @@ dependencies = [
  "pin-project-lite",
  "quick-xml",
  "regex",
+ "s3s-multipart",
  "s3s-rfc2047",
  "s3s-sigv2",
  "s3s-sigv4",
@@ -11315,11 +11319,23 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "s3s-multipart"
+version = "0.17.0-alpha.1"
+source = "git+https://github.com/s3s-project/s3s.git?rev=50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6#50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6"
+dependencies = [
+ "arrayvec",
+ "bytes",
+ "futures-core",
+ "httparse",
+ "memchr",
+ "thiserror 2.0.20",
+]
+
 [[package]]
 name = "s3s-rfc2047"
 version = "0.16.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9370a2f353a7929b79ae7b5f692b4758c59aa19526b18feff6a977e8ff17da5a"
+source = "git+https://github.com/s3s-project/s3s.git?rev=50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6#50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6"
 dependencies = [
  "base64-simd",
  "thiserror 2.0.20",
@@ -11328,8 +11344,7 @@ dependencies = [
 [[package]]
 name = "s3s-sigv2"
 version = "0.16.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f277b63ffc9a2135c49d52726199b7436c2f86746d625496984f5bb5cabef050"
+source = "git+https://github.com/s3s-project/s3s.git?rev=50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6#50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6"
 dependencies = [
  "base64-simd",
  "hmac 0.13.0",
@@ -11342,8 +11357,7 @@ dependencies = [
 [[package]]
 name = "s3s-sigv4"
 version = "0.16.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a979a401f6ae4035bc22e4ac0f7a9a5923ba93841c931e8e28652fe211d991bc"
+source = "git+https://github.com/s3s-project/s3s.git?rev=50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6#50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6"
 dependencies = [
  "arrayvec",
  "base64-simd",
@@ -11575,7 +11589,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 3.0.5",
+ "syn 3.0.6",
 ]
 
 [[package]]
@@ -11639,7 +11653,7 @@ checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 3.0.5",
+ "syn 3.0.6",
 ]
 
 [[package]]
@@ -11684,7 +11698,7 @@ dependencies = [
  "darling 0.24.1",
  "proc-macro2",
  "quote",
- "syn 3.0.5",
+ "syn 3.0.6",
 ]
 
 [[package]]
@@ -11719,7 +11733,7 @@ checksum = "a22144e767da4ddd8416dbf383700542ffd8a5dc493dfecedfe1fe3ad03c98ae"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 3.0.5",
+ "syn 3.0.6",
 ]
 
 [[package]]
@@ -12316,9 +12330,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "3.0.5"
+version = "3.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9"
+checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -12365,7 +12379,7 @@ checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 3.0.5",
+ "syn 3.0.6",
 ]
 
 [[package]]
@@ -12559,7 +12573,7 @@ checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 3.0.5",
+ "syn 3.0.6",
 ]
 
 [[package]]
@@ -12696,7 +12710,7 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 3.0.5",
+ "syn 3.0.6",
 ]
 
 [[package]]
@@ -13212,9 +13226,9 @@ checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5"
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.24"
+version = "1.0.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
+checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954"
 
 [[package]]
 name = "unicode-normalization"
@@ -13456,7 +13470,7 @@ dependencies = [
  "bumpalo",
  "proc-macro2",
  "quote",
- "syn 3.0.5",
+ "syn 3.0.6",
  "wasm-bindgen-shared",
 ]
 
@@ -14031,7 +14045,7 @@ checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 3.0.5",
+ "syn 3.0.6",
  "synstructure 0.14.0",
 ]
 
@@ -14072,7 +14086,7 @@ checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 3.0.5",
+ "syn 3.0.6",
  "synstructure 0.14.0",
 ]
 
@@ -14126,7 +14140,7 @@ checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 3.0.5",
+ "syn 3.0.6",
 ]
 
 [[package]]

Cargo.toml

@@ -145,7 +145,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.1" }
 async-channel = "2.5.0"
 async_zip = { default-features = false, version = "0.0.19" }
 mysql_async = { default-features = false, version = "0.37.1" }
-async-compression = { version = "0.4.47" }
+async-compression = { version = "0.4.48" }
 async-recursion = "1.1.1"
 async-trait = "0.1.92"
 async-nats = { version = "0.50.0", default-features = false }
@@ -244,9 +244,9 @@ atoi = "3.1.0"
 atomic_enum = "0.3.0"
 aws-config = { version = "1.12.0" }
 aws-credential-types = { version = "1.3.0" }
-aws-sdk-kms = { default-features = false, version = "1.118.0" }
-aws-sdk-s3 = { default-features = false, version = "1.146.1" }
-aws-sdk-sts = { default-features = false, version = "1.114.0" }
+aws-sdk-kms = { default-features = false, version = "1.120.0" }
+aws-sdk-s3 = { default-features = false, version = "1.148.0" }
+aws-sdk-sts = { default-features = false, version = "1.116.0" }
 aws-smithy-async = { version = "1.3.0" }
 aws-smithy-http-client = { default-features = false, version = "1.4.2" }
 aws-smithy-runtime-api = { version = "1.16.2" }
@@ -310,11 +310,11 @@ regex = { version = "1.13.1" }
 rumqttc = { package = "rumqttc-next", version = "0.34.0" }
 redis = { version = "1.7.0" }
 rustify = { version = "0.7", default-features = false }
-rustix = { version = "1.1.4" }
+rustix = { version = "1.1.5" }
 rust-embed = { version = "8.12.0" }
 rustc-hash = { version = "2.1.3" }
-s3s = { version = "0.16.0", features = ["minio"] }
-s3s-sigv4 = { version = "0.16.0" }
+s3s = { git = "https://github.com/s3s-project/s3s.git", rev = "50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6", features = ["minio"] }
+s3s-sigv4 = { git = "https://github.com/s3s-project/s3s.git", rev = "50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6" }
 serial_test = "4.0.1"
 shadow-rs = { default-features = false, version = "2.0.0" }
 siphasher = "1.0.3"
@@ -351,7 +351,7 @@ zstd = "0.14.0"
 # Observability and Metrics
 metrics = "0.24.6"
 metrics-util = "0.20"
-dial9 = { version = "0.5.0", default-features = false }
+dial9 = { version = "0.5.1", default-features = false }
 opentelemetry = { version = "0.32.0" }
 opentelemetry-appender-tracing = { version = "0.32.0" }
 opentelemetry-otlp = { version = "0.32.0" }
@@ -375,7 +375,7 @@ dav-server = "0.11.0"
 # Performance Analysis and Memory Profiling
 rustfs-mimalloc = { version = "0.5.4" }
 # Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
-hotpath = { version = "0.25.1", default-features = false }
+hotpath = { version = "0.26.0", default-features = false }
 # Snapshot testing for output format regression detection
 insta = { version = "1.48" }
 

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2024 Beijing Henghesha Technology Co., Ltd. 
+   Copyright 2024-2026 RustFS, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

crates/config/src/constants/object.rs

@@ -663,6 +663,15 @@ pub const ENV_OBJECT_LOCK_RPC_DETACHED_LIMIT: &str = "RUSTFS_OBJECT_LOCK_RPC_DET
 /// Default per-peer budget of detached (timed-out but still running) lock RPCs: 256.
 pub const DEFAULT_OBJECT_LOCK_RPC_DETACHED_LIMIT: usize = 256;
 
+/// Environment variable for the maximum number of in-flight lock acquisition
+/// RPCs admitted to one peer. Requests beyond this bound fail fast as
+/// retryable contention so a slow endpoint cannot accumulate an unbounded
+/// queue while its channel remains healthy.
+pub const ENV_OBJECT_LOCK_RPC_REQUEST_LIMIT: &str = "RUSTFS_OBJECT_LOCK_RPC_REQUEST_LIMIT";
+
+/// Default per-peer in-flight lock acquisition RPC admission limit.
+pub const DEFAULT_OBJECT_LOCK_RPC_REQUEST_LIMIT: usize = 128;
+
 /// Environment variable to enable object namespace lock diagnostics.
 ///
 /// When enabled, RustFS emits slow lock acquisition and long lock hold

crates/config/src/observability/mod.rs

@@ -30,6 +30,9 @@ pub const ENV_OBS_ENDPOINT_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_TIMEOUT_M
 pub const ENV_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS";
 pub const ENV_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS";
 pub const ENV_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS";
+/// Absolute path to a PEM bundle whose certificates are added to the system
+/// trust roots used by OTLP/HTTP exporters.
+pub const ENV_OBS_TLS_CA_FILE: &str = "RUSTFS_OBS_TLS_CA_FILE";
 pub const ENV_OBS_PROFILING_ENDPOINT: &str = "RUSTFS_OBS_PROFILING_ENDPOINT";
 pub const ENV_OBS_USE_STDOUT: &str = "RUSTFS_OBS_USE_STDOUT";
 pub const ENV_OBS_SAMPLE_RATIO: &str = "RUSTFS_OBS_SAMPLE_RATIO";
@@ -55,6 +58,13 @@ pub const ENV_OBS_LOG_FILENAME: &str = "RUSTFS_OBS_LOG_FILENAME";
 pub const ENV_OBS_LOG_ROTATION_TIME: &str = "RUSTFS_OBS_LOG_ROTATION_TIME";
 pub const ENV_OBS_LOG_KEEP_FILES: &str = "RUSTFS_OBS_LOG_KEEP_FILES";
 
+/// Span lifecycle events emitted by the JSON log layers (`none`|`close`|`full`).
+///
+/// Defaults to `none`: a record per span lifecycle multiplies log volume on
+/// object hot paths, and emitting the ancestor span list on every record
+/// produced multi-megabyte lines that journald truncated (backlog#2642).
+pub const ENV_OBS_LOG_SPAN_EVENTS: &str = "RUSTFS_OBS_LOG_SPAN_EVENTS";
+
 /// Log cleanup related configurations
 pub const ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES: &str = "RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES";
 pub const ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES: &str = "RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES";
@@ -99,6 +109,8 @@ pub const DEFAULT_OBS_LOG_DRY_RUN: bool = false;
 pub const DEFAULT_OBS_LOG_MATCH_MODE_PREFIX: &str = "prefix";
 pub const DEFAULT_OBS_LOG_MATCH_MODE: &str = "suffix";
 
+pub const DEFAULT_OBS_LOG_SPAN_EVENTS: &str = "none";
+
 /// Default values for observability configuration
 // ### Supported Environment Values
 // - `production` - Secure file-only logging
@@ -141,6 +153,7 @@ mod tests {
         assert_eq!(ENV_OBS_LOG_FILENAME, "RUSTFS_OBS_LOG_FILENAME");
         assert_eq!(ENV_OBS_LOG_ROTATION_TIME, "RUSTFS_OBS_LOG_ROTATION_TIME");
         assert_eq!(ENV_OBS_LOG_KEEP_FILES, "RUSTFS_OBS_LOG_KEEP_FILES");
+        assert_eq!(ENV_OBS_LOG_SPAN_EVENTS, "RUSTFS_OBS_LOG_SPAN_EVENTS");
         assert_eq!(ENV_OBS_TRACES_EXPORT_ENABLED, "RUSTFS_OBS_TRACES_EXPORT_ENABLED");
         assert_eq!(ENV_OBS_METRICS_EXPORT_ENABLED, "RUSTFS_OBS_METRICS_EXPORT_ENABLED");
         assert_eq!(ENV_OBS_LOGS_EXPORT_ENABLED, "RUSTFS_OBS_LOGS_EXPORT_ENABLED");
@@ -178,6 +191,7 @@ mod tests {
         assert_eq!(DEFAULT_OBS_LOG_COMPRESSION_ALGORITHM_GZIP, "gzip");
         assert_eq!(DEFAULT_OBS_LOG_COMPRESSION_ALGORITHM_ZSTD, "zstd");
         assert_eq!(DEFAULT_OBS_LOG_MATCH_MODE_PREFIX, "prefix");
+        assert_eq!(DEFAULT_OBS_LOG_SPAN_EVENTS, "none");
         assert_eq!(DEFAULT_OBS_LOG_MATCH_MODE, "suffix");
         assert_eq!(DEFAULT_OBS_LOG_COMPRESSION_ALGORITHM, "zstd");
     }

crates/e2e_test/src/copy_object_metadata_test.rs

@@ -298,7 +298,11 @@ mod tests {
         assert_eq!(empty_head_resp.content_disposition(), None);
         assert_eq!(empty_head_resp.content_encoding(), None);
         assert_eq!(empty_head_resp.content_language(), None);
-        assert_eq!(empty_head_resp.content_type(), None);
+        assert_eq!(
+            empty_head_resp.content_type(),
+            Some("binary/octet-stream"),
+            "REPLACE without an explicit content type must default to binary/octet-stream, matching S3"
+        );
         assert_eq!(empty_head_resp.expires_string(), None);
         assert_eq!(empty_head_resp.website_redirect_location(), None);
 
@@ -394,7 +398,15 @@ mod tests {
             assert_eq!(head.content_disposition(), (field == "content-disposition").then_some("attachment"));
             assert_eq!(head.content_encoding(), (field == "content-encoding").then_some("gzip"));
             assert_eq!(head.content_language(), (field == "content-language").then_some("de"));
-            assert_eq!(head.content_type(), (field == "content-type").then_some("text/field"));
+            assert_eq!(
+                head.content_type(),
+                Some(if field == "content-type" {
+                    "text/field"
+                } else {
+                    "binary/octet-stream"
+                }),
+                "REPLACE without an explicit content type must default to binary/octet-stream, matching S3"
+            );
             assert_eq!(
                 head.expires_string(),
                 (field == "expires").then_some(replacement_expires_http_date.as_str())
@@ -420,7 +432,11 @@ mod tests {
             .send()
             .await
             .expect("HEAD failed for metadata collision case");
-        assert_eq!(collision_head.content_type(), None);
+        assert_eq!(
+            collision_head.content_type(),
+            Some("binary/octet-stream"),
+            "user-metadata under the reserved 'content-type' key must not satisfy the system content-type default"
+        );
         assert_eq!(collision_head.content_encoding(), None);
         assert_eq!(
             collision_head.metadata().and_then(|metadata| metadata.get("content-type")),

crates/e2e_test/src/distributed/observability_test.rs

@@ -184,7 +184,7 @@ async fn four_node_health_inventory_metrics_and_audit_delivery_are_consistent()
         let (status, metrics_body) = admin_request(
             &node.url,
             Method::GET,
-            "/rustfs/admin/v3/metrics?n=1&by-host=true&by-disk=true",
+            "/rustfs/admin/v3/realtime?n=1&by-host=true&by-disk=true",
             None,
             &dist.cluster.access_key,
             &dist.cluster.secret_key,
@@ -257,7 +257,7 @@ async fn node_admin_body(dist: &DistCluster, node: usize, path: &str) -> TestRes
 }
 
 async fn http_put_counts(dist: &DistCluster, node: usize) -> TestResult<[u64; 2]> {
-    let body = node_admin_body(dist, node, "/rustfs/admin/v3/metrics?types=512&by-host=true&n=1").await?;
+    let body = node_admin_body(dist, node, "/rustfs/admin/v3/realtime?types=512&by-host=true&n=1").await?;
     let sample: RealtimeMetrics = serde_json::from_str(body.lines().next().ok_or("empty HTTP metrics stream")?)?;
     assert!(sample.errors.is_empty(), "HTTP metrics returned errors: {:?}", sample.errors);
     let http = sample.aggregated.http.ok_or("HTTP metrics missing at WARN log level")?;
@@ -400,17 +400,36 @@ async fn verify_write_observations_during_peer_failure(dist: &DistCluster, bucke
             let observations = storage["info"]["observations"]
                 .as_array()
                 .ok_or("recovery omitted observations")?;
+            let disks = storage["info"]["disks"].as_array().ok_or("recovery omitted disks")?;
             Ok(observations.len() == 4
                 && observations
                     .iter()
-                    .all(|item| item["status"] == "succeeded" && item["cached"] == false))
+                    .all(|item| item["status"] == "succeeded" && item["cached"] == false)
+                && disks.len() == 16
+                && disks.iter().all(|disk| {
+                    disk["state"].as_str().is_some_and(|state| state.eq_ignore_ascii_case("ok"))
+                        && disk["runtimeState"]
+                            .as_str()
+                            .is_some_and(|state| state.eq_ignore_ascii_case("online"))
+                }))
         },
-        "peer probes recover to fresh successful observations",
+        "peer probes and remote disks recover to fresh successful observations",
     )
     .await?;
     wait_for_ready(&dist.cluster).await?;
-    assert!(observed_put(dist, 0, bucket, "recovered").await?.is_success());
-    assert_eq!(http_put_counts(dist, 0).await?, [baseline[0][0] + 1, baseline[0][1] + 2]);
+    let recovered_before = http_put_counts(dist, 0).await?;
+    wait_until(
+        Duration::from_secs(90),
+        || async { Ok(observed_put(dist, 0, bucket, "recovered").await?.is_success()) },
+        "recovered PUT after remote disk and lock convergence",
+    )
+    .await?;
+    let recovered_after = http_put_counts(dist, 0).await?;
+    assert_eq!(recovered_after[0], recovered_before[0] + 1);
+    assert!(
+        recovered_after[1] >= baseline[0][1] + 2,
+        "recovery retries must retain the two intentional sub-quorum failures: before={recovered_before:?}, after={recovered_after:?}"
+    );
     for node in 0..dist.cluster.nodes.len() {
         assert_object_bytes(&dist.client(node)?, bucket, "healthy-0", b"write-observation").await?;
         assert_object_bytes(&dist.client(node)?, bucket, "recovered", b"write-observation").await?;

crates/ecstore/Cargo.toml

@@ -282,5 +282,9 @@ harness = false
 name = "single_block_non_inline_benchmark"
 harness = false
 
+[[bench]]
+name = "multipart_read_parts_benchmark"
+harness = false
+
 [lib]
 doctest = false

crates/ecstore/benches/multipart_read_parts_benchmark.rs

@@ -0,0 +1,92 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
+use rustfs_filemeta::ObjectPartInfo;
+use std::hint::black_box;
+use std::time::Duration;
+
+mod storage_api;
+use storage_api::multipart::{DiskAPI, DiskOption, Endpoint, new_disk};
+
+fn bench_multipart_read_parts(c: &mut Criterion) {
+    let runtime = tokio::runtime::Builder::new_multi_thread()
+        .worker_threads(4)
+        .enable_all()
+        .build()
+        .expect("benchmark runtime");
+    let root = tempfile::tempdir().expect("benchmark disk");
+    let mut endpoint = Endpoint::try_from(root.path().to_str().expect("UTF-8 path")).expect("endpoint");
+    endpoint.set_pool_index(0);
+    endpoint.set_set_index(0);
+    endpoint.set_disk_index(0);
+    let disk = runtime
+        .block_on(new_disk(
+            &endpoint,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+        ))
+        .expect("local disk");
+    let bucket = "multipart-bench";
+    runtime.block_on(disk.make_volume(bucket)).expect("benchmark volume");
+    let upload = root.path().join(bucket).join("upload");
+    std::fs::create_dir_all(&upload).expect("upload directory");
+    let mut paths = Vec::with_capacity(1024);
+    for number in 1..=1024 {
+        let part = ObjectPartInfo {
+            number,
+            etag: format!("{number:032x}"),
+            size: 1024,
+            actual_size: 1024,
+            ..Default::default()
+        };
+        std::fs::write(upload.join(format!("part.{number}")), b"data").expect("part data");
+        std::fs::write(upload.join(format!("part.{number}.meta")), part.marshal_msg().expect("metadata")).expect("part metadata");
+        paths.push(format!("upload/part.{number}.meta"));
+    }
+
+    // Measures the real DiskAPI path against warm local files. Fixture creation
+    // and correctness checks stay outside the timed region; this is not a cold-disk benchmark.
+    let mut group = c.benchmark_group("multipart_read_parts");
+    group.sample_size(20);
+    group.warm_up_time(Duration::from_secs(1));
+    group.measurement_time(Duration::from_secs(2));
+    for count in [1, 32, 256, 1024] {
+        let paths = &paths[..count];
+        let parts = runtime.block_on(disk.read_parts(bucket, paths)).expect("read parts");
+        assert_eq!(parts.len(), count);
+        assert!(
+            parts
+                .iter()
+                .enumerate()
+                .all(|(i, part)| part.number == i + 1 && part.error.is_none())
+        );
+        group.throughput(Throughput::Elements(u64::try_from(count).expect("part count")));
+        group.bench_with_input(BenchmarkId::from_parameter(count), &paths, |b, paths| {
+            b.iter(|| {
+                black_box(
+                    runtime
+                        .block_on(disk.read_parts(bucket, black_box(paths)))
+                        .expect("read parts"),
+                )
+            });
+        });
+    }
+    group.finish();
+}
+
+criterion_group!(benches, bench_multipart_read_parts);
+criterion_main!(benches);

crates/ecstore/benches/storage_api/mod.rs

@@ -29,3 +29,7 @@ pub(crate) mod erasure {
 pub(crate) mod single_block_non_inline {
     pub(crate) use super::{BitrotWriterWrapper, CustomWriter, Erasure};
 }
+
+pub(crate) mod multipart {
+    pub(crate) use rustfs_ecstore::api::disk::{DiskAPI, DiskOption, Endpoint, new_disk};
+}

crates/ecstore/src/api/mod.rs

@@ -391,8 +391,9 @@ pub mod data_usage {
     #[cfg(feature = "test-util")]
     pub use crate::data_movement::scanner_backlog::test_util::NativeScannerPauseBacklogWriteFault;
     pub use crate::data_movement::scanner_backlog::{
-        MAX_SCANNER_PAUSE_BACKLOG_BYTES, ScannerPauseBacklogRetirementPlan, ScannerPauseBacklogRetirementPlanner,
-        ScannerPauseBacklogRetirementReplica, register_scanner_pause_backlog_retirement_planner,
+        MAX_SCANNER_PAUSE_BACKLOG_BYTES, ScannerPauseBacklogRetirementError, ScannerPauseBacklogRetirementPlan,
+        ScannerPauseBacklogRetirementPlanner, ScannerPauseBacklogRetirementReplica,
+        register_scanner_pause_backlog_retirement_planner,
     };
     pub use crate::data_usage::{
         DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
@@ -503,6 +504,13 @@ pub mod notification {
     };
 }
 
+pub mod integrity {
+    pub use crate::services::integrity::{
+        IntegrityError, InventoryItem, InventoryPage, ItemRequest, ItemResult, ItemState, Job, JobMode, JobRequest, JobState,
+        Protection, Readiness, control_job, create_job, get_job, inventory, readiness, resume_job,
+    };
+}
+
 pub mod object {
     pub use crate::object_api::{
         BLOCK_SIZE_V2, ERASURE_ALGORITHM, EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook,

crates/ecstore/src/bucket/lifecycle/transition_transaction.rs

@@ -35,6 +35,7 @@ use crate::bucket::lifecycle::tier_sweeper::{
 use crate::disk::RUSTFS_META_BUCKET;
 use crate::error::{Error, Result as EcstoreResult};
 use crate::object_api::{ObjectInfo, ObjectOptions};
+use crate::services::notification_sys::acquire_remote_version_state_writer_fleet_proof;
 use crate::services::tier::{tier::TierConfigMgr, warm_backend::TransitionCandidateProbe};
 use crate::storage_api_contracts::{
     list::ListOperations as _,
@@ -1237,6 +1238,12 @@ pub async fn delete_transition_candidate_for_operator(
     lease
         .validate_remote_version_id(remote_version_id)
         .map_err(TransitionOperatorError::Remote)?;
+    if remote_version_requires_fleet_proof(remote_version_id) && acquire_remote_version_state_writer_fleet_proof().is_none() {
+        return Err(TransitionOperatorError::Remote(std::io::Error::new(
+            std::io::ErrorKind::Unsupported,
+            "opaque remote tier version cleanup requires the operator-attested live fleet capability gate",
+        )));
+    }
     let before_delete_probe = lease
         .probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
         .await
@@ -1845,6 +1852,11 @@ async fn recover_unknown_upload_outcome(
             ))
         }
         TransitionCandidateProbe::VersionedPresent(version_id) => {
+            if remote_version_requires_fleet_proof(&version_id) && acquire_remote_version_state_writer_fleet_proof().is_none() {
+                return Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(
+                    IlmRecoveryErrorCode::RemoteVersionUnknown,
+                ));
+            }
             cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::versioned(version_id)).await
         }
         TransitionCandidateProbe::Ambiguous => Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(
@@ -1940,6 +1952,14 @@ fn transition_source_lookup_options(transaction: &TransitionTransaction) -> Obje
 async fn delete_transition_remote_candidate(api: Arc<ECStore>, transaction: &TransitionTransaction) -> EcstoreResult<()> {
     let version_id = transaction.remote_version.tier_delete_version_id().unwrap_or_default();
     let version_id_exact = transaction.remote_version.kind == TransitionRemoteVersionKind::Versioned;
+    if version_id_exact
+        && remote_version_requires_fleet_proof(version_id)
+        && acquire_remote_version_state_writer_fleet_proof().is_none()
+    {
+        return Err(Error::other(
+            "opaque remote tier version cleanup requires the operator-attested live fleet capability gate",
+        ));
+    }
     delete_object_from_remote_tier_idempotent_with_manager_and_identity(
         &transaction.remote_object,
         version_id,
@@ -1953,6 +1973,10 @@ async fn delete_transition_remote_candidate(api: Arc<ECStore>, transaction: &Tra
     .map_err(Error::other)
 }
 
+fn remote_version_requires_fleet_proof(version_id: &str) -> bool {
+    !version_id.is_empty() && Uuid::parse_str(version_id).is_err()
+}
+
 pub async fn recover_transition_transaction_records(
     api: Arc<ECStore>,
     limit: usize,
@@ -2439,6 +2463,14 @@ mod tests {
         }
     }
 
+    #[test]
+    fn opaque_remote_versions_require_the_fleet_proof_boundary() {
+        assert!(!remote_version_requires_fleet_proof(""));
+        assert!(!remote_version_requires_fleet_proof(&Uuid::new_v4().to_string()));
+        assert!(remote_version_requires_fleet_proof("null"));
+        assert!(remote_version_requires_fleet_proof("b2-opaque-version"));
+    }
+
     #[test]
     fn remote_version_distinguishes_unknown_unversioned_and_versioned() {
         assert_eq!(TransitionRemoteVersion::known_from_put_response("").tier_delete_version_id(), None);

crates/ecstore/src/bucket/metadata.rs

@@ -1464,9 +1464,13 @@ pub(crate) async fn load_bucket_metadata_parse_with_presence(
                 "bucket incarnation sidecar is missing for new-format metadata: {bucket}"
             )));
         }
-    } else if incarnation.is_some() {
-        return Err(Error::other("bucket incarnation sidecar exists without bucket metadata"));
     }
+    // A sidecar without `.metadata.bin` is the window between the two legacy
+    // migration writes (rustfs/rustfs#8003): the sidecar lands first, and a
+    // crash or lost namespace lease before the metadata write leaves the
+    // bucket in this state on every node. Report it as not persisted so the
+    // migration runs again; the writers adopt the stored incarnation instead
+    // of minting a new one, so the sidecar keeps its authority.
 
     bm.default_timestamps();
 

crates/ecstore/src/bucket/metadata_sys.rs

@@ -1291,6 +1291,28 @@ pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_in(
     }
 }
 
+/// Inspect all migration-relevant settings while the caller holds an Object Lock
+/// snapshot's lifecycle and metadata transaction guards. Cached settings are not
+/// sufficient to authorize a direct storage writer that cannot apply S3 defaults.
+pub(crate) async fn integrity_migration_metadata_in(
+    ctx: &crate::runtime::instance::InstanceContext,
+    bucket: &str,
+) -> Result<Arc<BucketMetadata>> {
+    let sys = bucket_metadata_sys_of(ctx)?.read().await.clone();
+    match sys
+        .read_authoritative_metadata_from_disk_under_transaction_lock(bucket)
+        .await?
+    {
+        BucketMetadataAuthority::Authoritative(metadata)
+            if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
+        {
+            Ok(metadata)
+        }
+        BucketMetadataAuthority::MissingBucket => Err(Error::BucketNotFound(bucket.to_string())),
+        _ => Err(Error::other("migration requires authoritative bucket metadata")),
+    }
+}
+
 /// Re-read the quota configuration and bucket incarnation from the same
 /// authoritative metadata blob while the caller holds the bucket metadata
 /// transaction read lock.
@@ -2504,6 +2526,16 @@ impl BucketMetadataSys {
         if !persisted {
             metadata = BucketMetadata::new(bucket);
             metadata.created = bucket_info.created.unwrap_or(OffsetDateTime::UNIX_EPOCH);
+            // An interrupted migration may already have published this
+            // bucket's incarnation; re-read it under the transaction lock so
+            // the retry never replaces an identity other nodes fenced on. A
+            // retired incarnation is residue of a deleted bucket and must not
+            // come back, or heal would reclaim the bucket's new objects.
+            if let Some(stored) = load_bucket_incarnation(self.object_store(), bucket).await?
+                && !crate::bucket::retirement::is_retired(self.object_store(), bucket, stored).await?
+            {
+                metadata.bucket_incarnation_id = stored;
+            }
         } else if metadata.bucket_incarnation_id.is_nil() {
             metadata.bucket_incarnation_id = Uuid::new_v4();
         }
@@ -3556,6 +3588,98 @@ mod tests {
         assert!(err.to_string().contains("sidecar is missing"));
     }
 
+    /// rustfs/rustfs#8003: the legacy migration writes the incarnation sidecar
+    /// before `.metadata.bin`. A crash or lost namespace lease between the two
+    /// left every pre-existing bucket answering 500 on every request, with no
+    /// path back. The sidecar-only state must load as a legacy bucket, and the
+    /// retried migration must keep the stored incarnation.
+    #[tokio::test]
+    async fn issue_8003_sidecar_without_metadata_loads_as_legacy_and_migrates_with_its_incarnation() {
+        let (dirs, ecstore) = isolated_store_over_temp_disks().await;
+        let sys = BucketMetadataSys::new(ecstore.clone());
+        let bucket = "issue-8003-sidecar-only";
+        for dir in &dirs {
+            std::fs::create_dir_all(dir.path().join(bucket)).unwrap();
+        }
+        let stored = Uuid::new_v4();
+        save_bucket_incarnation(ecstore.clone(), bucket, stored)
+            .await
+            .expect("simulate the interrupted migration's first write");
+
+        let (fabricated, persisted) = load_bucket_metadata_parse_with_presence(ecstore.clone(), bucket, true)
+            .await
+            .expect("a sidecar without metadata must read as a legacy bucket, not fail closed");
+        assert!(!persisted);
+        assert!(!fabricated.bucket_incarnation_sidecar);
+
+        let (_, fabricated_read) = sys
+            .get_config(bucket)
+            .await
+            .expect("request-path metadata reads must not fail closed on the sidecar-only state");
+        assert!(fabricated_read);
+
+        let migrated = sys
+            .get_authoritative_metadata(bucket)
+            .await
+            .expect("the retried legacy migration must complete");
+        assert!(migrated.bucket_incarnation_sidecar);
+        assert_eq!(
+            migrated.bucket_incarnation_id, stored,
+            "the retry must adopt the published incarnation instead of minting a new one"
+        );
+        assert_eq!(sys.get_bucket_incarnation_id(bucket).await.unwrap(), stored);
+
+        let on_disk = sys.get_config_from_disk(bucket).await.expect("metadata is now persisted");
+        assert!(on_disk.bucket_incarnation_sidecar);
+        assert_eq!(on_disk.bucket_incarnation_id, stored);
+        assert_eq!(load_bucket_incarnation(ecstore, bucket).await.unwrap(), Some(stored));
+    }
+
+    /// A sidecar left behind by a deleted bucket names a retired incarnation.
+    /// The migration must mint a new identity for a same-name volume instead
+    /// of re-publishing the retired one, or heal would treat the bucket's new
+    /// objects as reclaimable residue.
+    #[tokio::test]
+    async fn issue_8003_sidecar_only_migration_does_not_adopt_a_retired_incarnation() {
+        let (dirs, ecstore) = isolated_store_over_temp_disks().await;
+        let sys = BucketMetadataSys::new(ecstore.clone());
+        let bucket = "issue-8003-retired-sidecar";
+        for dir in &dirs {
+            std::fs::create_dir_all(dir.path().join(bucket)).unwrap();
+        }
+        let retired = Uuid::new_v4();
+        save_bucket_incarnation(ecstore.clone(), bucket, retired)
+            .await
+            .expect("residual sidecar");
+        crate::bucket::retirement::commit_retirement(
+            ecstore.clone(),
+            bucket,
+            retired,
+            &ObjectOptions {
+                max_parity: true,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("retirement record");
+
+        let migrated = sys
+            .get_authoritative_metadata(bucket)
+            .await
+            .expect("the migration must still complete for the same-name volume");
+        assert!(migrated.bucket_incarnation_sidecar);
+        assert_ne!(
+            migrated.bucket_incarnation_id, retired,
+            "a retired incarnation must never be re-published"
+        );
+        assert!(!migrated.bucket_incarnation_id.is_nil());
+        assert_eq!(
+            load_bucket_incarnation(ecstore, bucket).await.unwrap(),
+            Some(migrated.bucket_incarnation_id),
+            "the sidecar must be rewritten to the new incarnation"
+        );
+    }
+
     /// Concurrent cache misses for one bucket must collapse into a single disk
     /// load.
     ///

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -199,6 +199,38 @@ fn has_raw_status(err: &SdkError<HeadObjectError>, status: u16) -> bool {
     err.raw_response().is_some_and(|r| r.status().as_u16() == status)
 }
 
+fn raw_header_value(err: &SdkError<HeadObjectError>, name: &str) -> Option<String> {
+    err.raw_response()
+        .and_then(|response| response.headers().get(name))
+        .filter(|value| !value.is_empty())
+        .map(ToOwned::to_owned)
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum DeleteMarkerHeadOutcome {
+    Missing,
+    Exists,
+    Ambiguous,
+}
+
+fn classify_delete_marker_head_error(err: &SdkError<HeadObjectError>) -> DeleteMarkerHeadOutcome {
+    let (is_not_found, code) = err
+        .as_service_error()
+        .map(|service_err| (service_err.is_not_found(), service_err.code()))
+        .unwrap_or((false, None));
+
+    // Prefer the raw HTTP status when present. HEAD responses commonly have no
+    // body, and a contradictory XML code must not turn a 403/5xx into success.
+    match err.raw_response().map(|response| response.status().as_u16()) {
+        Some(404) => DeleteMarkerHeadOutcome::Missing,
+        Some(405) => DeleteMarkerHeadOutcome::Exists,
+        Some(_) => DeleteMarkerHeadOutcome::Ambiguous,
+        None if is_not_found => DeleteMarkerHeadOutcome::Missing,
+        None if !is_retryable_delete_replication_head_error(false, code) => DeleteMarkerHeadOutcome::Exists,
+        None => DeleteMarkerHeadOutcome::Ambiguous,
+    }
+}
+
 fn metadata_requires_existing_target(op_type: ReplicationType, object_info: &ObjectInfo) -> bool {
     op_type == ReplicationType::Metadata
         && object_info
@@ -1809,32 +1841,14 @@ async fn verify_resync_head_result(
             (roi.size, None)
         }
         Err(err) if roi.delete_marker => {
-            // Verifying a replicated delete marker: only a
-            // definitive 404/NoSuchKey or 405/MethodNotAllowed
-            // confirms the marker propagated. Any other
-            // (retryable/ambiguous) HEAD error leaves the outcome
-            // unverified, so it must count as failed — not as a
-            // blanket success (backlog#862 / #799 B13).
-            // A HEAD carries no body, so the SDK only synthesizes an error
-            // code for 404; a target answering the delete marker's version
-            // with 405 (RustFS, MinIO, AWS) leaves `code()` empty and the
-            // raw status is the only evidence (rustfs/backlog#2479).
-            // A marker *version purge* (`version_purge_status` set) asks
-            // for the opposite outcome: the marker must be gone from the
-            // target, so only absence verifies it and a 405 proves the
-            // marker is still there.
-            let retryable = {
-                let (is_not_found, code) = err
-                    .as_service_error()
-                    .map(|se| (se.is_not_found(), se.code()))
-                    .unwrap_or((false, None));
-                let not_found = is_not_found || has_raw_status(&err, 404);
-                if roi.version_purge_status.is_empty() {
-                    let code = code.or_else(|| has_raw_status(&err, 405).then_some("405"));
-                    is_retryable_delete_replication_head_error(not_found, code)
-                } else {
-                    !not_found
-                }
+            // A marker creation is verified by either absence (404) or
+            // presence (405). A marker-version purge has the opposite goal:
+            // only absence proves completion. All other outcomes remain
+            // failed (backlog#862 / #799 B13).
+            let retryable = if roi.version_purge_status.is_empty() {
+                matches!(classify_delete_marker_head_error(&err), DeleteMarkerHeadOutcome::Ambiguous)
+            } else {
+                !matches!(classify_delete_marker_head_error(&err), DeleteMarkerHeadOutcome::Missing)
             };
             if retryable {
                 st.failed_count += 1;
@@ -3553,7 +3567,7 @@ async fn replicate_delete_to_target(
         }
     }
 
-    if dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() {
+    if !is_version_purge && dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() {
         match head_object_for_worker(
             tgt_client.as_ref(),
             &tgt_client.bucket,
@@ -3562,20 +3576,48 @@ async fn replicate_delete_to_target(
         )
         .await
         {
-            Ok(_) => {}
+            Ok(head) if head.delete_marker == Some(true) => {
+                // Some targets expose an existing marker as a successful HEAD
+                // with the marker bit instead of the usual 405. Treat that as
+                // positive convergence evidence and retain the target version
+                // id for any later marker purge.
+                rinfo.target_delete_marker_version_id = head.version_id.filter(|version_id| !version_id.is_empty());
+                rinfo.replication_status = ReplicationStatusType::Completed;
+                return rinfo;
+            }
+            Ok(_) => {
+                // A successful HEAD without the delete-marker bit proves that
+                // the addressed version is a regular object (or that the
+                // target omitted the marker contract). Creating a marker in
+                // that state would hide a live version, so fail closed.
+                rinfo.replication_status = ReplicationStatusType::Failed;
+                rinfo.error = Some("target version is not a delete marker".to_string());
+                return rinfo;
+            }
             Err(e) => {
-                let non_retryable = matches!(
-                    e.as_ref(),
-                    SdkError::ServiceError(service_err)
-                        if is_retryable_delete_replication_head_error(
-                            service_err.err().is_not_found(),
-                            service_err.err().code(),
-                        )
-                );
-                if non_retryable {
-                    rinfo.replication_status = ReplicationStatusType::Failed;
-                    rinfo.error = Some(e.to_string());
-                    return rinfo;
+                match classify_delete_marker_head_error(e.as_ref()) {
+                    DeleteMarkerHeadOutcome::Exists => {
+                        // An explicit HEAD of a delete-marker version is expected
+                        // to answer 405 (often without an SDK error code). That
+                        // is positive evidence the marker already exists, so the
+                        // creation is complete and must not issue another
+                        // versionless DELETE that would mint a second marker.
+                        if let Some(version_id) = raw_header_value(e.as_ref(), "x-amz-version-id") {
+                            rinfo.target_delete_marker_version_id = Some(version_id);
+                        }
+                        rinfo.replication_status = ReplicationStatusType::Completed;
+                        return rinfo;
+                    }
+                    DeleteMarkerHeadOutcome::Missing => {}
+                    DeleteMarkerHeadOutcome::Ambiguous => {
+                        // Only a definitive not-found means the marker is absent
+                        // and may be created. Keep transport and ambiguous
+                        // failures fail-closed instead of creating blindly.
+                        rinfo.replication_status = ReplicationStatusType::Failed;
+                        rinfo.error = Some(e.to_string());
+                        mark_replication_target_offline_if_needed(&tgt_client, &e).await;
+                        return rinfo;
+                    }
                 }
             }
         }
@@ -6529,6 +6571,110 @@ mod tests {
         }
     }
 
+    fn marker_creation_dobj(target_arn: &str) -> DeletedObjectReplicationInfo {
+        let mut state = ReplicationState::default();
+        state.targets.insert(target_arn.to_string(), ReplicationStatusType::Pending);
+        DeletedObjectReplicationInfo {
+            bucket: "source".to_string(),
+            target_arn: target_arn.to_string(),
+            op_type: ReplicationType::Heal,
+            delete_object: ReplicationDeletedObject {
+                object_name: "gone.txt".to_string(),
+                delete_marker: true,
+                delete_marker_version_id: Some(Uuid::new_v4()),
+                replication_state: Some(state),
+                ..Default::default()
+            },
+            ..Default::default()
+        }
+    }
+
+    /// A target that already has the marker answers a versioned HEAD with a
+    /// bodiless 405. The live worker must settle the marker as completed and
+    /// avoid minting another marker with a versionless DELETE.
+    #[tokio::test]
+    async fn live_delete_marker_worker_accepts_bodiless_405_without_delete() {
+        let (endpoint, server) = spawn_scripted_target_server(1, |_| {
+            "HTTP/1.1 405 Method Not Allowed\r\nX-Amz-Version-Id: target-marker\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
+                .to_string()
+        });
+        let target = test_target_client(endpoint);
+        register_test_target(&target).await;
+        let dobj = marker_creation_dobj(&target.arn);
+
+        let rinfo = replicate_delete_to_target(&dobj, target.clone(), None).await;
+
+        assert_eq!(rinfo.replication_status, ReplicationStatusType::Completed, "{rinfo:?}");
+        assert!(rinfo.error.is_none(), "{rinfo:?}");
+        assert_eq!(rinfo.target_delete_marker_version_id.as_deref(), Some("target-marker"));
+        let seen = server.join().expect("test HTTP server should finish");
+        assert_eq!(seen.len(), 1, "an existing marker must not be re-created: {seen:?}");
+        assert!(seen[0].starts_with("HEAD "), "{seen:?}");
+    }
+
+    /// A successful HEAD that does not identify a delete marker is not
+    /// convergence evidence: creating a marker would hide a live version.
+    #[tokio::test]
+    async fn live_delete_marker_worker_rejects_successful_regular_head_without_delete() {
+        let (endpoint, server) = spawn_scripted_target_server(1, |_| empty_response("200 OK"));
+        let target = test_target_client(endpoint);
+        register_test_target(&target).await;
+        let dobj = marker_creation_dobj(&target.arn);
+
+        let rinfo = replicate_delete_to_target(&dobj, target.clone(), None).await;
+
+        assert_eq!(rinfo.replication_status, ReplicationStatusType::Failed, "{rinfo:?}");
+        assert_eq!(rinfo.error.as_deref(), Some("target version is not a delete marker"));
+        let seen = server.join().expect("test HTTP server should finish");
+        assert_eq!(seen.len(), 1, "a regular object must not be hidden by a new marker: {seen:?}");
+        assert!(seen[0].starts_with("HEAD "), "{seen:?}");
+    }
+
+    /// A definitive 404 means the marker is absent, so exactly one marker
+    /// creation DELETE is still required.
+    #[tokio::test]
+    async fn live_delete_marker_worker_creates_marker_after_404() {
+        let (endpoint, server) = spawn_scripted_target_server(2, |line| {
+            if line.starts_with("HEAD ") {
+                empty_response("404 Not Found")
+            } else if line.starts_with("DELETE ") {
+                empty_response("204 No Content")
+            } else {
+                empty_response("500 Unexpected")
+            }
+        });
+        let target = test_target_client(endpoint);
+        register_test_target(&target).await;
+        let dobj = marker_creation_dobj(&target.arn);
+
+        let rinfo = replicate_delete_to_target(&dobj, target.clone(), None).await;
+
+        assert_eq!(rinfo.replication_status, ReplicationStatusType::Completed, "{rinfo:?}");
+        let seen = server.join().expect("test HTTP server should finish");
+        assert_eq!(seen.len(), 2, "a missing marker should be created once: {seen:?}");
+        assert!(seen[0].starts_with("HEAD "), "{seen:?}");
+        assert!(seen[1].starts_with("DELETE "), "{seen:?}");
+        assert!(!seen[1].contains("versionId="), "marker creation must remain versionless: {seen:?}");
+    }
+
+    /// Ambiguous service errors must remain failed and must not fall through
+    /// to a blind marker-creation DELETE.
+    #[tokio::test]
+    async fn live_delete_marker_worker_keeps_ambiguous_head_failure_closed() {
+        let (endpoint, server) = spawn_scripted_target_server(1, |_| empty_response("503 Service Unavailable"));
+        let target = test_target_client(endpoint);
+        register_test_target(&target).await;
+        let dobj = marker_creation_dobj(&target.arn);
+
+        let rinfo = replicate_delete_to_target(&dobj, target.clone(), None).await;
+
+        assert_eq!(rinfo.replication_status, ReplicationStatusType::Failed, "{rinfo:?}");
+        assert!(rinfo.error.is_some(), "{rinfo:?}");
+        let seen = server.join().expect("test HTTP server should finish");
+        assert_eq!(seen.len(), 1, "an ambiguous HEAD must not trigger DELETE: {seen:?}");
+        assert!(seen[0].starts_with("HEAD "), "{seen:?}");
+    }
+
     /// A target answers `HEAD ?versionId=<delete marker>` with a bodiless
     /// 405, so the SDK reports no error code. That is the marker having
     /// propagated, not a failure (rustfs/backlog#2479).

crates/ecstore/src/cache_value/metacache_set.rs

@@ -23,7 +23,7 @@ use std::{
     future::Future,
     io::ErrorKind,
     pin::Pin,
-    sync::{Arc, OnceLock},
+    sync::{Arc, OnceLock, atomic::AtomicBool},
     task::{Context, Poll},
     time::Duration,
 };
@@ -234,6 +234,8 @@ pub struct ListPathRawOptions {
     pub skip_walkdir_total_timeout: bool,
     pub walkdir_timeout: Option<Duration>,
     pub walkdir_stall_timeout: Option<Duration>,
+    /// Shared terminal state for bounded producer walks.
+    pub producer_limit_reached: Option<Arc<AtomicBool>>,
     pub agreed: Option<AgreedFn>,
     pub partial: Option<PartialFn>,
     pub finished: Option<FinishedFn>,
@@ -267,6 +269,7 @@ impl Clone for ListPathRawOptions {
             skip_walkdir_total_timeout: self.skip_walkdir_total_timeout,
             walkdir_timeout: self.walkdir_timeout,
             walkdir_stall_timeout: self.walkdir_stall_timeout,
+            producer_limit_reached: self.producer_limit_reached.clone(),
             #[cfg(test)]
             test_reader_behaviors: self.test_reader_behaviors.clone(),
             #[cfg(test)]
@@ -292,6 +295,7 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
         skip_total_timeout: opts.skip_walkdir_total_timeout,
         timeout_ms: opts.walkdir_timeout.map(duration_millis),
         stall_timeout_ms: opts.walkdir_stall_timeout.map(duration_millis),
+        producer_limit_reached: opts.producer_limit_reached.clone(),
         ..Default::default()
     }
 }

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -1170,6 +1170,11 @@ impl RemoteDisk {
         self.health.force_runtime_state_for_test(state);
     }
 
+    #[cfg(test)]
+    pub fn force_offline_for_test(&self) {
+        self.health.force_offline_for_test();
+    }
+
     /// Same as [`DiskHealthTracker::reset_for_store_init_retry`]: undo a transient faulty mark before another format load attempt.
     pub fn reset_health_for_store_init_retry(&self) {
         self.health.reset_for_store_init_retry(&self.endpoint);
@@ -2136,7 +2141,7 @@ impl RemoteDisk {
                 let file_info_bin = encode_file_info_msgpack(fi)?;
                 let mut client = self.get_client().await?;
                 let mut request = Request::new(RenameDataRequest {
-                    bucket_incarnation_id: crate::store::bucket_heal_scope(dst_volume)
+                    bucket_incarnation_id: crate::store::bucket_heal_scope_for_object(dst_volume, dst_path)
                         .map(|scope| scope.incarnation.as_bytes().to_vec().into())
                         .unwrap_or_default(),
                     disk: self.endpoint.to_string(),
@@ -4303,6 +4308,13 @@ mod tests {
         object_read_all_disks: Arc<StdMutex<Vec<String>>>,
         format_data: Bytes,
         read_all_data: Bytes,
+        next_format_read_gate: Arc<StdMutex<Option<Arc<TestFormatReadGate>>>>,
+    }
+
+    #[derive(Debug, Default)]
+    struct TestFormatReadGate {
+        entered: tokio::sync::Notify,
+        release: CancellationToken,
     }
 
     impl AuthenticatedReadPeer {
@@ -4314,9 +4326,16 @@ mod tests {
                 object_read_all_disks: Arc::default(),
                 format_data,
                 read_all_data,
+                next_format_read_gate: Arc::default(),
             }
         }
 
+        fn pause_next_format_read(&self) -> Arc<TestFormatReadGate> {
+            let gate = Arc::new(TestFormatReadGate::default());
+            *self.next_format_read_gate.lock().expect("format gate lock poisoned") = Some(Arc::clone(&gate));
+            gate
+        }
+
         fn disk_info_calls(&self) -> u32 {
             self.disk_info_calls.load(Ordering::Acquire)
         }
@@ -4424,6 +4443,11 @@ mod tests {
                                 let disk = request.disk;
                                 peer.read_all_calls.fetch_add(1, Ordering::AcqRel);
                                 let data = if is_format_read {
+                                    let gate = peer.next_format_read_gate.lock().expect("format gate lock poisoned").take();
+                                    if let Some(gate) = gate {
+                                        gate.entered.notify_one();
+                                        gate.release.cancelled().await;
+                                    }
                                     peer.format_data.clone()
                                 } else {
                                     peer.object_read_all_disks
@@ -7126,6 +7150,319 @@ mod tests {
         peer.stop().await;
     }
 
+    async fn missing_read_set(peer: &TestGrpcPeer, format: crate::layout::format::FormatV3) -> Arc<crate::set_disk::SetDisks> {
+        let count = format.erasure.sets[0].len();
+        let endpoints = (0..count)
+            .map(|index| Endpoint {
+                url: url::Url::parse(&format!("{}/data/rustfs{index}", peer.addr)).expect("endpoint should parse"),
+                is_local: false,
+                pool_idx: 0,
+                set_idx: 0,
+                disk_idx: i32::try_from(index).expect("fixture index fits"),
+            })
+            .collect();
+        crate::set_disk::SetDisks::new(
+            "missing-read-test".to_string(),
+            Arc::new(tokio::sync::RwLock::new(vec![None; count])),
+            count,
+            0,
+            0,
+            0,
+            endpoints,
+            format,
+            Vec::new(),
+        )
+        .await
+    }
+
+    #[tokio::test]
+    #[serial(remote_disk_recovery_probe)]
+    async fn missing_read_disks_reconnect_before_periodic_monitor_and_coalesce() {
+        runtime_sources::ensure_test_rpc_secret();
+        let mut format = crate::layout::format::FormatV3::new(1, 1);
+        let disk_id = format.erasure.sets[0][0];
+        format.erasure.this = disk_id;
+        let Some(peer) = TestGrpcPeer::spawn(
+            Bytes::from(format.to_json().expect("format should serialize")),
+            Bytes::from_static(b"startup-recovered-data"),
+        )
+        .await
+        else {
+            panic!("authenticated regression peer is required");
+        };
+        let set = missing_read_set(&peer, format).await;
+        let snapshots = futures::future::join_all((0..8).map(|_| set.get_disks_for_data_read())).await;
+        let disk = snapshots[0][0]
+            .as_ref()
+            .expect("a ready peer must be usable before the periodic reconnect");
+        for snapshot in &snapshots {
+            assert!(Arc::ptr_eq(
+                disk,
+                snapshot[0]
+                    .as_ref()
+                    .expect("every waiting read should observe the same handle")
+            ));
+        }
+        assert_eq!(peer.peer.read_all_calls(), 1, "concurrent requests must share the format probe");
+        assert_eq!(disk.get_disk_id().await.expect("disk identity should be available"), Some(disk_id));
+        let raw = disk
+            .read_all("bucket", "object")
+            .await
+            .expect("reconnected handle should read data");
+        assert_eq!(raw, Bytes::from_static(b"startup-recovered-data"));
+        assert_eq!(peer.peer.object_read_all_disks(), vec![disk_id.to_string()]);
+        let healthy = set.get_disks_for_data_read().await;
+        assert!(Arc::ptr_eq(disk, healthy[0].as_ref().expect("healthy handle must remain registered")));
+        assert_eq!(peer.peer.read_all_calls(), 2, "healthy reads must not re-probe the format");
+        disk.close().await.expect("reconnected disk should close");
+        peer.stop().await;
+    }
+
+    #[tokio::test]
+    #[serial(remote_disk_recovery_probe)]
+    async fn missing_read_disks_reject_foreign_format_and_back_off() {
+        runtime_sources::ensure_test_rpc_secret();
+        let reference = crate::layout::format::FormatV3::new(1, 1);
+        let mut foreign = reference.clone();
+        foreign.id = Uuid::new_v4();
+        foreign.erasure.this = reference.erasure.sets[0][0];
+        let Some(peer) = TestGrpcPeer::spawn(
+            Bytes::from(foreign.to_json().expect("foreign format should serialize")),
+            Bytes::from_static(b"foreign-data"),
+        )
+        .await
+        else {
+            panic!("authenticated regression peer is required");
+        };
+        let set = missing_read_set(&peer, reference).await;
+        assert!(
+            set.get_disks_for_data_read().await[0].is_none(),
+            "a matching disk ID cannot authorize a foreign deployment"
+        );
+        assert_eq!(peer.peer.read_all_calls(), 1, "the foreign format must actually be inspected");
+        assert!(set.get_disks_for_data_read().await[0].is_none());
+        assert_eq!(peer.peer.read_all_calls(), 1, "failed probes must not repeat on every degraded GET");
+        assert!(peer.peer.object_read_all_disks().is_empty(), "foreign object data must never be read");
+        peer.stop().await;
+    }
+
+    #[tokio::test]
+    #[serial(remote_disk_recovery_probe)]
+    async fn missing_read_disks_reject_wrong_slot_and_invalid_formats() {
+        runtime_sources::ensure_test_rpc_secret();
+        let mut reference = crate::layout::format::FormatV3::new(1, 2);
+        reference.erasure.this = reference.erasure.sets[0][0];
+        let mut nil = reference.clone();
+        nil.erasure.this = Uuid::nil();
+        let mut unknown = reference.clone();
+        unknown.erasure.this = Uuid::new_v4();
+        let mut future = reference.clone();
+        future.erasure.version = crate::layout::format::FormatErasureVersion::Unknown;
+        for (name, payload, first_accepted) in [
+            ("duplicate slot zero", reference.to_json().expect("format should serialize"), true),
+            ("nil drive", nil.to_json().expect("nil format should serialize"), false),
+            ("unknown drive", unknown.to_json().expect("unknown format should serialize"), false),
+            ("future version", future.to_json().expect("future format should serialize"), false),
+            ("malformed", "not-json".to_string(), false),
+        ] {
+            let peer = TestGrpcPeer::spawn(Bytes::from(payload), Bytes::new())
+                .await
+                .expect("authenticated regression peer is required");
+            let set = missing_read_set(&peer, reference.clone()).await;
+            let disks = set.get_disks_for_data_read().await;
+            assert_eq!(disks[0].is_some(), first_accepted, "{name}: slot zero identity decision");
+            assert!(disks[1].is_none(), "{name}: a peer cannot claim another topology slot");
+            assert_eq!(peer.peer.read_all_calls(), 2, "{name}: both missing slots must be probed");
+            assert!(peer.peer.object_read_all_disks().is_empty(), "{name}: no object payload may be read");
+            for disk in disks.into_iter().flatten() {
+                disk.close().await.expect("accepted disk should close");
+            }
+            peer.stop().await;
+        }
+    }
+
+    #[tokio::test]
+    #[serial(remote_disk_recovery_probe)]
+    async fn missing_read_disks_do_not_claim_local_or_misconfigured_endpoints() {
+        runtime_sources::ensure_test_rpc_secret();
+        let mut reference = crate::layout::format::FormatV3::new(1, 1);
+        reference.erasure.this = reference.erasure.sets[0][0];
+        let peer = TestGrpcPeer::spawn(Bytes::from(reference.to_json().expect("format should serialize")), Bytes::new())
+            .await
+            .expect("authenticated regression peer is required");
+        for (local, pool, set_index, disk_index) in [(true, 0, 0, 0), (false, -1, 0, 0), (false, 0, 1, 0), (false, 0, 0, 1)] {
+            let mut set = missing_read_set(&peer, reference.clone()).await;
+            let endpoint = &mut Arc::get_mut(&mut set)
+                .expect("fixture set must be uniquely owned")
+                .set_endpoints[0];
+            endpoint.is_local = local;
+            endpoint.pool_idx = pool;
+            endpoint.set_idx = set_index;
+            endpoint.disk_idx = disk_index;
+            assert!(set.get_disks_for_data_read().await[0].is_none());
+        }
+        assert_eq!(peer.peer.read_all_calls(), 0, "invalid endpoint ownership must fail before I/O");
+        peer.stop().await;
+    }
+
+    #[tokio::test]
+    #[serial(remote_disk_recovery_probe)]
+    async fn missing_read_disks_preserve_concurrently_published_handles() {
+        runtime_sources::ensure_test_rpc_secret();
+        let mut reference = crate::layout::format::FormatV3::new(1, 1);
+        reference.erasure.this = reference.erasure.sets[0][0];
+        for reader_first in [false, true] {
+            let peer = TestGrpcPeer::spawn(Bytes::from(reference.to_json().expect("format should serialize")), Bytes::new())
+                .await
+                .expect("authenticated regression peer is required");
+            let set = missing_read_set(&peer, reference.clone()).await;
+            let gate = peer.peer.pause_next_format_read();
+            let pending_set = Arc::clone(&set);
+            let pending = tokio::spawn(async move {
+                if reader_first {
+                    pending_set.get_disks_for_data_read().await;
+                } else {
+                    pending_set.renew_disk(&pending_set.set_endpoints[0]).await;
+                }
+            });
+            time::timeout(Duration::from_secs(2), gate.entered.notified())
+                .await
+                .expect("the first reconnect must reach the format barrier");
+            if reader_first {
+                set.renew_disk(&set.set_endpoints[0]).await;
+            } else {
+                set.get_disks_for_data_read().await;
+            }
+            let published = set.disks.read().await[0]
+                .clone()
+                .expect("the competing reconnect must publish");
+            gate.release.cancel();
+            pending.await.expect("paused reconnect must complete");
+            assert!(
+                Arc::ptr_eq(&published, set.disks.read().await[0].as_ref().expect("published slot must remain")),
+                "neither reconnect may replace a handle published while its format I/O was pending"
+            );
+            published.close().await.expect("published disk should close");
+            peer.stop().await;
+        }
+    }
+
+    #[tokio::test]
+    #[serial(remote_disk_recovery_probe, internode_metrics)]
+    async fn missing_read_disks_expired_waiter_does_not_start_another_probe() {
+        // Disable cooldown so this independently proves deadline admission,
+        // rather than passing only because the leader has just finished.
+        temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_RETURNING_PROBE_INTERVAL_SECS, Some("0"))], async {
+            runtime_sources::ensure_test_rpc_secret();
+            let mut reference = crate::layout::format::FormatV3::new(1, 1);
+            reference.erasure.this = reference.erasure.sets[0][0];
+            let peer = TestGrpcPeer::spawn(Bytes::from(reference.to_json().expect("format should serialize")), Bytes::new())
+                .await
+                .expect("authenticated regression peer is required");
+            let set = missing_read_set(&peer, reference).await;
+            let gate = peer.peer.pause_next_format_read();
+            let mut leader = Box::pin(set.get_disks_for_data_read());
+            time::timeout(Duration::from_secs(2), async {
+                tokio::select! {
+                    _ = gate.entered.notified() => {}
+                    _ = &mut leader => panic!("leader must remain blocked in its format RPC"),
+                }
+            })
+            .await
+            .expect("leader must reach the format barrier");
+            time::pause();
+            let mut waiter = Box::pin(set.get_disks_for_data_read());
+            assert!(futures::poll!(&mut waiter).is_pending(), "waiter must queue behind the leader");
+            let requests_before =
+                crate::cluster::rpc::runtime_sources::internode_metrics_snapshot_for_test().outgoing_requests_total;
+
+            time::advance(crate::disk::disk_store::get_drive_active_check_timeout()).await;
+            assert!(leader.await[0].is_none(), "expired leader must not publish a disk");
+            assert!(waiter.await[0].is_none(), "expired waiter must not publish a disk");
+            time::resume();
+
+            // Count client dispatches, not server arrivals: a cancelled RPC can still
+            // be buffered in the transport when the expired waiter returns.
+            assert_eq!(
+                crate::cluster::rpc::runtime_sources::internode_metrics_snapshot_for_test().outgoing_requests_total,
+                requests_before,
+                "an expired waiter must not dispatch another format RPC after the leader releases the lock"
+            );
+            gate.release.cancel();
+            peer.stop().await;
+        })
+        .await;
+    }
+
+    #[tokio::test]
+    #[serial(remote_disk_recovery_probe)]
+    async fn missing_read_disks_timeout_releases_singleflight_and_allows_retry() {
+        runtime_sources::ensure_test_rpc_secret();
+        let mut reference = crate::layout::format::FormatV3::new(1, 1);
+        reference.erasure.this = reference.erasure.sets[0][0];
+        let peer = TestGrpcPeer::spawn(Bytes::from(reference.to_json().expect("format should serialize")), Bytes::new())
+            .await
+            .expect("authenticated regression peer is required");
+        let set = missing_read_set(&peer, reference).await;
+        let gate = peer.peer.pause_next_format_read();
+        let deadline = crate::disk::disk_store::get_drive_active_check_timeout();
+        let snapshots = time::timeout(
+            deadline + Duration::from_secs(2),
+            futures::future::join_all((0..4).map(|_| set.get_disks_for_data_read())),
+        )
+        .await
+        .expect("all waiting requests must respect the bounded probe deadline");
+        assert!(snapshots.iter().all(|snapshot| snapshot[0].is_none()));
+        assert_eq!(peer.peer.read_all_calls(), 1, "only one format RPC may remain stalled");
+        gate.release.cancel();
+        time::pause();
+        time::advance(crate::disk::health_state::get_drive_returning_probe_interval()).await;
+        time::resume();
+        let disks = set.get_disks_for_data_read().await;
+        let disk = disks[0]
+            .as_ref()
+            .expect("a timed-out probe must not prevent a later read from reconnecting");
+        assert_eq!(peer.peer.read_all_calls(), 2, "retry must validate the format again");
+        disk.close().await.expect("reconnected disk should close");
+        peer.stop().await;
+    }
+
+    #[tokio::test]
+    #[serial(remote_disk_recovery_probe)]
+    async fn missing_read_disks_cancellation_leaves_no_published_handle() {
+        runtime_sources::ensure_test_rpc_secret();
+        let mut reference = crate::layout::format::FormatV3::new(1, 1);
+        reference.erasure.this = reference.erasure.sets[0][0];
+        let peer = TestGrpcPeer::spawn(Bytes::from(reference.to_json().expect("format should serialize")), Bytes::new())
+            .await
+            .expect("authenticated regression peer is required");
+        let set = missing_read_set(&peer, reference).await;
+        let gate = peer.peer.pause_next_format_read();
+        let pending_set = Arc::clone(&set);
+        let pending = tokio::spawn(async move { pending_set.get_disks_for_data_read().await });
+        time::timeout(Duration::from_secs(2), gate.entered.notified())
+            .await
+            .expect("probe must reach the format barrier");
+        // Cancellation can happen after the start-based cooldown has elapsed.
+        // It must still cool down before another caller probes the same peer.
+        time::pause();
+        time::advance(crate::disk::health_state::get_drive_returning_probe_interval()).await;
+        time::resume();
+        pending.abort();
+        assert!(pending.await.expect_err("probe must be cancelled").is_cancelled());
+        assert!(
+            set.disks.read().await[0].is_none(),
+            "cancelled format I/O must not publish an unvalidated handle"
+        );
+        let waiting = time::timeout(Duration::from_secs(1), set.get_disks_for_data_read())
+            .await
+            .expect("cancelled probe must release the singleflight lock");
+        assert!(waiting[0].is_none(), "retry cooldown must still apply after cancellation");
+        assert_eq!(peer.peer.read_all_calls(), 1);
+        gate.release.cancel();
+        peer.stop().await;
+    }
+
     #[tokio::test]
     async fn test_copy_stream_with_buffer_copies_full_payload() {
         let payload = b"walk-dir-stream".repeat(1024);

crates/ecstore/src/cluster/rpc/remote_locker.rs

@@ -47,10 +47,37 @@ fn attach_lock_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(req
 
 /// Work to run if an RPC that already timed out for its caller completes later.
 type LateCompletion<T> = Option<Box<dyn FnOnce(T) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send>>;
+type LateFailure = LateCompletion<()>;
+
+struct RpcCleanup<T> {
+    late: LateCompletion<T>,
+    late_failure: LateFailure,
+    admission: Option<LockRequestAdmission>,
+}
+
+impl<T> RpcCleanup<T> {
+    fn new(late: LateCompletion<T>, late_failure: LateFailure, admission: Option<LockRequestAdmission>) -> Self {
+        Self {
+            late,
+            late_failure,
+            admission,
+        }
+    }
+}
 
 /// The liveness window is this many RPC deadlines: a peer that completed a
 /// lock RPC within it is slow, not gone, and keeps its channel on a timeout.
 const LOCK_RPC_LIVENESS_WINDOW_DEADLINES: u32 = 2;
+const LOCK_RPC_REQUEST_BACKOFF_THRESHOLD: u32 = 3;
+const LOCK_RPC_REQUEST_BACKOFF_MAX: Duration = Duration::from_secs(30);
+const LATE_RELEASE_RETRY_DELAYS: [Duration; 3] = [Duration::from_millis(100), Duration::from_millis(500), Duration::from_secs(1)];
+const LATE_RELEASE_IN_FLIGHT_LIMIT: usize = 32;
+const TIMEOUT_LOG_STATE_MAX: usize = 1024;
+const TIMEOUT_LOG_STATE_RETENTION: Duration = Duration::from_secs(600);
+
+fn is_request_breaker_operation(op: &'static str) -> bool {
+    matches!(op, "lock" | "lock_batch")
+}
 
 /// Recent history of the shared lock channel to one peer (issue #7363).
 ///
@@ -64,15 +91,44 @@ struct LockPeerChannelHealth {
     last_success: Option<Instant>,
     last_eviction: Option<Instant>,
     consecutive_timeouts: u32,
+    request_consecutive_timeouts: u32,
+    request_backoff_until: Option<Instant>,
+    request_backoff: Duration,
+    request_probe_generation: Option<u64>,
+    next_probe_generation: u64,
+    request_in_flight: usize,
+    late_release_in_flight: usize,
     /// Timed-out RPCs still running in the background for this peer.
     detached_rpcs: usize,
 }
 
+#[derive(Debug, Clone, Copy, Default)]
+struct TimeoutLogHealth {
+    last_logged: Option<Instant>,
+    suppressed: u64,
+}
+
+#[derive(Debug, Clone, Copy)]
+struct LockRequestAdmission {
+    probe_generation: Option<u64>,
+}
+
+impl LockRequestAdmission {
+    fn is_probe(self) -> bool {
+        self.probe_generation.is_some()
+    }
+}
+
 fn lock_peer_channel_health() -> &'static Mutex<HashMap<String, LockPeerChannelHealth>> {
     static HEALTH: OnceLock<Mutex<HashMap<String, LockPeerChannelHealth>>> = OnceLock::new();
     HEALTH.get_or_init(Mutex::default)
 }
 
+fn timeout_log_health() -> &'static Mutex<HashMap<(String, &'static str), TimeoutLogHealth>> {
+    static HEALTH: OnceLock<Mutex<HashMap<(String, &'static str), TimeoutLogHealth>>> = OnceLock::new();
+    HEALTH.get_or_init(Mutex::default)
+}
+
 fn with_lock_peer_health<R>(addr: &str, update: impl FnOnce(&mut LockPeerChannelHealth) -> R) -> R {
     let mut peers = lock_peer_channel_health()
         .lock()
@@ -343,17 +399,163 @@ impl RemoteClient {
         )
     }
 
+    fn request_limit() -> usize {
+        rustfs_utils::get_env_usize(
+            rustfs_config::ENV_OBJECT_LOCK_RPC_REQUEST_LIMIT,
+            rustfs_config::DEFAULT_OBJECT_LOCK_RPC_REQUEST_LIMIT,
+        )
+        .max(1)
+    }
+
     fn liveness_window(deadline: Duration) -> Duration {
         deadline.saturating_mul(LOCK_RPC_LIVENESS_WINDOW_DEADLINES)
     }
 
-    fn record_rpc_success(&self) {
+    fn record_rpc_success(&self, op: &'static str, admission: Option<LockRequestAdmission>) {
         with_lock_peer_health(&self.addr, |health| {
             health.last_success = Some(Instant::now());
             health.consecutive_timeouts = 0;
+            if is_request_breaker_operation(op) {
+                let is_probe = admission.is_some_and(LockRequestAdmission::is_probe);
+                if health.request_backoff_until.is_none() || is_probe {
+                    health.request_consecutive_timeouts = 0;
+                    health.request_backoff_until = None;
+                    health.request_backoff = Duration::ZERO;
+                }
+                if is_probe && admission.is_some_and(|admission| health.request_probe_generation == admission.probe_generation) {
+                    health.request_probe_generation = None;
+                }
+            }
         });
     }
 
+    fn admit_late_release(&self) -> bool {
+        with_lock_peer_health(&self.addr, |health| {
+            if health.late_release_in_flight >= LATE_RELEASE_IN_FLIGHT_LIMIT {
+                false
+            } else {
+                health.late_release_in_flight += 1;
+                true
+            }
+        })
+    }
+
+    fn release_late_release(&self) {
+        with_lock_peer_health(&self.addr, |health| {
+            health.late_release_in_flight = health.late_release_in_flight.saturating_sub(1);
+        });
+    }
+
+    /// Admit a new lock acquisition unless this peer is in the request-level
+    /// backoff window. A single half-open probe is allowed after the window;
+    /// channel liveness and request admission are deliberately separate.
+    fn admit_lock_request(&self, op: &'static str) -> Option<LockRequestAdmission> {
+        if !is_request_breaker_operation(op) {
+            return Some(LockRequestAdmission { probe_generation: None });
+        }
+
+        let now = Instant::now();
+        let (admission, suppression_reason) = with_lock_peer_health(&self.addr, |health| {
+            let degraded = health.request_backoff_until.is_some() || health.request_consecutive_timeouts > 0;
+            if degraded && health.request_in_flight >= Self::request_limit() {
+                return (None, "in_flight_limit");
+            }
+            let Some(until) = health.request_backoff_until else {
+                health.request_in_flight += 1;
+                return (Some(LockRequestAdmission { probe_generation: None }), "none");
+            };
+            if now < until || health.request_probe_generation.is_some() {
+                return (None, "breaker_open");
+            }
+            health.next_probe_generation = health.next_probe_generation.saturating_add(1);
+            let generation = health.next_probe_generation;
+            health.request_probe_generation = Some(generation);
+            health.request_in_flight += 1;
+            (
+                Some(LockRequestAdmission {
+                    probe_generation: Some(generation),
+                }),
+                "none",
+            )
+        });
+        if admission.is_none() {
+            rustfs_io_metrics::lock_metrics::record_remote_lock_request_suppressed(&self.addr, op, suppression_reason);
+        }
+        admission
+    }
+
+    fn release_lock_request(&self, op: &'static str, admission: Option<LockRequestAdmission>) {
+        if is_request_breaker_operation(op) && admission.is_some() {
+            with_lock_peer_health(&self.addr, |health| {
+                health.request_in_flight = health.request_in_flight.saturating_sub(1);
+                if admission.is_some_and(|admission| health.request_probe_generation == admission.probe_generation) {
+                    health.request_probe_generation = None;
+                }
+            });
+        }
+    }
+
+    fn record_request_timeout(&self, op: &'static str, deadline: Duration) {
+        if !is_request_breaker_operation(op) {
+            return;
+        }
+
+        let now = Instant::now();
+        with_lock_peer_health(&self.addr, |health| {
+            health.request_consecutive_timeouts = health.request_consecutive_timeouts.saturating_add(1);
+            if health.request_consecutive_timeouts < LOCK_RPC_REQUEST_BACKOFF_THRESHOLD {
+                return;
+            }
+
+            let initial = deadline / 2;
+            let next = if health.request_backoff.is_zero() {
+                initial.max(Duration::from_millis(1))
+            } else {
+                health.request_backoff.saturating_mul(2)
+            };
+            health.request_backoff = next.min(LOCK_RPC_REQUEST_BACKOFF_MAX);
+            health.request_backoff_until = Some(now + health.request_backoff);
+        });
+    }
+
+    fn timeout_log_decision(&self, op: &'static str, interval: Duration) -> (bool, u64) {
+        let now = Instant::now();
+        let mut peers = timeout_log_health().lock().unwrap_or_else(std::sync::PoisonError::into_inner);
+        if peers.len() >= TIMEOUT_LOG_STATE_MAX {
+            peers.retain(|_, state| {
+                state
+                    .last_logged
+                    .is_some_and(|at| now.saturating_duration_since(at) < TIMEOUT_LOG_STATE_RETENTION)
+            });
+        }
+        let key = (self.addr.clone(), op);
+        if peers.len() >= TIMEOUT_LOG_STATE_MAX
+            && !peers.contains_key(&key)
+            && let Some(eviction_key) = peers.keys().next().cloned()
+        {
+            peers.remove(&eviction_key);
+        }
+        let state = peers.entry(key).or_default();
+        if state
+            .last_logged
+            .is_some_and(|at| now.saturating_duration_since(at) < interval)
+        {
+            state.suppressed = state.suppressed.saturating_add(1);
+            return (false, state.suppressed);
+        }
+
+        let suppressed = state.suppressed;
+        state.last_logged = Some(now);
+        state.suppressed = 0;
+        (true, suppressed)
+    }
+
+    fn bounded_acquire_request(request: &LockRequest, deadline: Duration) -> LockRequest {
+        let mut bounded = request.clone();
+        bounded.acquire_timeout = bounded.acquire_timeout.min(deadline);
+        bounded
+    }
+
     /// Apply the per-peer eviction policy after a failed RPC.
     async fn maybe_evict_connection(
         &self,
@@ -406,6 +608,8 @@ impl RemoteClient {
         resource_summary: &str,
         handle: JoinHandle<std::result::Result<T, tonic::Status>>,
         late: LateCompletion<T>,
+        late_failure: LateFailure,
+        admission: Option<LockRequestAdmission>,
     ) {
         let limit = Self::detached_rpc_limit();
         let admitted = with_lock_peer_health(&self.addr, |health| {
@@ -418,6 +622,10 @@ impl RemoteClient {
         });
         if !admitted {
             handle.abort();
+            self.release_lock_request(op, admission);
+            if let Some(late_failure) = late_failure {
+                drop(tokio::spawn(late_failure(())));
+            }
             rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_detached(op, "aborted");
             debug!(
                 addr = %self.addr,
@@ -432,12 +640,19 @@ impl RemoteClient {
         let addr = self.addr.clone();
         tokio::spawn(async move {
             let outcome = handle.await;
-            with_lock_peer_health(&addr, |health| health.detached_rpcs = health.detached_rpcs.saturating_sub(1));
+            with_lock_peer_health(&addr, |health| {
+                health.detached_rpcs = health.detached_rpcs.saturating_sub(1);
+                if is_request_breaker_operation(op) && admission.is_some() {
+                    health.request_in_flight = health.request_in_flight.saturating_sub(1);
+                    if admission.is_some_and(|admission| health.request_probe_generation == admission.probe_generation) {
+                        health.request_probe_generation = None;
+                    }
+                }
+            });
             match outcome {
                 Ok(Ok(response)) => {
                     with_lock_peer_health(&addr, |health| {
                         health.last_success = Some(Instant::now());
-                        health.consecutive_timeouts = 0;
                     });
                     rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "success");
                     if let Some(late) = late {
@@ -445,6 +660,9 @@ impl RemoteClient {
                     }
                 }
                 Ok(Err(status)) => {
+                    if let Some(late_failure) = late_failure {
+                        late_failure(()).await;
+                    }
                     rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "error");
                     debug!(
                         addr = %addr,
@@ -455,6 +673,9 @@ impl RemoteClient {
                     );
                 }
                 Err(join_error) => {
+                    if let Some(late_failure) = late_failure {
+                        late_failure(()).await;
+                    }
                     rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "join_error");
                     debug!(addr = %addr, op, error = %join_error, "Detached remote lock RPC task ended abnormally");
                 }
@@ -485,24 +706,79 @@ impl RemoteClient {
         }))
     }
 
+    fn late_release_failure_hook(&self, lock_ids: Vec<LockId>) -> LateFailure {
+        let client = self.clone();
+        Some(Box::new(move |_| {
+            Box::pin(async move {
+                client.release_late_acquisitions(lock_ids).await;
+            })
+        }))
+    }
+
     /// A lock granted after its caller stopped waiting is an orphan until its
     /// lease expires; hand it back right away, best effort.
-    async fn release_late_acquisitions(&self, lock_ids: Vec<LockId>) {
-        let outcome = match self.release_locks_batch(&lock_ids).await {
-            Ok(released) if released.iter().all(|released| *released) => "released",
-            Ok(_) => "partial",
-            Err(_) => "failed",
-        };
+    async fn release_late_acquisitions(&self, mut lock_ids: Vec<LockId>) {
+        if !self.admit_late_release() {
+            rustfs_io_metrics::lock_metrics::record_remote_lock_late_release("failed");
+            let (log_warning, suppressed_timeouts) = self.timeout_log_decision("late_release", Self::rpc_timeout());
+            if log_warning {
+                warn!(
+                    addr = %self.addr,
+                    count = lock_ids.len(),
+                    outcome = "budget_exhausted",
+                    suppressed_timeouts,
+                    "Skipped late remote lock release because the cleanup budget is exhausted"
+                );
+            }
+            return;
+        }
+
+        let original_count = lock_ids.len();
+        let mut outcome = "failed";
+        for (attempt, delay) in std::iter::once(Duration::ZERO).chain(LATE_RELEASE_RETRY_DELAYS).enumerate() {
+            if !delay.is_zero() {
+                tokio::time::sleep(delay).await;
+            }
+
+            match self.release_locks_batch(&lock_ids).await {
+                Ok(released) => {
+                    lock_ids = lock_ids
+                        .into_iter()
+                        .zip(released)
+                        .filter_map(|(lock_id, released)| (!released).then_some(lock_id))
+                        .collect();
+                    if lock_ids.is_empty() {
+                        outcome = "released";
+                        break;
+                    }
+                    outcome = if attempt == LATE_RELEASE_RETRY_DELAYS.len() {
+                        "partial"
+                    } else {
+                        "failed"
+                    };
+                }
+                Err(_) => {
+                    outcome = "failed";
+                }
+            }
+        }
+        self.release_late_release();
         rustfs_io_metrics::lock_metrics::record_remote_lock_late_release(outcome);
         if outcome == "released" {
-            debug!(addr = %self.addr, count = lock_ids.len(), "Released remote locks granted after their caller timed out");
+            debug!(addr = %self.addr, count = original_count, "Released remote locks granted after their caller timed out");
         } else {
-            warn!(
-                addr = %self.addr,
-                count = lock_ids.len(),
-                outcome,
-                "Could not release every remote lock granted after its caller timed out; the server lease will expire it"
-            );
+            let (log_warning, suppressed_timeouts) = self.timeout_log_decision("late_release", Self::rpc_timeout());
+            if log_warning {
+                warn!(
+                    addr = %self.addr,
+                    count = original_count,
+                    outcome,
+                    suppressed_timeouts,
+                    "Could not release every remote lock granted after its caller timed out; the server lease will expire it"
+                );
+            } else {
+                debug!(addr = %self.addr, count = original_count, outcome, suppressed_timeouts, "Suppressed repeated late remote lock release warning");
+            }
         }
     }
 
@@ -512,26 +788,74 @@ impl RemoteClient {
         resource_summary: &str,
         deadline: Duration,
         future: Fut,
-        late: LateCompletion<T>,
+        cleanup: RpcCleanup<T>,
     ) -> std::result::Result<T, LockError>
     where
         Fut: Future<Output = std::result::Result<T, tonic::Status>> + Send + 'static,
         T: Send + 'static,
     {
+        let RpcCleanup {
+            late,
+            late_failure,
+            admission,
+        } = cleanup;
         let mut handle = tokio::spawn(future);
         match timeout(deadline, &mut handle).await {
             Ok(Ok(Ok(response))) => {
-                self.record_rpc_success();
+                self.record_rpc_success(op, admission);
+                self.release_lock_request(op, admission);
                 Ok(response)
             }
             Ok(Ok(Err(err))) => {
                 let reason = err.to_string();
+                if err.code() == tonic::Code::DeadlineExceeded {
+                    rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_timeout(&self.addr, op);
+                    self.record_request_timeout(op, deadline);
+                    let (log_timeout, suppressed_timeouts) = self.timeout_log_decision(op, deadline);
+                    if Self::is_scanner_leader_lock(resource_summary) {
+                        debug!(
+                            addr = %self.addr,
+                            op,
+                            timeout_ms = deadline.as_millis(),
+                            resource_summary,
+                            "Remote lock RPC returned deadline exceeded for scanner leader lock"
+                        );
+                    } else if log_timeout {
+                        warn!(
+                            addr = %self.addr,
+                            op,
+                            timeout_ms = deadline.as_millis(),
+                            resource_summary,
+                            suppressed_timeouts,
+                            "Remote lock RPC returned deadline exceeded"
+                        );
+                    } else {
+                        debug!(
+                            addr = %self.addr,
+                            op,
+                            timeout_ms = deadline.as_millis(),
+                            resource_summary,
+                            suppressed_timeouts,
+                            "Suppressed repeated remote lock RPC deadline"
+                        );
+                    }
+                    self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
+                        .await;
+                    self.release_lock_request(op, admission);
+                    if let Some(late_failure) = late_failure {
+                        drop(tokio::spawn(late_failure(())));
+                    }
+                    return Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline));
+                }
                 // Only evict (and re-dial) the cached channel when the failure is a genuine
                 // transport problem. A server-produced application status (auth denied, peer
                 // lock service not ready, invalid args, ...) arrives on a perfectly healthy
                 // channel; evicting it just churns the connection and pushes the peer toward
                 // the offline threshold for no benefit. See issue #4567.
                 let transport_failure = Self::is_transport_failure(&err);
+                if transport_failure {
+                    self.record_request_timeout(op, deadline);
+                }
                 if Self::is_scanner_leader_lock(resource_summary) {
                     debug!(
                         addr = %self.addr,
@@ -543,6 +867,31 @@ impl RemoteClient {
                         transport_failure,
                         "Remote lock RPC returned tonic error for scanner leader lock"
                     );
+                } else if transport_failure {
+                    let (log_failure, suppressed_failures) = self.timeout_log_decision(op, deadline);
+                    if log_failure {
+                        warn!(
+                            addr = %self.addr,
+                            op,
+                            timeout_ms = deadline.as_millis(),
+                            resource_summary,
+                            tonic_code = ?err.code(),
+                            tonic_message = err.message(),
+                            transport_failure,
+                            suppressed_failures,
+                            "Remote lock RPC returned transport error"
+                        );
+                    } else {
+                        debug!(
+                            addr = %self.addr,
+                            op,
+                            timeout_ms = deadline.as_millis(),
+                            resource_summary,
+                            tonic_code = ?err.code(),
+                            suppressed_failures,
+                            "Suppressed repeated remote lock transport error"
+                        );
+                    }
                 } else {
                     warn!(
                         addr = %self.addr,
@@ -559,6 +908,10 @@ impl RemoteClient {
                     self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Transport, deadline)
                         .await;
                 }
+                self.release_lock_request(op, admission);
+                if transport_failure && let Some(late_failure) = late_failure {
+                    drop(tokio::spawn(late_failure(())));
+                }
                 Err(LockError::internal(format!("{op} RPC failed: {reason}")))
             }
             Ok(Err(join_error)) => {
@@ -569,11 +922,17 @@ impl RemoteClient {
                     error = %join_error,
                     "Remote lock RPC task ended abnormally"
                 );
+                self.release_lock_request(op, admission);
+                if let Some(late_failure) = late_failure {
+                    drop(tokio::spawn(late_failure(())));
+                }
                 Err(LockError::internal(format!("{op} RPC task failed: {join_error}")))
             }
             Err(_) => {
                 let reason = format!("RPC timed out after {deadline:?}");
                 rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_timeout(&self.addr, op);
+                self.record_request_timeout(op, deadline);
+                let (log_timeout, suppressed_timeouts) = self.timeout_log_decision(op, deadline);
                 if Self::is_scanner_leader_lock(resource_summary) {
                     debug!(
                         addr = %self.addr,
@@ -582,18 +941,28 @@ impl RemoteClient {
                         resource_summary,
                         "Remote lock RPC timed out for scanner leader lock"
                     );
-                } else {
+                } else if log_timeout {
                     warn!(
                         addr = %self.addr,
                         op,
                         timeout_ms = deadline.as_millis(),
                         resource_summary,
+                        suppressed_timeouts,
                         "Remote lock RPC timed out"
                     );
+                } else {
+                    debug!(
+                        addr = %self.addr,
+                        op,
+                        timeout_ms = deadline.as_millis(),
+                        resource_summary,
+                        suppressed_timeouts,
+                        "Suppressed repeated remote lock RPC timeout"
+                    );
                 }
                 self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
                     .await;
-                self.detach_timed_out_rpc(op, resource_summary, handle, late);
+                self.detach_timed_out_rpc(op, resource_summary, handle, late, late_failure, admission);
                 Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline))
             }
         }
@@ -621,6 +990,14 @@ impl RemoteClient {
             .collect()
     }
 
+    fn rpc_backoff_failure_response(request: &LockRequest) -> LockResponse {
+        LockResponse::failure("Remote lock RPC timed out: peer request breaker is open", request.acquire_timeout)
+    }
+
+    fn rpc_backoff_failure_batch(requests: &[LockRequest]) -> Vec<LockResponse> {
+        requests.iter().map(Self::rpc_backoff_failure_response).collect()
+    }
+
     fn build_lock_info(request: &LockRequest, lock_info_json: Option<String>) -> LockInfo {
         if let Some(lock_info_json) = lock_info_json {
             match serde_json::from_str::<LockInfo>(&lock_info_json) {
@@ -680,22 +1057,45 @@ impl RemoteClient {
 impl LockClient for RemoteClient {
     async fn acquire_lock(&self, request: &LockRequest) -> Result<LockResponse> {
         info!("remote acquire_exclusive for {}", request.resource);
-        let mut client = self.get_client().await?;
+        let admission = match self.admit_lock_request("lock") {
+            Some(admission) => admission,
+            None => return Ok(Self::rpc_backoff_failure_response(request)),
+        };
+
+        let rpc_timeout = Self::rpc_timeout();
+        let bounded_request = Self::bounded_acquire_request(request, rpc_timeout);
+        let mut client = match self.get_client().await {
+            Ok(client) => client,
+            Err(err) => {
+                self.record_request_timeout("lock", rpc_timeout);
+                self.release_lock_request("lock", Some(admission));
+                return Err(err);
+            }
+        };
         let resource_summary = request.resource.to_string();
-        let mut req = Request::new(GenerallyLockRequest {
-            args: serde_json::to_string(&request)
-                .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
-        });
-        attach_lock_mutation_body_digest(&mut req)?;
+        let args = match serde_json::to_string(&bounded_request) {
+            Ok(args) => args,
+            Err(err) => {
+                self.release_lock_request("lock", Some(admission));
+                return Err(LockError::internal(format!("Failed to serialize request: {err}")));
+            }
+        };
+        let mut req = Request::new(GenerallyLockRequest { args });
+        req.set_timeout(rpc_timeout);
+        if let Err(err) = attach_lock_mutation_body_digest(&mut req) {
+            self.release_lock_request("lock", Some(admission));
+            return Err(err.into());
+        }
         let late = self.late_release_hook(request.lock_id.clone());
+        let late_failure = self.late_release_failure_hook(vec![request.lock_id.clone()]);
 
         let resp = match self
             .execute_rpc(
                 "lock",
                 &resource_summary,
-                Self::rpc_timeout(),
+                rpc_timeout,
                 async move { client.lock(req).await },
-                late,
+                RpcCleanup::new(late, late_failure, Some(admission)),
             )
             .await
         {
@@ -725,26 +1125,54 @@ impl LockClient for RemoteClient {
             return Ok(Vec::new());
         }
 
-        let mut client = self.get_client().await?;
+        let admission = match self.admit_lock_request("lock_batch") {
+            Some(admission) => admission,
+            None => return Ok(Self::rpc_backoff_failure_batch(requests)),
+        };
+
+        let rpc_timeout = Self::rpc_timeout();
+        let bounded_requests = requests
+            .iter()
+            .map(|request| Self::bounded_acquire_request(request, rpc_timeout))
+            .collect::<Vec<_>>();
+        let mut client = match self.get_client().await {
+            Ok(client) => client,
+            Err(err) => {
+                self.record_request_timeout("lock_batch", rpc_timeout);
+                self.release_lock_request("lock_batch", Some(admission));
+                return Err(err);
+            }
+        };
         let resource_summary = Self::summarize_resources(requests);
-        let mut req = Request::new(BatchGenerallyLockRequest {
-            args: requests
-                .iter()
-                .map(|request| {
-                    serde_json::to_string(request).map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))
-                })
-                .collect::<Result<Vec<_>>>()?,
-        });
-        attach_lock_mutation_body_digest(&mut req)?;
+        let args = match bounded_requests
+            .iter()
+            .map(|request| {
+                serde_json::to_string(request).map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))
+            })
+            .collect::<Result<Vec<_>>>()
+        {
+            Ok(args) => args,
+            Err(err) => {
+                self.release_lock_request("lock_batch", Some(admission));
+                return Err(err);
+            }
+        };
+        let mut req = Request::new(BatchGenerallyLockRequest { args });
+        req.set_timeout(rpc_timeout);
+        if let Err(err) = attach_lock_mutation_body_digest(&mut req) {
+            self.release_lock_request("lock_batch", Some(admission));
+            return Err(err.into());
+        }
         let late = self.late_release_batch_hook(requests.iter().map(|request| request.lock_id.clone()).collect());
+        let late_failure = self.late_release_failure_hook(requests.iter().map(|request| request.lock_id.clone()).collect());
 
         let resp = match self
             .execute_rpc(
                 "lock_batch",
                 &resource_summary,
-                Self::rpc_timeout(),
+                rpc_timeout,
                 async move { client.lock_batch(req).await },
-                late,
+                RpcCleanup::new(late, late_failure, Some(admission)),
             )
             .await
         {
@@ -791,7 +1219,7 @@ impl LockClient for RemoteClient {
                 &resource_summary,
                 Self::rpc_timeout(),
                 async move { client.un_lock(req).await },
-                None,
+                RpcCleanup::new(None, None, None),
             )
             .await?
             .into_inner();
@@ -825,7 +1253,7 @@ impl LockClient for RemoteClient {
                 &resource_summary,
                 Self::rpc_timeout(),
                 async move { client.un_lock_batch(req).await },
-                None,
+                RpcCleanup::new(None, None, None),
             )
             .await?
             .into_inner();
@@ -853,7 +1281,7 @@ impl LockClient for RemoteClient {
                 &resource_summary,
                 Self::rpc_timeout(),
                 async move { client.refresh(req).await },
-                None,
+                RpcCleanup::new(None, None, None),
             )
             .await?
             .into_inner();
@@ -879,7 +1307,7 @@ impl LockClient for RemoteClient {
                 &resource_summary,
                 Self::rpc_timeout(),
                 async move { client.force_un_lock(req).await },
-                None,
+                RpcCleanup::new(None, None, None),
             )
             .await?
             .into_inner();
@@ -894,26 +1322,31 @@ impl LockClient for RemoteClient {
 
         // Since there's no direct status query in the gRPC service,
         // we attempt a non-blocking lock acquisition to check if the resource is available
-        let status_request = Self::create_unlock_request(lock_id);
+        let probe_lock_id = LockId::new_unique(&lock_id.resource);
+        let rpc_timeout = Self::rpc_timeout();
+        let status_request = Self::create_unlock_request(&probe_lock_id).with_ttl(rpc_timeout);
+        let bounded_status_request = Self::bounded_acquire_request(&status_request, rpc_timeout);
         let resource_summary = status_request.resource.to_string();
         let mut client = self.get_client().await?;
-        let args = serde_json::to_string(&status_request)
+        let args = serde_json::to_string(&bounded_status_request)
             .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?;
 
         // Try to acquire a very short-lived lock to test availability
         let mut req = Request::new(GenerallyLockRequest { args: args.clone() });
+        req.set_timeout(rpc_timeout);
         attach_lock_mutation_body_digest(&mut req)?;
         // A probe lock granted after the deadline must not linger on the peer.
-        let late = self.late_release_hook(lock_id.clone());
+        let late = self.late_release_hook(probe_lock_id.clone());
+        let late_failure = self.late_release_failure_hook(vec![probe_lock_id]);
 
         // Try exclusive lock first with very short timeout
         let resp = match self
             .execute_rpc(
                 "check_status",
                 &resource_summary,
-                Self::rpc_timeout(),
+                rpc_timeout,
                 async move { client.lock(req).await },
-                late,
+                RpcCleanup::new(late, late_failure, None),
             )
             .await
         {
@@ -925,6 +1358,7 @@ impl LockClient for RemoteClient {
             // If we successfully acquired the lock, the resource was free.
             // Immediately release it on a best-effort basis.
             let mut release_req = Request::new(GenerallyLockRequest { args });
+            release_req.set_timeout(rpc_timeout);
             attach_lock_mutation_body_digest(&mut release_req)?;
             if let Ok(mut client) = self.get_client().await {
                 let _ = self
@@ -933,7 +1367,7 @@ impl LockClient for RemoteClient {
                         &resource_summary,
                         Self::rpc_timeout(),
                         async move { client.un_lock(release_req).await },
-                        None,
+                        RpcCleanup::new(None, None, None),
                     )
                     .await;
             }
@@ -998,7 +1432,7 @@ impl LockClient for RemoteClient {
                 Self::ONLINE_CHECK_RESOURCE,
                 online_timeout,
                 async move { client.ping(ping_req).await },
-                None,
+                RpcCleanup::new(None, None, None),
             )
             .await
         {
@@ -1133,6 +1567,94 @@ mod tests {
         );
     }
 
+    #[test]
+    fn request_breaker_opens_after_three_timeouts_and_allows_one_probe() {
+        let addr = "http://breaker-test";
+        let client = RemoteClient::new(addr.to_string());
+        reset_lock_peer_health_for_test(addr);
+
+        for _ in 0..LOCK_RPC_REQUEST_BACKOFF_THRESHOLD {
+            client.record_request_timeout("lock", Duration::from_millis(100));
+        }
+
+        assert!(
+            client.admit_lock_request("lock").is_none(),
+            "open breaker must reject requests during backoff"
+        );
+        with_lock_peer_health(addr, |health| {
+            health.request_backoff_until = Some(Instant::now() - Duration::from_millis(1));
+            health.request_probe_generation = None;
+            health.request_in_flight = 0;
+        });
+
+        let probe = client
+            .admit_lock_request("lock")
+            .expect("half-open breaker must admit one probe");
+        assert!(probe.is_probe());
+        assert!(client.admit_lock_request("lock").is_none(), "only one half-open probe may run");
+        client.release_lock_request("lock", Some(probe));
+        assert!(lock_peer_health_for_test(addr).request_probe_generation.is_none());
+        reset_lock_peer_health_for_test(addr);
+    }
+
+    #[test]
+    fn old_request_completion_cannot_clear_a_new_probe() {
+        let addr = "http://breaker-token-test";
+        let client = RemoteClient::new(addr.to_string());
+        reset_lock_peer_health_for_test(addr);
+        with_lock_peer_health(addr, |health| {
+            health.request_backoff_until = Some(Instant::now() - Duration::from_millis(1));
+            health.request_probe_generation = None;
+            health.request_in_flight = 1;
+        });
+
+        let probe = client.admit_lock_request("lock").expect("probe should be admitted");
+        client.release_lock_request("lock", Some(LockRequestAdmission { probe_generation: None }));
+        assert!(lock_peer_health_for_test(addr).request_probe_generation.is_some());
+        client.record_rpc_success("lock", Some(LockRequestAdmission { probe_generation: None }));
+        assert!(lock_peer_health_for_test(addr).request_probe_generation.is_some());
+        client.release_lock_request("lock", Some(probe));
+        assert!(lock_peer_health_for_test(addr).request_probe_generation.is_none());
+        reset_lock_peer_health_for_test(addr);
+    }
+
+    #[test]
+    fn stale_probe_completion_cannot_clear_a_later_generation() {
+        let addr = "http://breaker-generation-test";
+        let client = RemoteClient::new(addr.to_string());
+        reset_lock_peer_health_for_test(addr);
+        with_lock_peer_health(addr, |health| {
+            health.request_backoff_until = Some(Instant::now() - Duration::from_millis(1));
+        });
+
+        let first = client.admit_lock_request("lock").expect("first probe should be admitted");
+        client.record_request_timeout("lock", Duration::from_millis(10));
+        with_lock_peer_health(addr, |health| {
+            health.request_backoff_until = Some(Instant::now() - Duration::from_millis(1));
+            health.request_in_flight = 0;
+            health.request_probe_generation = None;
+            health.request_consecutive_timeouts = 0;
+        });
+        let second = client.admit_lock_request("lock").expect("second probe should be admitted");
+        assert_ne!(first.probe_generation, second.probe_generation);
+
+        client.release_lock_request("lock", Some(first));
+        assert_eq!(lock_peer_health_for_test(addr).request_probe_generation, second.probe_generation);
+        client.release_lock_request("lock", Some(second));
+        reset_lock_peer_health_for_test(addr);
+    }
+
+    #[test]
+    fn bounded_acquire_request_matches_transport_deadline() {
+        let request = test_lock_request(Duration::from_secs(30));
+        let bounded = RemoteClient::bounded_acquire_request(&request, Duration::from_secs(3));
+        assert_eq!(bounded.acquire_timeout, Duration::from_secs(3));
+
+        let request = test_lock_request(Duration::from_secs(1));
+        let bounded = RemoteClient::bounded_acquire_request(&request, Duration::from_secs(3));
+        assert_eq!(bounded.acquire_timeout, Duration::from_secs(1));
+    }
+
     #[test]
     fn acquired_lock_ids_picks_only_granted_batch_entries() {
         let lock_ids = vec![

crates/ecstore/src/core/pools_test.rs

@@ -1185,14 +1185,18 @@ mod decommission_lock_order_tests {
         barrier.wait_until_paused().await;
 
         let probe_store = Arc::clone(&other_store);
-        tokio::time::timeout(std::time::Duration::from_secs(1), async move {
-            probe_store
-                .save_current_pool_meta_for_test(&[0])
-                .await
-                .expect("pool metadata mutation probe should commit before UploadPart admission");
-        })
+        let pool_meta_lock = probe_store
+            .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
+            .await
+            .expect("create the public UploadPart capacity guard probe");
+        let pool_meta_guard = tokio::time::timeout(
+            std::time::Duration::from_secs(1),
+            pool_meta_lock.get_write_lock(std::time::Duration::from_secs(30)),
+        )
         .await
-        .expect("public UploadPart must not hold a capacity read guard during staging");
+        .expect("public UploadPart must not hold a capacity read guard during staging")
+        .expect("public UploadPart capacity guard probe should acquire");
+        drop(pool_meta_guard);
 
         barrier.release();
         let part = tokio::time::timeout(std::time::Duration::from_secs(30), put)
@@ -4253,6 +4257,13 @@ mod decommission_lock_order_tests {
                 panic!("cancellation probe finished before observing target contention: {result:?}");
             }
         }
+        // The mutation absorbs the gate contention inline before the outer wait
+        // loop ever sees it, and that inline budget is bounded.
+        assert_eq!(
+            retry_observer.target_gate_inline_retries(),
+            crate::core::pools::DECOMMISSION_MUTATION_GATE_MAX_INLINE_ATTEMPTS - 1,
+            "inline target-gate retries must stop at their bounded attempt budget"
+        );
         tokio::time::sleep(Duration::from_millis(800)).await;
         assert_eq!(
             retry_observer.target_gate_exact_reloads(),

crates/ecstore/src/core/sets.rs

@@ -1493,12 +1493,12 @@ impl Sets {
         object: &str,
         version_id: &str,
         opts: &HealOpts,
-        retirement: Option<&crate::bucket::retirement::MarkerRetirementContext<'_>>,
+        proof: crate::set_disk::AbsenceProofRequest<'_>,
     ) -> Result<(HealResultItem, Option<Error>, Option<crate::set_disk::HealedObjectAbsence>)> {
         let mut absence = None;
         let (item, error) = self
             .get_disks_for_heal_object(object, opts)?
-            .heal_object_with_retirement(bucket, object, version_id, opts, &mut absence, retirement)
+            .heal_object_with_retirement(bucket, object, version_id, opts, &mut absence, proof)
             .await?;
         // A caller-owned lock does not expose its lease to this boundary.
         // Keep cleanup unverified when that lease cannot be checked here.

crates/ecstore/src/data_movement/mod.rs

@@ -642,6 +642,21 @@ pub(crate) fn data_movement_stage_source(err: &Error) -> Option<&Error> {
         .downcast_ref::<Error>()
 }
 
+/// Recover a concrete typed error wrapped by [`data_movement_stage_error`].
+pub(crate) fn data_movement_stage_source_as<T>(err: &Error) -> Option<&T>
+where
+    T: std::error::Error + 'static,
+{
+    let Error::Io(io_err) = err else {
+        return None;
+    };
+    io_err
+        .get_ref()?
+        .downcast_ref::<DataMovementStageError>()?
+        .source
+        .downcast_ref::<T>()
+}
+
 fn schedule_data_movement_multipart_abort_cleanup(
     store: Arc<ECStore>,
     target_pool_idx: usize,

crates/ecstore/src/data_movement/scanner_backlog.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::data_movement::data_movement_context_error;
 use crate::disk::RUSTFS_META_BUCKET;
 use crate::error::{Error, Result, is_err_object_not_found, is_err_version_not_found};
 use crate::object_api::ObjectOptions;
@@ -27,6 +28,28 @@ use tokio::io::AsyncReadExt;
 pub const MAX_SCANNER_PAUSE_BACKLOG_BYTES: u64 = 64 * 1024;
 pub(crate) const SCANNER_PAUSE_BACKLOG_PATH: &str = "buckets/.scanner-pause-backlog.json";
 
+/// Typed failure from the scanner-owned native retirement planner.
+///
+/// Decommission must preserve the source when the native authority cannot be
+/// proven. The handoff variant is deliberately narrower than other authority
+/// failures so the worker can retry without converting a transient membership
+/// transition into a terminal pool failure.
+#[derive(Debug, thiserror::Error)]
+pub enum ScannerPauseBacklogRetirementError {
+    #[error("scanner pause backlog retirement has a retryable handoff conflict: {reason}")]
+    HandoffConflict { reason: String },
+    #[error("scanner pause backlog retirement has an unsafe authority conflict: {reason}")]
+    AuthorityConflict { reason: String },
+    #[error("scanner pause backlog retirement found an invalid native record: {reason}")]
+    InvalidRecord { reason: String },
+}
+
+impl ScannerPauseBacklogRetirementError {
+    pub fn is_retryable_handoff_conflict(&self) -> bool {
+        matches!(self, Self::HandoffConflict { .. })
+    }
+}
+
 /// A bounded, storage-fenced native replica. Only a confirmed missing object
 /// has no payload; read failures never enter the Scanner verifier.
 pub struct ScannerPauseBacklogRetirementReplica {
@@ -44,7 +67,10 @@ pub struct ScannerPauseBacklogRetirementPlan {
 }
 
 pub type ScannerPauseBacklogRetirementPlanner =
-    fn(usize, &[ScannerPauseBacklogRetirementReplica]) -> std::result::Result<Option<ScannerPauseBacklogRetirementPlan>, String>;
+    fn(
+        usize,
+        &[ScannerPauseBacklogRetirementReplica],
+    ) -> std::result::Result<Option<ScannerPauseBacklogRetirementPlan>, ScannerPauseBacklogRetirementError>;
 
 static RETIREMENT_PLANNER: OnceLock<ScannerPauseBacklogRetirementPlanner> = OnceLock::new();
 
@@ -161,7 +187,8 @@ pub(crate) fn plan_scanner_pause_backlog_retirement(
             data: read.replica.data.clone(),
         })
         .collect::<Vec<_>>();
-    planner(source_pool_index, &snapshots).map_err(Error::other)
+    planner(source_pool_index, &snapshots)
+        .map_err(|err| data_movement_context_error(format!("scanner pause backlog retirement planner failed: {err}"), err))
 }
 
 /// The native writer and retirement handoff use the same conditional, full-tail

crates/ecstore/src/disk/disk_store.rs

@@ -30,6 +30,8 @@ use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
 use rustfs_madmin::{info_commands::DiskMetrics, metrics::TimedAction};
 #[cfg(not(test))]
 use std::sync::OnceLock;
+#[cfg(test)]
+use std::sync::atomic::AtomicBool;
 use std::{
     collections::HashMap,
     path::PathBuf,
@@ -628,6 +630,8 @@ pub struct DiskHealthTracker {
     /// Authoritative atomically published runtime/status pair.
     state_snapshot: AtomicU64,
     transition_lock: std::sync::Mutex<()>,
+    #[cfg(test)]
+    test_forced_offline: AtomicBool,
 }
 
 fn pack_health_state(runtime_state: RuntimeDriveHealthState, status: u32) -> u64 {
@@ -975,6 +979,8 @@ impl DiskHealthTracker {
             last_capacity_probe_unix_secs: AtomicI64::new(0),
             state_snapshot: AtomicU64::new(pack_health_state(RuntimeDriveHealthState::Online, DISK_HEALTH_OK)),
             transition_lock: std::sync::Mutex::new(()),
+            #[cfg(test)]
+            test_forced_offline: AtomicBool::new(false),
         }
     }
 
@@ -1045,6 +1051,13 @@ impl DiskHealthTracker {
         self.publish_state(state, status);
     }
 
+    #[cfg(test)]
+    pub fn force_offline_for_test(&self) {
+        self.test_forced_offline.store(true, Ordering::Release);
+        let _guard = self.transition_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
+        self.publish_state(RuntimeDriveHealthState::Offline, DISK_HEALTH_FAULTY);
+    }
+
     pub fn swap_ok_to_faulty(&self) -> bool {
         let _guard = self.transition_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
         let (_, status) = unpack_health_state(self.state_snapshot.load(Ordering::Acquire));
@@ -1123,6 +1136,8 @@ impl DiskHealthTracker {
     /// Remote disks are marked faulty on timeout/network errors; the init loop retries with the
     /// same [`DiskStore`] handles, which would otherwise fail immediately at `is_faulty()`.
     pub fn reset_for_store_init_retry(&self, endpoint: &Endpoint) {
+        #[cfg(test)]
+        self.test_forced_offline.store(false, Ordering::Release);
         self.reset_for_store_init_retry_at(endpoint, current_unix_time());
     }
 
@@ -1142,6 +1157,10 @@ impl DiskHealthTracker {
     }
 
     pub fn mark_recovery_success(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
+        #[cfg(test)]
+        if self.test_forced_offline.load(Ordering::Acquire) {
+            return false;
+        }
         let _guard = self.transition_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
         let current = self.runtime_state();
         let next = match current {
@@ -1440,6 +1459,11 @@ impl LocalDiskWrapper {
         self.health.force_runtime_state_for_test(state);
     }
 
+    #[cfg(test)]
+    pub fn force_offline_for_test(&self) {
+        self.health.force_offline_for_test();
+    }
+
     /// Same as [`DiskHealthTracker::reset_for_store_init_retry`]: undo a transient faulty mark before another format load attempt.
     pub fn reset_health_for_store_init_retry(&self) {
         self.health.reset_for_store_init_retry(&self.disk.endpoint());

crates/ecstore/src/disk/error.rs

@@ -244,13 +244,16 @@ impl StdError for ConditionalFileNotCommittedError {
     }
 }
 
-fn classify_internode_missing_error(error: &InternodeHttpError) -> Option<DiskError> {
+fn classify_internode_disk_error(error: &InternodeHttpError) -> Option<DiskError> {
     if error.is_remote_file_not_found() {
         return Some(DiskError::FileNotFound);
     }
     if error.is_remote_volume_not_found() {
         return Some(DiskError::VolumeNotFound);
     }
+    if error.is_remote_file_corrupt() {
+        return Some(DiskError::FileCorrupt);
+    }
     None
 }
 
@@ -526,13 +529,10 @@ fn io_error_chain_contains_kind(io_error: &std::io::Error, kind: std::io::ErrorK
 
 impl From<std::io::Error> for DiskError {
     fn from(e: std::io::Error) -> Self {
-        if let Some(error) = e.get_ref().and_then(|source| source.downcast_ref::<InternodeHttpError>()) {
-            if error.is_remote_file_not_found() {
-                return DiskError::FileNotFound;
-            }
-            if error.is_remote_volume_not_found() {
-                return DiskError::VolumeNotFound;
-            }
+        if let Some(error) = e.get_ref().and_then(|source| source.downcast_ref::<InternodeHttpError>())
+            && let Some(classified) = classify_internode_disk_error(error)
+        {
+            return classified;
         }
         let e = match e.downcast::<TerminalReadError>() {
             Ok(terminal_error) => {
@@ -541,7 +541,7 @@ impl From<std::io::Error> for DiskError {
                     && let Some(internode_error) = io_error
                         .get_ref()
                         .and_then(|source| source.downcast_ref::<InternodeHttpError>())
-                    && let Some(classified) = classify_internode_missing_error(internode_error)
+                    && let Some(classified) = classify_internode_disk_error(internode_error)
                 {
                     return classified;
                 }
@@ -963,6 +963,7 @@ mod tests {
         for (remote_error, expected) in [
             (rustfs_rio::new_test_remote_file_not_found_http_io_error(), DiskError::FileNotFound),
             (rustfs_rio::new_test_remote_volume_not_found_http_io_error(), DiskError::VolumeNotFound),
+            (rustfs_rio::new_test_remote_file_corrupt_http_io_error(), DiskError::FileCorrupt),
         ] {
             let wrapped = terminal_read_error_to_io(DiskError::Io(remote_error));
             assert_eq!(DiskError::from(wrapped), expected);
@@ -1533,25 +1534,23 @@ mod tests {
     }
 
     #[test]
-    fn test_internode_missing_errors_preserve_disk_error_types() {
+    fn test_internode_disk_errors_preserve_disk_error_types() {
         let file_missing = DiskError::from(rustfs_rio::new_test_remote_file_not_found_http_io_error());
         let volume_missing = DiskError::from(rustfs_rio::new_test_remote_volume_not_found_http_io_error());
+        let file_corrupt = DiskError::from(rustfs_rio::new_test_remote_file_corrupt_http_io_error());
         let unmarked_server_error = DiskError::from(rustfs_rio::new_test_internode_http_io_error(
             rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::INTERNAL_SERVER_ERROR),
         ));
 
         assert_eq!(file_missing, DiskError::FileNotFound);
         assert_eq!(volume_missing, DiskError::VolumeNotFound);
+        assert_eq!(file_corrupt, DiskError::FileCorrupt);
         assert!(matches!(unmarked_server_error, DiskError::Io(_)));
-        for missing in [file_missing, volume_missing] {
-            assert_eq!(missing.clone(), missing);
+        for error in [file_missing, volume_missing, file_corrupt] {
+            assert_eq!(error.clone(), error);
             assert_eq!(
-                crate::disk::error_reduce::reduce_write_quorum_errs(
-                    &[Some(missing.clone()), Some(missing.clone()), None],
-                    &[],
-                    2
-                ),
-                Some(missing)
+                crate::disk::error_reduce::reduce_write_quorum_errs(&[Some(error.clone()), Some(error.clone()), None], &[], 2),
+                Some(error)
             );
         }
     }

crates/ecstore/src/disk/mod.rs

@@ -70,7 +70,12 @@ use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
 use rustfs_madmin::info_commands::DiskMetrics;
 use rustfs_rio::ChunkReaderBox;
 use serde::{Deserialize, Serialize};
-use std::{fmt::Debug, path::PathBuf, sync::Arc, time::Duration};
+use std::{
+    fmt::Debug,
+    path::PathBuf,
+    sync::{Arc, atomic::AtomicBool},
+    time::Duration,
+};
 use time::OffsetDateTime;
 use tokio::io::{AsyncRead, AsyncWrite};
 use uuid::Uuid;
@@ -820,7 +825,7 @@ impl Disk {
         dst_volume: &str,
         dst_path: &str,
     ) -> Result<RenameDataResp> {
-        let Some(scope) = crate::store::bucket_heal_scope(dst_volume) else {
+        let Some(scope) = crate::store::bucket_heal_scope_for_object(dst_volume, dst_path) else {
             return self
                 .rename_data_borrowed_with_fence(src_volume, src_path, fi, dst_volume, dst_path, None)
                 .await;
@@ -979,6 +984,14 @@ impl Disk {
             Disk::Remote(remote_disk) => remote_disk.force_runtime_state_for_test(state),
         }
     }
+
+    #[cfg(test)]
+    pub fn force_offline_for_test(&self) {
+        match self {
+            Disk::Local(local_disk) => local_disk.force_offline_for_test(),
+            Disk::Remote(remote_disk) => remote_disk.force_offline_for_test(),
+        }
+    }
 }
 
 #[derive(Debug)]
@@ -1471,6 +1484,12 @@ pub struct WalkDirOptions {
     // Override the remote stream stall timeout for long background walks.
     #[serde(default)]
     pub stall_timeout_ms: Option<u64>,
+
+    /// In-process completion state for bounded local walks. This is skipped
+    /// from RPC serialization; remote peers retain the legacy natural-EOF
+    /// behavior until they support an explicit capability.
+    #[serde(skip)]
+    pub producer_limit_reached: Option<Arc<AtomicBool>>,
 }
 
 impl WalkDirOptions {
@@ -1794,6 +1813,7 @@ mod tests {
             skip_total_timeout: false,
             timeout_ms: Some(10_000),
             stall_timeout_ms: Some(20_000),
+            producer_limit_reached: None,
         };
 
         assert_eq!(opts.bucket, "test-bucket");

crates/ecstore/src/erasure/coding/bitrot.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::disk::error::Error as DiskError;
 use crate::erasure::codec::buffer_pool::{get_ec_buffer, return_ec_buffer};
 use pin_project_lite::pin_project;
 use rustfs_utils::HashAlgorithm;
@@ -239,7 +240,7 @@ fn short_shard_read(got: usize, want: usize) -> std::io::Error {
         want,
         "short shard read: got {got} of {want} bytes"
     );
-    std::io::Error::new(std::io::ErrorKind::UnexpectedEof, format!("short shard read: got {got} of {want} bytes"))
+    std::io::Error::new(std::io::ErrorKind::UnexpectedEof, DiskError::FileCorrupt)
 }
 
 /// Split a `[hash][data]` block, verify the hash (unless `skip_verify`), and
@@ -265,7 +266,7 @@ fn split_and_verify<'a>(hash_algo: &HashAlgorithm, skip_verify: bool, block: &'a
             data_len = data.len(),
             "bitrot hash mismatch"
         );
-        return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "bitrot hash mismatch"));
+        return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, DiskError::FileCorrupt));
     }
     Ok((data, verify))
 }
@@ -434,7 +435,7 @@ where
                             data_len = want,
                             "bitrot hash mismatch"
                         );
-                        return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "bitrot hash mismatch"));
+                        return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, DiskError::FileCorrupt));
                     }
                     hash_offset += take;
                     remaining -= take;
@@ -1141,6 +1142,26 @@ mod tests {
     use std::time::Duration;
     use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, ReadBuf};
 
+    #[test]
+    fn bitrot_damage_preserves_typed_repair_evidence() {
+        use crate::disk::error::Error as DiskError;
+        let algo = HashAlgorithm::HighwayHash256;
+        let data = b"repair evidence";
+        let mut frame = algo.hash_encode(data).as_ref().to_vec();
+        frame.extend_from_slice(data);
+        *frame.last_mut().expect("test frame has payload") ^= 1;
+        let error = super::split_and_verify(&algo, false, &frame).expect_err("a hash mismatch must reject the shard");
+        assert_eq!(error.kind(), io::ErrorKind::InvalidData);
+        assert_eq!(DiskError::from(error), DiskError::FileCorrupt);
+        let error = super::short_shard_read(3, 4);
+        assert_eq!(error.kind(), io::ErrorKind::UnexpectedEof);
+        assert_eq!(DiskError::from(error), DiskError::FileCorrupt);
+        assert!(matches!(
+            DiskError::from(io::Error::new(io::ErrorKind::InvalidData, "unrelated invalid data")),
+            DiskError::Io(_)
+        ));
+    }
+
     struct FragmentedSource {
         chunks: VecDeque<Bytes>,
     }

crates/ecstore/src/erasure/coding/decode.rs

@@ -26,7 +26,7 @@ use crate::erasure::codec::workspace::ShardBufferPool;
 use crate::erasure::coding::{BitrotReader, Erasure};
 use crate::io_support::bitrot::DeferredReaderStripeHandle;
 use crate::set_disk::shard_source::{
-    INLINE_SHARD_SLOTS, ShardBuffers, ShardErrors, ShardReadCost, ShardStripeSource, StripeReadState,
+    INLINE_SHARD_SLOTS, ShardBuffers, ShardErrors, ShardReadCost, ShardReadRepair, ShardStripeSource, StripeReadState,
 };
 use futures::FutureExt;
 use futures::stream::{FuturesUnordered, StreamExt};
@@ -46,7 +46,13 @@ type ShardReadFuture<'a> = Pin<Box<dyn Future<Output = (usize, ShardReadCost, Re
 type OwnedShardReadFuture<'a, R> =
     Pin<Box<dyn Future<Output = (usize, ShardReadCost, Result<Vec<u8>, Error>, Option<BitrotReader<R>>, bool)> + Send + 'a>>;
 pub(crate) type DeferredReaderReopener<R> = Arc<dyn Fn(usize) -> Option<BitrotReader<R>> + Send + Sync>;
-pub(crate) type DecodeOutcome = (usize, Option<std::io::Error>, bool);
+#[derive(Debug, Default)]
+pub(crate) struct DecodeOutcome {
+    pub(crate) written: usize,
+    pub(crate) error: Option<std::io::Error>,
+    pub(crate) exact_quorum: bool,
+    pub(crate) repair: ShardReadRepair,
+}
 
 type ShardIndexes = SmallVec<[usize; INLINE_SHARD_SLOTS]>;
 type ActiveReaders = SmallVec<[bool; INLINE_SHARD_SLOTS]>;
@@ -596,6 +602,7 @@ pub(crate) struct ParallelReader<R> {
     hedged_readers: ActiveReaders,
     stripe_index: usize,
     integrity: Option<std::sync::Arc<crate::io_support::shard_integrity::PartProofReader>>,
+    repair_evidence: Option<Arc<ShardReadRepair>>,
 }
 }
 
@@ -808,9 +815,15 @@ where
             deferred_reopeners: Vec::new(),
             hedged_readers,
             stripe_index: 0,
+            repair_evidence: None,
         }
     }
 
+    pub(crate) fn with_repair_evidence(mut self, evidence: Arc<ShardReadRepair>) -> Self {
+        self.repair_evidence = Some(evidence);
+        self
+    }
+
     /// Attach the per-slot deferred stripe handles produced during bitrot
     /// reader setup. Only parity slots are consulted: a handle lets the
     /// lockstep path open a parity shard aligned to the stripe where a data
@@ -2027,6 +2040,9 @@ where
             .take()
             .unwrap_or_else(|| Box::new(StripeReadState::with_slot_count(self.readers.len(), self.data_shards)));
         self.read_into_state(&mut state).await;
+        if let Some(evidence) = &self.repair_evidence {
+            evidence.observe(state.parts_mut().1);
+        }
         if state.can_decode()
             && let Some(proof) = &self.integrity
         {
@@ -2231,10 +2247,10 @@ impl Erasure {
         W: AsyncWrite + Send + Sync + Unpin,
         R: crate::erasure::coding::ShardSource,
     {
-        let (written, error, _) = self
+        let outcome = self
             .decode_inner(writer, readers, offset, length, total_length, None, Vec::new(), Vec::new())
             .await;
-        (written, error)
+        (outcome.written, outcome.error)
     }
 
     #[allow(dead_code, reason = "read-cost decode path asserted by this file's tests (backlog#1823)")]
@@ -2251,10 +2267,10 @@ impl Erasure {
         W: AsyncWrite + Send + Sync + Unpin,
         R: crate::erasure::coding::ShardSource,
     {
-        let (written, error, _) = self
+        let outcome = self
             .decode_inner(writer, readers, offset, length, total_length, Some(read_costs), Vec::new(), Vec::new())
             .await;
-        (written, error)
+        (outcome.written, outcome.error)
     }
 
     /// GET decode entry point that also carries the deferred-parity stripe
@@ -2311,7 +2327,7 @@ impl Erasure {
         W: AsyncWrite + Send + Sync + Unpin,
         R: crate::erasure::coding::ShardSource,
     {
-        let (written, error, _) = self
+        let outcome = self
             .decode_inner(
                 writer,
                 readers,
@@ -2323,7 +2339,7 @@ impl Erasure {
                 deferred_reopeners,
             )
             .await;
-        (written, error)
+        (outcome.written, outcome.error)
     }
 
     #[allow(clippy::too_many_arguments)]
@@ -2374,6 +2390,7 @@ impl Erasure {
         block_length: usize,
         written: &mut usize,
         ret_err: &mut Option<std::io::Error>,
+        repair: &ShardReadRepair,
         stage_metrics_enabled: bool,
         require_surplus_source: bool,
         integrity: Option<&std::sync::Arc<crate::io_support::shard_integrity::PartProofReader>>,
@@ -2382,6 +2399,7 @@ impl Erasure {
     where
         W: AsyncWrite + Send + Sync + Unpin,
     {
+        repair.observe(errs);
         if ret_err.is_none()
             && let (_, Some(err)) = reduce_errs(errs, &[])
             && (err == Error::FileNotFound || err == Error::FileCorrupt)
@@ -2516,41 +2534,42 @@ impl Erasure {
     {
         if readers.len() != self.data_shards + self.parity_shards {
             record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
-            return (0, Some(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers")), false);
+            return DecodeOutcome {
+                error: Some(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers")),
+                ..Default::default()
+            };
         }
 
         // block_size/data_shards come from on-disk metadata; a corrupt FileInfo with a
         // zero here must surface as an error, not a divide-by-zero panic on every GET.
         if self.block_size == 0 || self.data_shards == 0 {
             record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
-            return (
-                0,
-                Some(io::Error::new(ErrorKind::InvalidInput, "Invalid erasure coding parameters")),
-                false,
-            );
+            return DecodeOutcome {
+                error: Some(io::Error::new(ErrorKind::InvalidInput, "Invalid erasure coding parameters")),
+                ..Default::default()
+            };
         }
 
         let Some(end_offset) = offset.checked_add(length) else {
             record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
-            return (
-                0,
-                Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")),
-                false,
-            );
+            return DecodeOutcome {
+                error: Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")),
+                ..Default::default()
+            };
         };
         if end_offset > total_length {
             record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
-            return (
-                0,
-                Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")),
-                false,
-            );
+            return DecodeOutcome {
+                error: Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")),
+                ..Default::default()
+            };
         }
 
         let mut ret_err = None;
+        let repair = ShardReadRepair::default();
 
         if length == 0 {
-            return (0, ret_err, false);
+            return DecodeOutcome::default();
         }
 
         let mut written = 0;
@@ -2573,6 +2592,11 @@ impl Erasure {
         .with_deferred_parity_reopeners(deferred_reopeners);
 
         let integrity = reader.integrity.clone();
+        // Copy sources use demand-bound scheduling for backpressure, but retain
+        // the ordinary GET read quorum. Only the opt-in GET rollout requires a
+        // surplus source; available surplus shards and integrity proofs are
+        // still verified before any reconstructed copy bytes are emitted.
+        let require_surplus_source = reader.demand_bound_lockstep && matches!(decode_read_policy(), DecodeReadPolicy::Default);
 
         let start = offset / self.block_size;
         let end = end_offset.saturating_sub(1) / self.block_size;
@@ -2667,7 +2691,6 @@ impl Erasure {
                         // `shards` are borrowed again below. In the `Stop` case that
                         // drop is what cancels the still-in-flight prefetch read.
                         let (flow, next): (Option<StripeFlow>, Option<StripeReadOutput>) = {
-                            let require_surplus_source = reader.demand_bound_lockstep;
                             let read_fut = read_stripe_timed(&mut reader, stage_metrics_enabled);
                             let emit_fut = self.emit_decoded_stripe(
                                 writer,
@@ -2677,6 +2700,7 @@ impl Erasure {
                                 block_length,
                                 &mut written,
                                 &mut ret_err,
+                                &repair,
                                 stage_metrics_enabled,
                                 require_surplus_source,
                                 integrity.as_ref(),
@@ -2727,8 +2751,9 @@ impl Erasure {
                                 block_length,
                                 &mut written,
                                 &mut ret_err,
+                                &repair,
                                 stage_metrics_enabled,
-                                reader.demand_bound_lockstep,
+                                require_surplus_source,
                                 integrity.as_ref(),
                                 offset,
                             )
@@ -2770,8 +2795,9 @@ impl Erasure {
                         block_length,
                         &mut written,
                         &mut ret_err,
+                        &repair,
                         stage_metrics_enabled,
-                        reader.demand_bound_lockstep,
+                        require_surplus_source,
                         integrity.as_ref(),
                         offset,
                     )
@@ -2785,15 +2811,16 @@ impl Erasure {
             }
         }
 
-        if ret_err.is_some() {
-            return (written, ret_err, exact_quorum);
-        }
-
-        if written < length {
+        if ret_err.is_none() && written < length {
             ret_err = Some(Error::LessData.into());
         }
 
-        (written, ret_err, exact_quorum)
+        DecodeOutcome {
+            written,
+            error: ret_err,
+            exact_quorum,
+            repair,
+        }
     }
 }
 
@@ -3449,6 +3476,150 @@ mod tests {
         }
     }
 
+    async fn copy_source_bitrot_shards(erasure: &Erasure, payload: &[u8], corrupt_parity: bool) -> Vec<Vec<u8>> {
+        let mut writers: Vec<_> = (0..erasure.data_shards + erasure.parity_shards)
+            .map(|_| BitrotWriter::new(Cursor::new(Vec::new()), erasure.shard_size(), HashAlgorithm::HighwayHash256))
+            .collect();
+        for block in payload.chunks(erasure.block_size) {
+            let shards = erasure.encode_data(block).expect("copy source should encode");
+            for (index, shard) in shards.iter().enumerate() {
+                let mut bytes = shard.to_vec();
+                if corrupt_parity && index == erasure.data_shards {
+                    // A stale source can have a valid per-shard bitrot checksum.
+                    // Reframe the changed parity to exercise reconstruction verification.
+                    bytes[0] ^= 0x80;
+                }
+                writers[index].write(&bytes).await.expect("copy source should frame");
+            }
+        }
+        writers.into_iter().map(|writer| writer.into_inner().into_inner()).collect()
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn copy_source_reads_exact_quorum_across_stripes_and_ranges() {
+        const BLOCK_SIZE: usize = 768;
+        let payload: Vec<_> = (0..BLOCK_SIZE * 3 + 37)
+            .map(|index| (index.wrapping_mul(31) % 251) as u8)
+            .collect();
+
+        for uses_legacy in [false, true] {
+            let erasure = Erasure::new_with_options(12, 4, BLOCK_SIZE, uses_legacy);
+            let buffers = copy_source_bitrot_shards(&erasure, &payload, false).await;
+            for offline in [[0, 1, 2, 3], [0, 5, 12, 15]] {
+                for (offset, length) in [(0, payload.len()), (BLOCK_SIZE + 17, payload.len() - BLOCK_SIZE - 17)] {
+                    let frame_offset = offset / BLOCK_SIZE * (erasure.shard_size() + HashAlgorithm::HighwayHash256.size());
+                    let positioned: Vec<_> = buffers.iter().map(|bytes| bytes[frame_offset..].to_vec()).collect();
+                    let (mut readers, handles) =
+                        readers_with_deferred_parity(&positioned, 12, erasure.shard_size(), &HashAlgorithm::HighwayHash256, &[]);
+                    for index in offline {
+                        readers[index] = None;
+                    }
+                    let mut output = Vec::new();
+                    let (written, error) = crate::set_disk::with_get_object_read_policy(
+                        crate::set_disk::GetObjectReadPolicy::CopySource,
+                        erasure.decode_with_stripe_handles(&mut output, readers, offset, length, payload.len(), None, handles),
+                    )
+                    .await;
+
+                    assert!(error.is_none(), "legacy={uses_legacy}, offline={offline:?}, offset={offset}: {error:?}");
+                    assert_eq!(written, length);
+                    assert_eq!(output, payload[offset..offset + length]);
+                }
+            }
+        }
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn copy_source_recovers_exact_quorum_after_first_stripe() {
+        const BLOCK_SIZE: usize = 768;
+        let payload: Vec<_> = (0..BLOCK_SIZE * 3 + 37)
+            .map(|index| (index.wrapping_mul(31) % 251) as u8)
+            .collect();
+        for uses_legacy in [false, true] {
+            let erasure = Erasure::new_with_options(12, 4, BLOCK_SIZE, uses_legacy);
+            let buffers = copy_source_bitrot_shards(&erasure, &payload, false).await;
+            let first_frame = erasure.shard_size() + HashAlgorithm::HighwayHash256.size();
+            let (readers, handles) = readers_with_deferred_parity(
+                &buffers,
+                12,
+                erasure.shard_size(),
+                &HashAlgorithm::HighwayHash256,
+                &[(0, first_frame), (1, first_frame), (2, first_frame), (3, first_frame)],
+            );
+            let mut output = Vec::new();
+            let (written, error) = crate::set_disk::with_get_object_read_policy(
+                crate::set_disk::GetObjectReadPolicy::CopySource,
+                erasure.decode_with_stripe_handles(&mut output, readers, 0, payload.len(), payload.len(), None, handles),
+            )
+            .await;
+
+            assert!(error.is_none(), "legacy={uses_legacy}: late shard loss should recover: {error:?}");
+            assert_eq!(written, payload.len());
+            assert_eq!(output, payload);
+        }
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn copy_source_below_read_quorum_fails_before_output() {
+        const BLOCK_SIZE: usize = 768;
+        let payload = vec![0x5a; BLOCK_SIZE * 2 + 37];
+        for uses_legacy in [false, true] {
+            let erasure = Erasure::new_with_options(12, 4, BLOCK_SIZE, uses_legacy);
+            let buffers = copy_source_bitrot_shards(&erasure, &payload, false).await;
+            let (mut readers, handles) =
+                readers_with_deferred_parity(&buffers, 12, erasure.shard_size(), &HashAlgorithm::HighwayHash256, &[]);
+            for reader in readers.iter_mut().take(5) {
+                *reader = None;
+            }
+            let mut output = Vec::new();
+            let (written, error) = crate::set_disk::with_get_object_read_policy(
+                crate::set_disk::GetObjectReadPolicy::CopySource,
+                erasure.decode_with_stripe_handles(&mut output, readers, 0, payload.len(), payload.len(), None, handles),
+            )
+            .await;
+
+            assert_eq!(written, 0);
+            assert!(output.is_empty());
+            assert_eq!(
+                error
+                    .expect("eleven shards cannot reconstruct twelve data shards")
+                    .to_string(),
+                Error::ErasureReadQuorum.to_string()
+            );
+        }
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn copy_source_rejects_corrupt_surplus_parity_before_output() {
+        const BLOCK_SIZE: usize = 768;
+        let payload = vec![0x5a; BLOCK_SIZE * 2 + 37];
+        for uses_legacy in [false, true] {
+            let erasure = Erasure::new_with_options(12, 4, BLOCK_SIZE, uses_legacy);
+            let buffers = copy_source_bitrot_shards(&erasure, &payload, true).await;
+            let (mut readers, handles) =
+                readers_with_deferred_parity(&buffers, 12, erasure.shard_size(), &HashAlgorithm::HighwayHash256, &[]);
+            for reader in readers.iter_mut().take(3) {
+                *reader = None;
+            }
+            let mut output = Vec::new();
+            let (written, error) = crate::set_disk::with_get_object_read_policy(
+                crate::set_disk::GetObjectReadPolicy::CopySource,
+                erasure.decode_with_stripe_handles(&mut output, readers, 0, payload.len(), payload.len(), None, handles),
+            )
+            .await;
+
+            assert_eq!(written, 0);
+            assert!(output.is_empty());
+            let error = error.expect("copy must reject inconsistent surplus parity despite valid bitrot framing");
+            assert_eq!(error.kind(), ErrorKind::InvalidData);
+            assert!(error.to_string().contains("inconsistent read source shards"));
+        }
+    }
+
     /// Guards the shard-buffer reuse in `ParallelReader`: a multi-stripe
     /// `Erasure::decode` that reconstructs missing data shards on every stripe
     /// must still return byte-exact output. Reconstructed and parity buffers are
@@ -3834,10 +4005,45 @@ mod tests {
             temp_env::async_with_vars(vars, async {
                 let readers = build_prefetch_readers(&shard_bufs, shard_size, &hash_algo, BLOCK_SIZE, 0, &[], &[1]);
                 let mut output = Vec::new();
-                let (written, err) = erasure.decode(&mut output, readers, 0, total_len, total_len).await;
-                assert!(err.is_none(), "{label}: corrupt-shard read errored {err:?}");
-                assert_eq!(written, total_len, "{label}: corrupt-shard short write");
+                let outcome = erasure
+                    .decode_with_stripe_handles_and_reopeners_with_diagnostics(
+                        &mut output,
+                        readers,
+                        0,
+                        total_len,
+                        total_len,
+                        None,
+                        Vec::new(),
+                        Vec::new(),
+                    )
+                    .await;
+                assert!(outcome.error.is_none(), "{label}: corrupt-shard read errored {:?}", outcome.error);
+                assert_eq!(outcome.written, total_len, "{label}: corrupt-shard short write");
                 assert_eq!(output, total_data, "{label}: corrupt bytes leaked into output");
+                assert!(
+                    outcome.repair.needed(false),
+                    "{label}: minority corruption must survive successful reconstruction"
+                );
+                let readers = build_prefetch_readers(&shard_bufs, shard_size, &hash_algo, BLOCK_SIZE, 0, &[], &[]);
+                let mut healthy_output = Vec::new();
+                let healthy = erasure
+                    .decode_with_stripe_handles_and_reopeners_with_diagnostics(
+                        &mut healthy_output,
+                        readers,
+                        0,
+                        total_len,
+                        total_len,
+                        None,
+                        Vec::new(),
+                        Vec::new(),
+                    )
+                    .await;
+                assert!(healthy.error.is_none());
+                assert_eq!(healthy_output, total_data);
+                assert!(
+                    !healthy.repair.needed(true),
+                    "{label}: healthy/deferred shards cannot create repair evidence"
+                );
             })
             .await;
         }
@@ -4635,40 +4841,38 @@ mod tests {
                 (ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
             ],
             async {
-            const NUM_SHARDS: usize = 1;
-            const BLOCK_SIZE: usize = 64;
-            const DATA_SHARDS: usize = 4;
-            const PARITY_SHARDS: usize = 2;
-            const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
+                const NUM_SHARDS: usize = 1;
+                const BLOCK_SIZE: usize = 64;
+                const DATA_SHARDS: usize = 4;
+                const PARITY_SHARDS: usize = 2;
+                const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
 
-            let hash_algo = HashAlgorithm::HighwayHash256;
-            let readers = make_test_readers(DATA_SHARDS + PARITY_SHARDS, SHARD_SIZE, NUM_SHARDS, &hash_algo, &[], &[1]).await;
-            let read_costs = vec![
-                ShardReadCost::Local,
-                ShardReadCost::Local,
-                ShardReadCost::Local,
-                ShardReadCost::Local,
-                ShardReadCost::Remote,
-                ShardReadCost::Remote,
-            ];
-            let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
-            let mut parallel_reader = ParallelReader::new_with_metrics_path_and_read_costs(
-                readers,
-                erasure,
-                0,
-                NUM_SHARDS * BLOCK_SIZE,
-                None,
-                read_costs,
-            );
+                let hash_algo = HashAlgorithm::HighwayHash256;
+                let readers = make_test_readers(DATA_SHARDS + PARITY_SHARDS, SHARD_SIZE, NUM_SHARDS, &hash_algo, &[], &[1]).await;
+                let read_costs = vec![
+                    ShardReadCost::Local,
+                    ShardReadCost::Local,
+                    ShardReadCost::Local,
+                    ShardReadCost::Local,
+                    ShardReadCost::Remote,
+                    ShardReadCost::Remote,
+                ];
+                let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
+                let mut parallel_reader = ParallelReader::new_with_metrics_path_and_read_costs(
+                    readers,
+                    erasure,
+                    0,
+                    NUM_SHARDS * BLOCK_SIZE,
+                    None,
+                    read_costs,
+                );
 
-            let (bufs, errs) = parallel_reader.read().await;
+                let (bufs, errs) = parallel_reader.read().await;
 
-            assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
-            assert!(
-                matches!(&errs[1], Some(DiskError::Io(err)) if err.kind() == ErrorKind::InvalidData && err.to_string().contains("bitrot"))
-            );
-            assert_eq!(bufs[4].as_deref(), Some(&[4u8; SHARD_SIZE][..]));
-            assert!(bufs[5].is_none());
+                assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
+                assert_eq!(errs[1], Some(DiskError::FileCorrupt));
+                assert_eq!(bufs[4].as_deref(), Some(&[4u8; SHARD_SIZE][..]));
+                assert!(bufs[5].is_none());
             },
         )
         .await;
@@ -4814,16 +5018,7 @@ mod tests {
             assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
             assert_eq!(
                 BITROT_DISKS,
-                errs.iter()
-                    .filter(|err| {
-                        match err {
-                            Some(DiskError::Io(err)) => {
-                                err.kind() == std::io::ErrorKind::InvalidData && err.to_string().contains("bitrot")
-                            }
-                            _ => false,
-                        }
-                    })
-                    .count()
+                errs.iter().filter(|err| matches!(err, Some(DiskError::FileCorrupt))).count()
             );
         }
     }

crates/ecstore/src/io_support/shard_integrity.rs

@@ -587,7 +587,9 @@ pub(crate) async fn verify_deep_parts(
             return Err(DiskError::FileCorrupt);
         }
         let part_status = statuses.get_mut(&part_index).ok_or(DiskError::FileCorrupt)?;
-        let length = erasure.shard_file_offset(0, part.size, part.size);
+        // Deep verification must consume every encoded stripe, including a
+        // partial final stripe, before certifying the shard's integrity.
+        let length = usize::try_from(erasure.shard_file_size(part.size as i64)).map_err(|_| DiskError::FileCorrupt)?;
         let mut readers = Vec::with_capacity(disks.len());
         for (index, disk) in disks.iter().enumerate() {
             if part_status[index] != CHECK_PART_UNKNOWN {
@@ -631,6 +633,9 @@ pub(crate) async fn verify_deep_parts(
                     part_status[index] = CHECK_PART_FILE_CORRUPT;
                 }
                 Err(error) => {
+                    if !matches!(error, DiskError::FileNotFound | DiskError::FileVersionNotFound | DiskError::FileCorrupt) {
+                        return Err(error);
+                    }
                     readers.push(None);
                     part_status[index] = conv_part_err_to_int(&Some(error));
                 }

crates/ecstore/src/object_api/types.rs

@@ -935,6 +935,8 @@ pub struct ObjectOptions {
     /// Persisted bucket incarnation observed before authorization.
     pub expected_bucket_incarnation_id: Option<Uuid>,
     pub no_lock: bool,
+    /// Internal read-only inspection must not enqueue metadata or payload repairs.
+    pub suppress_read_repair: bool,
     /// Control-plane writers that immediately read or CAS the same namespace
     /// key use TailDrained without changing namespace lock ownership.
     #[doc(hidden)]

crates/ecstore/src/services/integrity/mod.rs

@@ -0,0 +1,290 @@
+// Copyright 2024 RustFS Team
+// Licensed under the Apache License, Version 2.0.
+
+//! Explicit, bounded integrity operations. This service does not participate in ordinary GET or Heal.
+//! Lock order: cluster worker lock, bucket incarnation fence, object commit lock.
+//! Control requests use checkpoint CAS without taking the worker lock; a worker that loses CAS stops.
+
+mod model;
+mod runner;
+#[cfg(test)]
+mod tests;
+
+pub use model::{
+    InventoryItem, InventoryPage, ItemRequest, ItemResult, ItemState, Job, JobMode, JobRequest, JobState, Protection, Readiness,
+    readiness,
+};
+
+use crate::config::com::{read_config_limited_preserve_empty_with_metadata, save_config_with_opts_quiet};
+use crate::disk::RUSTFS_META_BUCKET;
+use crate::error::Error as StorageError;
+use crate::object_api::{ObjectOptions, WriteCompletion};
+use crate::storage_api_contracts::{
+    list::ListOperations,
+    namespace::NamespaceLocking,
+    object::{HTTPPreconditions, ObjectOperations},
+};
+use crate::store::ECStore;
+use model::InventoryItem as Item;
+use std::sync::Arc;
+use std::time::Duration;
+use uuid::Uuid;
+
+type Result<T> = std::result::Result<T, IntegrityError>;
+
+#[derive(Debug, thiserror::Error)]
+pub enum IntegrityError {
+    #[error("invalid integrity request: {0}")]
+    Invalid(&'static str),
+    #[error("integrity job or target changed concurrently")]
+    Conflict,
+    #[error("an integrity worker is already active")]
+    Busy,
+    #[error("integrity job not found")]
+    NotFound,
+    #[error("integrity operation was stopped")]
+    Stopped,
+    #[error("protected writes are not enabled on this coordinator")]
+    NotActivated,
+    #[error("unsupported migration bucket configuration")]
+    UnsupportedBucket,
+    #[error("integrity storage operation failed: {0}")]
+    Storage(#[from] StorageError),
+    #[error("integrity stream operation failed: {0}")]
+    Io(#[from] std::io::Error),
+    #[error("integrity checkpoint is malformed: {0}")]
+    Json(#[from] serde_json::Error),
+}
+
+fn check_bucket(bucket: &str) -> Result<()> {
+    crate::bucket::utils::check_valid_bucket_name_strict(bucket).map_err(|_| IntegrityError::Invalid("invalid source bucket"))
+}
+
+fn path(bucket: &str, id: Uuid) -> String {
+    format!("buckets/{bucket}/integrity-jobs/{id}.json")
+}
+
+pub async fn inventory(
+    api: Arc<ECStore>,
+    bucket: &str,
+    prefix: &str,
+    key_marker: Option<String>,
+    version_marker: Option<String>,
+    limit: i32,
+) -> Result<InventoryPage> {
+    check_bucket(bucket)?;
+    if !(1..=100).contains(&limit) || prefix.len() > 1024 || (version_marker.is_some() && key_marker.is_none()) {
+        return Err(IntegrityError::Invalid("invalid inventory page"));
+    }
+    let incarnation = api.bucket_incarnation_id_from_disk(bucket).await?;
+    let page = api
+        .clone()
+        .list_object_versions(bucket, prefix, key_marker.clone(), version_marker.clone(), None, limit)
+        .await?;
+    if page.is_truncated && (page.next_marker.clone(), page.next_version_idmarker.clone()) == (key_marker, version_marker) {
+        return Err(IntegrityError::Invalid("inventory cursor did not advance"));
+    }
+    let mut items = Vec::with_capacity(page.objects.len());
+    for listed in page.objects {
+        if listed.delete_marker {
+            items.push(Item::from_info(&listed));
+            continue;
+        }
+        let opts = ObjectOptions {
+            version_id: Some(listed.version_id.unwrap_or_else(Uuid::nil).to_string()),
+            include_part_checksums: true,
+            suppress_read_repair: true,
+            ..Default::default()
+        };
+        match api.get_object_info(bucket, &listed.name, &opts).await {
+            Ok(info) => items.push(Item::from_info(&info)),
+            Err(_) => {
+                let mut item = Item::from_info(&listed);
+                item.observation_error = Some("metadata_unavailable_or_changed".into());
+                item.protection = Protection::Unknown;
+                item.audit_unsupported = Some("metadata_unavailable_or_changed".into());
+                items.push(item);
+            }
+        }
+    }
+    if api.bucket_incarnation_id_from_disk(bucket).await? != incarnation {
+        return Err(IntegrityError::Conflict);
+    }
+    Ok(InventoryPage {
+        observed_at: time::OffsetDateTime::now_utc().to_string(),
+        bucket_incarnation: incarnation,
+        items,
+        is_truncated: page.is_truncated,
+        next_key_marker: page.next_marker,
+        next_version_marker: page.next_version_idmarker,
+    })
+}
+
+pub async fn create_job(api: Arc<ECStore>, bucket: &str, request: JobRequest) -> Result<Job> {
+    check_bucket(bucket)?;
+    request.validate()?;
+    if request.mode == JobMode::Migrate && !readiness().new_writes_enabled_here {
+        return Err(IntegrityError::NotActivated);
+    }
+    let incarnation = api.bucket_incarnation_id_from_disk(bucket).await?;
+    let job = Job {
+        format_version: 1,
+        id: Uuid::new_v4(),
+        bucket: bucket.into(),
+        bucket_incarnation: incarnation,
+        results: vec![ItemResult::default(); request.items.len()],
+        request,
+        state: JobState::Paused,
+        revision: 0,
+        last_error: None,
+    };
+    save(&api, &job, None, None).await?;
+    Ok(job)
+}
+
+pub async fn get_job(api: Arc<ECStore>, bucket: &str, id: Uuid) -> Result<Job> {
+    Ok(load(&api, bucket, id).await?.0)
+}
+
+async fn load(api: &Arc<ECStore>, bucket: &str, id: Uuid) -> Result<(Job, String)> {
+    check_bucket(bucket)?;
+    let (bytes, info) = match read_config_limited_preserve_empty_with_metadata(api.clone(), &path(bucket, id), 1024 * 1024).await
+    {
+        Ok(value) => value,
+        Err(StorageError::ConfigNotFound | StorageError::FileNotFound) => return Err(IntegrityError::NotFound),
+        Err(error) => return Err(error.into()),
+    };
+    let job: Job = serde_json::from_slice(&bytes)?;
+    job.request.validate()?;
+    if job.format_version != 1
+        || job.bucket != bucket
+        || job.id != id
+        || job.results.len() != job.request.items.len()
+        || api.bucket_incarnation_id_from_disk(bucket).await? != job.bucket_incarnation
+    {
+        return Err(IntegrityError::Conflict);
+    }
+    Ok((job, info.etag.ok_or(IntegrityError::Conflict)?))
+}
+
+async fn save(
+    api: &Arc<ECStore>,
+    job: &Job,
+    etag: Option<&str>,
+    worker: Option<&rustfs_lock::NamespaceLockGuard>,
+) -> Result<String> {
+    let fence = api
+        .acquire_bucket_incarnation_fence(&job.bucket, job.bucket_incarnation)
+        .await?;
+    let mut opts = ObjectOptions {
+        max_parity: true,
+        write_completion: WriteCompletion::TailDrained,
+        http_preconditions: Some(match etag {
+            Some(value) => HTTPPreconditions {
+                if_match: Some(value.into()),
+                ..Default::default()
+            },
+            None => HTTPPreconditions {
+                if_none_match: Some("*".into()),
+                ..Default::default()
+            },
+        }),
+        ..Default::default()
+    };
+    fence.attach_to_object_options(&mut opts);
+    if let Some(guard) = worker {
+        opts.add_namespace_lock_guard(guard);
+    }
+    let api = api.clone();
+    let job = job.clone();
+    // Keep the lifecycle guard through the drained commit even if the HTTP waiter is cancelled.
+    tokio::spawn(async move {
+        save_config_with_opts_quiet(api.clone(), &path(&job.bucket, job.id), serde_json::to_vec(&job)?, &opts).await?;
+        let (stored, next) = load(&api, &job.bucket, job.id).await?;
+        if stored.revision != job.revision {
+            return Err(IntegrityError::Conflict);
+        }
+        drop(fence);
+        Ok(next)
+    })
+    .await
+    .map_err(|_| IntegrityError::Invalid("checkpoint commit task failed"))?
+}
+
+/// Pause/cancel acknowledge intent. An in-flight conditional publication is allowed to drain.
+pub async fn control_job(api: Arc<ECStore>, bucket: &str, id: Uuid, cancel: bool) -> Result<Job> {
+    let (mut job, etag) = load(&api, bucket, id).await?;
+    match job.state {
+        JobState::Complete | JobState::Cancelled | JobState::CancelRequested => return Ok(job),
+        JobState::Paused | JobState::Failed => job.state = if cancel { JobState::Cancelled } else { JobState::Paused },
+        _ => {
+            job.state = if cancel {
+                JobState::CancelRequested
+            } else {
+                JobState::PauseRequested
+            }
+        }
+    }
+    job.revision = job.revision.checked_add(1).ok_or(IntegrityError::Conflict)?;
+    save(&api, &job, Some(&etag), None).await?;
+    Ok(job)
+}
+
+/// Resume is explicit after restart. The distributed lock excludes another worker, including another node.
+pub async fn resume_job(api: Arc<ECStore>, bucket: &str, id: Uuid) -> Result<Job> {
+    check_bucket(bucket)?;
+    let lock = api.new_ns_lock(RUSTFS_META_BUCKET, "integrity-jobs/worker.lock").await?;
+    let guard = lock
+        .get_write_lock_quiet(Duration::from_secs(1))
+        .await
+        .map_err(|_| IntegrityError::Busy)?;
+    let (mut job, etag) = load(&api, bucket, id).await?;
+    if matches!(job.state, JobState::Complete | JobState::Cancelled) {
+        return Ok(job);
+    }
+    if job.state == JobState::CancelRequested {
+        job.state = JobState::Cancelled;
+        job.revision = job.revision.checked_add(1).ok_or(IntegrityError::Conflict)?;
+        save(&api, &job, Some(&etag), Some(&guard)).await?;
+        return Ok(job);
+    }
+    if job.request.mode == JobMode::Migrate && !readiness().new_writes_enabled_here {
+        return Err(IntegrityError::NotActivated);
+    }
+    job.state = JobState::Running;
+    job.last_error = None;
+    job.revision = job.revision.checked_add(1).ok_or(IntegrityError::Conflict)?;
+    let etag = save(&api, &job, Some(&etag), Some(&guard)).await?;
+    let response = job.clone();
+    tokio::spawn(async move {
+        let result = runner::run(&api, &mut job, etag, &guard).await;
+        if let Err(error) = result {
+            // A failed write/CAS stops publication. Never overwrite a concurrent control request.
+            if let Ok((mut latest, etag)) = load(&api, &job.bucket, job.id).await {
+                match latest.state {
+                    JobState::PauseRequested => latest.state = JobState::Paused,
+                    JobState::CancelRequested => latest.state = JobState::Cancelled,
+                    JobState::Running if latest.revision <= job.revision => {
+                        latest.state = JobState::Failed;
+                        latest.last_error = Some(
+                            match error {
+                                IntegrityError::Conflict => "concurrent_change",
+                                IntegrityError::Stopped => "worker_stopped",
+                                IntegrityError::UnsupportedBucket => "unsupported_destination_bucket_configuration",
+                                IntegrityError::NotActivated => "protected_writes_not_activated",
+                                _ => "operation_failed_retry_requires_resume",
+                            }
+                            .into(),
+                        );
+                    }
+                    _ => return,
+                }
+                if let Some(revision) = latest.revision.checked_add(1) {
+                    latest.revision = revision;
+                    let _ = save(&api, &latest, Some(&etag), Some(&guard)).await;
+                }
+            }
+        }
+    });
+    Ok(response)
+}

crates/ecstore/src/services/integrity/model.rs

@@ -0,0 +1,343 @@
+// Copyright 2024 RustFS Team
+// Licensed under the Apache License, Version 2.0.
+
+use super::{IntegrityError, Result};
+use crate::object_api::ObjectInfo;
+use rustfs_rio::{Checksum, read_checksums};
+use serde::{Deserialize, Serialize};
+use sha2::{Digest, Sha256};
+use std::collections::{BTreeMap, HashSet};
+use uuid::Uuid;
+
+pub const MAX_ITEMS: usize = 64;
+pub const MAX_OBJECT_BYTES: u64 = 5 * 1024 * 1024 * 1024;
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct ItemRequest {
+    pub key: String,
+    pub version_id: Option<String>,
+    /// Base64 SHA-256 supplied by the administrator from an independent source.
+    pub expected_sha256: Option<String>,
+    pub target_key: Option<String>,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum JobMode {
+    Audit,
+    Migrate,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct JobRequest {
+    pub mode: JobMode,
+    pub items: Vec<ItemRequest>,
+    /// Bounds each source read, destination PUT, and verification stream; one worker runs per cluster.
+    pub bytes_per_second: u64,
+    pub max_object_bytes: u64,
+}
+
+impl JobRequest {
+    pub(crate) fn validate(&self) -> Result<()> {
+        if self.items.is_empty() || self.items.len() > MAX_ITEMS {
+            return Err(IntegrityError::Invalid("a job requires 1..=64 explicit source versions"));
+        }
+        if !(64 * 1024..=1024 * 1024 * 1024).contains(&self.bytes_per_second)
+            || self.max_object_bytes == 0
+            || self.max_object_bytes > MAX_OBJECT_BYTES
+        {
+            return Err(IntegrityError::Invalid("invalid byte rate or staging limit"));
+        }
+        let mut targets = HashSet::new();
+        let sources: HashSet<_> = self.items.iter().map(|i| i.key.as_str()).collect();
+        for item in &self.items {
+            validate_key(&item.key)?;
+            if let Some(version) = &item.version_id
+                && version != "null"
+                && Uuid::parse_str(version).ok().is_none_or(|id| id.is_nil())
+            {
+                return Err(IntegrityError::Invalid("invalid source version"));
+            }
+            if let Some(value) = &item.expected_sha256 {
+                sha256_bytes(value)?;
+            }
+            match (self.mode, &item.target_key) {
+                (JobMode::Audit, None) => {}
+                (JobMode::Migrate, Some(target)) => {
+                    validate_key(target)?;
+                    if sources.contains(target.as_str()) || !targets.insert(target) {
+                        return Err(IntegrityError::Invalid("targets must be distinct from every source and target"));
+                    }
+                }
+                _ => return Err(IntegrityError::Invalid("only migrations require an explicit target key")),
+            }
+        }
+        Ok(())
+    }
+}
+
+fn validate_key(key: &str) -> Result<()> {
+    if key.is_empty() || key.len() > 1024 || key.ends_with('/') || key.contains('\0') {
+        return Err(IntegrityError::Invalid("invalid object key"));
+    }
+    Ok(())
+}
+
+pub(crate) fn sha256_bytes(value: &str) -> Result<Vec<u8>> {
+    let raw = base64_simd::STANDARD
+        .decode_to_vec(value)
+        .map_err(|_| IntegrityError::Invalid("SHA256 must be canonical Base64"))?;
+    if raw.len() != 32 || base64_simd::STANDARD.encode_to_string(&raw) != value {
+        return Err(IntegrityError::Invalid("SHA256 must be canonical Base64 of 32 bytes"));
+    }
+    Ok(raw)
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum Protection {
+    Unknown,
+    Legacy,
+    IndependentCommitment,
+    InvalidDeclaration,
+    NoPayload,
+}
+
+pub(crate) fn protection(info: &ObjectInfo) -> Protection {
+    if info.delete_marker {
+        return Protection::NoPayload;
+    }
+    match rustfs_filemeta::shard_integrity::descriptor_from_metadata(&info.user_defined) {
+        Ok(None) if info.parts.iter().all(|p| p.integrity.is_none()) => Protection::Legacy,
+        Ok(Some(parts))
+            if parts.len() == info.parts.len()
+                && !parts.is_empty()
+                && parts
+                    .iter()
+                    .zip(info.parts.iter())
+                    .all(|(declared, part)| part.integrity.as_ref() == Some(declared)) =>
+        {
+            Protection::IndependentCommitment
+        }
+        _ => Protection::InvalidDeclaration,
+    }
+}
+
+/// Checksums shown in inventory are not automatically accepted as migration evidence.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct InventoryItem {
+    pub key: String,
+    /// S3 selector, including the literal "null" for the null version slot.
+    pub version_id: String,
+    pub data_dir: Option<Uuid>,
+    pub size: i64,
+    pub protection: Protection,
+    pub checksums: BTreeMap<String, String>,
+    pub checksum_is_multipart: bool,
+    pub audit_unsupported: Option<String>,
+    pub source_fingerprint: String,
+    pub observation_error: Option<String>,
+}
+
+impl InventoryItem {
+    pub(crate) fn from_info(info: &ObjectInfo) -> Self {
+        let (checksums, checksum_is_multipart) = info.checksum.as_ref().map_or_else(Default::default, |b| read_checksums(b, 0));
+        Self {
+            key: info.name.clone(),
+            version_id: info
+                .version_id
+                .filter(|id| !id.is_nil())
+                .map_or_else(|| "null".into(), |id| id.to_string()),
+            data_dir: info.data_dir,
+            size: info.size,
+            protection: protection(info),
+            checksums: checksums.into_iter().collect(),
+            checksum_is_multipart,
+            audit_unsupported: unsupported(info).map(str::to_owned),
+            source_fingerprint: fingerprint(info),
+            observation_error: None,
+        }
+    }
+}
+
+pub(crate) fn unsupported(info: &ObjectInfo) -> Option<&'static str> {
+    if info.delete_marker {
+        Some("delete_marker")
+    } else if !info.transitioned_object.status.is_empty() {
+        Some("transitioned_object")
+    } else if info.is_encrypted() || info.is_compressed() {
+        Some("transformed_object")
+    } else if info.is_multipart() {
+        Some("multipart_object")
+    } else if protection(info) == Protection::InvalidDeclaration {
+        Some("invalid_protection_declaration")
+    } else {
+        None
+    }
+}
+
+pub(crate) fn stored_sha256(info: &ObjectInfo) -> Option<String> {
+    let bytes = info.checksum.as_ref()?;
+    let (sums, multipart) = read_checksums(bytes, 0);
+    let value = sums.get("SHA256")?;
+    let canonical = Checksum::new_from_string("SHA256", value)?.to_bytes(&[]);
+    // The general display decoder tolerates truncated suffixes. Eligibility does not.
+    (!multipart && canonical.as_ref() == bytes.as_ref()).then(|| value.clone())
+}
+
+pub(crate) fn fingerprint(info: &ObjectInfo) -> String {
+    let mut hash = Sha256::new();
+    // Ordered metadata includes transformation and protection declarations as well as user metadata.
+    for value in [
+        info.bucket.clone(),
+        info.name.clone(),
+        format!("{:?}", info.version_id),
+        format!("{:?}", info.data_dir),
+        format!("{:?}", info.mod_time),
+        info.size.to_string(),
+        info.actual_size.to_string(),
+        format!("{:?}", info.etag),
+        format!("{:?}", info.checksum),
+        format!("{:?}", info.transitioned_object),
+        info.user_tags.to_string(),
+        format!("{:?}", info.expires),
+        format!("{:?}", info.content_type),
+        format!("{:?}", info.content_encoding),
+        format!("{:?}", info.storage_class),
+    ] {
+        hash.update(value.len().to_le_bytes());
+        hash.update(value.as_bytes());
+    }
+    for part in info.parts.iter() {
+        let values = format!(
+            "{}:{}:{}:{:?}:{:?}:{:?}:{:?}",
+            part.number, part.size, part.actual_size, part.etag, part.mod_time, part.index, part.integrity
+        );
+        hash.update(values.len().to_le_bytes());
+        hash.update(values.as_bytes());
+        if let Some(checksums) = &part.checksums {
+            let mut checksums: Vec<_> = checksums.iter().collect();
+            checksums.sort_unstable_by_key(|(key, _)| *key);
+            for (key, value) in checksums {
+                hash.update(key.len().to_le_bytes());
+                hash.update(key.as_bytes());
+                hash.update(value.len().to_le_bytes());
+                hash.update(value.as_bytes());
+            }
+        }
+    }
+    let mut metadata: Vec<_> = info.user_defined.iter().collect();
+    metadata.sort_unstable_by_key(|(key, _)| *key);
+    for (key, value) in metadata {
+        hash.update(key.len().to_le_bytes());
+        hash.update(key.as_bytes());
+        hash.update(value.len().to_le_bytes());
+        hash.update(value.as_bytes());
+    }
+    hex_simd::encode_to_string(hash.finalize(), hex_simd::AsciiCase::Lower)
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum JobState {
+    Paused,
+    Running,
+    PauseRequested,
+    CancelRequested,
+    Cancelled,
+    Complete,
+    Failed,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ItemState {
+    Pending,
+    Prepared,
+    Verified,
+    Migrated,
+    Unsupported,
+    Unavailable,
+    Mismatch,
+    Stale,
+    Conflict,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct ItemResult {
+    pub state: ItemState,
+    pub source_fingerprint: Option<String>,
+    pub expected_sha256: Option<String>,
+    pub evidence_source: Option<String>,
+    pub source_size: Option<u64>,
+    pub target_version_id: Option<Uuid>,
+    pub detail: Option<String>,
+}
+
+impl Default for ItemResult {
+    fn default() -> Self {
+        Self {
+            state: ItemState::Pending,
+            source_fingerprint: None,
+            expected_sha256: None,
+            evidence_source: None,
+            source_size: None,
+            target_version_id: None,
+            detail: None,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct Job {
+    pub format_version: u32,
+    pub id: Uuid,
+    pub bucket: String,
+    pub bucket_incarnation: Uuid,
+    pub request: JobRequest,
+    pub state: JobState,
+    pub revision: u64,
+    pub results: Vec<ItemResult>,
+    pub last_error: Option<String>,
+}
+
+#[derive(Debug, Serialize)]
+pub struct InventoryPage {
+    pub observed_at: String,
+    pub bucket_incarnation: Uuid,
+    pub items: Vec<InventoryItem>,
+    pub is_truncated: bool,
+    pub next_key_marker: Option<String>,
+    pub next_version_marker: Option<String>,
+}
+
+#[derive(Debug, Serialize)]
+pub struct Readiness {
+    pub local_reader_supported: bool,
+    pub write_requested: bool,
+    pub fleet_operator_attested: bool,
+    pub new_writes_enabled_here: bool,
+    pub fleet_capability_verified: bool,
+    pub unverified_requirements: [&'static str; 3],
+}
+
+pub fn readiness() -> Readiness {
+    let write = rustfs_utils::get_env_bool(rustfs_config::ENV_SHARD_INTEGRITY_WRITE, false);
+    let fleet = rustfs_utils::get_env_bool(rustfs_config::ENV_SHARD_INTEGRITY_FLEET_CONFIRMED, false);
+    Readiness {
+        local_reader_supported: true,
+        write_requested: write,
+        fleet_operator_attested: fleet,
+        new_writes_enabled_here: write && fleet,
+        fleet_capability_verified: false,
+        unverified_requirements: [
+            "peer_reader_writer_and_repair_capabilities",
+            "old_process_storage_admission",
+            "deployment_failure_and_performance_qualification",
+        ],
+    }
+}

crates/ecstore/src/services/integrity/runner.rs

@@ -0,0 +1,468 @@
+// Copyright 2024 RustFS Team
+// Licensed under the Apache License, Version 2.0.
+
+use super::model::{Protection, fingerprint, protection, sha256_bytes, stored_sha256, unsupported};
+use super::{IntegrityError, ItemState, Job, JobMode, JobState, Result, load, readiness, save};
+use crate::object_api::{
+    ObjectInfo, ObjectOptions, PutObjReader, ShardIntegrityWriteMode, WriteCompletion, without_get_object_body_cache_hook,
+};
+use crate::storage_api_contracts::object::{HTTPPreconditions, ObjectIO, ObjectOperations};
+use crate::store::ECStore;
+use rustfs_rio::{Checksum, HashReader};
+use sha2::{Digest, Sha256};
+use std::{collections::HashMap, sync::Arc, time::Duration};
+use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
+
+const MARKER: &str = "integrity-migration-job-v1";
+
+pub(super) async fn run(
+    api: &Arc<ECStore>,
+    job: &mut Job,
+    mut etag: String,
+    guard: &rustfs_lock::NamespaceLockGuard,
+) -> Result<()> {
+    without_get_object_body_cache_hook(async {
+        for index in 0..job.results.len() {
+            ensure_running(api, job, guard).await?;
+            if !matches!(
+                job.results[index].state,
+                ItemState::Pending | ItemState::Prepared | ItemState::Unavailable
+            ) {
+                continue;
+            }
+            if job.request.mode == JobMode::Migrate {
+                validate_destination(api, job).await?;
+            }
+            if job.results[index].state == ItemState::Prepared
+                && job.request.mode == JobMode::Migrate
+                && reconcile_target(api, job, index, guard).await?
+            {
+                etag = checkpoint(api, job, &etag, guard).await?;
+                continue;
+            }
+            let request = job.request.items[index].clone();
+            let opts = ObjectOptions {
+                version_id: request.version_id.as_ref().map(|version| {
+                    if version == "null" {
+                        uuid::Uuid::nil().to_string()
+                    } else {
+                        version.clone()
+                    }
+                }),
+                include_part_checksums: true,
+                suppress_read_repair: true,
+                ..Default::default()
+            };
+            let info = match api.get_object_info(&job.bucket, &request.key, &opts).await {
+                Ok(info) => info,
+                Err(_) => {
+                    job.results[index].state = ItemState::Unavailable;
+                    job.results[index].detail = Some("source_metadata_unavailable".into());
+                    etag = checkpoint(api, job, &etag, guard).await?;
+                    continue;
+                }
+            };
+            if let Some(reason) = unsupported(&info) {
+                job.results[index].state = ItemState::Unsupported;
+                job.results[index].detail = Some(reason.into());
+                etag = checkpoint(api, job, &etag, guard).await?;
+                continue;
+            }
+            let identity = fingerprint(&info);
+            if job.results[index]
+                .source_fingerprint
+                .as_ref()
+                .is_some_and(|old| old != &identity)
+            {
+                job.results[index].state = ItemState::Stale;
+                etag = checkpoint(api, job, &etag, guard).await?;
+                continue;
+            }
+            let expected = request.expected_sha256.clone().or_else(|| stored_sha256(&info));
+            let Some(expected) = expected else {
+                job.results[index].state = ItemState::Unsupported;
+                job.results[index].detail = Some("no_canonical_single_part_sha256".into());
+                etag = checkpoint(api, job, &etag, guard).await?;
+                continue;
+            };
+            let size = u64::try_from(info.size).map_err(|_| IntegrityError::Invalid("negative source size"))?;
+            if size > job.request.max_object_bytes {
+                job.results[index].state = ItemState::Unsupported;
+                job.results[index].detail = Some("object_exceeds_staging_budget".into());
+                etag = checkpoint(api, job, &etag, guard).await?;
+                continue;
+            }
+            let result = &mut job.results[index];
+            result.source_fingerprint = Some(identity);
+            result.expected_sha256 = Some(expected.clone());
+            result.evidence_source = Some(
+                if request.expected_sha256.is_some() {
+                    "administrator_supplied_sha256"
+                } else {
+                    "stored_sha256"
+                }
+                .into(),
+            );
+            result.source_size = Some(size);
+            result.state = ItemState::Prepared;
+            result.detail = None;
+            etag = checkpoint(api, job, &etag, guard).await?;
+            let mut file = match stage(api, job, index, &info, &opts, guard).await {
+                Ok(file) => file,
+                Err(IntegrityError::Conflict) => {
+                    job.results[index].state = ItemState::Stale;
+                    etag = checkpoint(api, job, &etag, guard).await?;
+                    continue;
+                }
+                Err(IntegrityError::Invalid("content_sha256_mismatch")) => {
+                    job.results[index].state = ItemState::Mismatch;
+                    etag = checkpoint(api, job, &etag, guard).await?;
+                    continue;
+                }
+                Err(error) => return Err(error),
+            };
+            ensure_running(api, job, guard).await?;
+            if job.request.mode == JobMode::Audit {
+                job.results[index].state = ItemState::Verified;
+            } else {
+                let snapshot = validate_destination(api, job).await?;
+                file.rewind().await?;
+                let raw = sha256_bytes(&expected)?;
+                let expected_hex = hex_simd::encode_to_string(raw, hex_simd::AsciiCase::Lower);
+                let mut reader = HashReader::from_stream(
+                    ThrottledFile::new(file, job.request.bytes_per_second),
+                    info.size,
+                    info.size,
+                    None,
+                    Some(expected_hex),
+                    false,
+                )?;
+                reader.add_non_trailing_checksum(Checksum::new_from_string("SHA256", &expected), false)?;
+                let mut body = PutObjReader::new(reader);
+                let mut metadata: HashMap<String, String> = info
+                    .user_defined
+                    .iter()
+                    .filter(|(key, _)| {
+                        key.starts_with("x-amz-meta-")
+                            || matches!(key.as_str(), "cache-control" | "content-disposition" | "content-language" | "expires")
+                    })
+                    .map(|(key, value)| (key.clone(), value.clone()))
+                    .collect();
+                if let Some(content_type) = &info.content_type {
+                    metadata.insert("content-type".into(), content_type.clone());
+                }
+                if let Some(content_encoding) = &info.content_encoding {
+                    metadata.insert("content-encoding".into(), content_encoding.clone());
+                }
+                if let Some(expires) = info.expires {
+                    metadata.insert(
+                        "expires".into(),
+                        expires
+                            .format(&time::format_description::well_known::Rfc3339)
+                            .map_err(|_| IntegrityError::Invalid("invalid source expiration"))?,
+                    );
+                }
+                if let Some(storage_class) = &info.storage_class {
+                    metadata.insert("x-amz-storage-class".into(), storage_class.clone());
+                }
+                if !info.user_tags.is_empty() {
+                    metadata.insert(rustfs_utils::http::headers::AMZ_OBJECT_TAGGING.into(), info.user_tags.to_string());
+                }
+                rustfs_utils::http::insert_str(&mut metadata, MARKER, marker(job, index)?);
+                let mut write = ObjectOptions {
+                    user_defined: metadata,
+                    preserve_delete_marker: true,
+                    shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Protected),
+                    write_completion: WriteCompletion::TailDrained,
+                    http_preconditions: Some(HTTPPreconditions {
+                        if_none_match: Some("*".into()),
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                };
+                write.add_namespace_lock_guard(guard);
+                write.expected_bucket_incarnation_id = Some(job.bucket_incarnation);
+                snapshot.add_lock_fences(&mut write);
+                write.object_lock_config_snapshot = Some(snapshot.clone());
+                let target = request
+                    .target_key
+                    .as_deref()
+                    .ok_or(IntegrityError::Invalid("missing target"))?;
+                // Even an error may follow a committed PUT. Reconcile before retrying; never overwrite.
+                let put = api.put_object(&job.bucket, target, &mut body, &write).await;
+                drop(write);
+                drop(snapshot);
+                if !reconcile_target(api, job, index, guard).await? {
+                    put?;
+                    return Err(IntegrityError::Conflict);
+                }
+            }
+            etag = checkpoint(api, job, &etag, guard).await?;
+        }
+        job.state = if job.results.iter().any(|item| item.state == ItemState::Unavailable) {
+            job.last_error = Some("source_unavailable_retry_requires_resume".into());
+            JobState::Failed
+        } else {
+            JobState::Complete
+        };
+        checkpoint(api, job, &etag, guard).await?;
+        Ok(())
+    })
+    .await
+}
+
+async fn checkpoint(api: &Arc<ECStore>, job: &mut Job, etag: &str, guard: &rustfs_lock::NamespaceLockGuard) -> Result<String> {
+    ensure_running(api, job, guard).await?;
+    job.revision = job.revision.checked_add(1).ok_or(IntegrityError::Conflict)?;
+    save(api, job, Some(etag), Some(guard)).await
+}
+
+async fn ensure_running(api: &Arc<ECStore>, job: &Job, guard: &rustfs_lock::NamespaceLockGuard) -> Result<()> {
+    if guard.lock_lost_signal().is_some_and(|signal| signal.is_lost()) {
+        return Err(IntegrityError::Stopped);
+    }
+    let (current, _) = load(api, &job.bucket, job.id).await?;
+    if current.state != JobState::Running {
+        return Err(IntegrityError::Stopped);
+    }
+    if current.revision != job.revision {
+        return Err(IntegrityError::Conflict);
+    }
+    Ok(())
+}
+
+async fn stage(
+    api: &Arc<ECStore>,
+    job: &Job,
+    index: usize,
+    info: &ObjectInfo,
+    opts: &ObjectOptions,
+    guard: &rustfs_lock::NamespaceLockGuard,
+) -> Result<tokio::fs::File> {
+    let mut reader = api
+        .get_object_reader(&job.bucket, &job.request.items[index].key, None, Default::default(), opts)
+        .await?;
+    if fingerprint(&reader.object_info) != fingerprint(info) {
+        return Err(IntegrityError::Conflict);
+    }
+    let std_file = tokio::task::spawn_blocking(tempfile::tempfile)
+        .await
+        .map_err(|_| IntegrityError::Invalid("staging task failed"))??;
+    let mut file = tokio::fs::File::from_std(std_file);
+    let expected = job.results[index]
+        .expected_sha256
+        .as_deref()
+        .ok_or(IntegrityError::Invalid("missing expected digest"))?;
+    copy_verified(api, job, &mut reader.stream, Some(&mut file), info.size, expected, guard).await?;
+    file.flush().await?;
+    let after = api.get_object_info(&job.bucket, &job.request.items[index].key, opts).await?;
+    if fingerprint(&after) != fingerprint(info) {
+        return Err(IntegrityError::Conflict);
+    }
+    Ok(file)
+}
+
+async fn copy_verified(
+    api: &Arc<ECStore>,
+    job: &Job,
+    input: &mut (dyn tokio::io::AsyncRead + Unpin + Send + Sync),
+    mut output: Option<&mut tokio::fs::File>,
+    size: i64,
+    expected: &str,
+    guard: &rustfs_lock::NamespaceLockGuard,
+) -> Result<()> {
+    let expected = sha256_bytes(expected)?;
+    let expected_size = u64::try_from(size).map_err(|_| IntegrityError::Invalid("negative object size"))?;
+    let mut buffer = vec![0u8; 64 * 1024];
+    let mut count = 0u64;
+    let mut hash = Sha256::new();
+    let started = tokio::time::Instant::now();
+    let mut checked = started;
+    loop {
+        if checked.elapsed() >= Duration::from_secs(1) {
+            ensure_running(api, job, guard).await?;
+            checked = tokio::time::Instant::now();
+        }
+        let n = tokio::time::timeout(Duration::from_secs(30), input.read(&mut buffer))
+            .await
+            .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "integrity read timed out"))??;
+        if n == 0 {
+            break;
+        }
+        count = count
+            .checked_add(u64::try_from(n).map_err(|_| IntegrityError::Invalid("read size overflow"))?)
+            .ok_or(IntegrityError::Invalid("read size overflow"))?;
+        if count > expected_size || count > job.request.max_object_bytes {
+            return Err(IntegrityError::Invalid("content_sha256_mismatch"));
+        }
+        hash.update(&buffer[..n]);
+        if let Some(file) = output.as_deref_mut() {
+            file.write_all(&buffer[..n]).await?;
+        }
+        let delay = Duration::from_secs_f64(count as f64 / job.request.bytes_per_second as f64);
+        tokio::time::sleep_until(started + delay).await;
+    }
+    if count != expected_size || hash.finalize().as_slice() != expected {
+        return Err(IntegrityError::Invalid("content_sha256_mismatch"));
+    }
+    Ok(())
+}
+
+fn marker(job: &Job, index: usize) -> Result<String> {
+    let result = &job.results[index];
+    Ok(format!(
+        "{}:{}:{}:{}",
+        job.id,
+        index,
+        result
+            .source_fingerprint
+            .as_deref()
+            .ok_or(IntegrityError::Invalid("missing source identity"))?,
+        result
+            .expected_sha256
+            .as_deref()
+            .ok_or(IntegrityError::Invalid("missing source digest"))?
+    ))
+}
+
+async fn reconcile_target(
+    api: &Arc<ECStore>,
+    job: &mut Job,
+    index: usize,
+    guard: &rustfs_lock::NamespaceLockGuard,
+) -> Result<bool> {
+    let key = job.request.items[index]
+        .target_key
+        .as_deref()
+        .ok_or(IntegrityError::Invalid("missing target"))?;
+    let opts = ObjectOptions {
+        include_part_checksums: true,
+        suppress_read_repair: true,
+        ..Default::default()
+    };
+    let target = match api.get_object_info(&job.bucket, key, &opts).await {
+        Ok(info) => info,
+        Err(crate::error::Error::FileNotFound | crate::error::Error::ObjectNotFound(_, _)) => return Ok(false),
+        Err(error) => return Err(error.into()),
+    };
+    let expected_marker = marker(job, index)?;
+    if rustfs_utils::http::get_consistent_str(&target.user_defined, MARKER) != Some(expected_marker.as_str()) {
+        job.results[index].state = ItemState::Conflict;
+        job.results[index].detail = Some("target_owned_by_another_operation".into());
+        return Ok(true);
+    }
+    if protection(&target) != Protection::IndependentCommitment {
+        return Err(IntegrityError::Invalid("target_not_protected"));
+    }
+    let mut reader = api
+        .get_object_reader(&job.bucket, key, None, Default::default(), &opts)
+        .await?;
+    if fingerprint(&target) != fingerprint(&reader.object_info) {
+        return Err(IntegrityError::Conflict);
+    }
+    let size = job.results[index]
+        .source_size
+        .ok_or(IntegrityError::Invalid("missing source size"))?;
+    let expected = job.results[index]
+        .expected_sha256
+        .clone()
+        .ok_or(IntegrityError::Invalid("missing source digest"))?;
+    copy_verified(
+        api,
+        job,
+        &mut reader.stream,
+        None,
+        i64::try_from(size).map_err(|_| IntegrityError::Invalid("source size overflow"))?,
+        &expected,
+        guard,
+    )
+    .await?;
+    let after = api.get_object_info(&job.bucket, key, &opts).await?;
+    if fingerprint(&after) != fingerprint(&target) {
+        return Err(IntegrityError::Conflict);
+    }
+    job.results[index].state = ItemState::Migrated;
+    job.results[index].target_version_id = target.version_id;
+    Ok(true)
+}
+
+async fn validate_destination(api: &Arc<ECStore>, job: &Job) -> Result<Arc<crate::object_api::ObjectLockConfigSnapshot>> {
+    if !readiness().new_writes_enabled_here {
+        return Err(IntegrityError::NotActivated);
+    }
+    let snapshot = api.object_lock_config_snapshot(&job.bucket).await?;
+    if snapshot
+        .metadata_transaction_guard_for(api.id, &job.bucket, Some(job.bucket_incarnation))
+        .is_none()
+    {
+        return Err(IntegrityError::Conflict);
+    }
+    let metadata = crate::bucket::metadata_sys::integrity_migration_metadata_in(&api.ctx, &job.bucket).await?;
+    // The direct storage writer is not the S3 orchestration layer. Reject configurations it cannot preserve.
+    if metadata.bucket_incarnation_id != job.bucket_incarnation {
+        return Err(IntegrityError::Conflict);
+    }
+    if !metadata.versioning_config_xml.is_empty()
+        || !metadata.encryption_config_xml.is_empty()
+        || !metadata.object_lock_config_xml.is_empty()
+        || !metadata.replication_config_xml.is_empty()
+        || !metadata.notification_config_xml.is_empty()
+        || !metadata.quota_config_json.is_empty()
+        || !metadata.on_demand_migration_config_json.is_empty()
+        || !metadata.lifecycle_config_xml.is_empty()
+        || !metadata.table_bucket_config_json.is_empty()
+        || !metadata.bucket_acl_config_json.is_empty()
+        || !metadata.logging_config_xml.is_empty()
+    {
+        return Err(IntegrityError::UnsupportedBucket);
+    }
+    Ok(snapshot)
+}
+
+/// Bound publication throughput as well as the source and destination verification reads.
+struct ThrottledFile {
+    file: tokio::fs::File,
+    started: tokio::time::Instant,
+    count: u64,
+    rate: u64,
+    sleep: std::pin::Pin<Box<tokio::time::Sleep>>,
+}
+
+impl ThrottledFile {
+    fn new(file: tokio::fs::File, rate: u64) -> Self {
+        let started = tokio::time::Instant::now();
+        Self {
+            file,
+            started,
+            count: 0,
+            rate,
+            sleep: Box::pin(tokio::time::sleep_until(started)),
+        }
+    }
+}
+
+impl tokio::io::AsyncRead for ThrottledFile {
+    fn poll_read(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        output: &mut tokio::io::ReadBuf<'_>,
+    ) -> std::task::Poll<std::io::Result<()>> {
+        use std::future::Future;
+        if self.sleep.as_mut().poll(cx).is_pending() {
+            return std::task::Poll::Pending;
+        }
+        let capacity = output.remaining().min(64 * 1024);
+        let mut buffer = tokio::io::ReadBuf::new(output.initialize_unfilled_to(capacity));
+        match std::pin::Pin::new(&mut self.file).poll_read(cx, &mut buffer) {
+            std::task::Poll::Ready(Ok(())) => {
+                let count = buffer.filled().len();
+                output.advance(count);
+                self.count += count as u64;
+                let deadline = self.started + Duration::from_secs_f64(self.count as f64 / self.rate as f64);
+                self.sleep.as_mut().reset(deadline);
+                std::task::Poll::Ready(Ok(()))
+            }
+            result => result,
+        }
+    }
+}

crates/ecstore/src/services/integrity/tests.rs

@@ -0,0 +1,387 @@
+// Copyright 2024 RustFS Team
+// Licensed under the Apache License, Version 2.0.
+
+use super::*;
+use crate::object_api::{ObjectOptions, PutObjReader, ShardIntegrityWriteMode};
+use crate::storage_api_contracts::{
+    bucket::{BucketOperations, MakeBucketOptions},
+    object::{ObjectIO, ObjectOperations},
+};
+use tokio::io::AsyncReadExt;
+
+async fn fixture() -> (Vec<tempfile::TempDir>, Arc<crate::store::ECStore>, String) {
+    let (dirs, store) = crate::services::rebalance::test_store_with_persisted_rebalance_meta(Default::default()).await;
+    crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+    let bucket = format!("integrity-{}", Uuid::new_v4());
+    store
+        .make_bucket(&bucket, &MakeBucketOptions::default())
+        .await
+        .expect("fixture bucket");
+    (dirs, store, bucket)
+}
+
+fn request(mode: JobMode, digest: String) -> JobRequest {
+    JobRequest {
+        mode,
+        items: vec![ItemRequest {
+            key: "source".into(),
+            version_id: Some("null".into()),
+            expected_sha256: Some(digest),
+            target_key: (mode == JobMode::Migrate).then(|| "target".into()),
+        }],
+        bytes_per_second: 1024 * 1024,
+        max_object_bytes: 16 * 1024 * 1024,
+    }
+}
+
+async fn finish(store: Arc<crate::store::ECStore>, bucket: &str, id: Uuid) -> Job {
+    tokio::time::timeout(std::time::Duration::from_secs(60), async {
+        loop {
+            let job = get_job(store.clone(), bucket, id).await.expect("durable job");
+            if job.state != JobState::Running {
+                return job;
+            }
+            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
+        }
+    })
+    .await
+    .expect("job must finish")
+}
+
+#[tokio::test]
+#[serial_test::serial(shard_integrity_rollout)]
+async fn audit_migration_and_ack_recovery_preserve_source_and_reject_conflicts() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_SHARD_INTEGRITY_WRITE, Some("true")),
+            (rustfs_config::ENV_SHARD_INTEGRITY_FLEET_CONFIRMED, Some("true")),
+        ],
+        async {
+            let (_dirs, store, bucket) = fixture().await;
+            let payload = vec![42; 256 * 1024 + 17];
+            let expected = rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::SHA256, &payload)
+                .expect("digest")
+                .encoded;
+            store
+                .put_object(
+                    &bucket,
+                    "source",
+                    &mut PutObjReader::from_vec(payload.clone()),
+                    &ObjectOptions {
+                        user_defined: [
+                            ("cache-control".into(), "max-age=300".into()),
+                            ("content-disposition".into(), "attachment; filename=sample.bin".into()),
+                            ("expires".into(), "2030-01-01T00:00:00Z".into()),
+                            (rustfs_utils::http::headers::AMZ_OBJECT_TAGGING.into(), "kind=archive".into()),
+                        ]
+                        .into(),
+                        shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy),
+                        write_completion: crate::object_api::WriteCompletion::TailDrained,
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("source");
+            let source = store
+                .get_object_info(&bucket, "source", &ObjectOptions::default())
+                .await
+                .expect("source metadata");
+            assert_eq!(source.user_tags.as_str(), "kind=archive", "source fixture tags");
+            let before = model::fingerprint(&source);
+            let page = inventory(store.clone(), &bucket, "", None, None, 1).await.expect("inventory");
+            assert_eq!(page.items.len(), 1);
+            assert_eq!(page.items[0].protection, Protection::Legacy);
+            assert_eq!(page.items[0].version_id, "null", "inventory must round-trip an explicit null selector");
+            let audit = create_job(store.clone(), &bucket, request(JobMode::Audit, expected.clone()))
+                .await
+                .expect("create audit");
+            resume_job(store.clone(), &bucket, audit.id).await.expect("resume audit");
+            let audited = finish(store.clone(), &bucket, audit.id).await;
+            assert_eq!(audited.state, JobState::Complete, "{audited:?}");
+            assert_eq!(audited.results[0].state, ItemState::Verified);
+
+            let migration = create_job(store.clone(), &bucket, request(JobMode::Migrate, expected.clone()))
+                .await
+                .expect("create migration");
+            resume_job(store.clone(), &bucket, migration.id)
+                .await
+                .expect("resume migration");
+            let migrated = finish(store.clone(), &bucket, migration.id).await;
+            assert_eq!(migrated.state, JobState::Complete, "{migrated:?}");
+            assert_eq!(migrated.results[0].state, ItemState::Migrated);
+            let mut reader = store
+                .get_object_reader(&bucket, "target", None, Default::default(), &ObjectOptions::default())
+                .await
+                .expect("target");
+            assert_eq!(model::protection(&reader.object_info), Protection::IndependentCommitment);
+            assert_eq!(reader.object_info.user_tags.as_str(), "kind=archive");
+            assert_eq!(reader.object_info.expires, source.expires);
+            assert_eq!(
+                reader.object_info.user_defined.get("cache-control"),
+                source.user_defined.get("cache-control")
+            );
+            assert_eq!(
+                reader.object_info.user_defined.get("content-disposition"),
+                source.user_defined.get("content-disposition")
+            );
+            let mut actual = Vec::new();
+            reader.stream.read_to_end(&mut actual).await.expect("target content");
+            assert_eq!(actual, payload);
+            let after = store
+                .get_object_info(&bucket, "source", &ObjectOptions::default())
+                .await
+                .expect("source after");
+            assert_eq!(model::fingerprint(&after), before);
+
+            // Simulate a lost completion checkpoint after the create-only PUT committed.
+            let (mut replay, etag) = load(&store, &bucket, migration.id).await.expect("load receipt");
+            replay.state = JobState::Paused;
+            replay.results[0].state = ItemState::Prepared;
+            replay.revision += 1;
+            save(&store, &replay, Some(&etag), None)
+                .await
+                .expect("lost acknowledgement fixture");
+            resume_job(store.clone(), &bucket, replay.id)
+                .await
+                .expect("reconcile after restart");
+            let reconciled = finish(store.clone(), &bucket, replay.id).await;
+            assert_eq!(reconciled.results[0].state, ItemState::Migrated, "{reconciled:?}");
+
+            let conflict = create_job(store.clone(), &bucket, request(JobMode::Migrate, expected))
+                .await
+                .expect("conflicting job");
+            resume_job(store.clone(), &bucket, conflict.id)
+                .await
+                .expect("resume conflict");
+            let conflict = finish(store.clone(), &bucket, conflict.id).await;
+            assert_eq!(conflict.results[0].state, ItemState::Conflict, "{conflict:?}");
+            let target = store
+                .get_object_info(&bucket, "target", &ObjectOptions::default())
+                .await
+                .expect("unchanged target");
+            assert_eq!(target.data_dir, reader.object_info.data_dir);
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+async fn mismatch_never_publishes_and_paused_job_survives_reload() {
+    let (_dirs, store, bucket) = fixture().await;
+    store
+        .put_object(
+            &bucket,
+            "source",
+            &mut PutObjReader::from_vec(b"actual".to_vec()),
+            &ObjectOptions {
+                shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy),
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("source");
+    let wrong = rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::SHA256, b"different")
+        .expect("digest")
+        .encoded;
+    let job = create_job(store.clone(), &bucket, request(JobMode::Audit, wrong))
+        .await
+        .expect("job");
+    assert_eq!(get_job(store.clone(), &bucket, job.id).await.expect("reload").state, JobState::Paused);
+    resume_job(store.clone(), &bucket, job.id).await.expect("resume");
+    let job = finish(store.clone(), &bucket, job.id).await;
+    assert_eq!(job.results[0].state, ItemState::Mismatch, "{job:?}");
+    assert!(
+        store
+            .get_object_info(&bucket, "target", &ObjectOptions::default())
+            .await
+            .is_err()
+    );
+}
+
+#[test]
+fn fingerprint_is_independent_of_metadata_insertion_order() {
+    let mut a = crate::object_api::ObjectInfo::default();
+    let mut first = std::collections::HashMap::new();
+    first.insert("a".into(), "one".into());
+    first.insert("b".into(), "two".into());
+    a.user_defined = Arc::new(first);
+    let mut b = a.clone();
+    let mut second = std::collections::HashMap::new();
+    second.insert("b".into(), "two".into());
+    second.insert("a".into(), "one".into());
+    b.user_defined = Arc::new(second);
+    assert_eq!(model::fingerprint(&a), model::fingerprint(&b));
+    b.data_dir = Some(Uuid::new_v4());
+    assert_ne!(model::fingerprint(&a), model::fingerprint(&b));
+}
+
+#[test]
+fn fingerprint_is_independent_of_part_checksum_insertion_order() {
+    let mut checksums = [
+        ("SHA256", "sha256"),
+        ("SHA1", "sha1"),
+        ("CRC32", "crc32"),
+        ("CRC32C", "crc32c"),
+    ];
+    let a = crate::object_api::ObjectInfo {
+        parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
+            checksums: Some(checksums.map(|(key, value)| (key.to_string(), value.to_string())).into()),
+            ..Default::default()
+        }]),
+        ..Default::default()
+    };
+    let mut b = a.clone();
+    let expected = model::fingerprint(&a);
+    for _ in 0..checksums.len() {
+        checksums.rotate_left(1);
+        Arc::make_mut(&mut b.parts)[0].checksums =
+            Some(checksums.map(|(key, value)| (key.to_string(), value.to_string())).into());
+        assert_eq!(expected, model::fingerprint(&b));
+    }
+    Arc::make_mut(&mut b.parts)[0]
+        .checksums
+        .as_mut()
+        .expect("part checksums")
+        .insert("SHA256".into(), "changed".into());
+    assert_ne!(expected, model::fingerprint(&b));
+}
+
+#[test]
+fn rejects_migration_into_a_source_key() {
+    let request = JobRequest {
+        mode: JobMode::Migrate,
+        items: vec![ItemRequest {
+            key: "source".into(),
+            version_id: None,
+            expected_sha256: None,
+            target_key: Some("source".into()),
+        }],
+        bytes_per_second: 1024 * 1024,
+        max_object_bytes: 1024,
+    };
+    assert!(request.validate().is_err());
+}
+
+#[test]
+fn rejects_truncated_checksum_suffix_as_migration_evidence() {
+    let checksum = rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::SHA256, b"payload").expect("checksum");
+    let mut info = crate::object_api::ObjectInfo {
+        checksum: Some(checksum.to_bytes(&[])),
+        ..Default::default()
+    };
+    assert_eq!(model::stored_sha256(&info), Some(checksum.encoded));
+    let mut bytes = info.checksum.take().expect("bytes").to_vec();
+    bytes.push(0x80);
+    info.checksum = Some(bytes.into());
+    assert!(model::stored_sha256(&info).is_none());
+}
+
+#[tokio::test]
+async fn unavailable_source_can_resume_and_cancel_intent_cannot_restart() {
+    let (_dirs, store, bucket) = fixture().await;
+    let payload = b"recovered".to_vec();
+    let digest = rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::SHA256, &payload)
+        .expect("digest")
+        .encoded;
+    let job = create_job(store.clone(), &bucket, request(JobMode::Audit, digest))
+        .await
+        .expect("create");
+    resume_job(store.clone(), &bucket, job.id).await.expect("start");
+    let failed = finish(store.clone(), &bucket, job.id).await;
+    assert_eq!(failed.state, JobState::Failed);
+    assert_eq!(failed.results[0].state, ItemState::Unavailable);
+    store
+        .put_object(
+            &bucket,
+            "source",
+            &mut PutObjReader::from_vec(payload),
+            &ObjectOptions {
+                shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy),
+                write_completion: WriteCompletion::TailDrained,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("recovered source");
+    resume_job(store.clone(), &bucket, job.id).await.expect("retry");
+    let completed = finish(store.clone(), &bucket, job.id).await;
+    assert_eq!(completed.state, JobState::Complete, "{completed:?}");
+    assert_eq!(completed.results[0].state, ItemState::Verified);
+
+    // A process restart may leave the durable cancel intent without a live worker.
+    let (mut interrupted, etag) = load(&store, &bucket, job.id).await.expect("load");
+    interrupted.state = JobState::CancelRequested;
+    interrupted.revision += 1;
+    save(&store, &interrupted, Some(&etag), None).await.expect("persist intent");
+    assert_eq!(
+        control_job(store.clone(), &bucket, job.id, false).await.expect("pause").state,
+        JobState::CancelRequested
+    );
+    assert_eq!(
+        resume_job(store.clone(), &bucket, job.id)
+            .await
+            .expect("recover cancel")
+            .state,
+        JobState::Cancelled
+    );
+}
+
+#[tokio::test]
+async fn pause_and_cancel_are_durable_during_content_read() {
+    let (_dirs, store, bucket) = fixture().await;
+    let payload = vec![7; 512 * 1024];
+    let digest = rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::SHA256, &payload)
+        .expect("digest")
+        .encoded;
+    store
+        .put_object(
+            &bucket,
+            "source",
+            &mut PutObjReader::from_vec(payload),
+            &ObjectOptions {
+                shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy),
+                write_completion: WriteCompletion::TailDrained,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("source");
+    let mut req = request(JobMode::Audit, digest);
+    req.bytes_per_second = 64 * 1024;
+    let job = create_job(store.clone(), &bucket, req).await.expect("create");
+    resume_job(store.clone(), &bucket, job.id).await.expect("start");
+    tokio::time::timeout(std::time::Duration::from_secs(20), async {
+        loop {
+            if get_job(store.clone(), &bucket, job.id).await.expect("status").results[0].state == ItemState::Prepared {
+                break;
+            }
+            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
+        }
+    })
+    .await
+    .expect("prepared");
+    let intent = control_job(store.clone(), &bucket, job.id, false).await.expect("pause");
+    assert_eq!(intent.state, JobState::PauseRequested);
+    tokio::time::timeout(std::time::Duration::from_secs(20), async {
+        loop {
+            let paused = get_job(store.clone(), &bucket, job.id).await.expect("status");
+            if paused.state == JobState::Paused {
+                break;
+            }
+            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
+        }
+    })
+    .await
+    .expect("pause acknowledged");
+    assert_eq!(
+        control_job(store.clone(), &bucket, job.id, true).await.expect("cancel").state,
+        JobState::Cancelled
+    );
+    assert_eq!(
+        resume_job(store.clone(), &bucket, job.id)
+            .await
+            .expect("terminal resume")
+            .state,
+        JobState::Cancelled
+    );
+}

crates/ecstore/src/services/mod.rs

@@ -16,6 +16,7 @@
 
 pub(crate) mod batch_processor;
 pub(crate) mod event_notification;
+pub(crate) mod integrity;
 pub(crate) mod metrics_realtime;
 pub(crate) mod notification_sys;
 pub(crate) mod rebalance;

crates/ecstore/src/services/notification_sys.rs

@@ -42,6 +42,7 @@ use std::sync::{
     atomic::{AtomicBool, AtomicUsize, Ordering},
 };
 use std::time::{Duration, Instant, SystemTime};
+use tokio::sync::Notify;
 use tokio::time::{sleep, timeout};
 use tokio_util::sync::CancellationToken;
 use tracing::{debug, error, info, warn};
@@ -61,6 +62,14 @@ const TIER_CONFIG_RELOAD_RETRY_CAP: Duration = Duration::from_secs(5);
 const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10);
 const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
 const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
+/// Fallback poll cadence while startup has not yet published the notification
+/// system. IAM finalization starts the probe before `init_notification_runtime`
+/// runs, so the first pass always fails closed; waiting a full probe interval
+/// there left a single node without its durable quota capability for ten
+/// seconds after `/health` reported ok (rustfs/rustfs#8014). Publication wakes
+/// the probe immediately through `NOTIFICATION_SYS_PUBLISHED`; this bound only
+/// covers a wakeup that races the availability check.
+const REMOTE_VERSION_STATE_PROBE_BOOTSTRAP_INTERVAL: Duration = Duration::from_millis(100);
 const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 2;
 const TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION: u32 = 3;
 const DECOMMISSION_TARGET_FENCE_POLICY_SUPPORTED_VERSION: u32 = 4;
@@ -338,6 +347,9 @@ static LEGACY_TRANSITION_STATE_RECONCILE_FLEET_PROOF: OnceLock<std::sync::RwLock
 static ILM_RECOVERY_EXPORT_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
 static TRANSITION_TRANSACTION_COMPACTION_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
 static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
+/// Signalled once `GLOBAL_NOTIFICATION_SYS` is published so the fleet probe can
+/// run its first real pass without waiting for the bootstrap poll.
+static NOTIFICATION_SYS_PUBLISHED: Notify = Notify::const_new();
 static ILM_RECOVERY_EXPORT_LOCAL_PROCESS_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4);
 
 fn cross_pool_fence_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
@@ -368,6 +380,24 @@ fn transition_transaction_compaction_fleet_proof_slot() -> &'static std::sync::R
     TRANSITION_TRANSACTION_COMPACTION_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
 }
 
+fn all_fleet_capability_proof_slots() -> [&'static std::sync::RwLock<FleetCapabilityProofState>; 7] {
+    [
+        remote_version_state_fleet_proof_slot(),
+        cross_pool_fence_fleet_proof_slot(),
+        tier_delete_journal_fleet_proof_slot(),
+        decommission_target_fence_fleet_proof_slot(),
+        legacy_transition_state_reconcile_fleet_proof_slot(),
+        ilm_recovery_export_fleet_proof_slot(),
+        transition_transaction_compaction_fleet_proof_slot(),
+    ]
+}
+
+fn revoke_all_fleet_capability_proofs() {
+    for slot in all_fleet_capability_proof_slots() {
+        revoke_fleet_capability_proof(slot);
+    }
+}
+
 fn revoke_fleet_capability_proof_state(state: &mut FleetCapabilityProofState) {
     if let Some(proof) = state.proof.take() {
         proof.generation.revoke();
@@ -453,6 +483,20 @@ pub(crate) fn acquire_remote_version_state_fleet_proof() -> Option<RemoteVersion
     acquire_fleet_capability_proof_from(&state, expected_topology, Instant::now()).map(RemoteVersionStateFleetProofToken)
 }
 
+pub(crate) fn acquire_remote_version_state_writer_fleet_proof() -> Option<RemoteVersionStateFleetProofToken> {
+    let requested = rustfs_utils::get_env_bool(
+        rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE,
+        rustfs_config::DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE,
+    );
+    let fleet_confirmed = rustfs_utils::get_env_bool(
+        rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
+        rustfs_config::DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
+    );
+    (requested && fleet_confirmed)
+        .then(acquire_remote_version_state_fleet_proof)
+        .flatten()
+}
+
 fn acquire_fleet_capability_proof_from(
     state: &FleetCapabilityProofState,
     expected_topology: &str,
@@ -1212,15 +1256,7 @@ fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap<String, Uuid>, pe
 pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
     if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.clone()).is_err() {
         if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() != Some(&topology_fingerprint) {
-            for slot in [
-                remote_version_state_fleet_proof_slot(),
-                cross_pool_fence_fleet_proof_slot(),
-                tier_delete_journal_fleet_proof_slot(),
-                decommission_target_fence_fleet_proof_slot(),
-                legacy_transition_state_reconcile_fleet_proof_slot(),
-                ilm_recovery_export_fleet_proof_slot(),
-                transition_transaction_compaction_fleet_proof_slot(),
-            ] {
+            for slot in all_fleet_capability_proof_slots() {
                 mark_fleet_capability_topology_conflict(slot);
             }
         }
@@ -1228,53 +1264,59 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
     }
 
     tokio::spawn(async move {
+        let mut notification_sys_unavailable_logged = false;
         loop {
-            let notification_sys = get_global_notification_sys();
-            let remote_version_state_probe = async {
-                match notification_sys.as_ref() {
-                    Some(notification_sys) => timeout(
-                        REMOTE_VERSION_STATE_PROBE_TIMEOUT,
-                        notification_sys.probe_remote_version_state_fleet(&topology_fingerprint),
-                    )
-                    .await
-                    .unwrap_or_else(|_| Err(Error::other("remote version state fleet capability probe timed out"))),
-                    None => Err(Error::other("remote version state fleet capability notification system is unavailable")),
+            let Some(notification_sys) = get_global_notification_sys() else {
+                // Startup publishes the notification system after this probe
+                // is started. Stay failed closed, but retry on the bootstrap
+                // cadence so a single node gains its capabilities as soon as
+                // the system appears instead of one full probe interval later.
+                revoke_all_fleet_capability_proofs();
+                if !notification_sys_unavailable_logged {
+                    notification_sys_unavailable_logged = true;
+                    debug!(
+                        event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_NOTIFICATION,
+                        state = "waiting_for_notification_system",
+                        "notification capability probe"
+                    );
                 }
+                let _ = timeout(REMOTE_VERSION_STATE_PROBE_BOOTSTRAP_INTERVAL, NOTIFICATION_SYS_PUBLISHED.notified()).await;
+                continue;
+            };
+            notification_sys_unavailable_logged = false;
+            let remote_version_state_probe = async {
+                timeout(
+                    REMOTE_VERSION_STATE_PROBE_TIMEOUT,
+                    notification_sys.probe_remote_version_state_fleet(&topology_fingerprint),
+                )
+                .await
+                .unwrap_or_else(|_| Err(Error::other("remote version state fleet capability probe timed out")))
             };
             let cross_pool_fence_probe = async {
-                match notification_sys.as_ref() {
-                    Some(notification_sys) => timeout(
-                        REMOTE_VERSION_STATE_PROBE_TIMEOUT,
-                        notification_sys.probe_cross_pool_fence_fleet(&topology_fingerprint),
-                    )
-                    .await
-                    .unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))),
-                    None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")),
-                }
+                timeout(
+                    REMOTE_VERSION_STATE_PROBE_TIMEOUT,
+                    notification_sys.probe_cross_pool_fence_fleet(&topology_fingerprint),
+                )
+                .await
+                .unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out")))
             };
             let recovery_export_probe = async {
-                match notification_sys.as_ref() {
-                    Some(notification_sys) => timeout(
-                        REMOTE_VERSION_STATE_PROBE_TIMEOUT,
-                        notification_sys.probe_ilm_recovery_export_fleet(&topology_fingerprint),
-                    )
-                    .await
-                    .unwrap_or_else(|_| Err(Error::other("ILM recovery export fleet capability probe timed out"))),
-                    None => Err(Error::other("ILM recovery export fleet capability notification system is unavailable")),
-                }
+                timeout(
+                    REMOTE_VERSION_STATE_PROBE_TIMEOUT,
+                    notification_sys.probe_ilm_recovery_export_fleet(&topology_fingerprint),
+                )
+                .await
+                .unwrap_or_else(|_| Err(Error::other("ILM recovery export fleet capability probe timed out")))
             };
             let transition_transaction_compaction_probe = async {
-                match notification_sys.as_ref() {
-                    Some(notification_sys) => timeout(
-                        REMOTE_VERSION_STATE_PROBE_TIMEOUT,
-                        notification_sys.probe_transition_transaction_compaction_fleet(&topology_fingerprint),
-                    )
-                    .await
-                    .unwrap_or_else(|_| Err(Error::other("transition transaction compaction fleet capability probe timed out"))),
-                    None => Err(Error::other(
-                        "transition transaction compaction fleet capability notification system is unavailable",
-                    )),
-                }
+                timeout(
+                    REMOTE_VERSION_STATE_PROBE_TIMEOUT,
+                    notification_sys.probe_transition_transaction_compaction_fleet(&topology_fingerprint),
+                )
+                .await
+                .unwrap_or_else(|_| Err(Error::other("transition transaction compaction fleet capability probe timed out")))
             };
             let (result, fence_probe, recovery_export_result, transition_transaction_compaction_result) = tokio::join!(
                 remote_version_state_probe,
@@ -1299,13 +1341,7 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
                 .unwrap_or_else(std::sync::PoisonError::into_inner)
                 .topology_conflict;
             if topology_conflict {
-                revoke_fleet_capability_proof(remote_version_state_fleet_proof_slot());
-                revoke_fleet_capability_proof(cross_pool_fence_fleet_proof_slot());
-                revoke_fleet_capability_proof(tier_delete_journal_fleet_proof_slot());
-                revoke_fleet_capability_proof(decommission_target_fence_fleet_proof_slot());
-                revoke_fleet_capability_proof(legacy_transition_state_reconcile_fleet_proof_slot());
-                revoke_fleet_capability_proof(ilm_recovery_export_fleet_proof_slot());
-                revoke_fleet_capability_proof(transition_transaction_compaction_fleet_proof_slot());
+                revoke_all_fleet_capability_proofs();
             } else if let Some(err) = publish_fleet_capability_probe_result(
                 remote_version_state_fleet_proof_slot(),
                 &topology_fingerprint,
@@ -1428,9 +1464,11 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
 }
 
 pub async fn new_global_notification_sys(eps: EndpointServerPools) -> Result<()> {
-    let _ = GLOBAL_NOTIFICATION_SYS
-        .set(Arc::new(NotificationSys::new(eps).await))
-        .map_err(|_| Error::other("init notification_sys fail"));
+    if GLOBAL_NOTIFICATION_SYS.set(Arc::new(NotificationSys::new(eps).await)).is_ok() {
+        // `notify_one` stores a permit, so a probe that checks availability
+        // just before this publication still wakes without losing the signal.
+        NOTIFICATION_SYS_PUBLISHED.notify_one();
+    }
     Ok(())
 }
 
@@ -3905,6 +3943,49 @@ mod tests {
         stats
     }
 
+    /// rustfs/rustfs#8014: IAM finalization starts the fleet probe before the
+    /// notification system is published. The first probe therefore fails
+    /// closed, and a single node used to wait a full probe interval before the
+    /// durable quota capability appeared, even though `/health` was already
+    /// reporting ok. The probe must retry promptly while it waits for startup
+    /// to publish the notification system.
+    #[tokio::test]
+    async fn fleet_probe_publishes_promptly_once_the_notification_system_appears() {
+        if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get().is_some() || GLOBAL_NOTIFICATION_SYS.get().is_some() {
+            // Process-wide state was bound by another test in this binary. The
+            // startup ordering under test needs a fresh process; nextest
+            // (the authoritative runner) always provides one.
+            eprintln!("skipping: fleet probe globals already bound in this process");
+            return;
+        }
+
+        // Share the fingerprint every other proof-installing test in this crate
+        // binds, so the plain `cargo test` fallback cannot see two topologies.
+        start_remote_version_state_fleet_probe("object-transaction-fencing-test".to_string());
+        sleep(Duration::from_millis(300)).await;
+        assert!(
+            acquire_cross_pool_fence_fleet_proof().is_none(),
+            "no capability proof may exist before the notification system is published"
+        );
+
+        let published_at = Instant::now();
+        new_global_notification_sys(EndpointServerPools::default())
+            .await
+            .expect("single-node notification system initializes");
+
+        let observed = timeout(Duration::from_secs(2), async {
+            while acquire_cross_pool_fence_fleet_proof().is_none() {
+                sleep(Duration::from_millis(20)).await;
+            }
+        })
+        .await;
+        assert!(
+            observed.is_ok(),
+            "single-node cross-pool fence proof should publish within 2s of the notification system, waited {:?}",
+            published_at.elapsed()
+        );
+    }
+
     #[test]
     fn merging_peer_rings_sums_each_tier_without_double_counting() {
         let mut cluster = DailyAllTierStats::from([("WARM".to_string(), ring(10))]);

crates/ecstore/src/services/rebalance/control.rs

@@ -9,11 +9,12 @@ use super::meta::{
 };
 use super::worker::{
     rebalance_max_attempts, rebalance_meta_lock_error, resolve_load_rebalance_stats_update_result,
-    resolve_rebalance_meta_load_result, resolve_rebalance_meta_save_result, retry_rebalance_metadata_access,
+    resolve_rebalance_deferred_last_error, resolve_rebalance_meta_load_result, resolve_rebalance_meta_save_result,
+    retry_rebalance_metadata_access,
 };
 use super::{
     DiskStat, EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE, REBAL_META_NAME,
-    RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord,
+    RebalStatus, RebalanceDeferKind, RebalanceInfo, RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord,
     encode_rebalance_stop_propagation_record,
 };
 use crate::core::pools::{
@@ -1103,6 +1104,7 @@ impl ECStore {
         bucket: String,
         last_error: String,
         expected_id: &str,
+        kind: RebalanceDeferKind,
     ) -> Result<()> {
         let mut rebalance_meta = self.rebalance_meta.write().await;
         ensure_rebalance_worker_active(rebalance_meta.as_ref(), expected_id, "defer rebalance bucket")?;
@@ -1116,7 +1118,8 @@ impl ECStore {
         };
 
         defer_bucket_in_rebalance_queue(pool_stat, &bucket)?;
-        pool_stat.info.last_error = Some(last_error);
+        let pending_entry_defer = pool_stat.info.last_error.clone();
+        pool_stat.info.last_error = resolve_rebalance_deferred_last_error(kind, pending_entry_defer.as_deref(), &last_error);
         meta.last_refreshed_at = Some(OffsetDateTime::now_utc());
         Ok(())
     }

crates/ecstore/src/services/rebalance/entry.rs

@@ -1209,7 +1209,7 @@ mod tests {
             cancel: &warning_cancel,
         };
         let warning_result = store
-            .finish_rebalance_entry_after_cleanup(&warning_cleanup_context, async { Err(Error::SlowDown.into()) })
+            .finish_rebalance_entry_after_cleanup(&warning_cleanup_context, async { Err(Error::FileAccessDenied.into()) })
             .await
             .expect("cleanup warnings should not fail the completed migration");
         assert!(matches!(warning_result, RebalanceEntryCleanupResult::Completed { warning: Some(_) }));
@@ -1250,6 +1250,35 @@ mod tests {
         let pool_stats = &meta.as_ref().expect("rebalance metadata should exist").pool_stats[0];
         assert_eq!(pool_stats.bytes, 0, "deferred cleanup must not commit completion stats");
         assert_eq!(pool_stats.cleanup_warnings.count, 1, "deferred cleanup must not add a permanent warning");
+        drop(meta);
+
+        let transient_guard = store
+            .rebalance_run_guard(rebalance_id, "rebalance transient cleanup deferral test")
+            .await
+            .expect("rebalance transient cleanup deferral test guard should be acquired");
+        let transient_cancel = CancellationToken::new();
+        let transient_stats_updates = [&warning_version];
+        let transient_cleanup_context = RebalanceEntryCleanupContext {
+            run_guard: &transient_guard,
+            pool_index: 0,
+            bucket: "bucket",
+            object: "object.bin",
+            stats_updates: &transient_stats_updates,
+            expected_id: rebalance_id,
+            cancel: &transient_cancel,
+        };
+        let transient = store
+            .finish_rebalance_entry_after_cleanup(&transient_cleanup_context, async { Err(Error::SlowDown.into()) })
+            .await
+            .expect("retryable cleanup failures should defer without failing the worker");
+        assert!(matches!(transient, RebalanceEntryCleanupResult::Deferred { .. }));
+        let meta = store.rebalance_meta.read().await;
+        let pool_stats = &meta.as_ref().expect("rebalance metadata should exist").pool_stats[0];
+        assert_eq!(pool_stats.bytes, 0, "retryable cleanup failures must not commit completion stats");
+        assert_eq!(
+            pool_stats.cleanup_warnings.count, 1,
+            "retryable cleanup failures must not add a permanent warning"
+        );
     }
 
     #[tokio::test]

crates/ecstore/src/services/rebalance/mod.rs

@@ -29,7 +29,7 @@ const REBAL_META_FMT: u16 = 1; // Replace with actual format value
 const REBAL_META_VER: u16 = 1; // Replace with actual version value
 pub(crate) const REBAL_META_NAME: &str = "rebalance.bin";
 const DEFAULT_REBALANCE_MAX_ATTEMPTS: usize = 3;
-pub(crate) const REBALANCE_SOURCE_CLEANUP_MAX_DEFERS: usize = 3;
+pub(crate) const REBALANCE_SOURCE_CLEANUP_MAX_DEFERS: usize = 8;
 const REBALANCE_MAX_ATTEMPTS_ENV: &str = "RUSTFS_REBALANCE_MAX_ATTEMPTS";
 const REBALANCE_STOP_PROPAGATION_ERROR_PREFIX: &str = "rebalance stop propagation incomplete: ";
 const REBALANCE_LISTING_RETRY_BASE_DELAY: Duration = Duration::from_millis(250);
@@ -55,7 +55,7 @@ pub use types::{
     DiskStat, RebalSaveOpt, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceInfo, RebalanceMeta,
     RebalanceStats, RebalanceStopPropagationRecord,
 };
-use types::{RebalanceBucketConfigs, RebalanceBucketOutcome, RebalanceEntryOutcome};
+use types::{RebalanceBucketConfigs, RebalanceBucketOutcome, RebalanceDeferKind, RebalanceEntryOutcome};
 
 #[cfg(any(test, feature = "test-util"))]
 pub async fn test_store_with_persisted_rebalance_meta(

crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs

@@ -34,27 +34,29 @@ use super::migration::{
 };
 use super::runtime::{
     RebalanceLocalActivationOutcome, commit_local_rebalance_worker_activation,
-    commit_local_rebalance_worker_activation_candidate, should_fail_repeated_rebalance_bucket_defer,
-    source_cleanup_defer_attempt, stage_local_rebalance_worker_activation,
+    commit_local_rebalance_worker_activation_candidate, reached_rebalance_source_cleanup_defer_limit,
+    should_fail_repeated_rebalance_bucket_defer, source_cleanup_defer_attempt, stage_local_rebalance_worker_activation,
 };
 use super::worker::{
     RebalanceEntryCleanupResult, ensure_rebalance_listing_disks_available, is_transient_rebalance_error,
     parse_rebalance_max_attempts, rebalance_listing_retry_delay, rebalance_migration_retry_delay,
     resolve_load_rebalance_stats_update_result, resolve_rebalance_bucket_error, resolve_rebalance_bucket_result,
-    resolve_rebalance_entry_cleanup_delete_result, resolve_rebalance_file_info_versions_result,
-    resolve_rebalance_meta_load_result, resolve_rebalance_meta_save_result, resolve_rebalance_migrate_result_error,
-    resolve_rebalance_optional_bucket_config_result, resolve_rebalance_save_task_result, resolve_rebalance_stats_update_result,
-    resolve_rebalance_terminal_error, resolve_rebalance_worker_result, run_rebalance_listing_with_retry,
-    send_rebalance_done_signal, should_cleanup_rebalance_source_entry, should_count_rebalance_version_complete,
-    should_defer_rebalance_entry_failure, should_retry_rebalance_listing, should_skip_rebalance_delete_marker,
-    wait_rebalance_entry_tasks, wait_rebalance_listing_retry, with_rebalance_entry_context,
+    resolve_rebalance_deferred_last_error, resolve_rebalance_entry_cleanup_delete_result,
+    resolve_rebalance_file_info_versions_result, resolve_rebalance_meta_load_result, resolve_rebalance_meta_save_result,
+    resolve_rebalance_migrate_result_error, resolve_rebalance_optional_bucket_config_result, resolve_rebalance_save_task_result,
+    resolve_rebalance_stats_update_result, resolve_rebalance_terminal_error, resolve_rebalance_worker_result,
+    run_rebalance_listing_with_retry, send_rebalance_done_signal, should_cleanup_rebalance_source_entry,
+    should_count_rebalance_version_complete, should_defer_rebalance_entry_failure, should_retry_rebalance_listing,
+    should_skip_rebalance_delete_marker, wait_rebalance_entry_tasks, wait_rebalance_listing_retry, with_rebalance_entry_context,
 };
 use super::{
     DiskStat, GetObjectReader, ObjectInfo, ObjectOptions, RebalSaveOpt, RebalStatus, RebalanceBucketConfigs,
-    RebalanceBucketOutcome, RebalanceCleanupWarnings, RebalanceEntryOutcome, RebalanceInfo, RebalanceMeta, RebalanceStats,
-    RebalanceStopPropagationRecord,
+    RebalanceBucketOutcome, RebalanceCleanupWarnings, RebalanceDeferKind, RebalanceEntryOutcome, RebalanceInfo, RebalanceMeta,
+    RebalanceStats, RebalanceStopPropagationRecord,
+};
+use super::{
+    REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX, REBALANCE_SOURCE_CLEANUP_MAX_DEFERS,
 };
-use super::{REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX};
 use crate::bucket::replication::{ReplicationState, ReplicationStatusType, replication_state_to_filemeta};
 use crate::data_movement;
 use crate::data_movement::SourceCleanupError;
@@ -1775,7 +1777,7 @@ fn test_resolve_rebalance_entry_cleanup_delete_result_ignores_not_found() {
 
 #[test]
 fn test_resolve_rebalance_entry_cleanup_delete_result_returns_warning_for_failures() {
-    let result = resolve_rebalance_entry_cleanup_delete_result(Err(Error::SlowDown.into()), "bucket-a", "obj.txt");
+    let result = resolve_rebalance_entry_cleanup_delete_result(Err(Error::FileAccessDenied.into()), "bucket-a", "obj.txt");
     assert!(matches!(
         result,
         RebalanceEntryCleanupResult::Completed { warning: Some(ref message) }
@@ -1783,6 +1785,122 @@ fn test_resolve_rebalance_entry_cleanup_delete_result_returns_warning_for_failur
     ));
 }
 
+#[test]
+fn test_resolve_rebalance_entry_cleanup_delete_result_defers_transient_failures() {
+    let cases = [
+        (Error::SlowDown, "slow down"),
+        (
+            Error::Lock(rustfs_lock::LockError::timeout("bucket-a/obj.txt@latest", Duration::from_secs(5))),
+            "object lock timeout",
+        ),
+        (
+            Error::Lock(rustfs_lock::LockError::network(
+                "peer unavailable",
+                std::io::Error::from(std::io::ErrorKind::ConnectionReset),
+            )),
+            "object lock network failure",
+        ),
+        (Error::ErasureWriteQuorum, "write quorum"),
+        (Error::Io(std::io::Error::from(std::io::ErrorKind::TimedOut)), "io timeout"),
+        (
+            Error::other("Lock error: Lock acquisition timeout for resource 'bucket-a/obj.txt@latest' after 5s"),
+            "rendered lock timeout text",
+        ),
+    ];
+
+    for (err, label) in cases {
+        match resolve_rebalance_entry_cleanup_delete_result(Err(err.into()), "bucket-a", "obj.txt") {
+            RebalanceEntryCleanupResult::Deferred { last_error } => {
+                assert!(
+                    last_error.starts_with(REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX),
+                    "{label}: {last_error}"
+                );
+                assert!(last_error.contains("bucket-a/obj.txt"), "{label}: {last_error}");
+            }
+            RebalanceEntryCleanupResult::Completed { warning } => {
+                panic!("{label} must defer source cleanup instead of completing the entry with warning {warning:?}")
+            }
+        }
+    }
+}
+
+#[test]
+fn test_resolve_rebalance_entry_cleanup_delete_result_defers_stage_wrapped_lock_timeout() {
+    let err = data_movement::data_movement_stage_error_for_test(
+        "rebalance",
+        "delete_object",
+        "bucket-a",
+        "obj.txt",
+        Error::Lock(rustfs_lock::LockError::timeout("bucket-a/obj.txt@latest", Duration::from_secs(5))),
+    );
+
+    assert!(matches!(
+        resolve_rebalance_entry_cleanup_delete_result(Err(err.into()), "bucket-a", "obj.txt"),
+        RebalanceEntryCleanupResult::Deferred { .. }
+    ));
+}
+
+#[test]
+fn test_resolve_rebalance_entry_cleanup_delete_result_ignores_stage_wrapped_not_found() {
+    let err = data_movement::data_movement_stage_error_for_test(
+        "rebalance",
+        "delete_object",
+        "bucket-a",
+        "obj.txt",
+        Error::ObjectNotFound("bucket-a".to_string(), "obj.txt".to_string()),
+    );
+
+    assert_eq!(
+        resolve_rebalance_entry_cleanup_delete_result(Err(err.into()), "bucket-a", "obj.txt"),
+        RebalanceEntryCleanupResult::Completed { warning: None }
+    );
+}
+
+#[test]
+fn test_resolve_rebalance_deferred_last_error_hides_retryable_cleanup_conflicts() {
+    let entry_error = format!("{REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX} timeout");
+    assert_eq!(
+        resolve_rebalance_deferred_last_error(RebalanceDeferKind::Entry, None, entry_error.as_str()),
+        Some(entry_error.clone()),
+        "transient migration deferrals must stay visible to the completion guards"
+    );
+
+    let cleanup_error = format!("{REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX} lock acquisition timeout");
+    assert_eq!(
+        resolve_rebalance_deferred_last_error(RebalanceDeferKind::SourceCleanup, None, cleanup_error.as_str()),
+        None,
+        "a retryable source cleanup conflict is progress, not a pool failure"
+    );
+
+    assert_eq!(
+        resolve_rebalance_deferred_last_error(
+            RebalanceDeferKind::SourceCleanup,
+            Some(entry_error.as_str()),
+            cleanup_error.as_str()
+        ),
+        Some(entry_error.clone()),
+        "a cleanup deferral for one bucket must not erase an unresolved migration deferral of the same pool"
+    );
+    assert_eq!(
+        resolve_rebalance_deferred_last_error(
+            RebalanceDeferKind::SourceCleanup,
+            Some(cleanup_error.as_str()),
+            cleanup_error.as_str()
+        ),
+        None,
+        "a stale retryable cleanup message must not survive as a pool failure"
+    );
+    assert_eq!(
+        resolve_rebalance_deferred_last_error(
+            RebalanceDeferKind::SourceCleanup,
+            Some(entry_error.as_str()),
+            entry_error.as_str()
+        ),
+        Some(entry_error),
+        "repeated cleanup deferrals must keep the pending migration deferral visible"
+    );
+}
+
 #[test]
 fn test_resolve_rebalance_entry_cleanup_delete_result_defers_source_change() {
     let result = resolve_rebalance_entry_cleanup_delete_result(Err(SourceCleanupError::SourceChanged), "bucket-a", "obj.txt");
@@ -1817,6 +1935,89 @@ fn test_source_cleanup_defer_does_not_fail_repeated_bucket_retry() {
     assert_eq!(source_cleanup_defer_attempt(&mut source_attempts, "bucket-c"), 1);
     assert_eq!(source_cleanup_defer_attempt(&mut source_attempts, "bucket-c"), 2);
     assert_eq!(source_cleanup_defer_attempt(&mut source_attempts, "bucket-c"), 3);
+
+    let mut bounded_attempts = std::collections::HashMap::new();
+    for expected in 1..REBALANCE_SOURCE_CLEANUP_MAX_DEFERS {
+        assert_eq!(source_cleanup_defer_attempt(&mut bounded_attempts, "bucket-d"), expected);
+        assert!(
+            !reached_rebalance_source_cleanup_defer_limit(expected),
+            "a retryable cleanup conflict must stay retryable at deferral {expected}"
+        );
+    }
+    assert_eq!(
+        source_cleanup_defer_attempt(&mut bounded_attempts, "bucket-d"),
+        REBALANCE_SOURCE_CLEANUP_MAX_DEFERS
+    );
+    assert!(
+        reached_rebalance_source_cleanup_defer_limit(REBALANCE_SOURCE_CLEANUP_MAX_DEFERS),
+        "an unreclaimable source replica must fail the bucket after the bounded deferral budget"
+    );
+}
+
+#[tokio::test]
+async fn test_defer_rebalance_bucket_keeps_pending_entry_defer_without_surfacing_cleanup_conflicts() {
+    let id = "rebalance-defer-last-error";
+    let meta = RebalanceMeta {
+        id: id.to_string(),
+        pool_stats: vec![RebalanceStats {
+            participating: true,
+            buckets: vec!["bucket-a".to_string(), "bucket-b".to_string()],
+            info: RebalanceInfo {
+                status: RebalStatus::Started,
+                ..Default::default()
+            },
+            ..Default::default()
+        }],
+        ..Default::default()
+    };
+    let store = test_store_with_rebalance_meta(meta);
+    let cleanup_error = format!("{REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX} lock acquisition timeout");
+
+    store
+        .defer_rebalance_bucket(0, "bucket-a".to_string(), cleanup_error.clone(), id, RebalanceDeferKind::SourceCleanup)
+        .await
+        .expect("a retryable cleanup conflict must be deferrable");
+    {
+        let meta = store.rebalance_meta.read().await;
+        let pool_stat = &meta.as_ref().expect("rebalance metadata should exist").pool_stats[0];
+        assert_eq!(
+            pool_stat.info.last_error, None,
+            "a retryable cleanup conflict is progress and must not surface as a pool failure"
+        );
+        assert_eq!(pool_stat.buckets, vec!["bucket-b".to_string(), "bucket-a".to_string()]);
+    }
+
+    let entry_error = format!("{REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX} slow down");
+    {
+        let mut meta = store.rebalance_meta.write().await;
+        meta.as_mut().expect("rebalance metadata should exist").pool_stats[0]
+            .info
+            .last_error = Some(entry_error.clone());
+    }
+    store
+        .defer_rebalance_bucket(0, "bucket-b".to_string(), cleanup_error, id, RebalanceDeferKind::SourceCleanup)
+        .await
+        .expect("a cleanup deferral must be accepted while another bucket still defers an entry");
+    {
+        let meta = store.rebalance_meta.read().await;
+        let pool_stat = &meta.as_ref().expect("rebalance metadata should exist").pool_stats[0];
+        assert_eq!(
+            pool_stat.info.last_error,
+            Some(entry_error),
+            "a cleanup deferral must not erase the pending migration deferral that blocks goal completion"
+        );
+        assert!(has_deferred_rebalance_error(pool_stat));
+        assert_eq!(pool_stat.buckets, vec!["bucket-a".to_string(), "bucket-b".to_string()]);
+    }
+
+    let migration_error = format!("{REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX} i/o timeout");
+    store
+        .defer_rebalance_bucket(0, "bucket-a".to_string(), migration_error.clone(), id, RebalanceDeferKind::Entry)
+        .await
+        .expect("migration deferrals must keep their last error");
+    let meta = store.rebalance_meta.read().await;
+    let pool_stat = &meta.as_ref().expect("rebalance metadata should exist").pool_stats[0];
+    assert_eq!(pool_stat.info.last_error, Some(migration_error));
 }
 
 #[test]