fix(heal): detect stale delete-marker metadata (#8102)

A flat (recursive) listing resumed inside "s/" re-emitted everything in
its sibling "s-x/". scan_dir dropped the entries before forward_to by
comparing directory names without their trailing slash, where "s" sorts
before "s-x", while the keys they stand for sort the other way round:
'-' (0x2d) is below '/' (0x2f), so all of "s-x/..." precedes "s/...".
The drain stopped at "s" and kept "s-x".

The next page then started with keys at or before the marker; the
listing layer filtered all of them out, found no more candidates and
answered IsTruncated=false. A bucket of 104,137 objects with backup
directories named "<id>" and "<id>-rollbacks" listed as 5,000; two such
directories of 1,200 keys each listed as 2,000.

Compare every remaining entry as the key prefix it stands for, slash
included, and keep it only when it sorts at or after forward_to or
contains it. The remainder of forward_to is taken before `current` is
trimmed, so this also holds below the bucket root; retain also drops
entries when all of them precede forward_to, which the drain never did.

.agents/skills/security-advisory-lessons/references/advisory-patterns.md

@@ -101,6 +101,7 @@ Update this file only when an advisory adds or changes a reusable lesson, affect
 ### SSE and on-disk storage invariants
 
 - `GHSA-xrrf-67jm-3c2r`: SSE metadata reported encryption while reader composition bypassed `EncryptReader` and stored plaintext. Lesson: test actual bytes on disk and wrapper order, not only API metadata.
+- `GHSA-wqmc-vjgv-jrpw`: SSE-C persisted and listed the plaintext MD5 as the ETag, allowing list-only users to confirm or brute-force low-entropy content and correlate equal plaintext across keys. Lesson: encrypted-object ETags must not expose deterministic plaintext fingerprints; protect them with per-object key material and keep listing, conditional, copy, and multipart semantics consistent.
 
 ### Object Lock and retention invariants
 
@@ -126,7 +127,7 @@ rg -n "normalize_extract_entry_key|Snowball|auto-extract|PathBuf::join|canonical
 rg -n "DEFAULT_SECRET|DEFAULT_ACCESS|TEST_PRIVATE_KEY|rustfs rpc|RUSTFS_RPC_SECRET" rustfs crates
 rg -n "TONIC_RPC_PREFIX|verify_rpc_signature|check_auth|NodeServiceServer|x-rustfs-signature" rustfs crates
 rg -n "debug!|trace!|info!|error!|\\?resp|\\?merged_config|session_token|secret_key" rustfs crates
-rg -n "HashReader|EncryptReader|SSE|server-side encryption|Access-Control-Allow-Credentials|Origin" rustfs crates
+rg -n "HashReader|EncryptReader|try_resolve_etag|ETag|SSE|server-side encryption|Access-Control-Allow-Credentials|Origin" rustfs crates
 rg -n "ObjectLock|object_lock|retention|COMPLIANCE|GOVERNANCE|delete_prefix|lifecycle|scanner" rustfs crates
 rg -n "deny_unknown_fields|serde.default|as u32|as usize|as i32" rustfs crates
 ```
@@ -145,5 +146,5 @@ rg -n "deny_unknown_fields|serde.default|as u32|as usize|as i32" rustfs crates
 - IAM export fixes: assert exported archives omit plaintext user and service-account secrets unless the format deliberately encrypts or seals them.
 - RPC auth fixes: include captured metadata replay across two concrete methods, stale timestamps, wrong path, wrong method surrogate, wrong secret, and valid same-method calls.
 - Browser/CORS fixes: assert no credentials on reflected/default origins, correct behavior for explicit allowlists, and no same-origin script execution for previewed object content.
-- SSE fixes: inspect stored bytes and verify API metadata, read-back behavior, and on-disk ciphertext together.
+- SSE fixes: inspect stored bytes and verify API metadata, read-back behavior, and on-disk ciphertext together; for ETags, upload equal plaintext under different keys and assert list-only callers cannot derive or correlate the plaintext digest, including multipart objects.
 - Object Lock fixes: include unreadable metadata, fabricated metadata defaults, unparsable config, confirmed absent config, COMPLIANCE/GOVERNANCE retention, lifecycle expiry, scanner sweeps, and force-delete paths.

.config/make/tests.mak

@@ -29,6 +29,7 @@ script-tests: ## Run shell script tests
 	./scripts/test_entrypoint_credentials.sh
 	./scripts/test_internode_grpc_ab_bench.sh
 	./scripts/test_object_batch_bench_enhanced.sh
+	./scripts/test_package_service_scripts.sh
 	./scripts/test_hotpath_warp_ab_gate.sh
 	./scripts/test_hotpath_warp_abba.sh
 	./scripts/test_scanner_validation_harness.sh

.github/actions/quick-checks/action.yml

@@ -19,7 +19,7 @@ runs:
   using: composite
   steps:
     - name: Install quality tools
-      uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
+      uses: taiki-e/install-action@7623a79cdfecb99d681017af368ca353d9f49bb5 # v2
       with:
         tool: |
           ripgrep@15.2.0

.github/actions/setup/action.yml

@@ -109,7 +109,7 @@ runs:
       uses: taiki-e/install-action@96c7780c1d8a2b8723e12031def873a434d39d8d # nextest
 
     - name: Setup Rust cache
-      uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
+      uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
       with:
         # false is rust-cache's own default. With true, cleanup.ts returns
         # *before* pruning ~/.cargo/registry/src, and config.ts archives the

.github/workflows/architecture-migration-rules.yml

@@ -48,12 +48,12 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
       - name: Install ripgrep
-        uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
+        uses: taiki-e/install-action@7623a79cdfecb99d681017af368ca353d9f49bb5 # v2
         with:
           tool: ripgrep@15.2.0
 

.github/workflows/audit.yml

@@ -24,8 +24,11 @@ on:
       - '.github/actions/**'
       - '.github/workflows/**'
       - 'scripts/release/create_or_update_release.sh'
+      - 'scripts/release/package_service_scripts.sh'
       - 'scripts/release/package_versions.sh'
+      - 'scripts/test_package_service_scripts.sh'
       - 'scripts/test_package_versions.sh'
+      - 'scripts/test_docker_workflow.py'
       - 'scripts/security/check_performance_ab_workflow.sh'
       - 'scripts/security/check_preview_release_workflow.sh'
       - 'scripts/security/check_tier_artifact_workflow.sh'
@@ -40,8 +43,11 @@ on:
       - '.github/actions/**'
       - '.github/workflows/**'
       - 'scripts/release/create_or_update_release.sh'
+      - 'scripts/release/package_service_scripts.sh'
       - 'scripts/release/package_versions.sh'
+      - 'scripts/test_package_service_scripts.sh'
       - 'scripts/test_package_versions.sh'
+      - 'scripts/test_docker_workflow.py'
       - 'scripts/security/check_performance_ab_workflow.sh'
       - 'scripts/security/check_preview_release_workflow.sh'
       - 'scripts/security/check_tier_artifact_workflow.sh'
@@ -87,7 +93,7 @@ jobs:
     timeout-minutes: 20
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -106,7 +112,7 @@ jobs:
       # default is now "false", but state it explicitly — see
       # scripts/security/check_cache_save_if.sh.
       - name: Setup Rust cache
-        uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
+        uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
         with:
           # Same reasoning as the setup composite: true archives every
           # dependency's unpacked source tree.
@@ -116,7 +122,7 @@ jobs:
           save-if: ${{ github.ref == 'refs/heads/main' }}
 
       - name: Install cargo-deny
-        uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
+        uses: taiki-e/install-action@7623a79cdfecb99d681017af368ca353d9f49bb5 # v2
         with:
           tool: cargo-deny
 
@@ -130,7 +136,7 @@ jobs:
     timeout-minutes: 5
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -158,6 +164,9 @@ jobs:
       - name: Check package version contract
         run: ./scripts/test_package_versions.sh
 
+      - name: Check package service lifecycle contract
+        run: ./scripts/test_package_service_scripts.sh
+
   dependency-review:
     name: Dependency Review
     runs-on: ubuntu-latest
@@ -168,7 +177,7 @@ jobs:
       pull-requests: write
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -201,7 +210,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/build.yml

@@ -98,7 +98,7 @@ jobs:
       is_prerelease: ${{ steps.check.outputs.is_prerelease }}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -257,7 +257,7 @@ jobs:
       matrix: ${{ fromJson(needs.prepare-platform-matrix.outputs.matrix) }}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
           fetch-depth: 0
@@ -702,7 +702,7 @@ jobs:
           exit 1
 
       - name: Upload to GitHub artifacts
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: ${{ steps.package.outputs.package_name }}
           path: "rustfs-*.zip"
@@ -818,9 +818,7 @@ jobs:
 
           echo ""
           echo "🐳 Docker Images:"
-          if [[ "$BUILD_TYPE" == "preview" ]]; then
-            echo "⏭️  Preview tags do not publish Docker images"
-          elif [[ "$INPUT_BUILD_DOCKER" == "false" ]]; then
+          if [[ "$INPUT_BUILD_DOCKER" == "false" ]]; then
             echo "⏭️  Docker image build was skipped (binary only build)"
           elif [[ "$BUILD_STATUS" == "success" ]]; then
             echo "🔄 Docker images will be built and pushed automatically via workflow_run event"
@@ -842,7 +840,7 @@ jobs:
       release_url: ${{ steps.create.outputs.release_url }}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
           fetch-depth: 0
@@ -901,12 +899,12 @@ jobs:
       actions: read
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
       - name: Download all build artifacts
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           path: ./artifacts
           pattern: rustfs-*
@@ -1140,7 +1138,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/cache-warm.yml

@@ -109,7 +109,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -141,7 +141,7 @@ jobs:
 
       - name: Upload cargo timings report
         if: inputs.emit_timings
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: cargo-timings-ci-dev
           path: target/cargo-timings/
@@ -199,7 +199,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -230,7 +230,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -260,7 +260,7 @@ jobs:
     timeout-minutes: 60
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 

.github/workflows/ci.yml

@@ -77,7 +77,7 @@ jobs:
     outputs:
       mode: ${{ steps.scope.outputs.mode }}
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           fetch-depth: 2
           persist-credentials: false
@@ -101,11 +101,11 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Typos check with custom config file
-        uses: crate-ci/typos@37bb98842b0d8c4ffebdb75301a13db0267cef89 # master
+        uses: crate-ci/typos@512fc24f32f44ab01972217aaaf3dc86ec234d53 # v1.50.2
 
   # Fail early with compile-free checks for every pull request.
   quick-checks:
@@ -115,7 +115,7 @@ jobs:
     timeout-minutes: 10
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -132,7 +132,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           # Checkout otherwise writes the token into .git/config, where a PR's
           # own build.rs or proc-macro could read it back out.
@@ -261,7 +261,7 @@ jobs:
 
       - name: Upload test reports and diagnostics
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: junit-test-and-lint-${{ github.run_number }}
           path: |
@@ -305,7 +305,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -344,7 +344,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -391,7 +391,7 @@ jobs:
 
       - name: Upload ILM test diagnostics
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: ilm-integration-${{ github.run_number }}-${{ github.run_attempt }}
           path: |
@@ -408,7 +408,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -449,7 +449,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -521,7 +521,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -561,7 +561,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -621,7 +621,7 @@ jobs:
           PYBUILD
 
       - name: Upload debug binary
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: rustfs-debug-binary
           path: |
@@ -646,7 +646,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -662,7 +662,7 @@ jobs:
         run: python3 scripts/e2e_binary.py build --bins --features rio-v2,e2e-test-hooks
 
       - name: Upload debug binary
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: rustfs-debug-binary-rio-v2
           path: |
@@ -690,7 +690,7 @@ jobs:
     timeout-minutes: 30
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -754,7 +754,7 @@ jobs:
     timeout-minutes: 30
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -770,7 +770,7 @@ jobs:
           install-build-packaging-tools: 'false'
 
       - name: Set up Python
-        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: "3.12"
 
@@ -785,7 +785,7 @@ jobs:
       # Download after the cache restore so the freshly built binary from the
       # build job always wins over anything restored into target/debug.
       - name: Download debug binary
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: rustfs-debug-binary
           path: target/debug
@@ -821,7 +821,7 @@ jobs:
 
       - name: Upload e2e smoke diagnostics
         if: failure()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: e2e-smoke-diagnostics-${{ github.run_number }}
           path: |
@@ -831,7 +831,7 @@ jobs:
 
       - name: Upload e2e smoke JUnit report
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: e2e-smoke-junit-${{ github.run_number }}
           path: target/nextest/e2e-smoke/junit.xml
@@ -856,7 +856,7 @@ jobs:
 
       - name: Upload test logs
         if: failure()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: e2e-test-logs-${{ github.run_number }}
           path: ${{ runner.temp }}/rustfs-e2e-*/rustfs.log
@@ -879,7 +879,7 @@ jobs:
     timeout-minutes: 55
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -903,7 +903,7 @@ jobs:
           fi
 
       - name: Set up Python
-        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: "3.12"
 
@@ -945,7 +945,7 @@ jobs:
       # Download after the cache restore so the freshly built binary from the
       # build job always wins over anything restored into target/debug.
       - name: Download debug binary
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: rustfs-debug-binary
           path: target/debug
@@ -1005,7 +1005,7 @@ jobs:
 
       - name: Upload junit
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: e2e-full-junit-${{ github.run_number }}
           path: |
@@ -1015,7 +1015,7 @@ jobs:
 
       - name: Upload startup CAS evidence
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: fresh-startup-cas-evidence-${{ github.run_number }}
           path: |
@@ -1034,7 +1034,7 @@ jobs:
     timeout-minutes: 30
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -1044,7 +1044,7 @@ jobs:
           rm -f /tmp/rustfs.log
 
       - name: Download debug binary
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: rustfs-debug-binary-rio-v2
           path: target/debug
@@ -1077,7 +1077,7 @@ jobs:
 
       - name: Upload test logs
         if: failure()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: e2e-test-logs-rio-v2-${{ github.run_number }}
           path: /tmp/rustfs.log
@@ -1090,12 +1090,12 @@ jobs:
     timeout-minutes: 60
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
       - name: Download debug binary
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: rustfs-debug-binary
           path: target/debug
@@ -1120,7 +1120,7 @@ jobs:
 
       - name: Upload s3 test artifacts
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: s3tests-implemented-${{ github.run_number }}
           path: artifacts/s3tests-single/**
@@ -1164,12 +1164,12 @@ jobs:
     timeout-minutes: 30
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
       - name: Download debug binary
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: rustfs-debug-binary
           path: target/debug
@@ -1202,7 +1202,7 @@ jobs:
 
       - name: Upload s3 test artifacts
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: s3tests-lifecycle-behavior-${{ github.run_number }}
           path: artifacts/s3tests-single/**
@@ -1233,7 +1233,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Require the expected result of every CI lane
@@ -1272,7 +1272,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/cla.yml

@@ -66,7 +66,7 @@ jobs:
     steps:
       - name: Report CLA result for merge queue
         if: github.event_name == 'merge_group'
-        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         with:
           script: |
             await github.rest.checks.create({

.github/workflows/connect-diagnostic-schedule-acceptance.yml

@@ -0,0 +1,198 @@
+# Copyright 2026 RustFS Team
+# SPDX-License-Identifier: Apache-2.0
+
+name: Connect diagnostic scheduler receipt acceptance
+
+on:
+  workflow_dispatch:
+    inputs:
+      build_run_id:
+        description: Main-branch Build and Release run ID
+        required: true
+        type: string
+      artifact_id:
+        description: Linux x86_64 GNU artifact from that run
+        required: true
+        type: string
+      source_sha:
+        description: Exact RustFS source commit
+        required: true
+        type: string
+      artifact_digest:
+        description: GitHub artifact digest including sha256 prefix
+        required: true
+        type: string
+      binary_sha256:
+        description: Independently verified rustfs binary SHA-256
+        required: true
+        type: string
+      connect_sha:
+        description: Exact Connect scheduler acceptance harness commit
+        required: true
+        type: string
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  diagnostic-schedule:
+    name: Verify production scheduler receipts over mTLS
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - name: Checkout exact RustFS source
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          path: rustfs-source
+          persist-credentials: false
+          ref: ${{ inputs.source_sha }}
+
+      - name: Checkout exact Connect harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/connect
+          path: connect-harness
+          persist-credentials: false
+          ref: ${{ inputs.connect_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 22.22.2
+          cache: npm
+          cache-dependency-path: connect-harness/web/package-lock.json
+
+      - name: Verify official source and artifact identity
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          BINARY_SHA256: ${{ inputs.binary_sha256 }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+        run: |
+          set -euo pipefail
+          [[ "$BUILD_RUN_ID" =~ ^[0-9]+$ && "$ARTIFACT_ID" =~ ^[0-9]+$ ]]
+          [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ && "$CONNECT_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ "$ARTIFACT_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
+          [[ "$BINARY_SHA256" =~ ^[0-9a-f]{64}$ ]]
+          [[ $(git -C rustfs-source rev-parse HEAD) == "$SOURCE_SHA" ]]
+          [[ $(git -C connect-harness rev-parse HEAD) == "$CONNECT_SHA" ]]
+          [[ $(git -C rustfs-source remote get-url origin) == https://github.com/rustfs/rustfs ]]
+          [[ $(git -C connect-harness remote get-url origin) == https://github.com/rustfs/connect ]]
+
+          run=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}")
+          jq -e --arg source "$SOURCE_SHA" --arg repository "$GITHUB_REPOSITORY" '
+            .head_sha == $source and .head_branch == "main"
+            and .head_repository.full_name == $repository
+            and .name == "Build and Release" and .path == ".github/workflows/build.yml"
+            and .status == "completed" and .conclusion == "success"
+          ' <<<"$run" >/dev/null
+          jobs=$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
+          jq -e '
+            [.[].jobs[] | select(.name | test("^Build RustFS \\(linux-x86_64-gnu, [a-z0-9-]+, x86_64-unknown-linux-gnu, false, linux, pyroscope\\)$"))] as $matches
+            | ($matches | length) == 1 and $matches[0].conclusion == "success"
+          ' <<<"$jobs" >/dev/null
+          artifact=$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}")
+          jq -e --argjson run "$BUILD_RUN_ID" --arg source "$SOURCE_SHA" --arg digest "$ARTIFACT_DIGEST" '
+            .workflow_run.id == $run and .workflow_run.head_sha == $source
+            and .name == ("rustfs-linux-x86_64-gnu-dev-" + $source[0:7])
+            and .digest == $digest and .expired == false
+            and (.expires_at | fromdateiso8601) > now
+            and .size_in_bytes > 0 and .size_in_bytes <= 2147483648
+          ' <<<"$artifact" >/dev/null
+
+          gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}/zip" > official-artifact.zip
+          printf '%s  official-artifact.zip\n' "${ARTIFACT_DIGEST#sha256:}" | sha256sum --check --strict
+
+      - name: Extract the verified official binary
+        shell: bash
+        env:
+          SOURCE_SHA: ${{ inputs.source_sha }}
+          BINARY_SHA256: ${{ inputs.binary_sha256 }}
+        run: |
+          set -euo pipefail
+          python3 - <<'PY'
+          import io
+          import os
+          import pathlib
+          import stat
+          import zipfile
+
+          def regular(entry):
+              mode = entry.external_attr >> 16
+              return not entry.is_dir() and (stat.S_IFMT(mode) in (0, stat.S_IFREG))
+
+          package = f"rustfs-linux-x86_64-gnu-dev-{os.environ['SOURCE_SHA'][:7]}.zip"
+          with zipfile.ZipFile('official-artifact.zip') as outer:
+              entries = outer.infolist()
+              assert 1 <= len(entries) <= 16
+              assert len({item.filename for item in entries}) == len(entries)
+              assert all(regular(item) and pathlib.PurePosixPath(item.filename).name == item.filename for item in entries)
+              assert sum(item.file_size for item in entries) <= 2147483648
+              assert outer.testzip() is None
+              with zipfile.ZipFile(io.BytesIO(outer.read(package))) as inner:
+                  entries = inner.infolist()
+                  assert {item.filename for item in entries} == {'rustfs', 'rustfs-cli'} and len(entries) == 2
+                  assert all(regular(item) and 0 < item.file_size <= 1073741824 for item in entries)
+                  assert inner.testzip() is None
+                  pathlib.Path('binary').mkdir()
+                  with inner.open('rustfs') as source, open('binary/rustfs', 'xb') as destination:
+                      import shutil
+                      shutil.copyfileobj(source, destination)
+          PY
+          chmod 0755 binary/rustfs
+          printf '%s  binary/rustfs\n' "$BINARY_SHA256" | sha256sum --check --strict
+          file binary/rustfs | grep -Eq 'ELF 64-bit LSB.*x86-64'
+
+      - name: Run the real scheduler and receipt delivery scenario
+        shell: bash
+        env:
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          git -C connect-harness diff --exit-code
+          printf '%s\n' "$SOURCE_SHA" >connect-harness/tests/e2e/connected/rustfs-ref
+          [[ $(git -C connect-harness diff --name-only) == tests/e2e/connected/rustfs-ref ]]
+          npm --prefix connect-harness/web ci
+          connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
+          make -C connect-harness e2e-connected-dispatch-check
+          (cd connect-harness && ./tests/connectivity/diagnostic-schedule-evidence.test.sh)
+          RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
+            RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
+            CONNECT_E2E_DIAGNOSTIC_SCHEDULE_EVIDENCE="$GITHUB_WORKSPACE/diagnostic-schedule-evidence.json" \
+            make -C connect-harness e2e-connected E2E_SCENARIO=diagnostic-schedule
+
+      - name: Bind and validate sanitized evidence
+        shell: bash
+        env:
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          BINARY_SHA256: ${{ inputs.binary_sha256 }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          jq -e --arg connect "$CONNECT_SHA" --arg source "$SOURCE_SHA" --arg binary "$BINARY_SHA256" '
+            .provenance.connectSha == $connect and .provenance.sourceSha == $source
+            and .provenance.binarySha256 == $binary
+          ' diagnostic-schedule-evidence.json >/dev/null
+          jq --arg run "$GITHUB_RUN_ID" --arg build "$BUILD_RUN_ID" \
+            --arg artifact "$ARTIFACT_ID" --arg digest "$ARTIFACT_DIGEST" '
+            .provenance += {workflowRunId:$run,buildRunId:$build,artifactId:$artifact,artifactDigest:$digest}
+          ' diagnostic-schedule-evidence.json > diagnostic-schedule-evidence.bound.json
+          mv diagnostic-schedule-evidence.bound.json diagnostic-schedule-evidence.json
+          node connect-harness/tests/connectivity/verify-diagnostic-schedule-evidence.mjs diagnostic-schedule-evidence.json --bound
+
+      - name: Upload verified receipt evidence
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: connect-diagnostic-schedule-evidence-${{ github.run_id }}
+          path: diagnostic-schedule-evidence.json
+          retention-days: 14
+          if-no-files-found: error

.github/workflows/connect-performance-drive-acceptance.yml

@@ -53,19 +53,19 @@ jobs:
     timeout-minutes: 45
     steps:
       - name: Checkout acceptance harness
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
       - name: Checkout exact RustFS source
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           path: rustfs-source
           persist-credentials: false
           ref: ${{ inputs.source_sha }}
 
       - name: Checkout exact Connect harness
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/connect
           path: connect-harness
@@ -122,7 +122,7 @@ jobs:
           [[ $(jq -r '.expired' <<<"$artifact") == false ]]
 
       - name: Download exact build artifact
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           artifact-ids: ${{ inputs.artifact_id }}
           path: artifact
@@ -183,7 +183,7 @@ jobs:
 
       - name: Upload acceptance evidence
         if: ${{ always() && hashFiles('performance-drive-service-job-evidence.json') != '' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: connect-performance-drive-evidence-${{ github.run_id }}
           path: |

.github/workflows/connect-performance-network-acceptance.yml

@@ -53,19 +53,19 @@ jobs:
     timeout-minutes: 45
     steps:
       - name: Checkout acceptance harness
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
       - name: Checkout exact RustFS source
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           path: rustfs-source
           persist-credentials: false
           ref: ${{ inputs.source_sha }}
 
       - name: Checkout exact Connect harness
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/connect
           path: connect-harness
@@ -122,7 +122,7 @@ jobs:
           [[ $(jq -r '.expired' <<<"$artifact") == false ]]
 
       - name: Download exact build artifact
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           artifact-ids: ${{ inputs.artifact_id }}
           path: artifact
@@ -183,7 +183,7 @@ jobs:
 
       - name: Upload acceptance evidence
         if: ${{ always() && hashFiles('performance-network-service-job-evidence.json') != '' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: connect-performance-network-evidence-${{ github.run_id }}
           path: |

.github/workflows/connect-profile-cpu-acceptance.yml

@@ -53,19 +53,19 @@ jobs:
     timeout-minutes: 45
     steps:
       - name: Checkout acceptance harness
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
       - name: Checkout exact RustFS source
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           path: rustfs-source
           persist-credentials: false
           ref: ${{ inputs.source_sha }}
 
       - name: Checkout exact Connect harness
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/connect
           path: connect-harness
@@ -122,7 +122,7 @@ jobs:
           [[ $(jq -r '.expired' <<<"$artifact") == false ]]
 
       - name: Download exact build artifact
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           artifact-ids: ${{ inputs.artifact_id }}
           path: artifact
@@ -183,7 +183,7 @@ jobs:
 
       - name: Upload acceptance evidence
         if: ${{ always() && hashFiles('profile-service-job-evidence.json') != '' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: connect-profile-cpu-evidence-${{ github.run_id }}
           path: |

.github/workflows/connect-profile-memory-acceptance.yml

@@ -0,0 +1,201 @@
+# Copyright 2026 RustFS Team
+# SPDX-License-Identifier: Apache-2.0
+
+name: Connect service memory profile acceptance
+
+on:
+  workflow_dispatch:
+    inputs:
+      build_run_id:
+        description: Successful main-branch Build and Release run ID
+        required: true
+        type: string
+      artifact_id:
+        description: Linux x86_64 GNU artifact from that run
+        required: true
+        type: string
+      source_sha:
+        description: Exact RustFS source commit
+        required: true
+        type: string
+      artifact_digest:
+        description: GitHub artifact digest including sha256 prefix
+        required: true
+        type: string
+      binary_sha256:
+        description: Independently verified rustfs binary SHA-256
+        required: true
+        type: string
+      connect_sha:
+        description: Exact Connect memory acceptance harness commit
+        required: true
+        type: string
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  profile-memory:
+    name: Verify signed service memory profile over mTLS
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - name: Checkout exact RustFS source
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          path: rustfs-source
+          persist-credentials: false
+          ref: ${{ inputs.source_sha }}
+
+      - name: Checkout exact Connect harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/connect
+          path: connect-harness
+          persist-credentials: false
+          ref: ${{ inputs.connect_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 22.22.2
+          cache: npm
+          cache-dependency-path: connect-harness/web/package-lock.json
+
+      - name: Verify official source and artifact identity
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          BINARY_SHA256: ${{ inputs.binary_sha256 }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+        run: |
+          set -euo pipefail
+          [[ "$GITHUB_REPOSITORY" == rustfs/rustfs ]]
+          [[ "$BUILD_RUN_ID" =~ ^[1-9][0-9]*$ && "$ARTIFACT_ID" =~ ^[1-9][0-9]*$ ]]
+          [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ && "$CONNECT_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ "$ARTIFACT_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
+          [[ "$BINARY_SHA256" =~ ^[0-9a-f]{64}$ ]]
+          [[ $(git -C rustfs-source rev-parse HEAD) == "$SOURCE_SHA" ]]
+          [[ $(git -C connect-harness rev-parse HEAD) == "$CONNECT_SHA" ]]
+          [[ $(git -C rustfs-source remote get-url origin) == https://github.com/rustfs/rustfs ]]
+          [[ $(git -C connect-harness remote get-url origin) == https://github.com/rustfs/connect ]]
+          run=$(gh api "repos/rustfs/rustfs/actions/runs/${BUILD_RUN_ID}")
+          jq -e --arg source "$SOURCE_SHA" '
+            .head_sha == $source and .head_branch == "main"
+            and .head_repository.full_name == "rustfs/rustfs"
+            and .name == "Build and Release" and .path == ".github/workflows/build.yml"
+            and .status == "completed" and .conclusion == "success"
+          ' <<<"$run" >/dev/null
+          jobs=$(gh api --paginate --slurp "repos/rustfs/rustfs/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
+          jq -e '
+            [.[].jobs[] | select(.name | test("^Build RustFS \\(linux-x86_64-gnu, [a-z0-9-]+, x86_64-unknown-linux-gnu, false, linux, pyroscope\\)$"))] as $matches
+            | ($matches | length) == 1 and $matches[0].conclusion == "success"
+          ' <<<"$jobs" >/dev/null
+          artifact=$(gh api "repos/rustfs/rustfs/actions/artifacts/${ARTIFACT_ID}")
+          jq -e --argjson run "$BUILD_RUN_ID" --arg source "$SOURCE_SHA" --arg digest "$ARTIFACT_DIGEST" '
+            .workflow_run.id == $run and .workflow_run.head_sha == $source
+            and .name == ("rustfs-linux-x86_64-gnu-dev-" + $source[0:7])
+            and .digest == $digest and .expired == false
+            and (.expires_at | fromdateiso8601) > now
+            and .size_in_bytes > 0 and .size_in_bytes <= 2147483648
+          ' <<<"$artifact" >/dev/null
+          jq -r '.expires_at' <<<"$artifact" > artifact-expires-at
+          gh api "repos/rustfs/rustfs/actions/artifacts/${ARTIFACT_ID}/zip" > official-artifact.zip
+          [[ $(stat --format=%s official-artifact.zip) == $(jq -r '.size_in_bytes' <<<"$artifact") ]]
+          printf '%s  official-artifact.zip\n' "${ARTIFACT_DIGEST#sha256:}" | sha256sum --check --strict
+
+      - name: Extract only the verified official service binary
+        shell: bash
+        env:
+          SOURCE_SHA: ${{ inputs.source_sha }}
+          BINARY_SHA256: ${{ inputs.binary_sha256 }}
+        run: |
+          set -euo pipefail
+          python3 - <<'PY'
+          import io
+          import os
+          import pathlib
+          import shutil
+          import stat
+          import zipfile
+
+          def regular(entry):
+              mode = entry.external_attr >> 16
+              return not entry.is_dir() and stat.S_IFMT(mode) in (0, stat.S_IFREG)
+
+          package = f"rustfs-linux-x86_64-gnu-dev-{os.environ['SOURCE_SHA'][:7]}.zip"
+          with zipfile.ZipFile('official-artifact.zip') as outer:
+              entries = outer.infolist()
+              assert 1 <= len(entries) <= 16
+              assert len({entry.filename for entry in entries}) == len(entries)
+              assert all(regular(entry) and pathlib.PurePosixPath(entry.filename).name == entry.filename for entry in entries)
+              assert sum(entry.file_size for entry in entries) <= 2147483648
+              assert outer.testzip() is None
+              with zipfile.ZipFile(io.BytesIO(outer.read(package))) as inner:
+                  entries = inner.infolist()
+                  assert {entry.filename for entry in entries} == {'rustfs', 'rustfs-cli'} and len(entries) == 2
+                  assert all(regular(entry) and 0 < entry.file_size <= 1073741824 for entry in entries)
+                  assert inner.testzip() is None
+                  pathlib.Path('binary').mkdir()
+                  with inner.open('rustfs') as source, open('binary/rustfs', 'xb') as destination:
+                      shutil.copyfileobj(source, destination)
+          PY
+          chmod 0755 binary/rustfs
+          printf '%s  binary/rustfs\n' "$BINARY_SHA256" | sha256sum --check --strict
+          file binary/rustfs | grep -Eq 'ELF 64-bit LSB.*x86-64'
+
+      - name: Run the real service job under S3 workload
+        shell: bash
+        env:
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          git -C connect-harness diff --exit-code
+          printf '%s\n' "$SOURCE_SHA" >connect-harness/tests/e2e/connected/rustfs-ref
+          changed=$(git -C connect-harness diff --name-only)
+          [[ -z "$changed" || "$changed" == tests/e2e/connected/rustfs-ref ]]
+          npm --prefix connect-harness/web ci
+          connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
+          make -C connect-harness e2e-connected-dispatch-check
+          (cd connect-harness && ./tests/connectivity/profile-memory-evidence.test.sh)
+          RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
+            RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
+            CONNECT_E2E_PROFILE_MEMORY_EVIDENCE="$GITHUB_WORKSPACE/profile-memory-evidence.json" \
+            make -C connect-harness e2e-connected E2E_SCENARIO=profile-memory
+
+      - name: Bind and validate sanitized evidence
+        shell: bash
+        env:
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          BINARY_SHA256: ${{ inputs.binary_sha256 }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          jq -e --arg connect "$CONNECT_SHA" --arg source "$SOURCE_SHA" --arg binary "$BINARY_SHA256" '
+            .provenance.connectSha == $connect and .provenance.sourceSha == $source
+            and .provenance.binarySha256 == $binary
+          ' profile-memory-evidence.json >/dev/null
+          jq --arg run "$GITHUB_RUN_ID" --arg build "$BUILD_RUN_ID" \
+            --arg artifact "$ARTIFACT_ID" --arg digest "$ARTIFACT_DIGEST" \
+            --arg expiry "$(cat artifact-expires-at)" '
+            .provenance += {workflowRunId:$run,buildRunId:$build,artifactId:$artifact,artifactDigest:$digest,artifactExpiresAt:$expiry}
+          ' profile-memory-evidence.json > profile-memory-evidence.bound.json
+          mv profile-memory-evidence.bound.json profile-memory-evidence.json
+          node connect-harness/tests/connectivity/profile-memory-evidence.mjs profile-memory-evidence.json --bound
+
+      - name: Upload only verified sanitized evidence
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: connect-profile-memory-evidence-${{ github.run_id }}
+          path: profile-memory-evidence.json
+          retention-days: 14
+          if-no-files-found: error

.github/workflows/connect-profile-threads-acceptance.yml

@@ -53,19 +53,19 @@ jobs:
     timeout-minutes: 45
     steps:
       - name: Checkout acceptance harness
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
       - name: Checkout exact RustFS source
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           path: rustfs-source
           persist-credentials: false
           ref: ${{ inputs.source_sha }}
 
       - name: Checkout exact Connect harness
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/connect
           path: connect-harness
@@ -122,7 +122,7 @@ jobs:
           [[ $(jq -r '.expired' <<<"$artifact") == false ]]
 
       - name: Download exact build artifact
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           artifact-ids: ${{ inputs.artifact_id }}
           path: artifact
@@ -183,7 +183,7 @@ jobs:
 
       - name: Upload acceptance evidence
         if: ${{ always() && hashFiles('profile-threads-service-job-evidence.json') != '' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: connect-profile-threads-evidence-${{ github.run_id }}
           path: |

.github/workflows/connect-top-api-acceptance.yml

@@ -53,19 +53,19 @@ jobs:
     timeout-minutes: 45
     steps:
       - name: Checkout acceptance harness
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
       - name: Checkout exact RustFS source
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           path: rustfs-source
           persist-credentials: false
           ref: ${{ inputs.source_sha }}
 
       - name: Checkout exact Connect harness
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/connect
           path: connect-harness
@@ -122,7 +122,7 @@ jobs:
           [[ $(jq -r '.expired' <<<"$artifact") == false ]]
 
       - name: Download exact build artifact
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           artifact-ids: ${{ inputs.artifact_id }}
           path: artifact
@@ -183,7 +183,7 @@ jobs:
 
       - name: Upload acceptance evidence
         if: ${{ always() && hashFiles('top-api-service-job-evidence.json') != '' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: connect-top-api-evidence-${{ github.run_id }}
           path: |

.github/workflows/connect-top-disk-acceptance.yml

@@ -49,7 +49,7 @@ jobs:
     timeout-minutes: 15
     steps:
       - name: Checkout acceptance harness
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -83,7 +83,7 @@ jobs:
           [[ $(jq -r '.expired' <<<"$artifact") == false ]]
 
       - name: Download exact build artifact
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           artifact-ids: ${{ inputs.artifact_id }}
           path: artifact
@@ -132,7 +132,7 @@ jobs:
           mv top-disk-runtime-evidence.bound.json top-disk-runtime-evidence.json
 
       - name: Upload acceptance evidence
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: connect-top-disk-evidence-${{ github.run_id }}
           path: |

.github/workflows/connect-top-locks-acceptance.yml

@@ -53,19 +53,19 @@ jobs:
     timeout-minutes: 45
     steps:
       - name: Checkout acceptance harness
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
       - name: Checkout exact RustFS source
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           path: rustfs-source
           persist-credentials: false
           ref: ${{ inputs.source_sha }}
 
       - name: Checkout exact Connect harness
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/connect
           path: connect-harness
@@ -122,7 +122,7 @@ jobs:
           [[ $(jq -r '.expired' <<<"$artifact") == false ]]
 
       - name: Download exact build artifact
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           artifact-ids: ${{ inputs.artifact_id }}
           path: artifact
@@ -183,7 +183,7 @@ jobs:
 
       - name: Upload acceptance evidence
         if: ${{ always() && hashFiles('top-locks-service-job-evidence.json') != '' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: connect-top-locks-evidence-${{ github.run_id }}
           path: |

.github/workflows/connect-top-rpc-acceptance.yml

@@ -53,19 +53,19 @@ jobs:
     timeout-minutes: 45
     steps:
       - name: Checkout acceptance harness
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
       - name: Checkout exact RustFS source
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           path: rustfs-source
           persist-credentials: false
           ref: ${{ inputs.source_sha }}
 
       - name: Checkout exact Connect harness
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/connect
           path: connect-harness
@@ -122,7 +122,7 @@ jobs:
           [[ $(jq -r '.expired' <<<"$artifact") == false ]]
 
       - name: Download exact build artifact
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           artifact-ids: ${{ inputs.artifact_id }}
           path: artifact
@@ -183,7 +183,7 @@ jobs:
 
       - name: Upload acceptance evidence
         if: ${{ always() && hashFiles('top-rpc-service-job-evidence.json') != '' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: connect-top-rpc-evidence-${{ github.run_id }}
           path: |

.github/workflows/coverage.yml

@@ -76,7 +76,7 @@ jobs:
       NEXTEST_PROFILE: ci
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -89,7 +89,7 @@ jobs:
           install-build-packaging-tools: 'false'
 
       - name: Install cargo-llvm-cov
-        uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
+        uses: taiki-e/install-action@7623a79cdfecb99d681017af368ca353d9f49bb5 # v2
         with:
           tool: cargo-llvm-cov
 
@@ -112,7 +112,7 @@ jobs:
 
       - name: Upload coverage artifact
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: coverage-lcov-${{ github.run_number }}
           path: |
@@ -134,7 +134,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/docker.yml

@@ -47,7 +47,7 @@ on:
         default: true
         type: boolean
       version:
-        description: "Version to build (latest for stable release, or specific version like v1.0.0, v1.0.0-alpha1)"
+        description: "Version to build (latest, v1.0.0, v1.0.0-alpha1, or v1.0.1-preview.1)"
         required: false
         default: "latest"
         type: string
@@ -82,8 +82,7 @@ jobs:
       github.event_name == 'workflow_dispatch' ||
       (github.event.workflow_run.conclusion == 'success' &&
        github.event.workflow_run.event == 'push' &&
-       github.event.workflow_run.head_branch != 'main' &&
-       !contains(github.event.workflow_run.head_branch, '-preview'))
+       github.event.workflow_run.head_branch != 'main')
     runs-on: ubuntu-latest
     timeout-minutes: 30
     outputs:
@@ -97,7 +96,7 @@ jobs:
       source_ref: ${{ steps.check.outputs.source_ref }}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
           # For workflow_run events, checkout the specific commit that triggered the workflow
@@ -197,7 +196,14 @@ jobs:
                   version="${version#v}"
                 fi
 
-                if [[ "$version" == *"alpha"* ]] || [[ "$version" == *"beta"* ]] || [[ "$version" == *"rc"* ]]; then
+                if [[ "$version" =~ -preview\.[0-9]+$ ]]; then
+                  build_type="preview"
+                  is_prerelease=true
+                  echo "🔍 Building Docker image for preview: $version (version tags only)"
+                elif [[ "$version" == *"-preview"* ]]; then
+                  echo "❌ Invalid preview tag: $version (expected suffix: -preview.<number>)" >&2
+                  exit 1
+                elif [[ "$version" == *"alpha"* ]] || [[ "$version" == *"beta"* ]] || [[ "$version" == *"rc"* ]]; then
                   build_type="prerelease"
                   is_prerelease=true
                   # Pre-GA policy: prereleases update latest until the first stable tag exists.
@@ -249,11 +255,14 @@ jobs:
                 echo "🚀 Building with latest stable release version"
                 ;;
               *-preview*)
+                if [[ ! "$input_version" =~ ^v?[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?-preview\.[0-9]+$ ]]; then
+                  echo "❌ Invalid preview version: $input_version" >&2
+                  exit 1
+                fi
                 build_type="preview"
                 is_prerelease=true
-                should_build=false
-                should_push=false
-                echo "⏭️  Preview tags do not publish Docker images"
+                version="${input_version#v}"
+                echo "🔍 Building with preview version: $version (version tags only)"
                 ;;
               # Prerelease versions (must match first, more specific)
               v*alpha*|v*beta*|v*rc*|*alpha*|*beta*|*rc*)
@@ -343,7 +352,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
           ref: ${{ needs.build-check.outputs.source_ref }}
@@ -535,13 +544,61 @@ jobs:
           category: container-image-${{ matrix.variant }}
 
       - name: Upload container scan report
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: container-image-scan-${{ matrix.variant }}
           path: trivy-${{ matrix.variant }}.sarif
           if-no-files-found: error
           retention-days: 30
 
+  # Docker Hub's overview is a separate `full_description` field that
+  # `docker push` never touches, so it drifted from README.md and kept
+  # linking to pages that no longer exist (#7995). Republish it from the
+  # commit the images were built from, only when images were pushed.
+  sync-dockerhub-description:
+    name: Sync Docker Hub Description
+    needs: [ build-check, build-docker ]
+    if: needs.build-check.outputs.should_build == 'true' && needs.build-check.outputs.should_push == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+          ref: ${{ needs.build-check.outputs.source_ref }}
+          sparse-checkout: README.md
+          sparse-checkout-cone-mode: false
+
+      - name: Sync Docker Hub description
+        uses: peter-evans/dockerhub-description@1b9a80c056b620d92cedb9d9b5a223409c68ddfa # v5.0.0
+        with:
+          username: ${{ env.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+          repository: ${{ env.REGISTRY_DOCKERHUB }}
+          readme-filepath: ./README.md
+          # README.md links to docs/ and CONTRIBUTING.md by relative path;
+          # rewrite those to github.com URLs so they resolve on Docker Hub.
+          enable-url-completion: true
+
+      # Docker Hub caps `full_description` at 25,000 bytes and the action
+      # truncates to fit with only a warning, which would publish a README
+      # cut off mid-sentence. README.md is already near the cap, so read the
+      # published overview back and fail if it hit the limit.
+      - name: Check published description was not truncated
+        env:
+          REPOSITORY: ${{ env.REGISTRY_DOCKERHUB }}
+        run: |
+          limit=25000
+          published=$(curl -sSf "https://hub.docker.com/v2/repositories/${REPOSITORY}/" \
+            | jq -j '.full_description' | wc -c | tr -d ' ')
+          echo "Published overview: ${published} bytes (Docker Hub limit: ${limit})"
+          # Truncation lands within one UTF-8 character of the cap.
+          if [ "$published" -ge $((limit - 4)) ]; then
+            echo "::error file=README.md::Docker Hub overview was truncated at ${limit} bytes; trim README.md before the next release."
+            exit 1
+          fi
+
   # Docker build summary
   docker-summary:
     name: Docker Build Summary
@@ -563,6 +620,10 @@ jobs:
           echo ""
 
           case "$BUILD_TYPE" in
+            "preview")
+              echo "🔍 Preview Docker image has been built with ${VERSION} tags"
+              echo "⏭️  Preview images do not update latest or prerelease channels"
+              ;;
             "release")
               echo "🚀 Release Docker image has been built with ${VERSION} tags"
               echo "✅ This image is ready for production use"

.github/workflows/e2e-distributed.yml

@@ -93,7 +93,7 @@ jobs:
       UPGRADE_SOURCE_SHA256: 7c789386bf85278f865b8e0d359bf4edb84d5aa408cc3fa54a18c25ca74cd6e7
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -174,7 +174,7 @@ jobs:
 
       - name: Upload distributed e2e diagnostics
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: e2e-distributed-${{ github.run_number }}
           path: |
@@ -206,7 +206,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/e2e-replication-nightly.yml

@@ -63,7 +63,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -77,7 +77,7 @@ jobs:
 
       # The STS dual-node test requires awscurl and fails if it is unavailable.
       - name: Set up Python
-        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: "3.12"
 
@@ -108,7 +108,7 @@ jobs:
 
       - name: Upload nextest junit report
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: e2e-replication-nightly-junit-${{ github.run_number }}
           path: |
@@ -126,7 +126,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -157,7 +157,7 @@ jobs:
 
       - name: Upload cluster fault diagnostics
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: e2e-cluster-nightly-${{ github.run_number }}
           path: |
@@ -176,7 +176,7 @@ jobs:
       RUSTFS_BUILD_FEATURES: ftps,webdav,sftp
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -213,7 +213,7 @@ jobs:
 
       - name: Upload protocol diagnostics
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: e2e-protocol-nightly-${{ github.run_number }}
           path: |
@@ -236,7 +236,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/e2e-s3tests.yml

@@ -159,7 +159,7 @@ jobs:
       TEST_MODE: ${{ matrix.test-mode }}
       S3_SHARD_INDEX: ${{ matrix.shard-index }}
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -167,7 +167,7 @@ jobs:
       # ship a working pip (ci-1: a bare python3 without pip is what broke the
       # scheduled sweep). Keeps the workflow correct across runner drift.
       - name: Set up Python
-        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: "3.12"
 
@@ -264,7 +264,7 @@ jobs:
               hostname: rustfs4
               volumes: [rustfs4-data:/data]
             lb:
-              image: haproxy:2.9
+              image: haproxy:3.4.4
               hostname: lb
               networks: [rustfs-net]
               ports:
@@ -308,6 +308,11 @@ jobs:
 
           docker compose -f compose.yml up -d
 
+          # Validate the pinned HAProxy image parses the exact config before
+          # the S3 compatibility sweep starts.
+          docker compose -f compose.yml exec -T lb \
+            haproxy -c -f /usr/local/etc/haproxy/haproxy.cfg
+
       - name: Wait for RustFS ready
         run: |
           for _ in {1..120}; do
@@ -357,7 +362,7 @@ jobs:
 
       - name: Upload artifacts
         if: always() && env.ACT != 'true'
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: s3tests-${{ env.TEST_MODE }}-shard-${{ matrix.shard-index }}
           path: artifacts/**
@@ -369,12 +374,12 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
       - name: Set up Python
-        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: "3.12"
 
@@ -435,7 +440,7 @@ jobs:
 
       - name: Upload canary artifacts
         if: always() && env.ACT != 'true'
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: s3tests-upstream-head
           path: artifacts/s3tests-upstream-head/**
@@ -454,7 +459,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/e2e-upgrade.yml

@@ -96,7 +96,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -137,7 +137,7 @@ jobs:
 
       - name: Upload scanner/heal G09 evidence
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: ${{ matrix.artifact }}-g09-evidence-${{ github.run_number }}
           path: ${{ runner.temp }}/rustfs-upgrade-g09-evidence/${{ matrix.artifact }}
@@ -146,7 +146,7 @@ jobs:
 
       - name: Upload server logs
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: ${{ matrix.artifact }}-server-logs-${{ github.run_number }}
           path: ${{ runner.temp }}/rustfs-upgrade-logs

.github/workflows/functional-chain-health.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 15
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Collect and publish verified chain health
@@ -24,7 +24,7 @@ jobs:
         run: python3 scripts/functional_chain_health.py --source-ref "${NIGHTLY_SOURCE_REF}" --publish --output "${RUNNER_TEMP}/chain-health.json"
       - name: Retain health observation
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-chain-health-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-health.json

.github/workflows/fuzz.yml

@@ -81,7 +81,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -93,7 +93,7 @@ jobs:
           cache-save-if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'schedule' }}
 
       - name: Install cargo-fuzz
-        uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
+        uses: taiki-e/install-action@7623a79cdfecb99d681017af368ca353d9f49bb5 # v2
         with:
           tool: cargo-fuzz
 
@@ -113,7 +113,7 @@ jobs:
           done
 
       - name: Upload prebuilt fuzz binaries
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: fuzz-prebuilt-binaries-${{ github.run_number }}
           path: fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/
@@ -143,12 +143,12 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
       - name: Download prebuilt fuzz binaries
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: fuzz-prebuilt-binaries-${{ github.run_number }}
           path: fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release
@@ -167,7 +167,7 @@ jobs:
 
       - name: Upload fuzz smoke artifacts
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: fuzz-smoke-${{ matrix.target }}-${{ github.run_number }}
           path: |
@@ -197,12 +197,12 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
       - name: Download prebuilt fuzz binaries
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: fuzz-prebuilt-binaries-${{ github.run_number }}
           path: fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release
@@ -221,7 +221,7 @@ jobs:
 
       - name: Upload nightly fuzz artifacts
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: fuzz-nightly-${{ matrix.target }}-${{ github.run_number }}
           path: |
@@ -246,7 +246,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/helm-package.yml

@@ -49,7 +49,7 @@ jobs:
 
     steps:
       - name: Checkout helm chart repo
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -107,7 +107,7 @@ jobs:
             --version "${{ steps.version.outputs.chart_version }}"
 
       - name: Upload helm package as artifact
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: helm-package
           path: helm/rustfs/*.tgz
@@ -121,7 +121,7 @@ jobs:
 
     steps:
       - name: Checkout helm package repo
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           # persist-credentials-exempt: this checkout's token IS the push credential —
           # the job git-pushes to rustfs/helm below. Clearing it breaks chart publishing.
@@ -129,7 +129,7 @@ jobs:
           token: ${{ secrets.RUSTFS_HELM_PACKAGE }}
 
       - name: Download helm package
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: helm-package
           path: ./

.github/workflows/minio-interop.yml

@@ -83,7 +83,7 @@ jobs:
       INTEROP_REQUIRED_TESTS: '["reads_minio_generated_sse_s3_multipart_fixture", "reads_minio_generated_sse_kms_multipart_fixture", "rejects_minio_generated_sse_s3_fixture_with_wrong_kms_key", "rejects_minio_generated_sse_s3_fixture_with_truncated_ciphertext"]'
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -137,7 +137,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/mint.yml

@@ -117,7 +117,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 120
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -246,7 +246,7 @@ jobs:
 
       - name: Upload artifacts
         if: always() && env.ACT != 'true'
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: mint
           path: artifacts/**
@@ -264,7 +264,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/nightly-gnu.yml

@@ -51,7 +51,7 @@ jobs:
       source_ref: ${{ steps.source.outputs.ref }}
     steps:
       - name: Checkout selected source
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
           ref: ${{ env.NIGHTLY_BUILD_REF }}
@@ -71,7 +71,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
           ref: ${{ needs.resolve-source.outputs.source_sha }}
@@ -117,10 +117,15 @@ jobs:
 
           cat > "${PKG_DIR}/etc/default/rustfs" << 'ENVEOF'
           # RustFS Environment Configuration
-          # See https://rustfs.com/docs/ for more information
+          # See https://docs.rustfs.com for more information
+          # RUSTFS_ACCESS_KEY=<changeme>
+          # RUSTFS_SECRET_KEY=<changeme>
           # RUSTFS_VOLUMES=""
-          # RUSTFS_ROOT_USER=""
-          # RUSTFS_ROOT_PASSWORD=""
+          # RUSTFS_ADDRESS=":9000"
+          # RUSTFS_CONSOLE_ADDRESS=":9001"
+          # RUSTFS_CONSOLE_ENABLE=true
+          # RUSTFS_OBS_LOGGER_LEVEL=error
+          # RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"
           ENVEOF
 
           # dpkg versions must start with a digit and cannot contain hyphens;
@@ -146,37 +151,13 @@ jobs:
           /etc/default/rustfs
           CONFFILES
 
-          cat > "${PKG_DIR}/DEBIAN/postinst" << 'POSTINST'
-          #!/bin/bash
-          set -e
-          if ! getent passwd rustfs > /dev/null 2>&1; then
-              useradd -r -s /bin/false -d /opt/rustfs rustfs
-          fi
-          mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs
-          chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs
-          if [ -d /run/systemd/system ]; then
-              systemctl daemon-reload
-          fi
-          echo "RustFS installed. Configure /etc/default/rustfs then: systemctl start rustfs"
-          POSTINST
+          ./scripts/release/package_service_scripts.sh after-install > "${PKG_DIR}/DEBIAN/postinst"
           chmod 755 "${PKG_DIR}/DEBIAN/postinst"
 
-          cat > "${PKG_DIR}/DEBIAN/prerm" << 'PRERM'
-          #!/bin/bash
-          set -e
-          if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then
-              systemctl stop rustfs
-          fi
-          PRERM
+          ./scripts/release/package_service_scripts.sh before-remove > "${PKG_DIR}/DEBIAN/prerm"
           chmod 755 "${PKG_DIR}/DEBIAN/prerm"
 
-          cat > "${PKG_DIR}/DEBIAN/postrm" << 'POSTRM'
-          #!/bin/bash
-          set -e
-          if [ -d /run/systemd/system ]; then
-              systemctl daemon-reload
-          fi
-          POSTRM
+          ./scripts/release/package_service_scripts.sh after-remove > "${PKG_DIR}/DEBIAN/postrm"
           chmod 755 "${PKG_DIR}/DEBIAN/postrm"
 
           cp LICENSE "${PKG_DIR}/usr/share/doc/rustfs/"
@@ -214,10 +195,15 @@ jobs:
           mkdir -p ./tmp-pkg/etc/default
           cat > ./tmp-pkg/etc/default/rustfs << 'ENVEOF'
           # RustFS Environment Configuration
-          # See https://rustfs.com/docs/ for more information
+          # See https://docs.rustfs.com for more information
+          # RUSTFS_ACCESS_KEY=<changeme>
+          # RUSTFS_SECRET_KEY=<changeme>
           # RUSTFS_VOLUMES=""
-          # RUSTFS_ROOT_USER=""
-          # RUSTFS_ROOT_PASSWORD=""
+          # RUSTFS_ADDRESS=":9000"
+          # RUSTFS_CONSOLE_ADDRESS=":9001"
+          # RUSTFS_CONSOLE_ENABLE=true
+          # RUSTFS_OBS_LOGGER_LEVEL=error
+          # RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"
           ENVEOF
 
           fpm -s dir -t rpm \
@@ -231,35 +217,13 @@ jobs:
             --description "High-performance distributed object storage" \
             --url "https://rustfs.com" \
             --license "Apache-2.0" \
-            --after-install <(cat << 'POSTINST'
-          #!/bin/bash
-          set -e
-          if ! getent passwd rustfs > /dev/null 2>&1; then
-              useradd -r -s /bin/false -d /opt/rustfs rustfs
-          fi
-          mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs
-          chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs
-          if [ -d /run/systemd/system ]; then
-              systemctl daemon-reload
-          fi
-          POSTINST
-            ) \
-            --before-remove <(cat << 'PRERM'
-          #!/bin/bash
-          set -e
-          if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then
-              systemctl stop rustfs
-          fi
-          PRERM
-            ) \
-            --after-remove <(cat << 'POSTRM'
-          #!/bin/bash
-          set -e
-          if [ -d /run/systemd/system ]; then
-              systemctl daemon-reload
-          fi
-          POSTRM
-            ) \
+            --before-install <(./scripts/release/package_service_scripts.sh rpm-before-install) \
+            --before-upgrade <(./scripts/release/package_service_scripts.sh rpm-before-upgrade) \
+            --after-install <(./scripts/release/package_service_scripts.sh after-install) \
+            --after-upgrade <(./scripts/release/package_service_scripts.sh after-install) \
+            --before-remove <(./scripts/release/package_service_scripts.sh before-remove) \
+            --after-remove <(./scripts/release/package_service_scripts.sh after-remove) \
+            --rpm-posttrans <(./scripts/release/package_service_scripts.sh rpm-posttrans) \
             --config-files /etc/default/rustfs \
             "rustfs-nightly-${DEB_DATE}/usr/bin/rustfs=/usr/bin/rustfs" \
             ./tmp-pkg/etc/default/rustfs=/etc/default/rustfs \
@@ -273,12 +237,12 @@ jobs:
           echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
 
       - name: Upload DEB artifact
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: ${{ steps.deb.outputs.deb_file }}
           path: ${{ steps.deb.outputs.deb_file }}
       - name: Upload RPM artifact
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: ${{ steps.rpm.outputs.rpm_file }}
           path: ${{ steps.rpm.outputs.rpm_file }}
@@ -364,7 +328,7 @@ jobs:
 
       - name: Upload nightly candidate manifest
         if: ${{ steps.publish.outputs.candidate_file != '' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: nightly-candidate-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ steps.publish.outputs.candidate_file }}
@@ -419,7 +383,7 @@ jobs:
       NO_PROXY: 127.0.0.1,localhost
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
           ref: ${{ needs.resolve-source.outputs.source_sha }}
@@ -509,7 +473,7 @@ jobs:
       NO_PROXY: 127.0.0.1,localhost
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
           ref: ${{ needs.resolve-source.outputs.source_sha }}
@@ -537,7 +501,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/nix-flake-update.yml

@@ -39,24 +39,24 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         # persist-credentials-exempt: update-flake-lock pushes the branch and opens
         # the PR. It passes FLAKE_UPDATE_TOKEN to create-pull-request itself rather
         # than reusing .git/config, but that is unverified — exempt until a
         # workflow_dispatch run confirms it (rustfs/backlog#1602).
 
       - name: Install Nix
-        uses: DeterminateSystems/determinate-nix-action@629b284231c2a82554b724e357e47fc6020833c8 # v3
+        uses: DeterminateSystems/determinate-nix-action@8d87e8d5e5b8a8309d4281094560f127d9a265f1 # v3.22.5
 
       - name: Cache Nix
-        uses: DeterminateSystems/flakehub-cache-action@1f9a51a2959d3e26c7838c6f3bf9f48acae525ea # v3.20.0
+        uses: DeterminateSystems/flakehub-cache-action@83282a8aef353db1d659d3bacf4a28b6683de0b3 # v3.22.5
 
       - name: Check Nix flake inputs
-        uses: DeterminateSystems/flake-checker-action@3164002371bc90729c68af0e24d5aacf20d7c9f6 # v12
+        uses: DeterminateSystems/flake-checker-action@786422608c7bded2bbc9741ad9f91356842bf520 # v14
 
       - name: Update flake.lock
         id: update
-        uses: DeterminateSystems/update-flake-lock@fd9359ac79d0e912f1b4b947a48470b3e2799b56 # main
+        uses: DeterminateSystems/update-flake-lock@da03c0f078bc4b2c37ee4f7e072d34bf8f188bb3 # v29
         with:
           git-author-name: houseme
           git-author-email: housemecn@gmail.com

.github/workflows/nix.yml

@@ -72,12 +72,12 @@ jobs:
       NIX_CURL_FLAGS: -A cargo/stable
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
       - name: Install Nix
-        uses: DeterminateSystems/determinate-nix-action@4eea0b33e3d1f02ecfe37cf16e7204c424009606 # v3.21.0
+        uses: DeterminateSystems/determinate-nix-action@8d87e8d5e5b8a8309d4281094560f127d9a265f1 # v3.22.5
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           extra-conf: |
@@ -86,10 +86,10 @@ jobs:
             max-jobs = 1
 
       - name: Cache Nix
-        uses: DeterminateSystems/flakehub-cache-action@c01e819d047464c3edf6ba778f075952af5a3aa7 # v3.21.0
+        uses: DeterminateSystems/flakehub-cache-action@83282a8aef353db1d659d3bacf4a28b6683de0b3 # v3.22.5
 
       - name: Check Nix Flake Inputs
-        uses: DeterminateSystems/flake-checker-action@3164002371bc90729c68af0e24d5aacf20d7c9f6 # v12
+        uses: DeterminateSystems/flake-checker-action@786422608c7bded2bbc9741ad9f91356842bf520 # v14
         with:
           fail-mode: true
           ignore-missing-flake-lock: false

.github/workflows/oidc-keycloak.yml

@@ -57,7 +57,7 @@ jobs:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -82,7 +82,7 @@ jobs:
 
       - name: Upload service logs
         if: failure()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: oidc-keycloak-live-${{ github.run_number }}
           path: ${{ runner.temp }}/rustfs-keycloak-live-*/**/*.log
@@ -101,7 +101,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/on-demand-migration-interop.yml

@@ -93,7 +93,7 @@ jobs:
       NEXTEST_LISTING: ${{ github.workspace }}/artifacts/odm-interop/minio/selection.json
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -166,7 +166,7 @@ jobs:
 
       - name: Upload the MinIO interop report
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: odm-interop-minio-${{ github.run_number }}-${{ github.run_attempt }}
           path: |
@@ -237,7 +237,7 @@ jobs:
 
       - name: Checkout repository
         if: steps.credentials.outputs.present == 'true'
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -286,7 +286,7 @@ jobs:
 
       - name: Upload the ${{ matrix.provider }} interop report
         if: always() && steps.credentials.outputs.present == 'true'
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: odm-interop-${{ matrix.provider }}-${{ github.run_number }}-${{ github.run_attempt }}
           path: |
@@ -306,7 +306,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/package.yml

@@ -25,9 +25,10 @@
 #
 # Flow:
 # 1. Resolve and validate the selected Build workflow run and source identity
-# 2. Download Linux binaries (x86_64-gnu, aarch64-gnu) from build artifacts
-# 3. Build DEB packages for amd64 and arm64
-# 4. Build RPM packages for x86_64 and aarch64
+# 2. Download Linux binaries (x86_64-gnu, aarch64-gnu, x86_64-musl,
+#    aarch64-musl) from build artifacts
+# 3. Build DEB packages for amd64 and arm64, gnu and musl variants
+# 4. Build RPM packages for x86_64 and aarch64, gnu and musl variants
 # 5. Upload all packages to Cloudflare R2 and the GitHub release
 
 name: Package DEB/RPM
@@ -224,9 +225,9 @@ jobs:
           echo "  Build run ID: $BUILD_RUN_ID"
           echo "  Build run number: $RUN_NUMBER"
 
-  # Build DEB and RPM packages for each architecture
+  # Build DEB and RPM packages for each architecture and libc variant
   package:
-    name: Package (${{ matrix.arch }})
+    name: Package (${{ matrix.arch }}-${{ matrix.libc }})
     needs: resolve
     runs-on: ubuntu-latest
     timeout-minutes: 30
@@ -236,16 +237,38 @@ jobs:
       matrix:
         include:
           - arch: x86_64
+            libc: gnu
             deb_arch: amd64
             rpm_arch: x86_64
             artifact_name: "rustfs-linux-x86_64-gnu"
+            # gnu binaries dynamically link glibc; musl binaries are static
+            # and run without a libc dependency.
+            deb_depends: "libc6 (>= 2.31)"
+            rpm_depends: "glibc >= 2.31"
           - arch: aarch64
+            libc: gnu
             deb_arch: arm64
             rpm_arch: aarch64
             artifact_name: "rustfs-linux-aarch64-gnu"
+            deb_depends: "libc6 (>= 2.31)"
+            rpm_depends: "glibc >= 2.31"
+          - arch: x86_64
+            libc: musl
+            deb_arch: amd64
+            rpm_arch: x86_64
+            artifact_name: "rustfs-linux-x86_64-musl"
+            deb_depends: ""
+            rpm_depends: ""
+          - arch: aarch64
+            libc: musl
+            deb_arch: arm64
+            rpm_arch: aarch64
+            artifact_name: "rustfs-linux-aarch64-musl"
+            deb_depends: ""
+            rpm_depends: ""
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -256,17 +279,17 @@ jobs:
           BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
           SOURCE_VERSION: ${{ needs.resolve.outputs.version }}
           DEV_SEQUENCE: ${{ needs.resolve.outputs.dev_sequence }}
-          DEB_ARCH: ${{ matrix.deb_arch }}
-          RPM_ARCH: ${{ matrix.rpm_arch }}
+          ARCH: ${{ matrix.rpm_arch }}
+          LIBC: ${{ matrix.libc }}
         run: |
           set -euo pipefail
 
           normalized=$(./scripts/release/package_versions.sh \
-            "$BUILD_TYPE" "$SOURCE_VERSION" "$DEV_SEQUENCE" "$DEB_ARCH" "$RPM_ARCH")
+            "$BUILD_TYPE" "$SOURCE_VERSION" "$DEV_SEQUENCE" "$ARCH" "$LIBC")
           printf '%s\n' "$normalized" >> "$GITHUB_OUTPUT"
 
       - name: Download binary artifact from build run
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           pattern: ${{ matrix.artifact_name }}*
           path: ./binary-artifact
@@ -308,6 +331,8 @@ jobs:
           DEB_VERSION: ${{ steps.versions.outputs.deb_version }}
           DEB_ARCH: ${{ matrix.deb_arch }}
           DEB_FILE: ${{ steps.versions.outputs.deb_file }}
+          DEB_DEPENDS: ${{ matrix.deb_depends }}
+          LIBC: ${{ matrix.libc }}
         run: |
           set -euo pipefail
 
@@ -328,10 +353,15 @@ jobs:
 
           cat > "${PKG_DIR}/etc/default/rustfs" << 'ENVEOF'
           # RustFS Environment Configuration
-          # See https://rustfs.com/docs/ for more information
+          # See https://docs.rustfs.com for more information
+          # RUSTFS_ACCESS_KEY=<changeme>
+          # RUSTFS_SECRET_KEY=<changeme>
           # RUSTFS_VOLUMES=""
-          # RUSTFS_ROOT_USER=""
-          # RUSTFS_ROOT_PASSWORD=""
+          # RUSTFS_ADDRESS=":9000"
+          # RUSTFS_CONSOLE_ADDRESS=":9001"
+          # RUSTFS_CONSOLE_ENABLE=true
+          # RUSTFS_OBS_LOGGER_LEVEL=error
+          # RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"
           ENVEOF
 
           cat > "${PKG_DIR}/DEBIAN/control" << EOF
@@ -340,9 +370,17 @@ jobs:
           Section: utils
           Priority: optional
           Architecture: ${DEB_ARCH}
-          Depends: libc6 (>= 2.31)
+          EOF
+
+          # gnu binaries dynamically link glibc and must declare it; musl
+          # binaries are statically linked and need no libc dependency.
+          if [[ -n "$DEB_DEPENDS" ]]; then
+            echo "Depends: ${DEB_DEPENDS}" >> "${PKG_DIR}/DEBIAN/control"
+          fi
+
+          cat >> "${PKG_DIR}/DEBIAN/control" << EOF
           Maintainer: RustFS Team <support@rustfs.com>
-          Description: High-performance distributed object storage
+          Description: High-performance distributed object storage (${LIBC} build)
            RustFS is a high-performance distributed object storage software
            built using Rust. It is compatible with MinIO and S3 API.
           Homepage: https://rustfs.com
@@ -354,37 +392,13 @@ jobs:
           /etc/default/rustfs
           CONFFILES
 
-          cat > "${PKG_DIR}/DEBIAN/postinst" << 'POSTINST'
-          #!/bin/bash
-          set -e
-          if ! getent passwd rustfs > /dev/null 2>&1; then
-              useradd -r -s /bin/false -d /opt/rustfs rustfs
-          fi
-          mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs
-          chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs
-          if [ -d /run/systemd/system ]; then
-              systemctl daemon-reload
-          fi
-          echo "RustFS installed. Configure /etc/default/rustfs then: systemctl start rustfs"
-          POSTINST
+          ./scripts/release/package_service_scripts.sh after-install > "${PKG_DIR}/DEBIAN/postinst"
           chmod 755 "${PKG_DIR}/DEBIAN/postinst"
 
-          cat > "${PKG_DIR}/DEBIAN/prerm" << 'PRERM'
-          #!/bin/bash
-          set -e
-          if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then
-              systemctl stop rustfs
-          fi
-          PRERM
+          ./scripts/release/package_service_scripts.sh before-remove > "${PKG_DIR}/DEBIAN/prerm"
           chmod 755 "${PKG_DIR}/DEBIAN/prerm"
 
-          cat > "${PKG_DIR}/DEBIAN/postrm" << 'POSTRM'
-          #!/bin/bash
-          set -e
-          if [ -d /run/systemd/system ]; then
-              systemctl daemon-reload
-          fi
-          POSTRM
+          ./scripts/release/package_service_scripts.sh after-remove > "${PKG_DIR}/DEBIAN/postrm"
           chmod 755 "${PKG_DIR}/DEBIAN/postrm"
 
           cp LICENSE "${PKG_DIR}/usr/share/doc/rustfs/"
@@ -408,24 +422,38 @@ jobs:
           RPM_RELEASE: ${{ steps.versions.outputs.rpm_release }}
           RPM_ARCH: ${{ matrix.rpm_arch }}
           RPM_FILE: ${{ steps.versions.outputs.rpm_file }}
+          RPM_DEPENDS: ${{ matrix.rpm_depends }}
+          LIBC: ${{ matrix.libc }}
         run: |
           set -euo pipefail
 
-          echo "Building RPM for ${RPM_ARCH}"
+          echo "Building RPM for ${RPM_ARCH} (${LIBC})"
 
           sudo apt-get update && sudo apt-get install -y ruby ruby-dev build-essential rpm
           sudo gem install fpm
           ./scripts/test_package_versions.sh --require-package-managers
 
+          # gnu binaries dynamically link glibc and must declare it; musl
+          # binaries are statically linked and need no libc dependency.
+          depends_args=()
+          if [[ -n "$RPM_DEPENDS" ]]; then
+            depends_args+=(--depends "$RPM_DEPENDS")
+          fi
+
           # Create config file for fpm (DEB build creates it in its package dir structure,
           # but fpm needs the file to exist before packaging)
           mkdir -p ./tmp-pkg/etc/default
           cat > ./tmp-pkg/etc/default/rustfs << 'ENVEOF'
           # RustFS Environment Configuration
-          # See https://rustfs.com/docs/ for more information
+          # See https://docs.rustfs.com for more information
+          # RUSTFS_ACCESS_KEY=<changeme>
+          # RUSTFS_SECRET_KEY=<changeme>
           # RUSTFS_VOLUMES=""
-          # RUSTFS_ROOT_USER=""
-          # RUSTFS_ROOT_PASSWORD=""
+          # RUSTFS_ADDRESS=":9000"
+          # RUSTFS_CONSOLE_ADDRESS=":9001"
+          # RUSTFS_CONSOLE_ENABLE=true
+          # RUSTFS_OBS_LOGGER_LEVEL=error
+          # RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"
           ENVEOF
 
           fpm -s dir -t rpm \
@@ -434,40 +462,18 @@ jobs:
             --iteration "$RPM_RELEASE" \
             --architecture "$RPM_ARCH" \
             --package "$RPM_FILE" \
-            --depends "glibc >= 2.31" \
+            "${depends_args[@]}" \
+            --description "High-performance distributed object storage (${LIBC} build)" \
             --maintainer "RustFS Team <support@rustfs.com>" \
-            --description "High-performance distributed object storage" \
             --url "https://rustfs.com" \
             --license "Apache-2.0" \
-            --after-install <(cat <<'POSTINST'
-          #!/bin/bash
-          set -e
-          if ! getent passwd rustfs > /dev/null 2>&1; then
-              useradd -r -s /bin/false -d /opt/rustfs rustfs
-          fi
-          mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs
-          chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs
-          if [ -d /run/systemd/system ]; then
-              systemctl daemon-reload
-          fi
-          POSTINST
-            ) \
-            --before-remove <(cat <<'PRERM'
-          #!/bin/bash
-          set -e
-          if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then
-              systemctl stop rustfs
-          fi
-          PRERM
-            ) \
-            --after-remove <(cat <<'POSTRM'
-          #!/bin/bash
-          set -e
-          if [ -d /run/systemd/system ]; then
-              systemctl daemon-reload
-          fi
-          POSTRM
-            ) \
+            --before-install <(./scripts/release/package_service_scripts.sh rpm-before-install) \
+            --before-upgrade <(./scripts/release/package_service_scripts.sh rpm-before-upgrade) \
+            --after-install <(./scripts/release/package_service_scripts.sh after-install) \
+            --after-upgrade <(./scripts/release/package_service_scripts.sh after-install) \
+            --before-remove <(./scripts/release/package_service_scripts.sh before-remove) \
+            --after-remove <(./scripts/release/package_service_scripts.sh after-remove) \
+            --rpm-posttrans <(./scripts/release/package_service_scripts.sh rpm-posttrans) \
             --config-files /etc/default/rustfs \
             ./bin/rustfs=/usr/bin/rustfs \
             ./tmp-pkg/etc/default/rustfs=/etc/default/rustfs \
@@ -489,9 +495,9 @@ jobs:
           echo "✅ RPM built: $RPM_FILE"
 
       - name: Upload packages to artifacts
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: packages-${{ matrix.arch }}
+          name: packages-${{ matrix.arch }}-${{ matrix.libc }}
           path: |
             *.deb
             *.rpm
@@ -587,10 +593,11 @@ jobs:
             for f in "$DEB_FILE" "$RPM_FILE"; do
               if [[ -n "$f" && -f "$f" ]]; then
                 base="$(basename "$f")"
-                # GitHub stores release asset names with '~' normalized to '.'
-                # (e.g. rustfs_1.0.0~rc.2_amd64.deb is stored as
-                # rustfs_1.0.0.rc.2_amd64.deb), so checksum entries must
-                # reference the name as stored on the release.
+                # GitHub stores release asset names with '~' normalized to '.'.
+                # Current package file names embed the raw tag and contain no
+                # '~', but older releases may still ship such assets, so the
+                # checksum entries must reference the name as stored on the
+                # release.
                 github_base="${base//\~/.}"
                 # Remove any stale entry (both naming variants), then append
                 grep -Fv -- "$base" "$checksum_file" > "${checksum_file}.tmp" || true

.github/workflows/performance-ab.yml

@@ -57,7 +57,7 @@ jobs:
     timeout-minutes: 180
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
           fetch-depth: 0 # baseline may be an earlier successful scheduled head
@@ -94,7 +94,7 @@ jobs:
       - name: Find last successful scheduled baseline
         id: scheduled_baseline
         if: github.event_name == 'schedule'
-        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         with:
           result-encoding: string
           script: |
@@ -271,7 +271,7 @@ jobs:
 
       - name: Upload A/B results
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: hotpath-warp-ab-${{ github.run_number }}
           # Includes per-cell median_summary.csv / baseline_compare.csv, both gates,
@@ -372,7 +372,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/runner-hygiene.yml

@@ -47,7 +47,7 @@ jobs:
     timeout-minutes: 15
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -72,7 +72,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/rustfs-fault-tolerance-matrix.yml

@@ -0,0 +1,221 @@
+# RustFS Fault-Tolerance (degradation) Matrix Sweep
+#
+# Exhaustive topology x EC outage sweep, complementary to the scenario suite
+# (rustfs-fault-tolerance-test.yml). For each topology (nodes x drives per
+# node) and each EC level P in 1..N*drives/2 the runner installs that exact
+# cluster shape and drives progressive node outages plus recovery:
+#
+#   M-<topo>EC<P>-full   full-capacity reads/writes must succeed
+#   M-<topo>EC<P>-n<k>   with k nodes online: reads succeed iff the online
+#                        drives cover the read quorum (N-P), writes iff they
+#                        cover the write quorum (N-P+1); probes below quorum
+#                        assert the expected outage
+#   M-<topo>EC<P>-rec    after restoring every node, reads/writes succeed
+#
+# 10 topologies x EC levels = 38 combinations, 320 cases. The full sweep runs
+# 3-5 hours and is dispatched on demand (e.g. after a nightly chain goes
+# green), not nightly. Case ids and expectations come from product source
+# (default_parity_count, erasure set sizing) via rustfs/auto-testing.
+
+name: RustFS Fault-Tolerance Matrix
+
+on:
+  workflow_dispatch:
+    inputs:
+      package_url:
+        description: 'Direct .deb URL. Required unless the nightly default is wanted.'
+        required: false
+        type: string
+      auto_testing_ref:
+        description: 'auto-testing branch, tag, or SHA (defaults to main)'
+        required: false
+        default: main
+        type: string
+      matrix_topo:
+        description: 'Topology filter like 4x2; empty sweeps all ten topologies'
+        required: false
+        default: ''
+        type: string
+      matrix_ec:
+        description: 'EC/parity filter; empty sweeps every EC level'
+        required: false
+        default: ''
+        type: string
+      cleanup_before:
+        description: 'Reset the nodes before the test (DESTROYS existing data/config)'
+        type: boolean
+        default: true
+      cleanup_after:
+        description: 'Reset the nodes after the test (DESTROYS test data/config)'
+        type: boolean
+        default: true
+
+permissions:
+  contents: read
+
+# The suite stops services and revokes drive access on the shared fleet; only one
+# functional suite may touch the environment at a time.
+concurrency:
+  group: rustfs-shared-functional-tests-v2
+  cancel-in-progress: false
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
+  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
+  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
+  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
+  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
+
+jobs:
+  fault-tolerance-matrix:
+    runs-on: smoke-testing
+    # The full 38-combination sweep takes 3-5h; 360 is the runner job ceiling.
+    timeout-minutes: 360
+    steps:
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-fault-tolerance-matrix-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch" "${FUNCTIONAL_ARTIFACTS_DIR}/evidence"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'RESULTS_FILE=%s/results.json\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'EVIDENCE_DIR=%s/evidence\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
+      - name: Checkout auto-testing scripts
+        timeout-minutes: 5
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ inputs.auto_testing_ref }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
+
+      - name: Record auto-testing revision
+        run: |
+          set -euo pipefail
+          AUTO_TESTING_SHA="$(git -C auto-testing rev-parse HEAD)"
+          printf 'AUTO_TESTING_SHA=%s\n' "${AUTO_TESTING_SHA}" >> "${GITHUB_ENV}"
+          echo "auto-testing revision: ${AUTO_TESTING_SHA}"
+
+      - name: Show environment
+        run: |
+          uname -a
+          jq --version
+          aws --version
+          echo "auto-testing: ${AUTO_TESTING_SHA}"
+          df -h /data | tail -1
+
+      - name: Cleanup environment (before)
+        timeout-minutes: 5
+        if: ${{ inputs.cleanup_before != 'false' }}
+        run: |
+          ./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}"
+
+      - name: Run matrix sweep (topology x EC outage combinations)
+        timeout-minutes: 350
+        id: test
+        # No continue-on-error: an UNEXPECTED verdict (a probe that should
+        # fail by quorum math but passes, or vice versa) is the finding this
+        # sweep exists to surface, and the red run is the signal until matrix
+        # runs get backlog integration like the scenario suite has.
+        env:
+          MATRIX_EC: ${{ inputs.matrix_ec }}
+        run: |
+          set -euo pipefail
+          ARGS=(--matrix -y --package-url "${{ inputs.package_url || env.RUSTFS_NIGHTLY_PACKAGE_URL }}" --log-file "${LOG_FILE}" --results-file "${RESULTS_FILE}")
+          if [ -n "${{ inputs.matrix_topo }}" ]; then
+            # Trailing value form: consumed as `--matrix-topo <topo> [ec]`.
+            ARGS+=(--matrix-topo "${{ inputs.matrix_topo }}")
+            if [ -n "${MATRIX_EC}" ]; then
+              ARGS+=("${MATRIX_EC}")
+            fi
+          fi
+          ./auto-testing/rustfs-fault-tolerance-test.sh "${ARGS[@]}"
+
+      - name: Generate report
+        timeout-minutes: 2
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        run: |
+          set -euo pipefail
+          {
+            echo "# RustFS fault-tolerance matrix report"
+            echo ""
+            echo "- Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+            echo "- Package: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
+            echo "- auto-testing: \`${AUTO_TESTING_SHA}\`"
+            echo "- Topology filter: ${{ inputs.matrix_topo || 'all (10 topologies)' }}"
+            echo "- EC filter: ${{ inputs.matrix_ec || 'all levels' }}"
+            echo ""
+            echo "## Per-probe results"
+            echo ""
+            echo '```'
+            grep -E '^FT-(CASE|COMPLETE|SUMMARY|REPORT)' "${LOG_FILE}" || echo "(no FT-CASE lines found)"
+            echo '```'
+            echo ""
+            echo "## Health snapshots"
+            echo ""
+            for f in "${FUNCTIONAL_ARTIFACTS_DIR}"/evidence/*.code; do
+              [ -e "${f}" ] || continue
+              printf '%s -> %s\n' "$(basename "${f}" .code)" "$(cat "${f}")"
+            done
+          } > "${REPORT_FILE}"
+          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+
+          # An unfiltered sweep must land exactly 320 cases (38 topology x EC
+          # combinations at 2*(nodes+1) cases each); filtered sweeps are held
+          # to internal consistency. Missing/harness-error rows always fail.
+          EXPECTED_TOTAL=""
+          if [ -z "${{ inputs.matrix_topo }}" ] && [ -z "${MATRIX_EC}" ]; then
+            EXPECTED_TOTAL=320
+          fi
+          jq -e --arg testing_sha "${AUTO_TESTING_SHA}" --argjson expected_total "${EXPECTED_TOTAL:-null}" '
+            .schema_version == 1 and
+            .auto_testing_sha == $testing_sha and
+            .complete == true and
+            .counts.expected == ($expected_total // .counts.expected) and
+            .counts.expected == .counts.seen and
+            .counts.harness_errors == 0 and
+            (.expected_cases | length) == .counts.expected and
+            (.expected_cases | unique | length) == .counts.expected and
+            (.seen_cases | length) == .counts.seen and
+            (.missing_cases | length) == 0 and
+            (.harness_errors | length) == 0 and
+            (.cases | length) == .counts.seen and
+            ([.cases[].case_id] | unique | length) == .counts.seen and
+            ([.cases[].case_id] | sort) == (.expected_cases | sort) and
+            all(.cases[]; (.case_id | startswith("M-")) and (.verdict == "pass" or .verdict == "known-divergence" or .verdict == "UNEXPECTED")) and
+            .counts.unexpected == ([.cases[] | select(.verdict == "UNEXPECTED")] | length) and
+            .counts.known_divergence == ([.cases[] | select(.verdict == "known-divergence")] | length)
+          ' "${RESULTS_FILE}" >/dev/null
+
+      - name: Upload test logs & evidence
+        timeout-minutes: 2
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: rustfs-fault-tolerance-matrix-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results.json
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/evidence/
+          if-no-files-found: warn
+
+      - name: Cleanup environment (after)
+        timeout-minutes: 5
+        if: ${{ always() && inputs.cleanup_after != 'false' }}
+        run: |
+          ./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}" || true

.github/workflows/rustfs-fault-tolerance-test.yml

@@ -83,7 +83,7 @@ jobs:
     if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -113,7 +113,7 @@ jobs:
 
       - name: Checkout auto-testing scripts
         timeout-minutes: 5
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/auto-testing
           ref: ${{ steps.chain.outputs.testing_sha || inputs.auto_testing_ref || 'main' }}
@@ -149,7 +149,7 @@ jobs:
           CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
         run: python3 scripts/prepare_functional_package.py prepare
 
-      - name: Run fault-tolerance scenarios (A, B, C, C2, D)
+      - name: Run fault-tolerance scenarios (A, B, C, C2, D, E)
         timeout-minutes: 45
         id: test
         # Standalone case failures are reported by the backlog manager.
@@ -194,21 +194,21 @@ jobs:
           jq -e --arg testing_sha "${AUTO_TESTING_SHA}" '
             .schema_version == 1 and
             .auto_testing_sha == $testing_sha and
-            .selected_scenarios == ["A", "B", "C", "C2", "D"] and
+            .selected_scenarios == ["A", "B", "C", "C2", "D", "E"] and
             .complete == true and
-            .counts.expected == 38 and
-            .counts.seen == 38 and
+            .counts.expected == 44 and
+            .counts.seen == 44 and
             .counts.harness_errors == 0 and
-            (.expected_cases | length) == 38 and
-            (.expected_cases | unique | length) == 38 and
-            (.seen_cases | length) == 38 and
-            (.seen_cases | unique | length) == 38 and
+            (.expected_cases | length) == 44 and
+            (.expected_cases | unique | length) == 44 and
+            (.seen_cases | length) == 44 and
+            (.seen_cases | unique | length) == 44 and
             (.seen_cases | sort) == (.expected_cases | sort) and
             (.missing_cases | length) == 0 and
             (.harness_errors | length) == 0 and
-            (.cases | length) == 38 and
+            (.cases | length) == 44 and
             ([.cases[].case_id] | sort) == (.expected_cases | sort) and
-            ([.cases[].case_id] | unique | length) == 38 and
+            ([.cases[].case_id] | unique | length) == 44 and
             all(.cases[]; .verdict == "pass" or .verdict == "known-divergence" or .verdict == "UNEXPECTED") and
             .counts.unexpected == ([.cases[] | select(.verdict == "UNEXPECTED")] | length) and
             .counts.known_divergence == ([.cases[] | select(.verdict == "known-divergence")] | length)
@@ -294,7 +294,7 @@ jobs:
       - name: Upload test logs & evidence
         timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: rustfs-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
@@ -386,7 +386,7 @@ jobs:
 
       - name: Upload chain evidence
         if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-chain-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}/fault-tolerance.json

.github/workflows/rustfs-functional-chain.yml

@@ -48,19 +48,22 @@ jobs:
       manifest: ${{ steps.candidate.outputs.manifest }}
       performance_ready: ${{ steps.perf_probe.outputs.performance_ready }}
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Resolve published candidate
         id: candidate
         env:
-          GH_TOKEN: ${{ github.token }}
+          # The staleness probe in resolve_functional_candidate.py reads the
+          # private rustfs/auto-testing repo; github.token cannot, and a probe
+          # failure silently keeps a stale testing_sha pinned for the chain.
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
           BUILD_RUN_ID: ${{ inputs.build_run_id }}
           BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
           CHAIN_OUTPUT: ${{ runner.temp }}/chain-candidate.json
         run: python3 scripts/resolve_functional_candidate.py
       - name: Retain candidate identity
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-candidate-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-candidate.json
@@ -188,13 +191,13 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Download suite evidence
         id: download
         continue-on-error: true
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           pattern: functional-chain-*-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-evidence
@@ -210,7 +213,7 @@ jobs:
           --output "${RUNNER_TEMP}/chain-report.json"
       - name: Retain chain report regardless of test verdict
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-chain-report-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
@@ -228,7 +231,7 @@ jobs:
           --directory "${RUNNER_TEMP}/chain-evidence"
           --output "${RUNNER_TEMP}/chain-complete.json"
       - name: Upload complete-chain evidence
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-chain-complete-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-complete.json

.github/workflows/rustfs-heal-test.yml

@@ -66,7 +66,7 @@ jobs:
     if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout chain tooling
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -93,7 +93,7 @@ jobs:
         run: python3 scripts/functional_chain_evidence.py consume
 
       - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/auto-testing
           ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
@@ -366,7 +366,7 @@ jobs:
       - name: Upload test logs
         timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: rustfs-heal-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
@@ -467,7 +467,7 @@ jobs:
 
       - name: Upload chain evidence
         if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-chain-heal-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-heal-${{ github.run_id }}-${{ github.run_attempt }}/heal.json

.github/workflows/rustfs-kms-test.yml

@@ -59,7 +59,7 @@ jobs:
     if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -85,7 +85,7 @@ jobs:
         run: python3 scripts/functional_chain_evidence.py consume
 
       - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/auto-testing
           ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
@@ -315,7 +315,7 @@ jobs:
       - name: Upload report and logs
         timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: rustfs-kms-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
@@ -414,7 +414,7 @@ jobs:
 
       - name: Upload chain evidence
         if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-chain-kms-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-kms-${{ github.run_id }}-${{ github.run_attempt }}/kms.json

.github/workflows/rustfs-performance-test.yml

@@ -93,7 +93,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Check performance runner before entering its queue
@@ -116,7 +116,7 @@ jobs:
     if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout chain tooling
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -144,7 +144,7 @@ jobs:
         run: python3 scripts/functional_chain_evidence.py consume
 
       - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/auto-testing
           ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
@@ -307,7 +307,7 @@ jobs:
 
       - name: Upload test logs & results
         if: ${{ always() && steps.evidence.outcome == 'success' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: rustfs-perf-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
@@ -349,7 +349,7 @@ jobs:
 
       - name: Upload chain evidence
         if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-chain-performance-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-performance-${{ github.run_id }}-${{ github.run_attempt }}/performance.json

.github/workflows/rustfs-pool-expand-test.yml

@@ -92,7 +92,7 @@ jobs:
       RUSTFS_POOL_NODE_ENDPOINTS: ${{ secrets.RUSTFS_POOL_NODE_ENDPOINTS || vars.RUSTFS_POOL_NODE_ENDPOINTS || 'http://rustfs-node1:9000 http://rustfs-node2:9000 http://rustfs-node3:9000 http://rustfs-node4:9000' }}
     steps:
       - name: Checkout chain tooling
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -104,7 +104,7 @@ jobs:
         run: python3 scripts/functional_chain_evidence.py consume
 
       - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/auto-testing
           ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
@@ -656,7 +656,7 @@ jobs:
       - name: Upload test logs
         timeout-minutes: 2
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: rustfs-pool-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/rustfs-pool-${{ github.run_id }}-${{ github.run_attempt }}
@@ -764,7 +764,7 @@ jobs:
 
       - name: Upload chain evidence
         if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-chain-pool-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-pool-${{ github.run_id }}-${{ github.run_attempt }}/pool.json

.github/workflows/rustfs-replication-test.yml

@@ -71,7 +71,7 @@ jobs:
     if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -97,7 +97,7 @@ jobs:
         run: python3 scripts/functional_chain_evidence.py consume
 
       - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/auto-testing
           ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
@@ -323,7 +323,7 @@ jobs:
       - name: Upload report and logs
         timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: rustfs-replication-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
@@ -419,7 +419,7 @@ jobs:
 
       - name: Upload chain evidence
         if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-chain-replication-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-replication-${{ github.run_id }}-${{ github.run_attempt }}/replication.json

.github/workflows/rustfs-s3-compat-test.yml

@@ -47,7 +47,7 @@ jobs:
     if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -73,7 +73,7 @@ jobs:
         run: python3 scripts/functional_chain_evidence.py consume
 
       - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/auto-testing
           ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
@@ -292,7 +292,7 @@ jobs:
       - name: Upload report and logs
         timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: rustfs-s3-compat-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
@@ -391,7 +391,7 @@ jobs:
 
       - name: Upload chain evidence
         if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-chain-s3-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-s3-${{ github.run_id }}-${{ github.run_attempt }}/s3.json

.github/workflows/rustfs-security-test.yml

@@ -87,7 +87,7 @@ jobs:
       # the workspace root would wipe the auto-testing clone above (that is
       # exactly how run 33934141181 lost rustfs-security-test.sh).
       - name: Checkout repository (for the OIDC live gate script)
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
           path: rustfs-repo
@@ -109,7 +109,7 @@ jobs:
         run: python3 scripts/functional_chain_evidence.py consume
 
       - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/auto-testing
           ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
@@ -325,7 +325,7 @@ jobs:
       - name: Upload report and logs
         timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: rustfs-security-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
@@ -395,7 +395,7 @@ jobs:
 
       - name: Upload chain evidence
         if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-chain-security-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-security-${{ github.run_id }}-${{ github.run_attempt }}/security.json

.github/workflows/rustfs-storage-test.yml

@@ -56,7 +56,7 @@ jobs:
     if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -82,7 +82,7 @@ jobs:
         run: python3 scripts/functional_chain_evidence.py consume
 
       - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/auto-testing
           ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
@@ -307,7 +307,7 @@ jobs:
       - name: Upload report and logs
         timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: rustfs-storage-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
@@ -406,7 +406,7 @@ jobs:
 
       - name: Upload chain evidence
         if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-chain-storage-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-storage-${{ github.run_id }}-${{ github.run_attempt }}/storage.json

.github/workflows/rustfs-table-test.yml

@@ -46,7 +46,7 @@ jobs:
     timeout-minutes: 60
     steps:
       - name: Checkout repository (for report parser)
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -73,7 +73,7 @@ jobs:
 
       - name: Checkout auto-testing scripts
         timeout-minutes: 5
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/auto-testing
           ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
@@ -133,7 +133,7 @@ jobs:
           if [ -n "${PACKAGE_URL}" ]; then
             ARGS+=(--package-url "${PACKAGE_URL}")
           elif [ -n "${RUSTFS_VERSION}" ] && [ "${RUSTFS_VERSION}" != "null" ]; then
-            ARGS+=(--package-url "https://github.com/rustfs/rustfs/releases/download/${RUSTFS_VERSION}/rustfs_${RUSTFS_VERSION}_amd64.deb")
+            ARGS+=(--package-url "https://github.com/rustfs/rustfs/releases/download/${RUSTFS_VERSION}/rustfs-linux-x86_64-gnu-v${RUSTFS_VERSION}.deb")
           else
             ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
           fi
@@ -281,7 +281,7 @@ jobs:
       - name: Upload report and logs
         timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: rustfs-table-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
@@ -353,7 +353,7 @@ jobs:
 
       - name: Upload chain evidence
         if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-chain-table-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-table-${{ github.run_id }}-${{ github.run_attempt }}/table.json

.github/workflows/rustfs-tier-test.yml

@@ -71,7 +71,7 @@ jobs:
     if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout chain tooling
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -95,7 +95,7 @@ jobs:
         run: python3 scripts/functional_chain_evidence.py consume
 
       - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/auto-testing
           ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
@@ -460,7 +460,7 @@ jobs:
       - name: Upload report and logs
         timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: rustfs-tier-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ env.TIER_ARTIFACTS_DIR }}/
@@ -734,7 +734,7 @@ jobs:
 
       - name: Upload chain evidence
         if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-chain-tier-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-tier-${{ github.run_id }}-${{ github.run_attempt }}/tier.json

.github/workflows/rustfs-upgrade-test.yml

@@ -89,7 +89,7 @@ jobs:
     if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -115,7 +115,7 @@ jobs:
         run: python3 scripts/functional_chain_evidence.py consume
 
       - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           repository: rustfs/auto-testing
           ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
@@ -207,7 +207,7 @@ jobs:
             local version="$1" tag asset url
             [ -n "${version}" ] && [ "${version}" != "null" ] || return 0
             tag="${version#v}"
-            asset="rustfs_${tag//-/.}_amd64.deb"
+            asset="rustfs-linux-x86_64-gnu-v${tag}.deb"
             url="https://github.com/rustfs/rustfs/releases/download/${tag}/${asset}"
             if ! gh api "repos/rustfs/rustfs/releases/tags/${tag}" --jq '.assets[].name' 2>/dev/null | grep -qxF "${asset}"; then
               echo "ERROR: release ${tag} has no downloadable asset ${asset}:" >&2
@@ -400,7 +400,7 @@ jobs:
       - name: Upload report and logs
         timeout-minutes: 2
         if: ${{ always() && steps.evidence.outcome == 'success' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: rustfs-upgrade-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
@@ -503,7 +503,7 @@ jobs:
 
       - name: Upload chain evidence
         if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: functional-chain-upgrade-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-upgrade-${{ github.run_id }}-${{ github.run_attempt }}/upgrade.json

.github/workflows/schedule-failure-alert-drill.yml

@@ -64,7 +64,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/scheduled-validation-freshness.yml

@@ -36,7 +36,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Check latest scheduled runs

.github/workflows/scheduled-validation-watchdog.yml

@@ -49,7 +49,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update incomplete-run issue

.github/workflows/stale.yml

@@ -29,7 +29,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 30
     steps:
-      - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9
+      - uses: actions/stale@4391f3da665fdf50b6810c1a66712fb9ba21aa93 # v11.0.0
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           stale-issue-message: 'This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs.'

.github/workflows/targets-integration.yml

@@ -52,7 +52,7 @@ jobs:
       RUSTFS_TEST_NATS_URL: nats://127.0.0.1:4222
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 
@@ -166,7 +166,7 @@ jobs:
 
       - name: Upload target integration diagnostics
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: targets-integration-${{ github.run_number }}-${{ github.run_attempt }}
           path: artifacts/targets-live
@@ -183,7 +183,7 @@ jobs:
       contents: read
       issues: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
       - name: Open or update failure-tracking issue

.github/workflows/windows-filesystem.yml

@@ -56,7 +56,7 @@ jobs:
     timeout-minutes: 60
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
 

Cargo.lock

@@ -1984,9 +1984,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.4.6"
+version = "1.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a3eb0f42d6c360dc3f8a821f6bf2fdea7f72bfd36b3076eb0e6d1e9e0752fff4"
+checksum = "54413ede23c2daf518f35156dfde027feb2374004d63bd497f983c8db9c0e313"
 dependencies = [
  "find-msvc-tools",
  "jobserver",
@@ -4410,9 +4410,9 @@ dependencies = [
 
 [[package]]
 name = "find-msvc-tools"
-version = "0.1.12"
+version = "0.1.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d"
+checksum = "ef25905e51abafe4dcea6c15fec58c57b601cdbd0ee53d22ea1d3016c587d39b"
 
 [[package]]
 name = "findshlibs"
@@ -10408,6 +10408,8 @@ dependencies = [
  "opentelemetry_sdk",
  "percent-encoding",
  "pyroscope",
+ "rcgen",
+ "reqwest",
  "rustfs-audit",
  "rustfs-common",
  "rustfs-config",
@@ -10421,6 +10423,8 @@ dependencies = [
  "rustfs-storage-api",
  "rustfs-utils",
  "rustix",
+ "rustls",
+ "rustls-pki-types",
  "serde",
  "serde_json",
  "sysinfo",
@@ -10428,6 +10432,7 @@ dependencies = [
  "tempfile",
  "thiserror 2.0.20",
  "tokio",
+ "tokio-rustls",
  "tokio-util",
  "tracing",
  "tracing-appender",
@@ -11258,9 +11263,8 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
 
 [[package]]
 name = "s3s"
-version = "0.16.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87d22507a8ebd0ea64972c76dfedbc477418dc70afd98bfce928263894067d2f"
+version = "0.16.0"
+source = "git+https://github.com/s3s-project/s3s.git?rev=50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6#50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6"
 dependencies = [
  "arc-swap",
  "arrayvec",
@@ -11278,7 +11282,6 @@ dependencies = [
  "http 1.5.0",
  "http-body 1.1.0",
  "http-body-util",
- "httparse",
  "hyper",
  "itoa",
  "jiff",
@@ -11291,6 +11294,7 @@ dependencies = [
  "pin-project-lite",
  "quick-xml",
  "regex",
+ "s3s-multipart",
  "s3s-rfc2047",
  "s3s-sigv2",
  "s3s-sigv4",
@@ -11315,11 +11319,23 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "s3s-multipart"
+version = "0.17.0-alpha.1"
+source = "git+https://github.com/s3s-project/s3s.git?rev=50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6#50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6"
+dependencies = [
+ "arrayvec",
+ "bytes",
+ "futures-core",
+ "httparse",
+ "memchr",
+ "thiserror 2.0.20",
+]
+
 [[package]]
 name = "s3s-rfc2047"
-version = "0.16.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9d3e6b36be7cca3da377b4cabcd668ffe5b16c3b73a83f7a8f7b63ff6a178ee"
+version = "0.16.0"
+source = "git+https://github.com/s3s-project/s3s.git?rev=50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6#50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6"
 dependencies = [
  "base64-simd",
  "thiserror 2.0.20",
@@ -11327,9 +11343,8 @@ dependencies = [
 
 [[package]]
 name = "s3s-sigv2"
-version = "0.16.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d96880b5ad8d639c9224155126741708aeb62ad3682e0476d2fd14290af808ef"
+version = "0.16.0"
+source = "git+https://github.com/s3s-project/s3s.git?rev=50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6#50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6"
 dependencies = [
  "base64-simd",
  "hmac 0.13.0",
@@ -11341,9 +11356,8 @@ dependencies = [
 
 [[package]]
 name = "s3s-sigv4"
-version = "0.16.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fed605cbaaa1cf4d82e8ee15098fb3e42d49c92bb6215eb1359f63bfc0c6b020"
+version = "0.16.0"
+source = "git+https://github.com/s3s-project/s3s.git?rev=50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6#50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6"
 dependencies = [
  "arrayvec",
  "base64-simd",

Cargo.toml

@@ -313,8 +313,8 @@ rustify = { version = "0.7", default-features = false }
 rustix = { version = "1.1.5" }
 rust-embed = { version = "8.12.0" }
 rustc-hash = { version = "2.1.3" }
-s3s = { version = "0.16.1", features = ["minio"] }
-s3s-sigv4 = { version = "0.16.1" }
+s3s = { git = "https://github.com/s3s-project/s3s.git", rev = "50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6", features = ["minio"] }
+s3s-sigv4 = { git = "https://github.com/s3s-project/s3s.git", rev = "50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6" }
 serial_test = "4.0.1"
 shadow-rs = { default-features = false, version = "2.0.0" }
 siphasher = "1.0.3"

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2024 Beijing Henghesha Technology Co., Ltd. 
+   Copyright 2024-2026 RustFS, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

crates/config/src/observability/mod.rs

@@ -30,6 +30,9 @@ pub const ENV_OBS_ENDPOINT_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_TIMEOUT_M
 pub const ENV_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS";
 pub const ENV_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS";
 pub const ENV_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS";
+/// Absolute path to a PEM bundle whose certificates are added to the system
+/// trust roots used by OTLP/HTTP exporters.
+pub const ENV_OBS_TLS_CA_FILE: &str = "RUSTFS_OBS_TLS_CA_FILE";
 pub const ENV_OBS_PROFILING_ENDPOINT: &str = "RUSTFS_OBS_PROFILING_ENDPOINT";
 pub const ENV_OBS_USE_STDOUT: &str = "RUSTFS_OBS_USE_STDOUT";
 pub const ENV_OBS_SAMPLE_RATIO: &str = "RUSTFS_OBS_SAMPLE_RATIO";
@@ -55,6 +58,13 @@ pub const ENV_OBS_LOG_FILENAME: &str = "RUSTFS_OBS_LOG_FILENAME";
 pub const ENV_OBS_LOG_ROTATION_TIME: &str = "RUSTFS_OBS_LOG_ROTATION_TIME";
 pub const ENV_OBS_LOG_KEEP_FILES: &str = "RUSTFS_OBS_LOG_KEEP_FILES";
 
+/// Span lifecycle events emitted by the JSON log layers (`none`|`close`|`full`).
+///
+/// Defaults to `none`: a record per span lifecycle multiplies log volume on
+/// object hot paths, and emitting the ancestor span list on every record
+/// produced multi-megabyte lines that journald truncated (backlog#2642).
+pub const ENV_OBS_LOG_SPAN_EVENTS: &str = "RUSTFS_OBS_LOG_SPAN_EVENTS";
+
 /// Log cleanup related configurations
 pub const ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES: &str = "RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES";
 pub const ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES: &str = "RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES";
@@ -99,6 +109,8 @@ pub const DEFAULT_OBS_LOG_DRY_RUN: bool = false;
 pub const DEFAULT_OBS_LOG_MATCH_MODE_PREFIX: &str = "prefix";
 pub const DEFAULT_OBS_LOG_MATCH_MODE: &str = "suffix";
 
+pub const DEFAULT_OBS_LOG_SPAN_EVENTS: &str = "none";
+
 /// Default values for observability configuration
 // ### Supported Environment Values
 // - `production` - Secure file-only logging
@@ -141,6 +153,7 @@ mod tests {
         assert_eq!(ENV_OBS_LOG_FILENAME, "RUSTFS_OBS_LOG_FILENAME");
         assert_eq!(ENV_OBS_LOG_ROTATION_TIME, "RUSTFS_OBS_LOG_ROTATION_TIME");
         assert_eq!(ENV_OBS_LOG_KEEP_FILES, "RUSTFS_OBS_LOG_KEEP_FILES");
+        assert_eq!(ENV_OBS_LOG_SPAN_EVENTS, "RUSTFS_OBS_LOG_SPAN_EVENTS");
         assert_eq!(ENV_OBS_TRACES_EXPORT_ENABLED, "RUSTFS_OBS_TRACES_EXPORT_ENABLED");
         assert_eq!(ENV_OBS_METRICS_EXPORT_ENABLED, "RUSTFS_OBS_METRICS_EXPORT_ENABLED");
         assert_eq!(ENV_OBS_LOGS_EXPORT_ENABLED, "RUSTFS_OBS_LOGS_EXPORT_ENABLED");
@@ -178,6 +191,7 @@ mod tests {
         assert_eq!(DEFAULT_OBS_LOG_COMPRESSION_ALGORITHM_GZIP, "gzip");
         assert_eq!(DEFAULT_OBS_LOG_COMPRESSION_ALGORITHM_ZSTD, "zstd");
         assert_eq!(DEFAULT_OBS_LOG_MATCH_MODE_PREFIX, "prefix");
+        assert_eq!(DEFAULT_OBS_LOG_SPAN_EVENTS, "none");
         assert_eq!(DEFAULT_OBS_LOG_MATCH_MODE, "suffix");
         assert_eq!(DEFAULT_OBS_LOG_COMPRESSION_ALGORITHM, "zstd");
     }

crates/e2e_test/src/copy_object_metadata_test.rs

@@ -298,7 +298,11 @@ mod tests {
         assert_eq!(empty_head_resp.content_disposition(), None);
         assert_eq!(empty_head_resp.content_encoding(), None);
         assert_eq!(empty_head_resp.content_language(), None);
-        assert_eq!(empty_head_resp.content_type(), None);
+        assert_eq!(
+            empty_head_resp.content_type(),
+            Some("binary/octet-stream"),
+            "REPLACE without an explicit content type must default to binary/octet-stream, matching S3"
+        );
         assert_eq!(empty_head_resp.expires_string(), None);
         assert_eq!(empty_head_resp.website_redirect_location(), None);
 
@@ -394,7 +398,15 @@ mod tests {
             assert_eq!(head.content_disposition(), (field == "content-disposition").then_some("attachment"));
             assert_eq!(head.content_encoding(), (field == "content-encoding").then_some("gzip"));
             assert_eq!(head.content_language(), (field == "content-language").then_some("de"));
-            assert_eq!(head.content_type(), (field == "content-type").then_some("text/field"));
+            assert_eq!(
+                head.content_type(),
+                Some(if field == "content-type" {
+                    "text/field"
+                } else {
+                    "binary/octet-stream"
+                }),
+                "REPLACE without an explicit content type must default to binary/octet-stream, matching S3"
+            );
             assert_eq!(
                 head.expires_string(),
                 (field == "expires").then_some(replacement_expires_http_date.as_str())
@@ -420,7 +432,11 @@ mod tests {
             .send()
             .await
             .expect("HEAD failed for metadata collision case");
-        assert_eq!(collision_head.content_type(), None);
+        assert_eq!(
+            collision_head.content_type(),
+            Some("binary/octet-stream"),
+            "user-metadata under the reserved 'content-type' key must not satisfy the system content-type default"
+        );
         assert_eq!(collision_head.content_encoding(), None);
         assert_eq!(
             collision_head.metadata().and_then(|metadata| metadata.get("content-type")),

crates/e2e_test/src/distributed/observability_test.rs

@@ -184,7 +184,7 @@ async fn four_node_health_inventory_metrics_and_audit_delivery_are_consistent()
         let (status, metrics_body) = admin_request(
             &node.url,
             Method::GET,
-            "/rustfs/admin/v3/metrics?n=1&by-host=true&by-disk=true",
+            "/rustfs/admin/v3/realtime?n=1&by-host=true&by-disk=true",
             None,
             &dist.cluster.access_key,
             &dist.cluster.secret_key,
@@ -257,7 +257,7 @@ async fn node_admin_body(dist: &DistCluster, node: usize, path: &str) -> TestRes
 }
 
 async fn http_put_counts(dist: &DistCluster, node: usize) -> TestResult<[u64; 2]> {
-    let body = node_admin_body(dist, node, "/rustfs/admin/v3/metrics?types=512&by-host=true&n=1").await?;
+    let body = node_admin_body(dist, node, "/rustfs/admin/v3/realtime?types=512&by-host=true&n=1").await?;
     let sample: RealtimeMetrics = serde_json::from_str(body.lines().next().ok_or("empty HTTP metrics stream")?)?;
     assert!(sample.errors.is_empty(), "HTTP metrics returned errors: {:?}", sample.errors);
     let http = sample.aggregated.http.ok_or("HTTP metrics missing at WARN log level")?;

crates/ecstore/Cargo.toml

@@ -282,5 +282,9 @@ harness = false
 name = "single_block_non_inline_benchmark"
 harness = false
 
+[[bench]]
+name = "multipart_read_parts_benchmark"
+harness = false
+
 [lib]
 doctest = false

crates/ecstore/benches/multipart_read_parts_benchmark.rs

@@ -0,0 +1,92 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
+use rustfs_filemeta::ObjectPartInfo;
+use std::hint::black_box;
+use std::time::Duration;
+
+mod storage_api;
+use storage_api::multipart::{DiskAPI, DiskOption, Endpoint, new_disk};
+
+fn bench_multipart_read_parts(c: &mut Criterion) {
+    let runtime = tokio::runtime::Builder::new_multi_thread()
+        .worker_threads(4)
+        .enable_all()
+        .build()
+        .expect("benchmark runtime");
+    let root = tempfile::tempdir().expect("benchmark disk");
+    let mut endpoint = Endpoint::try_from(root.path().to_str().expect("UTF-8 path")).expect("endpoint");
+    endpoint.set_pool_index(0);
+    endpoint.set_set_index(0);
+    endpoint.set_disk_index(0);
+    let disk = runtime
+        .block_on(new_disk(
+            &endpoint,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+        ))
+        .expect("local disk");
+    let bucket = "multipart-bench";
+    runtime.block_on(disk.make_volume(bucket)).expect("benchmark volume");
+    let upload = root.path().join(bucket).join("upload");
+    std::fs::create_dir_all(&upload).expect("upload directory");
+    let mut paths = Vec::with_capacity(1024);
+    for number in 1..=1024 {
+        let part = ObjectPartInfo {
+            number,
+            etag: format!("{number:032x}"),
+            size: 1024,
+            actual_size: 1024,
+            ..Default::default()
+        };
+        std::fs::write(upload.join(format!("part.{number}")), b"data").expect("part data");
+        std::fs::write(upload.join(format!("part.{number}.meta")), part.marshal_msg().expect("metadata")).expect("part metadata");
+        paths.push(format!("upload/part.{number}.meta"));
+    }
+
+    // Measures the real DiskAPI path against warm local files. Fixture creation
+    // and correctness checks stay outside the timed region; this is not a cold-disk benchmark.
+    let mut group = c.benchmark_group("multipart_read_parts");
+    group.sample_size(20);
+    group.warm_up_time(Duration::from_secs(1));
+    group.measurement_time(Duration::from_secs(2));
+    for count in [1, 32, 256, 1024] {
+        let paths = &paths[..count];
+        let parts = runtime.block_on(disk.read_parts(bucket, paths)).expect("read parts");
+        assert_eq!(parts.len(), count);
+        assert!(
+            parts
+                .iter()
+                .enumerate()
+                .all(|(i, part)| part.number == i + 1 && part.error.is_none())
+        );
+        group.throughput(Throughput::Elements(u64::try_from(count).expect("part count")));
+        group.bench_with_input(BenchmarkId::from_parameter(count), &paths, |b, paths| {
+            b.iter(|| {
+                black_box(
+                    runtime
+                        .block_on(disk.read_parts(bucket, black_box(paths)))
+                        .expect("read parts"),
+                )
+            });
+        });
+    }
+    group.finish();
+}
+
+criterion_group!(benches, bench_multipart_read_parts);
+criterion_main!(benches);

crates/ecstore/benches/storage_api/mod.rs

@@ -29,3 +29,7 @@ pub(crate) mod erasure {
 pub(crate) mod single_block_non_inline {
     pub(crate) use super::{BitrotWriterWrapper, CustomWriter, Erasure};
 }
+
+pub(crate) mod multipart {
+    pub(crate) use rustfs_ecstore::api::disk::{DiskAPI, DiskOption, Endpoint, new_disk};
+}

crates/ecstore/src/api/mod.rs

@@ -504,6 +504,13 @@ pub mod notification {
     };
 }
 
+pub mod integrity {
+    pub use crate::services::integrity::{
+        IntegrityError, InventoryItem, InventoryPage, ItemRequest, ItemResult, ItemState, Job, JobMode, JobRequest, JobState,
+        Protection, Readiness, control_job, create_job, get_job, inventory, readiness, resume_job,
+    };
+}
+
 pub mod object {
     pub use crate::object_api::{
         BLOCK_SIZE_V2, ERASURE_ALGORITHM, EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook,

crates/ecstore/src/bucket/metadata_sys.rs

@@ -1291,6 +1291,28 @@ pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_in(
     }
 }
 
+/// Inspect all migration-relevant settings while the caller holds an Object Lock
+/// snapshot's lifecycle and metadata transaction guards. Cached settings are not
+/// sufficient to authorize a direct storage writer that cannot apply S3 defaults.
+pub(crate) async fn integrity_migration_metadata_in(
+    ctx: &crate::runtime::instance::InstanceContext,
+    bucket: &str,
+) -> Result<Arc<BucketMetadata>> {
+    let sys = bucket_metadata_sys_of(ctx)?.read().await.clone();
+    match sys
+        .read_authoritative_metadata_from_disk_under_transaction_lock(bucket)
+        .await?
+    {
+        BucketMetadataAuthority::Authoritative(metadata)
+            if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
+        {
+            Ok(metadata)
+        }
+        BucketMetadataAuthority::MissingBucket => Err(Error::BucketNotFound(bucket.to_string())),
+        _ => Err(Error::other("migration requires authoritative bucket metadata")),
+    }
+}
+
 /// Re-read the quota configuration and bucket incarnation from the same
 /// authoritative metadata blob while the caller holds the bucket metadata
 /// transaction read lock.

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -2141,7 +2141,7 @@ impl RemoteDisk {
                 let file_info_bin = encode_file_info_msgpack(fi)?;
                 let mut client = self.get_client().await?;
                 let mut request = Request::new(RenameDataRequest {
-                    bucket_incarnation_id: crate::store::bucket_heal_scope(dst_volume)
+                    bucket_incarnation_id: crate::store::bucket_heal_scope_for_object(dst_volume, dst_path)
                         .map(|scope| scope.incarnation.as_bytes().to_vec().into())
                         .unwrap_or_default(),
                     disk: self.endpoint.to_string(),
@@ -4308,6 +4308,13 @@ mod tests {
         object_read_all_disks: Arc<StdMutex<Vec<String>>>,
         format_data: Bytes,
         read_all_data: Bytes,
+        next_format_read_gate: Arc<StdMutex<Option<Arc<TestFormatReadGate>>>>,
+    }
+
+    #[derive(Debug, Default)]
+    struct TestFormatReadGate {
+        entered: tokio::sync::Notify,
+        release: CancellationToken,
     }
 
     impl AuthenticatedReadPeer {
@@ -4319,9 +4326,16 @@ mod tests {
                 object_read_all_disks: Arc::default(),
                 format_data,
                 read_all_data,
+                next_format_read_gate: Arc::default(),
             }
         }
 
+        fn pause_next_format_read(&self) -> Arc<TestFormatReadGate> {
+            let gate = Arc::new(TestFormatReadGate::default());
+            *self.next_format_read_gate.lock().expect("format gate lock poisoned") = Some(Arc::clone(&gate));
+            gate
+        }
+
         fn disk_info_calls(&self) -> u32 {
             self.disk_info_calls.load(Ordering::Acquire)
         }
@@ -4429,6 +4443,11 @@ mod tests {
                                 let disk = request.disk;
                                 peer.read_all_calls.fetch_add(1, Ordering::AcqRel);
                                 let data = if is_format_read {
+                                    let gate = peer.next_format_read_gate.lock().expect("format gate lock poisoned").take();
+                                    if let Some(gate) = gate {
+                                        gate.entered.notify_one();
+                                        gate.release.cancelled().await;
+                                    }
                                     peer.format_data.clone()
                                 } else {
                                     peer.object_read_all_disks
@@ -7131,6 +7150,319 @@ mod tests {
         peer.stop().await;
     }
 
+    async fn missing_read_set(peer: &TestGrpcPeer, format: crate::layout::format::FormatV3) -> Arc<crate::set_disk::SetDisks> {
+        let count = format.erasure.sets[0].len();
+        let endpoints = (0..count)
+            .map(|index| Endpoint {
+                url: url::Url::parse(&format!("{}/data/rustfs{index}", peer.addr)).expect("endpoint should parse"),
+                is_local: false,
+                pool_idx: 0,
+                set_idx: 0,
+                disk_idx: i32::try_from(index).expect("fixture index fits"),
+            })
+            .collect();
+        crate::set_disk::SetDisks::new(
+            "missing-read-test".to_string(),
+            Arc::new(tokio::sync::RwLock::new(vec![None; count])),
+            count,
+            0,
+            0,
+            0,
+            endpoints,
+            format,
+            Vec::new(),
+        )
+        .await
+    }
+
+    #[tokio::test]
+    #[serial(remote_disk_recovery_probe)]
+    async fn missing_read_disks_reconnect_before_periodic_monitor_and_coalesce() {
+        runtime_sources::ensure_test_rpc_secret();
+        let mut format = crate::layout::format::FormatV3::new(1, 1);
+        let disk_id = format.erasure.sets[0][0];
+        format.erasure.this = disk_id;
+        let Some(peer) = TestGrpcPeer::spawn(
+            Bytes::from(format.to_json().expect("format should serialize")),
+            Bytes::from_static(b"startup-recovered-data"),
+        )
+        .await
+        else {
+            panic!("authenticated regression peer is required");
+        };
+        let set = missing_read_set(&peer, format).await;
+        let snapshots = futures::future::join_all((0..8).map(|_| set.get_disks_for_data_read())).await;
+        let disk = snapshots[0][0]
+            .as_ref()
+            .expect("a ready peer must be usable before the periodic reconnect");
+        for snapshot in &snapshots {
+            assert!(Arc::ptr_eq(
+                disk,
+                snapshot[0]
+                    .as_ref()
+                    .expect("every waiting read should observe the same handle")
+            ));
+        }
+        assert_eq!(peer.peer.read_all_calls(), 1, "concurrent requests must share the format probe");
+        assert_eq!(disk.get_disk_id().await.expect("disk identity should be available"), Some(disk_id));
+        let raw = disk
+            .read_all("bucket", "object")
+            .await
+            .expect("reconnected handle should read data");
+        assert_eq!(raw, Bytes::from_static(b"startup-recovered-data"));
+        assert_eq!(peer.peer.object_read_all_disks(), vec![disk_id.to_string()]);
+        let healthy = set.get_disks_for_data_read().await;
+        assert!(Arc::ptr_eq(disk, healthy[0].as_ref().expect("healthy handle must remain registered")));
+        assert_eq!(peer.peer.read_all_calls(), 2, "healthy reads must not re-probe the format");
+        disk.close().await.expect("reconnected disk should close");
+        peer.stop().await;
+    }
+
+    #[tokio::test]
+    #[serial(remote_disk_recovery_probe)]
+    async fn missing_read_disks_reject_foreign_format_and_back_off() {
+        runtime_sources::ensure_test_rpc_secret();
+        let reference = crate::layout::format::FormatV3::new(1, 1);
+        let mut foreign = reference.clone();
+        foreign.id = Uuid::new_v4();
+        foreign.erasure.this = reference.erasure.sets[0][0];
+        let Some(peer) = TestGrpcPeer::spawn(
+            Bytes::from(foreign.to_json().expect("foreign format should serialize")),
+            Bytes::from_static(b"foreign-data"),
+        )
+        .await
+        else {
+            panic!("authenticated regression peer is required");
+        };
+        let set = missing_read_set(&peer, reference).await;
+        assert!(
+            set.get_disks_for_data_read().await[0].is_none(),
+            "a matching disk ID cannot authorize a foreign deployment"
+        );
+        assert_eq!(peer.peer.read_all_calls(), 1, "the foreign format must actually be inspected");
+        assert!(set.get_disks_for_data_read().await[0].is_none());
+        assert_eq!(peer.peer.read_all_calls(), 1, "failed probes must not repeat on every degraded GET");
+        assert!(peer.peer.object_read_all_disks().is_empty(), "foreign object data must never be read");
+        peer.stop().await;
+    }
+
+    #[tokio::test]
+    #[serial(remote_disk_recovery_probe)]
+    async fn missing_read_disks_reject_wrong_slot_and_invalid_formats() {
+        runtime_sources::ensure_test_rpc_secret();
+        let mut reference = crate::layout::format::FormatV3::new(1, 2);
+        reference.erasure.this = reference.erasure.sets[0][0];
+        let mut nil = reference.clone();
+        nil.erasure.this = Uuid::nil();
+        let mut unknown = reference.clone();
+        unknown.erasure.this = Uuid::new_v4();
+        let mut future = reference.clone();
+        future.erasure.version = crate::layout::format::FormatErasureVersion::Unknown;
+        for (name, payload, first_accepted) in [
+            ("duplicate slot zero", reference.to_json().expect("format should serialize"), true),
+            ("nil drive", nil.to_json().expect("nil format should serialize"), false),
+            ("unknown drive", unknown.to_json().expect("unknown format should serialize"), false),
+            ("future version", future.to_json().expect("future format should serialize"), false),
+            ("malformed", "not-json".to_string(), false),
+        ] {
+            let peer = TestGrpcPeer::spawn(Bytes::from(payload), Bytes::new())
+                .await
+                .expect("authenticated regression peer is required");
+            let set = missing_read_set(&peer, reference.clone()).await;
+            let disks = set.get_disks_for_data_read().await;
+            assert_eq!(disks[0].is_some(), first_accepted, "{name}: slot zero identity decision");
+            assert!(disks[1].is_none(), "{name}: a peer cannot claim another topology slot");
+            assert_eq!(peer.peer.read_all_calls(), 2, "{name}: both missing slots must be probed");
+            assert!(peer.peer.object_read_all_disks().is_empty(), "{name}: no object payload may be read");
+            for disk in disks.into_iter().flatten() {
+                disk.close().await.expect("accepted disk should close");
+            }
+            peer.stop().await;
+        }
+    }
+
+    #[tokio::test]
+    #[serial(remote_disk_recovery_probe)]
+    async fn missing_read_disks_do_not_claim_local_or_misconfigured_endpoints() {
+        runtime_sources::ensure_test_rpc_secret();
+        let mut reference = crate::layout::format::FormatV3::new(1, 1);
+        reference.erasure.this = reference.erasure.sets[0][0];
+        let peer = TestGrpcPeer::spawn(Bytes::from(reference.to_json().expect("format should serialize")), Bytes::new())
+            .await
+            .expect("authenticated regression peer is required");
+        for (local, pool, set_index, disk_index) in [(true, 0, 0, 0), (false, -1, 0, 0), (false, 0, 1, 0), (false, 0, 0, 1)] {
+            let mut set = missing_read_set(&peer, reference.clone()).await;
+            let endpoint = &mut Arc::get_mut(&mut set)
+                .expect("fixture set must be uniquely owned")
+                .set_endpoints[0];
+            endpoint.is_local = local;
+            endpoint.pool_idx = pool;
+            endpoint.set_idx = set_index;
+            endpoint.disk_idx = disk_index;
+            assert!(set.get_disks_for_data_read().await[0].is_none());
+        }
+        assert_eq!(peer.peer.read_all_calls(), 0, "invalid endpoint ownership must fail before I/O");
+        peer.stop().await;
+    }
+
+    #[tokio::test]
+    #[serial(remote_disk_recovery_probe)]
+    async fn missing_read_disks_preserve_concurrently_published_handles() {
+        runtime_sources::ensure_test_rpc_secret();
+        let mut reference = crate::layout::format::FormatV3::new(1, 1);
+        reference.erasure.this = reference.erasure.sets[0][0];
+        for reader_first in [false, true] {
+            let peer = TestGrpcPeer::spawn(Bytes::from(reference.to_json().expect("format should serialize")), Bytes::new())
+                .await
+                .expect("authenticated regression peer is required");
+            let set = missing_read_set(&peer, reference.clone()).await;
+            let gate = peer.peer.pause_next_format_read();
+            let pending_set = Arc::clone(&set);
+            let pending = tokio::spawn(async move {
+                if reader_first {
+                    pending_set.get_disks_for_data_read().await;
+                } else {
+                    pending_set.renew_disk(&pending_set.set_endpoints[0]).await;
+                }
+            });
+            time::timeout(Duration::from_secs(2), gate.entered.notified())
+                .await
+                .expect("the first reconnect must reach the format barrier");
+            if reader_first {
+                set.renew_disk(&set.set_endpoints[0]).await;
+            } else {
+                set.get_disks_for_data_read().await;
+            }
+            let published = set.disks.read().await[0]
+                .clone()
+                .expect("the competing reconnect must publish");
+            gate.release.cancel();
+            pending.await.expect("paused reconnect must complete");
+            assert!(
+                Arc::ptr_eq(&published, set.disks.read().await[0].as_ref().expect("published slot must remain")),
+                "neither reconnect may replace a handle published while its format I/O was pending"
+            );
+            published.close().await.expect("published disk should close");
+            peer.stop().await;
+        }
+    }
+
+    #[tokio::test]
+    #[serial(remote_disk_recovery_probe, internode_metrics)]
+    async fn missing_read_disks_expired_waiter_does_not_start_another_probe() {
+        // Disable cooldown so this independently proves deadline admission,
+        // rather than passing only because the leader has just finished.
+        temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_RETURNING_PROBE_INTERVAL_SECS, Some("0"))], async {
+            runtime_sources::ensure_test_rpc_secret();
+            let mut reference = crate::layout::format::FormatV3::new(1, 1);
+            reference.erasure.this = reference.erasure.sets[0][0];
+            let peer = TestGrpcPeer::spawn(Bytes::from(reference.to_json().expect("format should serialize")), Bytes::new())
+                .await
+                .expect("authenticated regression peer is required");
+            let set = missing_read_set(&peer, reference).await;
+            let gate = peer.peer.pause_next_format_read();
+            let mut leader = Box::pin(set.get_disks_for_data_read());
+            time::timeout(Duration::from_secs(2), async {
+                tokio::select! {
+                    _ = gate.entered.notified() => {}
+                    _ = &mut leader => panic!("leader must remain blocked in its format RPC"),
+                }
+            })
+            .await
+            .expect("leader must reach the format barrier");
+            time::pause();
+            let mut waiter = Box::pin(set.get_disks_for_data_read());
+            assert!(futures::poll!(&mut waiter).is_pending(), "waiter must queue behind the leader");
+            let requests_before =
+                crate::cluster::rpc::runtime_sources::internode_metrics_snapshot_for_test().outgoing_requests_total;
+
+            time::advance(crate::disk::disk_store::get_drive_active_check_timeout()).await;
+            assert!(leader.await[0].is_none(), "expired leader must not publish a disk");
+            assert!(waiter.await[0].is_none(), "expired waiter must not publish a disk");
+            time::resume();
+
+            // Count client dispatches, not server arrivals: a cancelled RPC can still
+            // be buffered in the transport when the expired waiter returns.
+            assert_eq!(
+                crate::cluster::rpc::runtime_sources::internode_metrics_snapshot_for_test().outgoing_requests_total,
+                requests_before,
+                "an expired waiter must not dispatch another format RPC after the leader releases the lock"
+            );
+            gate.release.cancel();
+            peer.stop().await;
+        })
+        .await;
+    }
+
+    #[tokio::test]
+    #[serial(remote_disk_recovery_probe)]
+    async fn missing_read_disks_timeout_releases_singleflight_and_allows_retry() {
+        runtime_sources::ensure_test_rpc_secret();
+        let mut reference = crate::layout::format::FormatV3::new(1, 1);
+        reference.erasure.this = reference.erasure.sets[0][0];
+        let peer = TestGrpcPeer::spawn(Bytes::from(reference.to_json().expect("format should serialize")), Bytes::new())
+            .await
+            .expect("authenticated regression peer is required");
+        let set = missing_read_set(&peer, reference).await;
+        let gate = peer.peer.pause_next_format_read();
+        let deadline = crate::disk::disk_store::get_drive_active_check_timeout();
+        let snapshots = time::timeout(
+            deadline + Duration::from_secs(2),
+            futures::future::join_all((0..4).map(|_| set.get_disks_for_data_read())),
+        )
+        .await
+        .expect("all waiting requests must respect the bounded probe deadline");
+        assert!(snapshots.iter().all(|snapshot| snapshot[0].is_none()));
+        assert_eq!(peer.peer.read_all_calls(), 1, "only one format RPC may remain stalled");
+        gate.release.cancel();
+        time::pause();
+        time::advance(crate::disk::health_state::get_drive_returning_probe_interval()).await;
+        time::resume();
+        let disks = set.get_disks_for_data_read().await;
+        let disk = disks[0]
+            .as_ref()
+            .expect("a timed-out probe must not prevent a later read from reconnecting");
+        assert_eq!(peer.peer.read_all_calls(), 2, "retry must validate the format again");
+        disk.close().await.expect("reconnected disk should close");
+        peer.stop().await;
+    }
+
+    #[tokio::test]
+    #[serial(remote_disk_recovery_probe)]
+    async fn missing_read_disks_cancellation_leaves_no_published_handle() {
+        runtime_sources::ensure_test_rpc_secret();
+        let mut reference = crate::layout::format::FormatV3::new(1, 1);
+        reference.erasure.this = reference.erasure.sets[0][0];
+        let peer = TestGrpcPeer::spawn(Bytes::from(reference.to_json().expect("format should serialize")), Bytes::new())
+            .await
+            .expect("authenticated regression peer is required");
+        let set = missing_read_set(&peer, reference).await;
+        let gate = peer.peer.pause_next_format_read();
+        let pending_set = Arc::clone(&set);
+        let pending = tokio::spawn(async move { pending_set.get_disks_for_data_read().await });
+        time::timeout(Duration::from_secs(2), gate.entered.notified())
+            .await
+            .expect("probe must reach the format barrier");
+        // Cancellation can happen after the start-based cooldown has elapsed.
+        // It must still cool down before another caller probes the same peer.
+        time::pause();
+        time::advance(crate::disk::health_state::get_drive_returning_probe_interval()).await;
+        time::resume();
+        pending.abort();
+        assert!(pending.await.expect_err("probe must be cancelled").is_cancelled());
+        assert!(
+            set.disks.read().await[0].is_none(),
+            "cancelled format I/O must not publish an unvalidated handle"
+        );
+        let waiting = time::timeout(Duration::from_secs(1), set.get_disks_for_data_read())
+            .await
+            .expect("cancelled probe must release the singleflight lock");
+        assert!(waiting[0].is_none(), "retry cooldown must still apply after cancellation");
+        assert_eq!(peer.peer.read_all_calls(), 1);
+        gate.release.cancel();
+        peer.stop().await;
+    }
+
     #[tokio::test]
     async fn test_copy_stream_with_buffer_copies_full_payload() {
         let payload = b"walk-dir-stream".repeat(1024);

crates/ecstore/src/core/pools_test.rs

@@ -1185,14 +1185,18 @@ mod decommission_lock_order_tests {
         barrier.wait_until_paused().await;
 
         let probe_store = Arc::clone(&other_store);
-        tokio::time::timeout(std::time::Duration::from_secs(1), async move {
-            probe_store
-                .save_current_pool_meta_for_test(&[0])
-                .await
-                .expect("pool metadata mutation probe should commit before UploadPart admission");
-        })
+        let pool_meta_lock = probe_store
+            .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
+            .await
+            .expect("create the public UploadPart capacity guard probe");
+        let pool_meta_guard = tokio::time::timeout(
+            std::time::Duration::from_secs(1),
+            pool_meta_lock.get_write_lock(std::time::Duration::from_secs(30)),
+        )
         .await
-        .expect("public UploadPart must not hold a capacity read guard during staging");
+        .expect("public UploadPart must not hold a capacity read guard during staging")
+        .expect("public UploadPart capacity guard probe should acquire");
+        drop(pool_meta_guard);
 
         barrier.release();
         let part = tokio::time::timeout(std::time::Duration::from_secs(30), put)
@@ -4253,6 +4257,13 @@ mod decommission_lock_order_tests {
                 panic!("cancellation probe finished before observing target contention: {result:?}");
             }
         }
+        // The mutation absorbs the gate contention inline before the outer wait
+        // loop ever sees it, and that inline budget is bounded.
+        assert_eq!(
+            retry_observer.target_gate_inline_retries(),
+            crate::core::pools::DECOMMISSION_MUTATION_GATE_MAX_INLINE_ATTEMPTS - 1,
+            "inline target-gate retries must stop at their bounded attempt budget"
+        );
         tokio::time::sleep(Duration::from_millis(800)).await;
         assert_eq!(
             retry_observer.target_gate_exact_reloads(),

crates/ecstore/src/disk/error.rs

@@ -244,13 +244,16 @@ impl StdError for ConditionalFileNotCommittedError {
     }
 }
 
-fn classify_internode_missing_error(error: &InternodeHttpError) -> Option<DiskError> {
+fn classify_internode_disk_error(error: &InternodeHttpError) -> Option<DiskError> {
     if error.is_remote_file_not_found() {
         return Some(DiskError::FileNotFound);
     }
     if error.is_remote_volume_not_found() {
         return Some(DiskError::VolumeNotFound);
     }
+    if error.is_remote_file_corrupt() {
+        return Some(DiskError::FileCorrupt);
+    }
     None
 }
 
@@ -526,13 +529,10 @@ fn io_error_chain_contains_kind(io_error: &std::io::Error, kind: std::io::ErrorK
 
 impl From<std::io::Error> for DiskError {
     fn from(e: std::io::Error) -> Self {
-        if let Some(error) = e.get_ref().and_then(|source| source.downcast_ref::<InternodeHttpError>()) {
-            if error.is_remote_file_not_found() {
-                return DiskError::FileNotFound;
-            }
-            if error.is_remote_volume_not_found() {
-                return DiskError::VolumeNotFound;
-            }
+        if let Some(error) = e.get_ref().and_then(|source| source.downcast_ref::<InternodeHttpError>())
+            && let Some(classified) = classify_internode_disk_error(error)
+        {
+            return classified;
         }
         let e = match e.downcast::<TerminalReadError>() {
             Ok(terminal_error) => {
@@ -541,7 +541,7 @@ impl From<std::io::Error> for DiskError {
                     && let Some(internode_error) = io_error
                         .get_ref()
                         .and_then(|source| source.downcast_ref::<InternodeHttpError>())
-                    && let Some(classified) = classify_internode_missing_error(internode_error)
+                    && let Some(classified) = classify_internode_disk_error(internode_error)
                 {
                     return classified;
                 }
@@ -963,6 +963,7 @@ mod tests {
         for (remote_error, expected) in [
             (rustfs_rio::new_test_remote_file_not_found_http_io_error(), DiskError::FileNotFound),
             (rustfs_rio::new_test_remote_volume_not_found_http_io_error(), DiskError::VolumeNotFound),
+            (rustfs_rio::new_test_remote_file_corrupt_http_io_error(), DiskError::FileCorrupt),
         ] {
             let wrapped = terminal_read_error_to_io(DiskError::Io(remote_error));
             assert_eq!(DiskError::from(wrapped), expected);
@@ -1533,25 +1534,23 @@ mod tests {
     }
 
     #[test]
-    fn test_internode_missing_errors_preserve_disk_error_types() {
+    fn test_internode_disk_errors_preserve_disk_error_types() {
         let file_missing = DiskError::from(rustfs_rio::new_test_remote_file_not_found_http_io_error());
         let volume_missing = DiskError::from(rustfs_rio::new_test_remote_volume_not_found_http_io_error());
+        let file_corrupt = DiskError::from(rustfs_rio::new_test_remote_file_corrupt_http_io_error());
         let unmarked_server_error = DiskError::from(rustfs_rio::new_test_internode_http_io_error(
             rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::INTERNAL_SERVER_ERROR),
         ));
 
         assert_eq!(file_missing, DiskError::FileNotFound);
         assert_eq!(volume_missing, DiskError::VolumeNotFound);
+        assert_eq!(file_corrupt, DiskError::FileCorrupt);
         assert!(matches!(unmarked_server_error, DiskError::Io(_)));
-        for missing in [file_missing, volume_missing] {
-            assert_eq!(missing.clone(), missing);
+        for error in [file_missing, volume_missing, file_corrupt] {
+            assert_eq!(error.clone(), error);
             assert_eq!(
-                crate::disk::error_reduce::reduce_write_quorum_errs(
-                    &[Some(missing.clone()), Some(missing.clone()), None],
-                    &[],
-                    2
-                ),
-                Some(missing)
+                crate::disk::error_reduce::reduce_write_quorum_errs(&[Some(error.clone()), Some(error.clone()), None], &[], 2),
+                Some(error)
             );
         }
     }

crates/ecstore/src/disk/local.rs

@@ -53,6 +53,7 @@ use crate::disk::{
 use crate::erasure::coding::{self, bitrot_verify};
 use crate::runtime::sources as runtime_sources;
 use bytes::Bytes;
+use futures::{StreamExt, TryStreamExt, stream};
 use metrics::counter;
 #[cfg(target_os = "linux")]
 use metrics::gauge;
@@ -88,6 +89,9 @@ use tokio::time::{Instant, Sleep, interval_at, timeout};
 use tracing::{debug, error, info, warn};
 use uuid::Uuid;
 
+// Bound outstanding filesystem jobs and metadata buffers per disk request.
+const PART_METADATA_READ_CONCURRENCY: usize = 8;
+
 const DELETED_OBJECTS_CLEANUP_INTERVAL: Duration = Duration::from_secs(60 * 5);
 const STALE_TMP_OBJECT_EXPIRY: Duration = Duration::from_secs(24 * 60 * 60);
 
@@ -6679,6 +6683,53 @@ impl LocalDisk {
         Ok(data)
     }
 
+    async fn read_part_metadata(&self, bucket: &str, volume_dir: &Path, path_str: &str) -> Result<ObjectPartInfo> {
+        let path = Path::new(path_str);
+        let num = path
+            .file_name()
+            .and_then(|v| v.to_str())
+            .unwrap_or_default()
+            .strip_prefix("part.")
+            .and_then(|v| v.strip_suffix(".meta"))
+            .and_then(|v| v.parse::<usize>().ok())
+            .unwrap_or_default();
+        let data_path = self.io_get_object_path(
+            bucket,
+            &path_join_buf(&[
+                path.parent().unwrap_or_else(|| Path::new("")).to_string_lossy().as_ref(),
+                &format!("part.{num}"),
+            ]),
+        )?;
+        let metadata_path = self.io_get_object_path(bucket, path.to_string_lossy().as_ref());
+        // A part's existence check, metadata read and decode share one dispatch.
+        // Keep open errors unmapped for the existing missing-volume fallback.
+        let result = tokio::task::spawn_blocking(move || -> Result<_> {
+            let part_error = |error: String| ObjectPartInfo {
+                number: num,
+                error: Some(error),
+                ..Default::default()
+            };
+            if let Err(err) = std::fs::metadata(data_path) {
+                return Ok(Ok(part_error(err.to_string())));
+            }
+            // Invalid metadata paths remain request errors, but missing data wins
+            // first, as it does in the serial reader.
+            let metadata_path = metadata_path?;
+            Ok(read_all_data_std(&metadata_path)
+                .map(|(data, _)| ObjectPartInfo::unmarshal(&data).unwrap_or_else(|err| part_error(err.to_string()))))
+        })
+        .await;
+        let result = match result {
+            Ok(result) => self.resolve_read_all_result(bucket, volume_dir, result?).await,
+            Err(err) => Err(DiskError::from(err)),
+        };
+        Ok(result.unwrap_or_else(|err| ObjectPartInfo {
+            number: num,
+            error: Some(err.to_string()),
+            ..Default::default()
+        }))
+    }
+
     async fn read_listing_metadata(&self, volume: &str, object_name: &str) -> Result<ListingMetadataRead> {
         let object_dir = self.io_get_object_path(volume, object_name)?;
         let metadata_path = object_dir.join(STORAGE_FORMAT_FILE);
@@ -7615,17 +7666,21 @@ impl LocalDisk {
     where
         W: AsyncWrite + Unpin + Send,
     {
+        // The part of forward_to below this directory, taken before `current`
+        // loses its trailing slash.
+        let forward_rest = opts
+            .forward_to
+            .as_ref()
+            .and_then(|v| v.strip_prefix(&current))
+            .map(str::to_owned);
         let forward = {
-            opts.forward_to
-                .as_ref()
-                .and_then(|v| v.strip_prefix(&current))
-                .map(|forward| {
-                    if let Some(idx) = forward.find('/') {
-                        forward[..idx].to_owned()
-                    } else {
-                        forward.to_owned()
-                    }
-                })
+            forward_rest.as_deref().map(|forward| {
+                if let Some(idx) = forward.find('/') {
+                    forward[..idx].to_owned()
+                } else {
+                    forward.to_owned()
+                }
+            })
         };
 
         if opts.limit > 0 && *objs_returned >= opts.limit {
@@ -7778,13 +7833,22 @@ impl LocalDisk {
 
         entries.sort();
 
-        if let Some(forward) = &forward {
-            for (i, entry) in entries.iter().enumerate() {
-                if entry >= forward || forward.starts_with(entry.as_str()) {
-                    entries.drain(..i);
-                    break;
+        // Every entry left here is a directory. Compare it as the key prefix it
+        // stands for, slash included: "s-x" sorts after "s" as a name, but all
+        // of "s-x/..." sorts before "s/..." ('-' < '/'), so a scan resuming
+        // inside "s/" has to leave "s-x" out even though it comes later.
+        if let Some(forward) = forward_rest.as_deref() {
+            entries.retain(|entry| {
+                if entry.is_empty() {
+                    return true;
                 }
-            }
+                let key = if entry.ends_with(SLASH_SEPARATOR) {
+                    std::borrow::Cow::Borrowed(entry.as_str())
+                } else {
+                    std::borrow::Cow::Owned(format!("{entry}{SLASH_SEPARATOR}"))
+                };
+                key.as_ref() >= forward || forward.starts_with(key.as_ref())
+            });
         }
 
         let mut dir_stack: Vec<(String, bool, Option<HashSet<String>>, bool)> = Vec::with_capacity(5);
@@ -9340,72 +9404,14 @@ impl DiskAPI for LocalDisk {
     #[tracing::instrument(level = "trace", skip_all)]
     async fn read_parts(&self, bucket: &str, paths: &[String]) -> Result<Vec<ObjectPartInfo>> {
         let volume_dir = self.io_get_bucket_path(bucket)?;
-
-        let mut ret = vec![ObjectPartInfo::default(); paths.len()];
-
-        for (i, path_str) in paths.iter().enumerate() {
-            let path = Path::new(path_str);
-            let file_name = path.file_name().and_then(|v| v.to_str()).unwrap_or_default();
-            let num = file_name
-                .strip_prefix("part.")
-                .and_then(|v| v.strip_suffix(".meta"))
-                .and_then(|v| v.parse::<usize>().ok())
-                .unwrap_or_default();
-
-            if let Err(err) = access(
-                self.io_get_object_path(
-                    bucket,
-                    path_join_buf(&[
-                        path.parent().unwrap_or_else(|| Path::new("")).to_string_lossy().as_ref(),
-                        &format!("part.{num}"),
-                    ])
-                    .as_str(),
-                )?,
-            )
+        stream::iter(0..paths.len())
+            .map(|index| self.read_part_metadata(bucket, &volume_dir, &paths[index]))
+            .buffered(PART_METADATA_READ_CONCURRENCY)
+            .try_fold(Vec::with_capacity(paths.len()), |mut parts, part| async move {
+                parts.push(part);
+                Ok(parts)
+            })
             .await
-            {
-                ret[i] = ObjectPartInfo {
-                    number: num,
-                    error: Some(err.to_string()),
-                    ..Default::default()
-                };
-                continue;
-            }
-
-            let data = match self
-                .read_all_data(
-                    bucket,
-                    volume_dir.clone(),
-                    self.io_get_object_path(bucket, path.to_string_lossy().as_ref())?,
-                )
-                .await
-            {
-                Ok(data) => data,
-                Err(err) => {
-                    ret[i] = ObjectPartInfo {
-                        number: num,
-                        error: Some(err.to_string()),
-                        ..Default::default()
-                    };
-                    continue;
-                }
-            };
-
-            match ObjectPartInfo::unmarshal(&data) {
-                Ok(meta) => {
-                    ret[i] = meta;
-                }
-                Err(err) => {
-                    ret[i] = ObjectPartInfo {
-                        number: num,
-                        error: Some(err.to_string()),
-                        ..Default::default()
-                    };
-                }
-            };
-        }
-
-        Ok(ret)
     }
     #[tracing::instrument(level = "trace", skip_all)]
     async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
@@ -12460,6 +12466,79 @@ mod test {
         );
     }
 
+    #[tokio::test]
+    async fn test_read_parts_preserves_order_across_concurrency_windows() {
+        let dir = tempfile::tempdir().expect("temporary disk");
+        let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint");
+        let disk = LocalDisk::new(&endpoint, false).await.expect("local disk");
+        let bucket = "bucket";
+        ensure_test_volume(&disk, bucket).await;
+        let count = PART_METADATA_READ_CONCURRENCY * 3 + 1;
+        let mut paths = Vec::with_capacity(count);
+        let mut expected = Vec::with_capacity(count);
+        for number in (1..=count).rev() {
+            let part = ObjectPartInfo {
+                number,
+                etag: format!("etag-{number}"),
+                size: number,
+                actual_size: i64::try_from(number).expect("small part"),
+                ..Default::default()
+            };
+            let data_path = format!("upload/part.{number}");
+            let meta_path = format!("{data_path}.meta");
+            disk.write_all(bucket, &data_path, Bytes::from_static(b"data"))
+                .await
+                .expect("part data");
+            disk.write_all(bucket, &meta_path, Bytes::from(part.marshal_msg().expect("metadata")))
+                .await
+                .expect("part metadata");
+            paths.push(meta_path);
+            expected.push(part);
+        }
+        assert_eq!(disk.read_parts(bucket, &paths).await.expect("read parts"), expected);
+        assert!(disk.read_parts(bucket, &[]).await.expect("empty batch").is_empty());
+        let missing_meta = "upload/part.99.meta".to_owned();
+        disk.write_all(bucket, "upload/part.99", Bytes::from_static(b"data"))
+            .await
+            .expect("data without metadata");
+        paths.insert(PART_METADATA_READ_CONCURRENCY, missing_meta.clone());
+        let parts = disk.read_parts(bucket, &paths).await.expect("per-part failures");
+        let error = disk
+            .read_all_data(
+                bucket,
+                disk.io_get_bucket_path(bucket).expect("bucket path"),
+                disk.io_get_object_path(bucket, &missing_meta).expect("metadata path"),
+            )
+            .await
+            .expect_err("missing metadata");
+        assert_eq!(parts[PART_METADATA_READ_CONCURRENCY].number, 99);
+        assert_eq!(parts[PART_METADATA_READ_CONCURRENCY].error.as_deref(), Some(error.to_string().as_str()));
+        let successful: Vec<_> = parts.into_iter().filter(|part| part.error.is_none()).collect();
+        assert_eq!(successful, expected, "later windows must survive an earlier part error");
+
+        #[cfg(unix)]
+        {
+            let meta_path = disk
+                .io_get_object_path(bucket, "upload")
+                .expect("upload path")
+                .join("part.100.meta");
+            std::os::unix::fs::symlink(dir.path(), meta_path).expect("invalid metadata symlink");
+            let paths = ["upload/part.100.meta".to_owned()];
+            let parts = disk
+                .read_parts(bucket, &paths)
+                .await
+                .expect("missing data remains a per-part error");
+            assert!(parts[0].error.is_some());
+            disk.write_all(bucket, "upload/part.100", Bytes::from_static(b"data"))
+                .await
+                .expect("data for invalid metadata path");
+            assert_eq!(
+                disk.read_parts(bucket, &paths).await.expect_err("reject metadata symlink"),
+                DiskError::InvalidPath
+            );
+        }
+    }
+
     #[tokio::test]
     async fn test_read_parts_reports_bad_metadata_and_missing_data_part() {
         use tempfile::tempdir;
@@ -18646,6 +18725,95 @@ mod test {
         assert_eq!(double_count as usize, double_names.len());
     }
 
+    #[tokio::test]
+    async fn test_scan_dir_orders_directory_before_its_dash_suffixed_sibling() {
+        use rustfs_filemeta::MetacacheReader;
+        use tempfile::tempdir;
+
+        // "s-x/..." sorts before "s/..." because '-' (0x2d) sorts before '/'
+        // (0x2f), while the directory names alone sort the other way round
+        // ("s" < "s-x"). Backup tools keep "<name>" and "<name>-rollbacks" side
+        // by side this way, which is how flat listings of real buckets ended
+        // early without being reported as truncated.
+        let dir = tempdir().expect("operation should succeed");
+        let bucket = "test-bucket";
+        let bucket_dir = dir.path().join(bucket);
+
+        let mut expected = Vec::new();
+        for parent in ["s", "s-x", "db/backup/s", "db/backup/s-x"] {
+            for n in 0..3 {
+                let name = format!("{parent}/l/l/{n:05}");
+                let object_dir = bucket_dir.join(&name);
+                fs::create_dir_all(&object_dir).await.expect("operation should succeed");
+                fs::write(object_dir.join(STORAGE_FORMAT_FILE), b"meta")
+                    .await
+                    .expect("operation should succeed");
+                expected.push(name);
+            }
+        }
+        expected.sort();
+
+        let endpoint =
+            Endpoint::try_from(dir.path().to_str().expect("operation should succeed")).expect("operation should succeed");
+        let disk = LocalDisk::new(&endpoint, false).await.expect("operation should succeed");
+
+        async fn scan_names(disk: &LocalDisk, bucket: &str, forward_to: Option<&str>, limit: i32) -> Vec<String> {
+            let (reader, mut writer) = tokio::io::duplex(1 << 16);
+            let mut out = MetacacheWriter::new(&mut writer);
+            let opts = WalkDirOptions {
+                bucket: bucket.to_string(),
+                base_dir: "".to_string(),
+                recursive: true,
+                forward_to: forward_to.map(str::to_string),
+                limit,
+                ..Default::default()
+            };
+            let mut objs_returned = 0;
+
+            disk.scan_dir("".to_string(), "".to_string(), &opts, &mut out, &mut objs_returned, false, None)
+                .await
+                .expect("operation should succeed");
+            out.close().await.expect("operation should succeed");
+            drop(out);
+            drop(writer);
+
+            let mut reader = MetacacheReader::new(reader);
+            let entries = reader.read_all().await.expect("operation should succeed");
+            entries
+                .into_iter()
+                .filter(|entry| !entry.metadata.is_empty())
+                .map(|entry| entry.name)
+                .collect()
+        }
+
+        assert_eq!(
+            scan_names(&disk, bucket, None, 0).await,
+            expected,
+            "a full recursive scan must emit keys in lexicographic order"
+        );
+
+        for (i, forward_to) in expected.iter().enumerate() {
+            assert_eq!(
+                scan_names(&disk, bucket, Some(forward_to), 0).await,
+                expected[i..].to_vec(),
+                "resuming at {forward_to} must return every later key"
+            );
+        }
+
+        let mut paged = Vec::new();
+        let mut forward_to: Option<String> = None;
+        while paged.len() < expected.len() {
+            let page = scan_names(&disk, bucket, forward_to.as_deref(), 2).await;
+            let page: Vec<String> = page.into_iter().filter(|name| Some(name) != forward_to.as_ref()).collect();
+            if page.is_empty() {
+                break;
+            }
+            forward_to = page.last().cloned();
+            paged.extend(page);
+        }
+        assert_eq!(paged, expected, "paging through the tree must not lose keys");
+    }
+
     #[tokio::test]
     async fn test_scan_dir_hidden_delete_markers_do_not_exhaust_limit() {
         use rustfs_filemeta::MetacacheReader;

crates/ecstore/src/disk/mod.rs

@@ -825,7 +825,7 @@ impl Disk {
         dst_volume: &str,
         dst_path: &str,
     ) -> Result<RenameDataResp> {
-        let Some(scope) = crate::store::bucket_heal_scope(dst_volume) else {
+        let Some(scope) = crate::store::bucket_heal_scope_for_object(dst_volume, dst_path) else {
             return self
                 .rename_data_borrowed_with_fence(src_volume, src_path, fi, dst_volume, dst_path, None)
                 .await;

crates/ecstore/src/erasure/coding/bitrot.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::disk::error::Error as DiskError;
 use crate::erasure::codec::buffer_pool::{get_ec_buffer, return_ec_buffer};
 use pin_project_lite::pin_project;
 use rustfs_utils::HashAlgorithm;
@@ -239,7 +240,7 @@ fn short_shard_read(got: usize, want: usize) -> std::io::Error {
         want,
         "short shard read: got {got} of {want} bytes"
     );
-    std::io::Error::new(std::io::ErrorKind::UnexpectedEof, format!("short shard read: got {got} of {want} bytes"))
+    std::io::Error::new(std::io::ErrorKind::UnexpectedEof, DiskError::FileCorrupt)
 }
 
 /// Split a `[hash][data]` block, verify the hash (unless `skip_verify`), and
@@ -265,7 +266,7 @@ fn split_and_verify<'a>(hash_algo: &HashAlgorithm, skip_verify: bool, block: &'a
             data_len = data.len(),
             "bitrot hash mismatch"
         );
-        return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "bitrot hash mismatch"));
+        return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, DiskError::FileCorrupt));
     }
     Ok((data, verify))
 }
@@ -434,7 +435,7 @@ where
                             data_len = want,
                             "bitrot hash mismatch"
                         );
-                        return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "bitrot hash mismatch"));
+                        return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, DiskError::FileCorrupt));
                     }
                     hash_offset += take;
                     remaining -= take;
@@ -1141,6 +1142,26 @@ mod tests {
     use std::time::Duration;
     use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, ReadBuf};
 
+    #[test]
+    fn bitrot_damage_preserves_typed_repair_evidence() {
+        use crate::disk::error::Error as DiskError;
+        let algo = HashAlgorithm::HighwayHash256;
+        let data = b"repair evidence";
+        let mut frame = algo.hash_encode(data).as_ref().to_vec();
+        frame.extend_from_slice(data);
+        *frame.last_mut().expect("test frame has payload") ^= 1;
+        let error = super::split_and_verify(&algo, false, &frame).expect_err("a hash mismatch must reject the shard");
+        assert_eq!(error.kind(), io::ErrorKind::InvalidData);
+        assert_eq!(DiskError::from(error), DiskError::FileCorrupt);
+        let error = super::short_shard_read(3, 4);
+        assert_eq!(error.kind(), io::ErrorKind::UnexpectedEof);
+        assert_eq!(DiskError::from(error), DiskError::FileCorrupt);
+        assert!(matches!(
+            DiskError::from(io::Error::new(io::ErrorKind::InvalidData, "unrelated invalid data")),
+            DiskError::Io(_)
+        ));
+    }
+
     struct FragmentedSource {
         chunks: VecDeque<Bytes>,
     }

crates/ecstore/src/erasure/coding/decode.rs

@@ -26,7 +26,7 @@ use crate::erasure::codec::workspace::ShardBufferPool;
 use crate::erasure::coding::{BitrotReader, Erasure};
 use crate::io_support::bitrot::DeferredReaderStripeHandle;
 use crate::set_disk::shard_source::{
-    INLINE_SHARD_SLOTS, ShardBuffers, ShardErrors, ShardReadCost, ShardStripeSource, StripeReadState,
+    INLINE_SHARD_SLOTS, ShardBuffers, ShardErrors, ShardReadCost, ShardReadRepair, ShardStripeSource, StripeReadState,
 };
 use futures::FutureExt;
 use futures::stream::{FuturesUnordered, StreamExt};
@@ -46,7 +46,13 @@ type ShardReadFuture<'a> = Pin<Box<dyn Future<Output = (usize, ShardReadCost, Re
 type OwnedShardReadFuture<'a, R> =
     Pin<Box<dyn Future<Output = (usize, ShardReadCost, Result<Vec<u8>, Error>, Option<BitrotReader<R>>, bool)> + Send + 'a>>;
 pub(crate) type DeferredReaderReopener<R> = Arc<dyn Fn(usize) -> Option<BitrotReader<R>> + Send + Sync>;
-pub(crate) type DecodeOutcome = (usize, Option<std::io::Error>, bool);
+#[derive(Debug, Default)]
+pub(crate) struct DecodeOutcome {
+    pub(crate) written: usize,
+    pub(crate) error: Option<std::io::Error>,
+    pub(crate) exact_quorum: bool,
+    pub(crate) repair: ShardReadRepair,
+}
 
 type ShardIndexes = SmallVec<[usize; INLINE_SHARD_SLOTS]>;
 type ActiveReaders = SmallVec<[bool; INLINE_SHARD_SLOTS]>;
@@ -596,6 +602,7 @@ pub(crate) struct ParallelReader<R> {
     hedged_readers: ActiveReaders,
     stripe_index: usize,
     integrity: Option<std::sync::Arc<crate::io_support::shard_integrity::PartProofReader>>,
+    repair_evidence: Option<Arc<ShardReadRepair>>,
 }
 }
 
@@ -808,9 +815,15 @@ where
             deferred_reopeners: Vec::new(),
             hedged_readers,
             stripe_index: 0,
+            repair_evidence: None,
         }
     }
 
+    pub(crate) fn with_repair_evidence(mut self, evidence: Arc<ShardReadRepair>) -> Self {
+        self.repair_evidence = Some(evidence);
+        self
+    }
+
     /// Attach the per-slot deferred stripe handles produced during bitrot
     /// reader setup. Only parity slots are consulted: a handle lets the
     /// lockstep path open a parity shard aligned to the stripe where a data
@@ -2027,6 +2040,9 @@ where
             .take()
             .unwrap_or_else(|| Box::new(StripeReadState::with_slot_count(self.readers.len(), self.data_shards)));
         self.read_into_state(&mut state).await;
+        if let Some(evidence) = &self.repair_evidence {
+            evidence.observe(state.parts_mut().1);
+        }
         if state.can_decode()
             && let Some(proof) = &self.integrity
         {
@@ -2231,10 +2247,10 @@ impl Erasure {
         W: AsyncWrite + Send + Sync + Unpin,
         R: crate::erasure::coding::ShardSource,
     {
-        let (written, error, _) = self
+        let outcome = self
             .decode_inner(writer, readers, offset, length, total_length, None, Vec::new(), Vec::new())
             .await;
-        (written, error)
+        (outcome.written, outcome.error)
     }
 
     #[allow(dead_code, reason = "read-cost decode path asserted by this file's tests (backlog#1823)")]
@@ -2251,10 +2267,10 @@ impl Erasure {
         W: AsyncWrite + Send + Sync + Unpin,
         R: crate::erasure::coding::ShardSource,
     {
-        let (written, error, _) = self
+        let outcome = self
             .decode_inner(writer, readers, offset, length, total_length, Some(read_costs), Vec::new(), Vec::new())
             .await;
-        (written, error)
+        (outcome.written, outcome.error)
     }
 
     /// GET decode entry point that also carries the deferred-parity stripe
@@ -2311,7 +2327,7 @@ impl Erasure {
         W: AsyncWrite + Send + Sync + Unpin,
         R: crate::erasure::coding::ShardSource,
     {
-        let (written, error, _) = self
+        let outcome = self
             .decode_inner(
                 writer,
                 readers,
@@ -2323,7 +2339,7 @@ impl Erasure {
                 deferred_reopeners,
             )
             .await;
-        (written, error)
+        (outcome.written, outcome.error)
     }
 
     #[allow(clippy::too_many_arguments)]
@@ -2374,6 +2390,7 @@ impl Erasure {
         block_length: usize,
         written: &mut usize,
         ret_err: &mut Option<std::io::Error>,
+        repair: &ShardReadRepair,
         stage_metrics_enabled: bool,
         require_surplus_source: bool,
         integrity: Option<&std::sync::Arc<crate::io_support::shard_integrity::PartProofReader>>,
@@ -2382,6 +2399,7 @@ impl Erasure {
     where
         W: AsyncWrite + Send + Sync + Unpin,
     {
+        repair.observe(errs);
         if ret_err.is_none()
             && let (_, Some(err)) = reduce_errs(errs, &[])
             && (err == Error::FileNotFound || err == Error::FileCorrupt)
@@ -2516,41 +2534,42 @@ impl Erasure {
     {
         if readers.len() != self.data_shards + self.parity_shards {
             record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
-            return (0, Some(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers")), false);
+            return DecodeOutcome {
+                error: Some(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers")),
+                ..Default::default()
+            };
         }
 
         // block_size/data_shards come from on-disk metadata; a corrupt FileInfo with a
         // zero here must surface as an error, not a divide-by-zero panic on every GET.
         if self.block_size == 0 || self.data_shards == 0 {
             record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
-            return (
-                0,
-                Some(io::Error::new(ErrorKind::InvalidInput, "Invalid erasure coding parameters")),
-                false,
-            );
+            return DecodeOutcome {
+                error: Some(io::Error::new(ErrorKind::InvalidInput, "Invalid erasure coding parameters")),
+                ..Default::default()
+            };
         }
 
         let Some(end_offset) = offset.checked_add(length) else {
             record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
-            return (
-                0,
-                Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")),
-                false,
-            );
+            return DecodeOutcome {
+                error: Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")),
+                ..Default::default()
+            };
         };
         if end_offset > total_length {
             record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
-            return (
-                0,
-                Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")),
-                false,
-            );
+            return DecodeOutcome {
+                error: Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")),
+                ..Default::default()
+            };
         }
 
         let mut ret_err = None;
+        let repair = ShardReadRepair::default();
 
         if length == 0 {
-            return (0, ret_err, false);
+            return DecodeOutcome::default();
         }
 
         let mut written = 0;
@@ -2573,6 +2592,11 @@ impl Erasure {
         .with_deferred_parity_reopeners(deferred_reopeners);
 
         let integrity = reader.integrity.clone();
+        // Copy sources use demand-bound scheduling for backpressure, but retain
+        // the ordinary GET read quorum. Only the opt-in GET rollout requires a
+        // surplus source; available surplus shards and integrity proofs are
+        // still verified before any reconstructed copy bytes are emitted.
+        let require_surplus_source = reader.demand_bound_lockstep && matches!(decode_read_policy(), DecodeReadPolicy::Default);
 
         let start = offset / self.block_size;
         let end = end_offset.saturating_sub(1) / self.block_size;
@@ -2667,7 +2691,6 @@ impl Erasure {
                         // `shards` are borrowed again below. In the `Stop` case that
                         // drop is what cancels the still-in-flight prefetch read.
                         let (flow, next): (Option<StripeFlow>, Option<StripeReadOutput>) = {
-                            let require_surplus_source = reader.demand_bound_lockstep;
                             let read_fut = read_stripe_timed(&mut reader, stage_metrics_enabled);
                             let emit_fut = self.emit_decoded_stripe(
                                 writer,
@@ -2677,6 +2700,7 @@ impl Erasure {
                                 block_length,
                                 &mut written,
                                 &mut ret_err,
+                                &repair,
                                 stage_metrics_enabled,
                                 require_surplus_source,
                                 integrity.as_ref(),
@@ -2727,8 +2751,9 @@ impl Erasure {
                                 block_length,
                                 &mut written,
                                 &mut ret_err,
+                                &repair,
                                 stage_metrics_enabled,
-                                reader.demand_bound_lockstep,
+                                require_surplus_source,
                                 integrity.as_ref(),
                                 offset,
                             )
@@ -2770,8 +2795,9 @@ impl Erasure {
                         block_length,
                         &mut written,
                         &mut ret_err,
+                        &repair,
                         stage_metrics_enabled,
-                        reader.demand_bound_lockstep,
+                        require_surplus_source,
                         integrity.as_ref(),
                         offset,
                     )
@@ -2785,15 +2811,16 @@ impl Erasure {
             }
         }
 
-        if ret_err.is_some() {
-            return (written, ret_err, exact_quorum);
-        }
-
-        if written < length {
+        if ret_err.is_none() && written < length {
             ret_err = Some(Error::LessData.into());
         }
 
-        (written, ret_err, exact_quorum)
+        DecodeOutcome {
+            written,
+            error: ret_err,
+            exact_quorum,
+            repair,
+        }
     }
 }
 
@@ -3449,6 +3476,150 @@ mod tests {
         }
     }
 
+    async fn copy_source_bitrot_shards(erasure: &Erasure, payload: &[u8], corrupt_parity: bool) -> Vec<Vec<u8>> {
+        let mut writers: Vec<_> = (0..erasure.data_shards + erasure.parity_shards)
+            .map(|_| BitrotWriter::new(Cursor::new(Vec::new()), erasure.shard_size(), HashAlgorithm::HighwayHash256))
+            .collect();
+        for block in payload.chunks(erasure.block_size) {
+            let shards = erasure.encode_data(block).expect("copy source should encode");
+            for (index, shard) in shards.iter().enumerate() {
+                let mut bytes = shard.to_vec();
+                if corrupt_parity && index == erasure.data_shards {
+                    // A stale source can have a valid per-shard bitrot checksum.
+                    // Reframe the changed parity to exercise reconstruction verification.
+                    bytes[0] ^= 0x80;
+                }
+                writers[index].write(&bytes).await.expect("copy source should frame");
+            }
+        }
+        writers.into_iter().map(|writer| writer.into_inner().into_inner()).collect()
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn copy_source_reads_exact_quorum_across_stripes_and_ranges() {
+        const BLOCK_SIZE: usize = 768;
+        let payload: Vec<_> = (0..BLOCK_SIZE * 3 + 37)
+            .map(|index| (index.wrapping_mul(31) % 251) as u8)
+            .collect();
+
+        for uses_legacy in [false, true] {
+            let erasure = Erasure::new_with_options(12, 4, BLOCK_SIZE, uses_legacy);
+            let buffers = copy_source_bitrot_shards(&erasure, &payload, false).await;
+            for offline in [[0, 1, 2, 3], [0, 5, 12, 15]] {
+                for (offset, length) in [(0, payload.len()), (BLOCK_SIZE + 17, payload.len() - BLOCK_SIZE - 17)] {
+                    let frame_offset = offset / BLOCK_SIZE * (erasure.shard_size() + HashAlgorithm::HighwayHash256.size());
+                    let positioned: Vec<_> = buffers.iter().map(|bytes| bytes[frame_offset..].to_vec()).collect();
+                    let (mut readers, handles) =
+                        readers_with_deferred_parity(&positioned, 12, erasure.shard_size(), &HashAlgorithm::HighwayHash256, &[]);
+                    for index in offline {
+                        readers[index] = None;
+                    }
+                    let mut output = Vec::new();
+                    let (written, error) = crate::set_disk::with_get_object_read_policy(
+                        crate::set_disk::GetObjectReadPolicy::CopySource,
+                        erasure.decode_with_stripe_handles(&mut output, readers, offset, length, payload.len(), None, handles),
+                    )
+                    .await;
+
+                    assert!(error.is_none(), "legacy={uses_legacy}, offline={offline:?}, offset={offset}: {error:?}");
+                    assert_eq!(written, length);
+                    assert_eq!(output, payload[offset..offset + length]);
+                }
+            }
+        }
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn copy_source_recovers_exact_quorum_after_first_stripe() {
+        const BLOCK_SIZE: usize = 768;
+        let payload: Vec<_> = (0..BLOCK_SIZE * 3 + 37)
+            .map(|index| (index.wrapping_mul(31) % 251) as u8)
+            .collect();
+        for uses_legacy in [false, true] {
+            let erasure = Erasure::new_with_options(12, 4, BLOCK_SIZE, uses_legacy);
+            let buffers = copy_source_bitrot_shards(&erasure, &payload, false).await;
+            let first_frame = erasure.shard_size() + HashAlgorithm::HighwayHash256.size();
+            let (readers, handles) = readers_with_deferred_parity(
+                &buffers,
+                12,
+                erasure.shard_size(),
+                &HashAlgorithm::HighwayHash256,
+                &[(0, first_frame), (1, first_frame), (2, first_frame), (3, first_frame)],
+            );
+            let mut output = Vec::new();
+            let (written, error) = crate::set_disk::with_get_object_read_policy(
+                crate::set_disk::GetObjectReadPolicy::CopySource,
+                erasure.decode_with_stripe_handles(&mut output, readers, 0, payload.len(), payload.len(), None, handles),
+            )
+            .await;
+
+            assert!(error.is_none(), "legacy={uses_legacy}: late shard loss should recover: {error:?}");
+            assert_eq!(written, payload.len());
+            assert_eq!(output, payload);
+        }
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn copy_source_below_read_quorum_fails_before_output() {
+        const BLOCK_SIZE: usize = 768;
+        let payload = vec![0x5a; BLOCK_SIZE * 2 + 37];
+        for uses_legacy in [false, true] {
+            let erasure = Erasure::new_with_options(12, 4, BLOCK_SIZE, uses_legacy);
+            let buffers = copy_source_bitrot_shards(&erasure, &payload, false).await;
+            let (mut readers, handles) =
+                readers_with_deferred_parity(&buffers, 12, erasure.shard_size(), &HashAlgorithm::HighwayHash256, &[]);
+            for reader in readers.iter_mut().take(5) {
+                *reader = None;
+            }
+            let mut output = Vec::new();
+            let (written, error) = crate::set_disk::with_get_object_read_policy(
+                crate::set_disk::GetObjectReadPolicy::CopySource,
+                erasure.decode_with_stripe_handles(&mut output, readers, 0, payload.len(), payload.len(), None, handles),
+            )
+            .await;
+
+            assert_eq!(written, 0);
+            assert!(output.is_empty());
+            assert_eq!(
+                error
+                    .expect("eleven shards cannot reconstruct twelve data shards")
+                    .to_string(),
+                Error::ErasureReadQuorum.to_string()
+            );
+        }
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn copy_source_rejects_corrupt_surplus_parity_before_output() {
+        const BLOCK_SIZE: usize = 768;
+        let payload = vec![0x5a; BLOCK_SIZE * 2 + 37];
+        for uses_legacy in [false, true] {
+            let erasure = Erasure::new_with_options(12, 4, BLOCK_SIZE, uses_legacy);
+            let buffers = copy_source_bitrot_shards(&erasure, &payload, true).await;
+            let (mut readers, handles) =
+                readers_with_deferred_parity(&buffers, 12, erasure.shard_size(), &HashAlgorithm::HighwayHash256, &[]);
+            for reader in readers.iter_mut().take(3) {
+                *reader = None;
+            }
+            let mut output = Vec::new();
+            let (written, error) = crate::set_disk::with_get_object_read_policy(
+                crate::set_disk::GetObjectReadPolicy::CopySource,
+                erasure.decode_with_stripe_handles(&mut output, readers, 0, payload.len(), payload.len(), None, handles),
+            )
+            .await;
+
+            assert_eq!(written, 0);
+            assert!(output.is_empty());
+            let error = error.expect("copy must reject inconsistent surplus parity despite valid bitrot framing");
+            assert_eq!(error.kind(), ErrorKind::InvalidData);
+            assert!(error.to_string().contains("inconsistent read source shards"));
+        }
+    }
+
     /// Guards the shard-buffer reuse in `ParallelReader`: a multi-stripe
     /// `Erasure::decode` that reconstructs missing data shards on every stripe
     /// must still return byte-exact output. Reconstructed and parity buffers are
@@ -3834,10 +4005,45 @@ mod tests {
             temp_env::async_with_vars(vars, async {
                 let readers = build_prefetch_readers(&shard_bufs, shard_size, &hash_algo, BLOCK_SIZE, 0, &[], &[1]);
                 let mut output = Vec::new();
-                let (written, err) = erasure.decode(&mut output, readers, 0, total_len, total_len).await;
-                assert!(err.is_none(), "{label}: corrupt-shard read errored {err:?}");
-                assert_eq!(written, total_len, "{label}: corrupt-shard short write");
+                let outcome = erasure
+                    .decode_with_stripe_handles_and_reopeners_with_diagnostics(
+                        &mut output,
+                        readers,
+                        0,
+                        total_len,
+                        total_len,
+                        None,
+                        Vec::new(),
+                        Vec::new(),
+                    )
+                    .await;
+                assert!(outcome.error.is_none(), "{label}: corrupt-shard read errored {:?}", outcome.error);
+                assert_eq!(outcome.written, total_len, "{label}: corrupt-shard short write");
                 assert_eq!(output, total_data, "{label}: corrupt bytes leaked into output");
+                assert!(
+                    outcome.repair.needed(false),
+                    "{label}: minority corruption must survive successful reconstruction"
+                );
+                let readers = build_prefetch_readers(&shard_bufs, shard_size, &hash_algo, BLOCK_SIZE, 0, &[], &[]);
+                let mut healthy_output = Vec::new();
+                let healthy = erasure
+                    .decode_with_stripe_handles_and_reopeners_with_diagnostics(
+                        &mut healthy_output,
+                        readers,
+                        0,
+                        total_len,
+                        total_len,
+                        None,
+                        Vec::new(),
+                        Vec::new(),
+                    )
+                    .await;
+                assert!(healthy.error.is_none());
+                assert_eq!(healthy_output, total_data);
+                assert!(
+                    !healthy.repair.needed(true),
+                    "{label}: healthy/deferred shards cannot create repair evidence"
+                );
             })
             .await;
         }
@@ -4635,40 +4841,38 @@ mod tests {
                 (ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE, None::<&str>),
             ],
             async {
-            const NUM_SHARDS: usize = 1;
-            const BLOCK_SIZE: usize = 64;
-            const DATA_SHARDS: usize = 4;
-            const PARITY_SHARDS: usize = 2;
-            const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
+                const NUM_SHARDS: usize = 1;
+                const BLOCK_SIZE: usize = 64;
+                const DATA_SHARDS: usize = 4;
+                const PARITY_SHARDS: usize = 2;
+                const SHARD_SIZE: usize = BLOCK_SIZE / DATA_SHARDS;
 
-            let hash_algo = HashAlgorithm::HighwayHash256;
-            let readers = make_test_readers(DATA_SHARDS + PARITY_SHARDS, SHARD_SIZE, NUM_SHARDS, &hash_algo, &[], &[1]).await;
-            let read_costs = vec![
-                ShardReadCost::Local,
-                ShardReadCost::Local,
-                ShardReadCost::Local,
-                ShardReadCost::Local,
-                ShardReadCost::Remote,
-                ShardReadCost::Remote,
-            ];
-            let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
-            let mut parallel_reader = ParallelReader::new_with_metrics_path_and_read_costs(
-                readers,
-                erasure,
-                0,
-                NUM_SHARDS * BLOCK_SIZE,
-                None,
-                read_costs,
-            );
+                let hash_algo = HashAlgorithm::HighwayHash256;
+                let readers = make_test_readers(DATA_SHARDS + PARITY_SHARDS, SHARD_SIZE, NUM_SHARDS, &hash_algo, &[], &[1]).await;
+                let read_costs = vec![
+                    ShardReadCost::Local,
+                    ShardReadCost::Local,
+                    ShardReadCost::Local,
+                    ShardReadCost::Local,
+                    ShardReadCost::Remote,
+                    ShardReadCost::Remote,
+                ];
+                let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
+                let mut parallel_reader = ParallelReader::new_with_metrics_path_and_read_costs(
+                    readers,
+                    erasure,
+                    0,
+                    NUM_SHARDS * BLOCK_SIZE,
+                    None,
+                    read_costs,
+                );
 
-            let (bufs, errs) = parallel_reader.read().await;
+                let (bufs, errs) = parallel_reader.read().await;
 
-            assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
-            assert!(
-                matches!(&errs[1], Some(DiskError::Io(err)) if err.kind() == ErrorKind::InvalidData && err.to_string().contains("bitrot"))
-            );
-            assert_eq!(bufs[4].as_deref(), Some(&[4u8; SHARD_SIZE][..]));
-            assert!(bufs[5].is_none());
+                assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
+                assert_eq!(errs[1], Some(DiskError::FileCorrupt));
+                assert_eq!(bufs[4].as_deref(), Some(&[4u8; SHARD_SIZE][..]));
+                assert!(bufs[5].is_none());
             },
         )
         .await;
@@ -4814,16 +5018,7 @@ mod tests {
             assert_eq!(DATA_SHARDS, bufs.iter().filter(|buf| buf.is_some()).count());
             assert_eq!(
                 BITROT_DISKS,
-                errs.iter()
-                    .filter(|err| {
-                        match err {
-                            Some(DiskError::Io(err)) => {
-                                err.kind() == std::io::ErrorKind::InvalidData && err.to_string().contains("bitrot")
-                            }
-                            _ => false,
-                        }
-                    })
-                    .count()
+                errs.iter().filter(|err| matches!(err, Some(DiskError::FileCorrupt))).count()
             );
         }
     }

crates/ecstore/src/io_support/shard_integrity.rs

@@ -587,10 +587,8 @@ pub(crate) async fn verify_deep_parts(
             return Err(DiskError::FileCorrupt);
         }
         let part_status = statuses.get_mut(&part_index).ok_or(DiskError::FileCorrupt)?;
-        // Deep verification reads the complete encoded shard. `shard_file_offset`
-        // is a range-read helper and can stop before the final encoded byte for
-        // a partial data block, which would let a one-byte tail truncation pass
-        // as healthy. Use the physical shard length for the integrity proof.
+        // Deep verification must consume every encoded stripe, including a
+        // partial final stripe, before certifying the shard's integrity.
         let length = usize::try_from(erasure.shard_file_size(part.size as i64)).map_err(|_| DiskError::FileCorrupt)?;
         let mut readers = Vec::with_capacity(disks.len());
         for (index, disk) in disks.iter().enumerate() {
@@ -635,6 +633,9 @@ pub(crate) async fn verify_deep_parts(
                     part_status[index] = CHECK_PART_FILE_CORRUPT;
                 }
                 Err(error) => {
+                    if !matches!(error, DiskError::FileNotFound | DiskError::FileVersionNotFound | DiskError::FileCorrupt) {
+                        return Err(error);
+                    }
                     readers.push(None);
                     part_status[index] = conv_part_err_to_int(&Some(error));
                 }

crates/ecstore/src/object_api/types.rs

@@ -935,6 +935,8 @@ pub struct ObjectOptions {
     /// Persisted bucket incarnation observed before authorization.
     pub expected_bucket_incarnation_id: Option<Uuid>,
     pub no_lock: bool,
+    /// Internal read-only inspection must not enqueue metadata or payload repairs.
+    pub suppress_read_repair: bool,
     /// Control-plane writers that immediately read or CAS the same namespace
     /// key use TailDrained without changing namespace lock ownership.
     #[doc(hidden)]

crates/ecstore/src/services/integrity/mod.rs

@@ -0,0 +1,290 @@
+// Copyright 2024 RustFS Team
+// Licensed under the Apache License, Version 2.0.
+
+//! Explicit, bounded integrity operations. This service does not participate in ordinary GET or Heal.
+//! Lock order: cluster worker lock, bucket incarnation fence, object commit lock.
+//! Control requests use checkpoint CAS without taking the worker lock; a worker that loses CAS stops.
+
+mod model;
+mod runner;
+#[cfg(test)]
+mod tests;
+
+pub use model::{
+    InventoryItem, InventoryPage, ItemRequest, ItemResult, ItemState, Job, JobMode, JobRequest, JobState, Protection, Readiness,
+    readiness,
+};
+
+use crate::config::com::{read_config_limited_preserve_empty_with_metadata, save_config_with_opts_quiet};
+use crate::disk::RUSTFS_META_BUCKET;
+use crate::error::Error as StorageError;
+use crate::object_api::{ObjectOptions, WriteCompletion};
+use crate::storage_api_contracts::{
+    list::ListOperations,
+    namespace::NamespaceLocking,
+    object::{HTTPPreconditions, ObjectOperations},
+};
+use crate::store::ECStore;
+use model::InventoryItem as Item;
+use std::sync::Arc;
+use std::time::Duration;
+use uuid::Uuid;
+
+type Result<T> = std::result::Result<T, IntegrityError>;
+
+#[derive(Debug, thiserror::Error)]
+pub enum IntegrityError {
+    #[error("invalid integrity request: {0}")]
+    Invalid(&'static str),
+    #[error("integrity job or target changed concurrently")]
+    Conflict,
+    #[error("an integrity worker is already active")]
+    Busy,
+    #[error("integrity job not found")]
+    NotFound,
+    #[error("integrity operation was stopped")]
+    Stopped,
+    #[error("protected writes are not enabled on this coordinator")]
+    NotActivated,
+    #[error("unsupported migration bucket configuration")]
+    UnsupportedBucket,
+    #[error("integrity storage operation failed: {0}")]
+    Storage(#[from] StorageError),
+    #[error("integrity stream operation failed: {0}")]
+    Io(#[from] std::io::Error),
+    #[error("integrity checkpoint is malformed: {0}")]
+    Json(#[from] serde_json::Error),
+}
+
+fn check_bucket(bucket: &str) -> Result<()> {
+    crate::bucket::utils::check_valid_bucket_name_strict(bucket).map_err(|_| IntegrityError::Invalid("invalid source bucket"))
+}
+
+fn path(bucket: &str, id: Uuid) -> String {
+    format!("buckets/{bucket}/integrity-jobs/{id}.json")
+}
+
+pub async fn inventory(
+    api: Arc<ECStore>,
+    bucket: &str,
+    prefix: &str,
+    key_marker: Option<String>,
+    version_marker: Option<String>,
+    limit: i32,
+) -> Result<InventoryPage> {
+    check_bucket(bucket)?;
+    if !(1..=100).contains(&limit) || prefix.len() > 1024 || (version_marker.is_some() && key_marker.is_none()) {
+        return Err(IntegrityError::Invalid("invalid inventory page"));
+    }
+    let incarnation = api.bucket_incarnation_id_from_disk(bucket).await?;
+    let page = api
+        .clone()
+        .list_object_versions(bucket, prefix, key_marker.clone(), version_marker.clone(), None, limit)
+        .await?;
+    if page.is_truncated && (page.next_marker.clone(), page.next_version_idmarker.clone()) == (key_marker, version_marker) {
+        return Err(IntegrityError::Invalid("inventory cursor did not advance"));
+    }
+    let mut items = Vec::with_capacity(page.objects.len());
+    for listed in page.objects {
+        if listed.delete_marker {
+            items.push(Item::from_info(&listed));
+            continue;
+        }
+        let opts = ObjectOptions {
+            version_id: Some(listed.version_id.unwrap_or_else(Uuid::nil).to_string()),
+            include_part_checksums: true,
+            suppress_read_repair: true,
+            ..Default::default()
+        };
+        match api.get_object_info(bucket, &listed.name, &opts).await {
+            Ok(info) => items.push(Item::from_info(&info)),
+            Err(_) => {
+                let mut item = Item::from_info(&listed);
+                item.observation_error = Some("metadata_unavailable_or_changed".into());
+                item.protection = Protection::Unknown;
+                item.audit_unsupported = Some("metadata_unavailable_or_changed".into());
+                items.push(item);
+            }
+        }
+    }
+    if api.bucket_incarnation_id_from_disk(bucket).await? != incarnation {
+        return Err(IntegrityError::Conflict);
+    }
+    Ok(InventoryPage {
+        observed_at: time::OffsetDateTime::now_utc().to_string(),
+        bucket_incarnation: incarnation,
+        items,
+        is_truncated: page.is_truncated,
+        next_key_marker: page.next_marker,
+        next_version_marker: page.next_version_idmarker,
+    })
+}
+
+pub async fn create_job(api: Arc<ECStore>, bucket: &str, request: JobRequest) -> Result<Job> {
+    check_bucket(bucket)?;
+    request.validate()?;
+    if request.mode == JobMode::Migrate && !readiness().new_writes_enabled_here {
+        return Err(IntegrityError::NotActivated);
+    }
+    let incarnation = api.bucket_incarnation_id_from_disk(bucket).await?;
+    let job = Job {
+        format_version: 1,
+        id: Uuid::new_v4(),
+        bucket: bucket.into(),
+        bucket_incarnation: incarnation,
+        results: vec![ItemResult::default(); request.items.len()],
+        request,
+        state: JobState::Paused,
+        revision: 0,
+        last_error: None,
+    };
+    save(&api, &job, None, None).await?;
+    Ok(job)
+}
+
+pub async fn get_job(api: Arc<ECStore>, bucket: &str, id: Uuid) -> Result<Job> {
+    Ok(load(&api, bucket, id).await?.0)
+}
+
+async fn load(api: &Arc<ECStore>, bucket: &str, id: Uuid) -> Result<(Job, String)> {
+    check_bucket(bucket)?;
+    let (bytes, info) = match read_config_limited_preserve_empty_with_metadata(api.clone(), &path(bucket, id), 1024 * 1024).await
+    {
+        Ok(value) => value,
+        Err(StorageError::ConfigNotFound | StorageError::FileNotFound) => return Err(IntegrityError::NotFound),
+        Err(error) => return Err(error.into()),
+    };
+    let job: Job = serde_json::from_slice(&bytes)?;
+    job.request.validate()?;
+    if job.format_version != 1
+        || job.bucket != bucket
+        || job.id != id
+        || job.results.len() != job.request.items.len()
+        || api.bucket_incarnation_id_from_disk(bucket).await? != job.bucket_incarnation
+    {
+        return Err(IntegrityError::Conflict);
+    }
+    Ok((job, info.etag.ok_or(IntegrityError::Conflict)?))
+}
+
+async fn save(
+    api: &Arc<ECStore>,
+    job: &Job,
+    etag: Option<&str>,
+    worker: Option<&rustfs_lock::NamespaceLockGuard>,
+) -> Result<String> {
+    let fence = api
+        .acquire_bucket_incarnation_fence(&job.bucket, job.bucket_incarnation)
+        .await?;
+    let mut opts = ObjectOptions {
+        max_parity: true,
+        write_completion: WriteCompletion::TailDrained,
+        http_preconditions: Some(match etag {
+            Some(value) => HTTPPreconditions {
+                if_match: Some(value.into()),
+                ..Default::default()
+            },
+            None => HTTPPreconditions {
+                if_none_match: Some("*".into()),
+                ..Default::default()
+            },
+        }),
+        ..Default::default()
+    };
+    fence.attach_to_object_options(&mut opts);
+    if let Some(guard) = worker {
+        opts.add_namespace_lock_guard(guard);
+    }
+    let api = api.clone();
+    let job = job.clone();
+    // Keep the lifecycle guard through the drained commit even if the HTTP waiter is cancelled.
+    tokio::spawn(async move {
+        save_config_with_opts_quiet(api.clone(), &path(&job.bucket, job.id), serde_json::to_vec(&job)?, &opts).await?;
+        let (stored, next) = load(&api, &job.bucket, job.id).await?;
+        if stored.revision != job.revision {
+            return Err(IntegrityError::Conflict);
+        }
+        drop(fence);
+        Ok(next)
+    })
+    .await
+    .map_err(|_| IntegrityError::Invalid("checkpoint commit task failed"))?
+}
+
+/// Pause/cancel acknowledge intent. An in-flight conditional publication is allowed to drain.
+pub async fn control_job(api: Arc<ECStore>, bucket: &str, id: Uuid, cancel: bool) -> Result<Job> {
+    let (mut job, etag) = load(&api, bucket, id).await?;
+    match job.state {
+        JobState::Complete | JobState::Cancelled | JobState::CancelRequested => return Ok(job),
+        JobState::Paused | JobState::Failed => job.state = if cancel { JobState::Cancelled } else { JobState::Paused },
+        _ => {
+            job.state = if cancel {
+                JobState::CancelRequested
+            } else {
+                JobState::PauseRequested
+            }
+        }
+    }
+    job.revision = job.revision.checked_add(1).ok_or(IntegrityError::Conflict)?;
+    save(&api, &job, Some(&etag), None).await?;
+    Ok(job)
+}
+
+/// Resume is explicit after restart. The distributed lock excludes another worker, including another node.
+pub async fn resume_job(api: Arc<ECStore>, bucket: &str, id: Uuid) -> Result<Job> {
+    check_bucket(bucket)?;
+    let lock = api.new_ns_lock(RUSTFS_META_BUCKET, "integrity-jobs/worker.lock").await?;
+    let guard = lock
+        .get_write_lock_quiet(Duration::from_secs(1))
+        .await
+        .map_err(|_| IntegrityError::Busy)?;
+    let (mut job, etag) = load(&api, bucket, id).await?;
+    if matches!(job.state, JobState::Complete | JobState::Cancelled) {
+        return Ok(job);
+    }
+    if job.state == JobState::CancelRequested {
+        job.state = JobState::Cancelled;
+        job.revision = job.revision.checked_add(1).ok_or(IntegrityError::Conflict)?;
+        save(&api, &job, Some(&etag), Some(&guard)).await?;
+        return Ok(job);
+    }
+    if job.request.mode == JobMode::Migrate && !readiness().new_writes_enabled_here {
+        return Err(IntegrityError::NotActivated);
+    }
+    job.state = JobState::Running;
+    job.last_error = None;
+    job.revision = job.revision.checked_add(1).ok_or(IntegrityError::Conflict)?;
+    let etag = save(&api, &job, Some(&etag), Some(&guard)).await?;
+    let response = job.clone();
+    tokio::spawn(async move {
+        let result = runner::run(&api, &mut job, etag, &guard).await;
+        if let Err(error) = result {
+            // A failed write/CAS stops publication. Never overwrite a concurrent control request.
+            if let Ok((mut latest, etag)) = load(&api, &job.bucket, job.id).await {
+                match latest.state {
+                    JobState::PauseRequested => latest.state = JobState::Paused,
+                    JobState::CancelRequested => latest.state = JobState::Cancelled,
+                    JobState::Running if latest.revision <= job.revision => {
+                        latest.state = JobState::Failed;
+                        latest.last_error = Some(
+                            match error {
+                                IntegrityError::Conflict => "concurrent_change",
+                                IntegrityError::Stopped => "worker_stopped",
+                                IntegrityError::UnsupportedBucket => "unsupported_destination_bucket_configuration",
+                                IntegrityError::NotActivated => "protected_writes_not_activated",
+                                _ => "operation_failed_retry_requires_resume",
+                            }
+                            .into(),
+                        );
+                    }
+                    _ => return,
+                }
+                if let Some(revision) = latest.revision.checked_add(1) {
+                    latest.revision = revision;
+                    let _ = save(&api, &latest, Some(&etag), Some(&guard)).await;
+                }
+            }
+        }
+    });
+    Ok(response)
+}

crates/ecstore/src/services/integrity/model.rs

@@ -0,0 +1,343 @@
+// Copyright 2024 RustFS Team
+// Licensed under the Apache License, Version 2.0.
+
+use super::{IntegrityError, Result};
+use crate::object_api::ObjectInfo;
+use rustfs_rio::{Checksum, read_checksums};
+use serde::{Deserialize, Serialize};
+use sha2::{Digest, Sha256};
+use std::collections::{BTreeMap, HashSet};
+use uuid::Uuid;
+
+pub const MAX_ITEMS: usize = 64;
+pub const MAX_OBJECT_BYTES: u64 = 5 * 1024 * 1024 * 1024;
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct ItemRequest {
+    pub key: String,
+    pub version_id: Option<String>,
+    /// Base64 SHA-256 supplied by the administrator from an independent source.
+    pub expected_sha256: Option<String>,
+    pub target_key: Option<String>,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum JobMode {
+    Audit,
+    Migrate,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct JobRequest {
+    pub mode: JobMode,
+    pub items: Vec<ItemRequest>,
+    /// Bounds each source read, destination PUT, and verification stream; one worker runs per cluster.
+    pub bytes_per_second: u64,
+    pub max_object_bytes: u64,
+}
+
+impl JobRequest {
+    pub(crate) fn validate(&self) -> Result<()> {
+        if self.items.is_empty() || self.items.len() > MAX_ITEMS {
+            return Err(IntegrityError::Invalid("a job requires 1..=64 explicit source versions"));
+        }
+        if !(64 * 1024..=1024 * 1024 * 1024).contains(&self.bytes_per_second)
+            || self.max_object_bytes == 0
+            || self.max_object_bytes > MAX_OBJECT_BYTES
+        {
+            return Err(IntegrityError::Invalid("invalid byte rate or staging limit"));
+        }
+        let mut targets = HashSet::new();
+        let sources: HashSet<_> = self.items.iter().map(|i| i.key.as_str()).collect();
+        for item in &self.items {
+            validate_key(&item.key)?;
+            if let Some(version) = &item.version_id
+                && version != "null"
+                && Uuid::parse_str(version).ok().is_none_or(|id| id.is_nil())
+            {
+                return Err(IntegrityError::Invalid("invalid source version"));
+            }
+            if let Some(value) = &item.expected_sha256 {
+                sha256_bytes(value)?;
+            }
+            match (self.mode, &item.target_key) {
+                (JobMode::Audit, None) => {}
+                (JobMode::Migrate, Some(target)) => {
+                    validate_key(target)?;
+                    if sources.contains(target.as_str()) || !targets.insert(target) {
+                        return Err(IntegrityError::Invalid("targets must be distinct from every source and target"));
+                    }
+                }
+                _ => return Err(IntegrityError::Invalid("only migrations require an explicit target key")),
+            }
+        }
+        Ok(())
+    }
+}
+
+fn validate_key(key: &str) -> Result<()> {
+    if key.is_empty() || key.len() > 1024 || key.ends_with('/') || key.contains('\0') {
+        return Err(IntegrityError::Invalid("invalid object key"));
+    }
+    Ok(())
+}
+
+pub(crate) fn sha256_bytes(value: &str) -> Result<Vec<u8>> {
+    let raw = base64_simd::STANDARD
+        .decode_to_vec(value)
+        .map_err(|_| IntegrityError::Invalid("SHA256 must be canonical Base64"))?;
+    if raw.len() != 32 || base64_simd::STANDARD.encode_to_string(&raw) != value {
+        return Err(IntegrityError::Invalid("SHA256 must be canonical Base64 of 32 bytes"));
+    }
+    Ok(raw)
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum Protection {
+    Unknown,
+    Legacy,
+    IndependentCommitment,
+    InvalidDeclaration,
+    NoPayload,
+}
+
+pub(crate) fn protection(info: &ObjectInfo) -> Protection {
+    if info.delete_marker {
+        return Protection::NoPayload;
+    }
+    match rustfs_filemeta::shard_integrity::descriptor_from_metadata(&info.user_defined) {
+        Ok(None) if info.parts.iter().all(|p| p.integrity.is_none()) => Protection::Legacy,
+        Ok(Some(parts))
+            if parts.len() == info.parts.len()
+                && !parts.is_empty()
+                && parts
+                    .iter()
+                    .zip(info.parts.iter())
+                    .all(|(declared, part)| part.integrity.as_ref() == Some(declared)) =>
+        {
+            Protection::IndependentCommitment
+        }
+        _ => Protection::InvalidDeclaration,
+    }
+}
+
+/// Checksums shown in inventory are not automatically accepted as migration evidence.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct InventoryItem {
+    pub key: String,
+    /// S3 selector, including the literal "null" for the null version slot.
+    pub version_id: String,
+    pub data_dir: Option<Uuid>,
+    pub size: i64,
+    pub protection: Protection,
+    pub checksums: BTreeMap<String, String>,
+    pub checksum_is_multipart: bool,
+    pub audit_unsupported: Option<String>,
+    pub source_fingerprint: String,
+    pub observation_error: Option<String>,
+}
+
+impl InventoryItem {
+    pub(crate) fn from_info(info: &ObjectInfo) -> Self {
+        let (checksums, checksum_is_multipart) = info.checksum.as_ref().map_or_else(Default::default, |b| read_checksums(b, 0));
+        Self {
+            key: info.name.clone(),
+            version_id: info
+                .version_id
+                .filter(|id| !id.is_nil())
+                .map_or_else(|| "null".into(), |id| id.to_string()),
+            data_dir: info.data_dir,
+            size: info.size,
+            protection: protection(info),
+            checksums: checksums.into_iter().collect(),
+            checksum_is_multipart,
+            audit_unsupported: unsupported(info).map(str::to_owned),
+            source_fingerprint: fingerprint(info),
+            observation_error: None,
+        }
+    }
+}
+
+pub(crate) fn unsupported(info: &ObjectInfo) -> Option<&'static str> {
+    if info.delete_marker {
+        Some("delete_marker")
+    } else if !info.transitioned_object.status.is_empty() {
+        Some("transitioned_object")
+    } else if info.is_encrypted() || info.is_compressed() {
+        Some("transformed_object")
+    } else if info.is_multipart() {
+        Some("multipart_object")
+    } else if protection(info) == Protection::InvalidDeclaration {
+        Some("invalid_protection_declaration")
+    } else {
+        None
+    }
+}
+
+pub(crate) fn stored_sha256(info: &ObjectInfo) -> Option<String> {
+    let bytes = info.checksum.as_ref()?;
+    let (sums, multipart) = read_checksums(bytes, 0);
+    let value = sums.get("SHA256")?;
+    let canonical = Checksum::new_from_string("SHA256", value)?.to_bytes(&[]);
+    // The general display decoder tolerates truncated suffixes. Eligibility does not.
+    (!multipart && canonical.as_ref() == bytes.as_ref()).then(|| value.clone())
+}
+
+pub(crate) fn fingerprint(info: &ObjectInfo) -> String {
+    let mut hash = Sha256::new();
+    // Ordered metadata includes transformation and protection declarations as well as user metadata.
+    for value in [
+        info.bucket.clone(),
+        info.name.clone(),
+        format!("{:?}", info.version_id),
+        format!("{:?}", info.data_dir),
+        format!("{:?}", info.mod_time),
+        info.size.to_string(),
+        info.actual_size.to_string(),
+        format!("{:?}", info.etag),
+        format!("{:?}", info.checksum),
+        format!("{:?}", info.transitioned_object),
+        info.user_tags.to_string(),
+        format!("{:?}", info.expires),
+        format!("{:?}", info.content_type),
+        format!("{:?}", info.content_encoding),
+        format!("{:?}", info.storage_class),
+    ] {
+        hash.update(value.len().to_le_bytes());
+        hash.update(value.as_bytes());
+    }
+    for part in info.parts.iter() {
+        let values = format!(
+            "{}:{}:{}:{:?}:{:?}:{:?}:{:?}",
+            part.number, part.size, part.actual_size, part.etag, part.mod_time, part.index, part.integrity
+        );
+        hash.update(values.len().to_le_bytes());
+        hash.update(values.as_bytes());
+        if let Some(checksums) = &part.checksums {
+            let mut checksums: Vec<_> = checksums.iter().collect();
+            checksums.sort_unstable_by_key(|(key, _)| *key);
+            for (key, value) in checksums {
+                hash.update(key.len().to_le_bytes());
+                hash.update(key.as_bytes());
+                hash.update(value.len().to_le_bytes());
+                hash.update(value.as_bytes());
+            }
+        }
+    }
+    let mut metadata: Vec<_> = info.user_defined.iter().collect();
+    metadata.sort_unstable_by_key(|(key, _)| *key);
+    for (key, value) in metadata {
+        hash.update(key.len().to_le_bytes());
+        hash.update(key.as_bytes());
+        hash.update(value.len().to_le_bytes());
+        hash.update(value.as_bytes());
+    }
+    hex_simd::encode_to_string(hash.finalize(), hex_simd::AsciiCase::Lower)
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum JobState {
+    Paused,
+    Running,
+    PauseRequested,
+    CancelRequested,
+    Cancelled,
+    Complete,
+    Failed,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ItemState {
+    Pending,
+    Prepared,
+    Verified,
+    Migrated,
+    Unsupported,
+    Unavailable,
+    Mismatch,
+    Stale,
+    Conflict,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct ItemResult {
+    pub state: ItemState,
+    pub source_fingerprint: Option<String>,
+    pub expected_sha256: Option<String>,
+    pub evidence_source: Option<String>,
+    pub source_size: Option<u64>,
+    pub target_version_id: Option<Uuid>,
+    pub detail: Option<String>,
+}
+
+impl Default for ItemResult {
+    fn default() -> Self {
+        Self {
+            state: ItemState::Pending,
+            source_fingerprint: None,
+            expected_sha256: None,
+            evidence_source: None,
+            source_size: None,
+            target_version_id: None,
+            detail: None,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct Job {
+    pub format_version: u32,
+    pub id: Uuid,
+    pub bucket: String,
+    pub bucket_incarnation: Uuid,
+    pub request: JobRequest,
+    pub state: JobState,
+    pub revision: u64,
+    pub results: Vec<ItemResult>,
+    pub last_error: Option<String>,
+}
+
+#[derive(Debug, Serialize)]
+pub struct InventoryPage {
+    pub observed_at: String,
+    pub bucket_incarnation: Uuid,
+    pub items: Vec<InventoryItem>,
+    pub is_truncated: bool,
+    pub next_key_marker: Option<String>,
+    pub next_version_marker: Option<String>,
+}
+
+#[derive(Debug, Serialize)]
+pub struct Readiness {
+    pub local_reader_supported: bool,
+    pub write_requested: bool,
+    pub fleet_operator_attested: bool,
+    pub new_writes_enabled_here: bool,
+    pub fleet_capability_verified: bool,
+    pub unverified_requirements: [&'static str; 3],
+}
+
+pub fn readiness() -> Readiness {
+    let write = rustfs_utils::get_env_bool(rustfs_config::ENV_SHARD_INTEGRITY_WRITE, false);
+    let fleet = rustfs_utils::get_env_bool(rustfs_config::ENV_SHARD_INTEGRITY_FLEET_CONFIRMED, false);
+    Readiness {
+        local_reader_supported: true,
+        write_requested: write,
+        fleet_operator_attested: fleet,
+        new_writes_enabled_here: write && fleet,
+        fleet_capability_verified: false,
+        unverified_requirements: [
+            "peer_reader_writer_and_repair_capabilities",
+            "old_process_storage_admission",
+            "deployment_failure_and_performance_qualification",
+        ],
+    }
+}

crates/ecstore/src/services/integrity/runner.rs

@@ -0,0 +1,468 @@
+// Copyright 2024 RustFS Team
+// Licensed under the Apache License, Version 2.0.
+
+use super::model::{Protection, fingerprint, protection, sha256_bytes, stored_sha256, unsupported};
+use super::{IntegrityError, ItemState, Job, JobMode, JobState, Result, load, readiness, save};
+use crate::object_api::{
+    ObjectInfo, ObjectOptions, PutObjReader, ShardIntegrityWriteMode, WriteCompletion, without_get_object_body_cache_hook,
+};
+use crate::storage_api_contracts::object::{HTTPPreconditions, ObjectIO, ObjectOperations};
+use crate::store::ECStore;
+use rustfs_rio::{Checksum, HashReader};
+use sha2::{Digest, Sha256};
+use std::{collections::HashMap, sync::Arc, time::Duration};
+use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
+
+const MARKER: &str = "integrity-migration-job-v1";
+
+pub(super) async fn run(
+    api: &Arc<ECStore>,
+    job: &mut Job,
+    mut etag: String,
+    guard: &rustfs_lock::NamespaceLockGuard,
+) -> Result<()> {
+    without_get_object_body_cache_hook(async {
+        for index in 0..job.results.len() {
+            ensure_running(api, job, guard).await?;
+            if !matches!(
+                job.results[index].state,
+                ItemState::Pending | ItemState::Prepared | ItemState::Unavailable
+            ) {
+                continue;
+            }
+            if job.request.mode == JobMode::Migrate {
+                validate_destination(api, job).await?;
+            }
+            if job.results[index].state == ItemState::Prepared
+                && job.request.mode == JobMode::Migrate
+                && reconcile_target(api, job, index, guard).await?
+            {
+                etag = checkpoint(api, job, &etag, guard).await?;
+                continue;
+            }
+            let request = job.request.items[index].clone();
+            let opts = ObjectOptions {
+                version_id: request.version_id.as_ref().map(|version| {
+                    if version == "null" {
+                        uuid::Uuid::nil().to_string()
+                    } else {
+                        version.clone()
+                    }
+                }),
+                include_part_checksums: true,
+                suppress_read_repair: true,
+                ..Default::default()
+            };
+            let info = match api.get_object_info(&job.bucket, &request.key, &opts).await {
+                Ok(info) => info,
+                Err(_) => {
+                    job.results[index].state = ItemState::Unavailable;
+                    job.results[index].detail = Some("source_metadata_unavailable".into());
+                    etag = checkpoint(api, job, &etag, guard).await?;
+                    continue;
+                }
+            };
+            if let Some(reason) = unsupported(&info) {
+                job.results[index].state = ItemState::Unsupported;
+                job.results[index].detail = Some(reason.into());
+                etag = checkpoint(api, job, &etag, guard).await?;
+                continue;
+            }
+            let identity = fingerprint(&info);
+            if job.results[index]
+                .source_fingerprint
+                .as_ref()
+                .is_some_and(|old| old != &identity)
+            {
+                job.results[index].state = ItemState::Stale;
+                etag = checkpoint(api, job, &etag, guard).await?;
+                continue;
+            }
+            let expected = request.expected_sha256.clone().or_else(|| stored_sha256(&info));
+            let Some(expected) = expected else {
+                job.results[index].state = ItemState::Unsupported;
+                job.results[index].detail = Some("no_canonical_single_part_sha256".into());
+                etag = checkpoint(api, job, &etag, guard).await?;
+                continue;
+            };
+            let size = u64::try_from(info.size).map_err(|_| IntegrityError::Invalid("negative source size"))?;
+            if size > job.request.max_object_bytes {
+                job.results[index].state = ItemState::Unsupported;
+                job.results[index].detail = Some("object_exceeds_staging_budget".into());
+                etag = checkpoint(api, job, &etag, guard).await?;
+                continue;
+            }
+            let result = &mut job.results[index];
+            result.source_fingerprint = Some(identity);
+            result.expected_sha256 = Some(expected.clone());
+            result.evidence_source = Some(
+                if request.expected_sha256.is_some() {
+                    "administrator_supplied_sha256"
+                } else {
+                    "stored_sha256"
+                }
+                .into(),
+            );
+            result.source_size = Some(size);
+            result.state = ItemState::Prepared;
+            result.detail = None;
+            etag = checkpoint(api, job, &etag, guard).await?;
+            let mut file = match stage(api, job, index, &info, &opts, guard).await {
+                Ok(file) => file,
+                Err(IntegrityError::Conflict) => {
+                    job.results[index].state = ItemState::Stale;
+                    etag = checkpoint(api, job, &etag, guard).await?;
+                    continue;
+                }
+                Err(IntegrityError::Invalid("content_sha256_mismatch")) => {
+                    job.results[index].state = ItemState::Mismatch;
+                    etag = checkpoint(api, job, &etag, guard).await?;
+                    continue;
+                }
+                Err(error) => return Err(error),
+            };
+            ensure_running(api, job, guard).await?;
+            if job.request.mode == JobMode::Audit {
+                job.results[index].state = ItemState::Verified;
+            } else {
+                let snapshot = validate_destination(api, job).await?;
+                file.rewind().await?;
+                let raw = sha256_bytes(&expected)?;
+                let expected_hex = hex_simd::encode_to_string(raw, hex_simd::AsciiCase::Lower);
+                let mut reader = HashReader::from_stream(
+                    ThrottledFile::new(file, job.request.bytes_per_second),
+                    info.size,
+                    info.size,
+                    None,
+                    Some(expected_hex),
+                    false,
+                )?;
+                reader.add_non_trailing_checksum(Checksum::new_from_string("SHA256", &expected), false)?;
+                let mut body = PutObjReader::new(reader);
+                let mut metadata: HashMap<String, String> = info
+                    .user_defined
+                    .iter()
+                    .filter(|(key, _)| {
+                        key.starts_with("x-amz-meta-")
+                            || matches!(key.as_str(), "cache-control" | "content-disposition" | "content-language" | "expires")
+                    })
+                    .map(|(key, value)| (key.clone(), value.clone()))
+                    .collect();
+                if let Some(content_type) = &info.content_type {
+                    metadata.insert("content-type".into(), content_type.clone());
+                }
+                if let Some(content_encoding) = &info.content_encoding {
+                    metadata.insert("content-encoding".into(), content_encoding.clone());
+                }
+                if let Some(expires) = info.expires {
+                    metadata.insert(
+                        "expires".into(),
+                        expires
+                            .format(&time::format_description::well_known::Rfc3339)
+                            .map_err(|_| IntegrityError::Invalid("invalid source expiration"))?,
+                    );
+                }
+                if let Some(storage_class) = &info.storage_class {
+                    metadata.insert("x-amz-storage-class".into(), storage_class.clone());
+                }
+                if !info.user_tags.is_empty() {
+                    metadata.insert(rustfs_utils::http::headers::AMZ_OBJECT_TAGGING.into(), info.user_tags.to_string());
+                }
+                rustfs_utils::http::insert_str(&mut metadata, MARKER, marker(job, index)?);
+                let mut write = ObjectOptions {
+                    user_defined: metadata,
+                    preserve_delete_marker: true,
+                    shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Protected),
+                    write_completion: WriteCompletion::TailDrained,
+                    http_preconditions: Some(HTTPPreconditions {
+                        if_none_match: Some("*".into()),
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                };
+                write.add_namespace_lock_guard(guard);
+                write.expected_bucket_incarnation_id = Some(job.bucket_incarnation);
+                snapshot.add_lock_fences(&mut write);
+                write.object_lock_config_snapshot = Some(snapshot.clone());
+                let target = request
+                    .target_key
+                    .as_deref()
+                    .ok_or(IntegrityError::Invalid("missing target"))?;
+                // Even an error may follow a committed PUT. Reconcile before retrying; never overwrite.
+                let put = api.put_object(&job.bucket, target, &mut body, &write).await;
+                drop(write);
+                drop(snapshot);
+                if !reconcile_target(api, job, index, guard).await? {
+                    put?;
+                    return Err(IntegrityError::Conflict);
+                }
+            }
+            etag = checkpoint(api, job, &etag, guard).await?;
+        }
+        job.state = if job.results.iter().any(|item| item.state == ItemState::Unavailable) {
+            job.last_error = Some("source_unavailable_retry_requires_resume".into());
+            JobState::Failed
+        } else {
+            JobState::Complete
+        };
+        checkpoint(api, job, &etag, guard).await?;
+        Ok(())
+    })
+    .await
+}
+
+async fn checkpoint(api: &Arc<ECStore>, job: &mut Job, etag: &str, guard: &rustfs_lock::NamespaceLockGuard) -> Result<String> {
+    ensure_running(api, job, guard).await?;
+    job.revision = job.revision.checked_add(1).ok_or(IntegrityError::Conflict)?;
+    save(api, job, Some(etag), Some(guard)).await
+}
+
+async fn ensure_running(api: &Arc<ECStore>, job: &Job, guard: &rustfs_lock::NamespaceLockGuard) -> Result<()> {
+    if guard.lock_lost_signal().is_some_and(|signal| signal.is_lost()) {
+        return Err(IntegrityError::Stopped);
+    }
+    let (current, _) = load(api, &job.bucket, job.id).await?;
+    if current.state != JobState::Running {
+        return Err(IntegrityError::Stopped);
+    }
+    if current.revision != job.revision {
+        return Err(IntegrityError::Conflict);
+    }
+    Ok(())
+}
+
+async fn stage(
+    api: &Arc<ECStore>,
+    job: &Job,
+    index: usize,
+    info: &ObjectInfo,
+    opts: &ObjectOptions,
+    guard: &rustfs_lock::NamespaceLockGuard,
+) -> Result<tokio::fs::File> {
+    let mut reader = api
+        .get_object_reader(&job.bucket, &job.request.items[index].key, None, Default::default(), opts)
+        .await?;
+    if fingerprint(&reader.object_info) != fingerprint(info) {
+        return Err(IntegrityError::Conflict);
+    }
+    let std_file = tokio::task::spawn_blocking(tempfile::tempfile)
+        .await
+        .map_err(|_| IntegrityError::Invalid("staging task failed"))??;
+    let mut file = tokio::fs::File::from_std(std_file);
+    let expected = job.results[index]
+        .expected_sha256
+        .as_deref()
+        .ok_or(IntegrityError::Invalid("missing expected digest"))?;
+    copy_verified(api, job, &mut reader.stream, Some(&mut file), info.size, expected, guard).await?;
+    file.flush().await?;
+    let after = api.get_object_info(&job.bucket, &job.request.items[index].key, opts).await?;
+    if fingerprint(&after) != fingerprint(info) {
+        return Err(IntegrityError::Conflict);
+    }
+    Ok(file)
+}
+
+async fn copy_verified(
+    api: &Arc<ECStore>,
+    job: &Job,
+    input: &mut (dyn tokio::io::AsyncRead + Unpin + Send + Sync),
+    mut output: Option<&mut tokio::fs::File>,
+    size: i64,
+    expected: &str,
+    guard: &rustfs_lock::NamespaceLockGuard,
+) -> Result<()> {
+    let expected = sha256_bytes(expected)?;
+    let expected_size = u64::try_from(size).map_err(|_| IntegrityError::Invalid("negative object size"))?;
+    let mut buffer = vec![0u8; 64 * 1024];
+    let mut count = 0u64;
+    let mut hash = Sha256::new();
+    let started = tokio::time::Instant::now();
+    let mut checked = started;
+    loop {
+        if checked.elapsed() >= Duration::from_secs(1) {
+            ensure_running(api, job, guard).await?;
+            checked = tokio::time::Instant::now();
+        }
+        let n = tokio::time::timeout(Duration::from_secs(30), input.read(&mut buffer))
+            .await
+            .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "integrity read timed out"))??;
+        if n == 0 {
+            break;
+        }
+        count = count
+            .checked_add(u64::try_from(n).map_err(|_| IntegrityError::Invalid("read size overflow"))?)
+            .ok_or(IntegrityError::Invalid("read size overflow"))?;
+        if count > expected_size || count > job.request.max_object_bytes {
+            return Err(IntegrityError::Invalid("content_sha256_mismatch"));
+        }
+        hash.update(&buffer[..n]);
+        if let Some(file) = output.as_deref_mut() {
+            file.write_all(&buffer[..n]).await?;
+        }
+        let delay = Duration::from_secs_f64(count as f64 / job.request.bytes_per_second as f64);
+        tokio::time::sleep_until(started + delay).await;
+    }
+    if count != expected_size || hash.finalize().as_slice() != expected {
+        return Err(IntegrityError::Invalid("content_sha256_mismatch"));
+    }
+    Ok(())
+}
+
+fn marker(job: &Job, index: usize) -> Result<String> {
+    let result = &job.results[index];
+    Ok(format!(
+        "{}:{}:{}:{}",
+        job.id,
+        index,
+        result
+            .source_fingerprint
+            .as_deref()
+            .ok_or(IntegrityError::Invalid("missing source identity"))?,
+        result
+            .expected_sha256
+            .as_deref()
+            .ok_or(IntegrityError::Invalid("missing source digest"))?
+    ))
+}
+
+async fn reconcile_target(
+    api: &Arc<ECStore>,
+    job: &mut Job,
+    index: usize,
+    guard: &rustfs_lock::NamespaceLockGuard,
+) -> Result<bool> {
+    let key = job.request.items[index]
+        .target_key
+        .as_deref()
+        .ok_or(IntegrityError::Invalid("missing target"))?;
+    let opts = ObjectOptions {
+        include_part_checksums: true,
+        suppress_read_repair: true,
+        ..Default::default()
+    };
+    let target = match api.get_object_info(&job.bucket, key, &opts).await {
+        Ok(info) => info,
+        Err(crate::error::Error::FileNotFound | crate::error::Error::ObjectNotFound(_, _)) => return Ok(false),
+        Err(error) => return Err(error.into()),
+    };
+    let expected_marker = marker(job, index)?;
+    if rustfs_utils::http::get_consistent_str(&target.user_defined, MARKER) != Some(expected_marker.as_str()) {
+        job.results[index].state = ItemState::Conflict;
+        job.results[index].detail = Some("target_owned_by_another_operation".into());
+        return Ok(true);
+    }
+    if protection(&target) != Protection::IndependentCommitment {
+        return Err(IntegrityError::Invalid("target_not_protected"));
+    }
+    let mut reader = api
+        .get_object_reader(&job.bucket, key, None, Default::default(), &opts)
+        .await?;
+    if fingerprint(&target) != fingerprint(&reader.object_info) {
+        return Err(IntegrityError::Conflict);
+    }
+    let size = job.results[index]
+        .source_size
+        .ok_or(IntegrityError::Invalid("missing source size"))?;
+    let expected = job.results[index]
+        .expected_sha256
+        .clone()
+        .ok_or(IntegrityError::Invalid("missing source digest"))?;
+    copy_verified(
+        api,
+        job,
+        &mut reader.stream,
+        None,
+        i64::try_from(size).map_err(|_| IntegrityError::Invalid("source size overflow"))?,
+        &expected,
+        guard,
+    )
+    .await?;
+    let after = api.get_object_info(&job.bucket, key, &opts).await?;
+    if fingerprint(&after) != fingerprint(&target) {
+        return Err(IntegrityError::Conflict);
+    }
+    job.results[index].state = ItemState::Migrated;
+    job.results[index].target_version_id = target.version_id;
+    Ok(true)
+}
+
+async fn validate_destination(api: &Arc<ECStore>, job: &Job) -> Result<Arc<crate::object_api::ObjectLockConfigSnapshot>> {
+    if !readiness().new_writes_enabled_here {
+        return Err(IntegrityError::NotActivated);
+    }
+    let snapshot = api.object_lock_config_snapshot(&job.bucket).await?;
+    if snapshot
+        .metadata_transaction_guard_for(api.id, &job.bucket, Some(job.bucket_incarnation))
+        .is_none()
+    {
+        return Err(IntegrityError::Conflict);
+    }
+    let metadata = crate::bucket::metadata_sys::integrity_migration_metadata_in(&api.ctx, &job.bucket).await?;
+    // The direct storage writer is not the S3 orchestration layer. Reject configurations it cannot preserve.
+    if metadata.bucket_incarnation_id != job.bucket_incarnation {
+        return Err(IntegrityError::Conflict);
+    }
+    if !metadata.versioning_config_xml.is_empty()
+        || !metadata.encryption_config_xml.is_empty()
+        || !metadata.object_lock_config_xml.is_empty()
+        || !metadata.replication_config_xml.is_empty()
+        || !metadata.notification_config_xml.is_empty()
+        || !metadata.quota_config_json.is_empty()
+        || !metadata.on_demand_migration_config_json.is_empty()
+        || !metadata.lifecycle_config_xml.is_empty()
+        || !metadata.table_bucket_config_json.is_empty()
+        || !metadata.bucket_acl_config_json.is_empty()
+        || !metadata.logging_config_xml.is_empty()
+    {
+        return Err(IntegrityError::UnsupportedBucket);
+    }
+    Ok(snapshot)
+}
+
+/// Bound publication throughput as well as the source and destination verification reads.
+struct ThrottledFile {
+    file: tokio::fs::File,
+    started: tokio::time::Instant,
+    count: u64,
+    rate: u64,
+    sleep: std::pin::Pin<Box<tokio::time::Sleep>>,
+}
+
+impl ThrottledFile {
+    fn new(file: tokio::fs::File, rate: u64) -> Self {
+        let started = tokio::time::Instant::now();
+        Self {
+            file,
+            started,
+            count: 0,
+            rate,
+            sleep: Box::pin(tokio::time::sleep_until(started)),
+        }
+    }
+}
+
+impl tokio::io::AsyncRead for ThrottledFile {
+    fn poll_read(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        output: &mut tokio::io::ReadBuf<'_>,
+    ) -> std::task::Poll<std::io::Result<()>> {
+        use std::future::Future;
+        if self.sleep.as_mut().poll(cx).is_pending() {
+            return std::task::Poll::Pending;
+        }
+        let capacity = output.remaining().min(64 * 1024);
+        let mut buffer = tokio::io::ReadBuf::new(output.initialize_unfilled_to(capacity));
+        match std::pin::Pin::new(&mut self.file).poll_read(cx, &mut buffer) {
+            std::task::Poll::Ready(Ok(())) => {
+                let count = buffer.filled().len();
+                output.advance(count);
+                self.count += count as u64;
+                let deadline = self.started + Duration::from_secs_f64(self.count as f64 / self.rate as f64);
+                self.sleep.as_mut().reset(deadline);
+                std::task::Poll::Ready(Ok(()))
+            }
+            result => result,
+        }
+    }
+}

crates/ecstore/src/services/integrity/tests.rs

@@ -0,0 +1,387 @@
+// Copyright 2024 RustFS Team
+// Licensed under the Apache License, Version 2.0.
+
+use super::*;
+use crate::object_api::{ObjectOptions, PutObjReader, ShardIntegrityWriteMode};
+use crate::storage_api_contracts::{
+    bucket::{BucketOperations, MakeBucketOptions},
+    object::{ObjectIO, ObjectOperations},
+};
+use tokio::io::AsyncReadExt;
+
+async fn fixture() -> (Vec<tempfile::TempDir>, Arc<crate::store::ECStore>, String) {
+    let (dirs, store) = crate::services::rebalance::test_store_with_persisted_rebalance_meta(Default::default()).await;
+    crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+    let bucket = format!("integrity-{}", Uuid::new_v4());
+    store
+        .make_bucket(&bucket, &MakeBucketOptions::default())
+        .await
+        .expect("fixture bucket");
+    (dirs, store, bucket)
+}
+
+fn request(mode: JobMode, digest: String) -> JobRequest {
+    JobRequest {
+        mode,
+        items: vec![ItemRequest {
+            key: "source".into(),
+            version_id: Some("null".into()),
+            expected_sha256: Some(digest),
+            target_key: (mode == JobMode::Migrate).then(|| "target".into()),
+        }],
+        bytes_per_second: 1024 * 1024,
+        max_object_bytes: 16 * 1024 * 1024,
+    }
+}
+
+async fn finish(store: Arc<crate::store::ECStore>, bucket: &str, id: Uuid) -> Job {
+    tokio::time::timeout(std::time::Duration::from_secs(60), async {
+        loop {
+            let job = get_job(store.clone(), bucket, id).await.expect("durable job");
+            if job.state != JobState::Running {
+                return job;
+            }
+            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
+        }
+    })
+    .await
+    .expect("job must finish")
+}
+
+#[tokio::test]
+#[serial_test::serial(shard_integrity_rollout)]
+async fn audit_migration_and_ack_recovery_preserve_source_and_reject_conflicts() {
+    temp_env::async_with_vars(
+        [
+            (rustfs_config::ENV_SHARD_INTEGRITY_WRITE, Some("true")),
+            (rustfs_config::ENV_SHARD_INTEGRITY_FLEET_CONFIRMED, Some("true")),
+        ],
+        async {
+            let (_dirs, store, bucket) = fixture().await;
+            let payload = vec![42; 256 * 1024 + 17];
+            let expected = rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::SHA256, &payload)
+                .expect("digest")
+                .encoded;
+            store
+                .put_object(
+                    &bucket,
+                    "source",
+                    &mut PutObjReader::from_vec(payload.clone()),
+                    &ObjectOptions {
+                        user_defined: [
+                            ("cache-control".into(), "max-age=300".into()),
+                            ("content-disposition".into(), "attachment; filename=sample.bin".into()),
+                            ("expires".into(), "2030-01-01T00:00:00Z".into()),
+                            (rustfs_utils::http::headers::AMZ_OBJECT_TAGGING.into(), "kind=archive".into()),
+                        ]
+                        .into(),
+                        shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy),
+                        write_completion: crate::object_api::WriteCompletion::TailDrained,
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("source");
+            let source = store
+                .get_object_info(&bucket, "source", &ObjectOptions::default())
+                .await
+                .expect("source metadata");
+            assert_eq!(source.user_tags.as_str(), "kind=archive", "source fixture tags");
+            let before = model::fingerprint(&source);
+            let page = inventory(store.clone(), &bucket, "", None, None, 1).await.expect("inventory");
+            assert_eq!(page.items.len(), 1);
+            assert_eq!(page.items[0].protection, Protection::Legacy);
+            assert_eq!(page.items[0].version_id, "null", "inventory must round-trip an explicit null selector");
+            let audit = create_job(store.clone(), &bucket, request(JobMode::Audit, expected.clone()))
+                .await
+                .expect("create audit");
+            resume_job(store.clone(), &bucket, audit.id).await.expect("resume audit");
+            let audited = finish(store.clone(), &bucket, audit.id).await;
+            assert_eq!(audited.state, JobState::Complete, "{audited:?}");
+            assert_eq!(audited.results[0].state, ItemState::Verified);
+
+            let migration = create_job(store.clone(), &bucket, request(JobMode::Migrate, expected.clone()))
+                .await
+                .expect("create migration");
+            resume_job(store.clone(), &bucket, migration.id)
+                .await
+                .expect("resume migration");
+            let migrated = finish(store.clone(), &bucket, migration.id).await;
+            assert_eq!(migrated.state, JobState::Complete, "{migrated:?}");
+            assert_eq!(migrated.results[0].state, ItemState::Migrated);
+            let mut reader = store
+                .get_object_reader(&bucket, "target", None, Default::default(), &ObjectOptions::default())
+                .await
+                .expect("target");
+            assert_eq!(model::protection(&reader.object_info), Protection::IndependentCommitment);
+            assert_eq!(reader.object_info.user_tags.as_str(), "kind=archive");
+            assert_eq!(reader.object_info.expires, source.expires);
+            assert_eq!(
+                reader.object_info.user_defined.get("cache-control"),
+                source.user_defined.get("cache-control")
+            );
+            assert_eq!(
+                reader.object_info.user_defined.get("content-disposition"),
+                source.user_defined.get("content-disposition")
+            );
+            let mut actual = Vec::new();
+            reader.stream.read_to_end(&mut actual).await.expect("target content");
+            assert_eq!(actual, payload);
+            let after = store
+                .get_object_info(&bucket, "source", &ObjectOptions::default())
+                .await
+                .expect("source after");
+            assert_eq!(model::fingerprint(&after), before);
+
+            // Simulate a lost completion checkpoint after the create-only PUT committed.
+            let (mut replay, etag) = load(&store, &bucket, migration.id).await.expect("load receipt");
+            replay.state = JobState::Paused;
+            replay.results[0].state = ItemState::Prepared;
+            replay.revision += 1;
+            save(&store, &replay, Some(&etag), None)
+                .await
+                .expect("lost acknowledgement fixture");
+            resume_job(store.clone(), &bucket, replay.id)
+                .await
+                .expect("reconcile after restart");
+            let reconciled = finish(store.clone(), &bucket, replay.id).await;
+            assert_eq!(reconciled.results[0].state, ItemState::Migrated, "{reconciled:?}");
+
+            let conflict = create_job(store.clone(), &bucket, request(JobMode::Migrate, expected))
+                .await
+                .expect("conflicting job");
+            resume_job(store.clone(), &bucket, conflict.id)
+                .await
+                .expect("resume conflict");
+            let conflict = finish(store.clone(), &bucket, conflict.id).await;
+            assert_eq!(conflict.results[0].state, ItemState::Conflict, "{conflict:?}");
+            let target = store
+                .get_object_info(&bucket, "target", &ObjectOptions::default())
+                .await
+                .expect("unchanged target");
+            assert_eq!(target.data_dir, reader.object_info.data_dir);
+        },
+    )
+    .await;
+}
+
+#[tokio::test]
+async fn mismatch_never_publishes_and_paused_job_survives_reload() {
+    let (_dirs, store, bucket) = fixture().await;
+    store
+        .put_object(
+            &bucket,
+            "source",
+            &mut PutObjReader::from_vec(b"actual".to_vec()),
+            &ObjectOptions {
+                shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy),
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("source");
+    let wrong = rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::SHA256, b"different")
+        .expect("digest")
+        .encoded;
+    let job = create_job(store.clone(), &bucket, request(JobMode::Audit, wrong))
+        .await
+        .expect("job");
+    assert_eq!(get_job(store.clone(), &bucket, job.id).await.expect("reload").state, JobState::Paused);
+    resume_job(store.clone(), &bucket, job.id).await.expect("resume");
+    let job = finish(store.clone(), &bucket, job.id).await;
+    assert_eq!(job.results[0].state, ItemState::Mismatch, "{job:?}");
+    assert!(
+        store
+            .get_object_info(&bucket, "target", &ObjectOptions::default())
+            .await
+            .is_err()
+    );
+}
+
+#[test]
+fn fingerprint_is_independent_of_metadata_insertion_order() {
+    let mut a = crate::object_api::ObjectInfo::default();
+    let mut first = std::collections::HashMap::new();
+    first.insert("a".into(), "one".into());
+    first.insert("b".into(), "two".into());
+    a.user_defined = Arc::new(first);
+    let mut b = a.clone();
+    let mut second = std::collections::HashMap::new();
+    second.insert("b".into(), "two".into());
+    second.insert("a".into(), "one".into());
+    b.user_defined = Arc::new(second);
+    assert_eq!(model::fingerprint(&a), model::fingerprint(&b));
+    b.data_dir = Some(Uuid::new_v4());
+    assert_ne!(model::fingerprint(&a), model::fingerprint(&b));
+}
+
+#[test]
+fn fingerprint_is_independent_of_part_checksum_insertion_order() {
+    let mut checksums = [
+        ("SHA256", "sha256"),
+        ("SHA1", "sha1"),
+        ("CRC32", "crc32"),
+        ("CRC32C", "crc32c"),
+    ];
+    let a = crate::object_api::ObjectInfo {
+        parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
+            checksums: Some(checksums.map(|(key, value)| (key.to_string(), value.to_string())).into()),
+            ..Default::default()
+        }]),
+        ..Default::default()
+    };
+    let mut b = a.clone();
+    let expected = model::fingerprint(&a);
+    for _ in 0..checksums.len() {
+        checksums.rotate_left(1);
+        Arc::make_mut(&mut b.parts)[0].checksums =
+            Some(checksums.map(|(key, value)| (key.to_string(), value.to_string())).into());
+        assert_eq!(expected, model::fingerprint(&b));
+    }
+    Arc::make_mut(&mut b.parts)[0]
+        .checksums
+        .as_mut()
+        .expect("part checksums")
+        .insert("SHA256".into(), "changed".into());
+    assert_ne!(expected, model::fingerprint(&b));
+}
+
+#[test]
+fn rejects_migration_into_a_source_key() {
+    let request = JobRequest {
+        mode: JobMode::Migrate,
+        items: vec![ItemRequest {
+            key: "source".into(),
+            version_id: None,
+            expected_sha256: None,
+            target_key: Some("source".into()),
+        }],
+        bytes_per_second: 1024 * 1024,
+        max_object_bytes: 1024,
+    };
+    assert!(request.validate().is_err());
+}
+
+#[test]
+fn rejects_truncated_checksum_suffix_as_migration_evidence() {
+    let checksum = rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::SHA256, b"payload").expect("checksum");
+    let mut info = crate::object_api::ObjectInfo {
+        checksum: Some(checksum.to_bytes(&[])),
+        ..Default::default()
+    };
+    assert_eq!(model::stored_sha256(&info), Some(checksum.encoded));
+    let mut bytes = info.checksum.take().expect("bytes").to_vec();
+    bytes.push(0x80);
+    info.checksum = Some(bytes.into());
+    assert!(model::stored_sha256(&info).is_none());
+}
+
+#[tokio::test]
+async fn unavailable_source_can_resume_and_cancel_intent_cannot_restart() {
+    let (_dirs, store, bucket) = fixture().await;
+    let payload = b"recovered".to_vec();
+    let digest = rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::SHA256, &payload)
+        .expect("digest")
+        .encoded;
+    let job = create_job(store.clone(), &bucket, request(JobMode::Audit, digest))
+        .await
+        .expect("create");
+    resume_job(store.clone(), &bucket, job.id).await.expect("start");
+    let failed = finish(store.clone(), &bucket, job.id).await;
+    assert_eq!(failed.state, JobState::Failed);
+    assert_eq!(failed.results[0].state, ItemState::Unavailable);
+    store
+        .put_object(
+            &bucket,
+            "source",
+            &mut PutObjReader::from_vec(payload),
+            &ObjectOptions {
+                shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy),
+                write_completion: WriteCompletion::TailDrained,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("recovered source");
+    resume_job(store.clone(), &bucket, job.id).await.expect("retry");
+    let completed = finish(store.clone(), &bucket, job.id).await;
+    assert_eq!(completed.state, JobState::Complete, "{completed:?}");
+    assert_eq!(completed.results[0].state, ItemState::Verified);
+
+    // A process restart may leave the durable cancel intent without a live worker.
+    let (mut interrupted, etag) = load(&store, &bucket, job.id).await.expect("load");
+    interrupted.state = JobState::CancelRequested;
+    interrupted.revision += 1;
+    save(&store, &interrupted, Some(&etag), None).await.expect("persist intent");
+    assert_eq!(
+        control_job(store.clone(), &bucket, job.id, false).await.expect("pause").state,
+        JobState::CancelRequested
+    );
+    assert_eq!(
+        resume_job(store.clone(), &bucket, job.id)
+            .await
+            .expect("recover cancel")
+            .state,
+        JobState::Cancelled
+    );
+}
+
+#[tokio::test]
+async fn pause_and_cancel_are_durable_during_content_read() {
+    let (_dirs, store, bucket) = fixture().await;
+    let payload = vec![7; 512 * 1024];
+    let digest = rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::SHA256, &payload)
+        .expect("digest")
+        .encoded;
+    store
+        .put_object(
+            &bucket,
+            "source",
+            &mut PutObjReader::from_vec(payload),
+            &ObjectOptions {
+                shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy),
+                write_completion: WriteCompletion::TailDrained,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("source");
+    let mut req = request(JobMode::Audit, digest);
+    req.bytes_per_second = 64 * 1024;
+    let job = create_job(store.clone(), &bucket, req).await.expect("create");
+    resume_job(store.clone(), &bucket, job.id).await.expect("start");
+    tokio::time::timeout(std::time::Duration::from_secs(20), async {
+        loop {
+            if get_job(store.clone(), &bucket, job.id).await.expect("status").results[0].state == ItemState::Prepared {
+                break;
+            }
+            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
+        }
+    })
+    .await
+    .expect("prepared");
+    let intent = control_job(store.clone(), &bucket, job.id, false).await.expect("pause");
+    assert_eq!(intent.state, JobState::PauseRequested);
+    tokio::time::timeout(std::time::Duration::from_secs(20), async {
+        loop {
+            let paused = get_job(store.clone(), &bucket, job.id).await.expect("status");
+            if paused.state == JobState::Paused {
+                break;
+            }
+            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
+        }
+    })
+    .await
+    .expect("pause acknowledged");
+    assert_eq!(
+        control_job(store.clone(), &bucket, job.id, true).await.expect("cancel").state,
+        JobState::Cancelled
+    );
+    assert_eq!(
+        resume_job(store.clone(), &bucket, job.id)
+            .await
+            .expect("terminal resume")
+            .state,
+        JobState::Cancelled
+    );
+}

crates/ecstore/src/services/mod.rs

@@ -16,6 +16,7 @@
 
 pub(crate) mod batch_processor;
 pub(crate) mod event_notification;
+pub(crate) mod integrity;
 pub(crate) mod metrics_realtime;
 pub(crate) mod notification_sys;
 pub(crate) mod rebalance;

crates/ecstore/src/services/rebalance/runtime.rs

@@ -7,8 +7,9 @@ use super::meta::{
     validate_start_rebalance_state,
 };
 use super::worker::{
-    resolve_rebalance_bucket_result, resolve_rebalance_meta_save_result, resolve_rebalance_save_task_result,
-    resolve_rebalance_terminal_error, send_rebalance_done_signal,
+    rebalance_max_attempts, resolve_rebalance_bucket_result, resolve_rebalance_meta_save_result,
+    resolve_rebalance_save_task_result, resolve_rebalance_terminal_error, retry_rebalance_metadata_access,
+    send_rebalance_done_signal,
 };
 use super::{
     EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE,
@@ -449,13 +450,14 @@ impl ECStore {
                         };
 
                         if terminal_state_present {
-                            if let Err(err) = store
-                                .save_rebalance_stats_inner(
+                            if let Err(err) = retry_rebalance_metadata_access(None, rebalance_max_attempts(), || {
+                                store.save_rebalance_stats_inner(
                                     pool_index,
                                     RebalSaveOpt::Stats,
                                     Some(save_rebalance_id.as_ref()),
                                 )
-                                .await
+                            })
+                            .await
                             {
                                 let mut rebalance_meta = store.rebalance_meta.write().await;
                                 *rebalance_meta = previous_meta;
@@ -475,9 +477,10 @@ impl ECStore {
                 }
 
                 if !terminal_state_saved
-                    && let Err(err) = store
-                        .save_rebalance_stats_for_id(pool_index, RebalSaveOpt::Stats, save_rebalance_id.as_ref())
-                        .await
+                    && let Err(err) = retry_rebalance_metadata_access(None, rebalance_max_attempts(), || {
+                        store.save_rebalance_stats_for_id(pool_index, RebalSaveOpt::Stats, save_rebalance_id.as_ref())
+                    })
+                    .await
                 {
                     let wrapped = Error::other(format!("rebalance save_task stats save failed for pool {pool_index}: {err}"));
                     error!("{} err: {:?}", msg, wrapped);

crates/ecstore/src/services/rebalance/worker.rs

@@ -109,7 +109,13 @@ pub(super) fn resolve_rebalance_save_task_result(
 }
 
 pub(super) fn resolve_rebalance_meta_save_result(result: Result<()>, stage: &str) -> Result<()> {
-    result.map_err(|err| Error::other(format!("rebalance meta save failed during {stage}: {err}")))
+    // Keep the source error reachable: the metadata retry policy classifies
+    // transient lock timeouts by inspecting the source chain, so collapsing the
+    // failure into a plain string here would make that retry a no-op.
+    result.map_err(|err| {
+        let rendered = format!("rebalance meta save failed during {stage}: {err}");
+        crate::data_movement::data_movement_context_error(rendered, err)
+    })
 }
 
 pub(super) fn rebalance_meta_lock_error(err: rustfs_lock::LockError, mode: &'static str) -> Error {
@@ -739,6 +745,29 @@ mod error_source_tests {
         }
     }
 
+    #[tokio::test]
+    async fn rebalance_metadata_retry_engages_for_wrapped_meta_save_lock_timeout() {
+        let mut attempts = 0;
+        let result = retry_rebalance_metadata_access(None, 3, || {
+            attempts += 1;
+            std::future::ready(resolve_rebalance_meta_save_result(
+                Err(rebalance_meta_lock_error(
+                    rustfs_lock::LockError::timeout(".rustfs.sys/rebalance.bin@latest", Duration::from_secs(5)),
+                    "write",
+                )),
+                "save_rebalance_stats for pool 0 opt Stats",
+            ))
+        })
+        .await;
+
+        assert_eq!(attempts, 3, "a wrapped meta save lock timeout must stay retryable");
+        let err = result.expect_err("persistent lock contention must not become success");
+        assert!(matches!(
+            rebalance_error_source(&err),
+            Error::Lock(rustfs_lock::LockError::Timeout { .. })
+        ));
+    }
+
     #[tokio::test]
     async fn rebalance_metadata_retry_cancels_a_pending_attempt() {
         struct DropProbe(Arc<std::sync::atomic::AtomicUsize>);

crates/ecstore/src/set_disk/core/io_primitives.rs

@@ -1163,13 +1163,10 @@ pub(in crate::set_disk) struct ReadRepairHealSubmission<'a> {
     pub(in crate::set_disk) set_index: usize,
     pub(in crate::set_disk) part_number: Option<usize>,
     pub(in crate::set_disk) reason: &'static str,
-    /// Durable MRF journal intent to file alongside the read-repair request
-    /// (backlog#1894 axis A): the intent kind plus its native `Uuid`
-    /// version id (the submission's string form stays display-only). Bound
-    /// to the reservation — the intent is only delivered when this sighting
-    /// wins the dedup TTL, so a burst of reads failing on the same object
-    /// books exactly one journal record instead of one per retry. `None`
-    /// keeps the historical no-intent behavior.
+    /// Best-effort ingress for the durable MRF consumer. Its identity coalescer
+    /// is independent of the heal reservation: an earlier metadata repair must
+    /// not suppress newly observed payload damage, and rejected ingress must
+    /// remain retryable. Channel admission does not prove checkpoint commit.
     pub(in crate::set_disk) mrf_intent: Option<(rustfs_common::mrf_channel::MrfKind, Option<uuid::Uuid>)>,
 }
 
@@ -1224,6 +1221,27 @@ pub(in crate::set_disk) async fn submit_read_repair_heal_with_submitter(
         mrf_intent,
     } = submission;
 
+    if let Some((kind, version_uuid)) = mrf_intent
+        && let (Ok(pool_index), Ok(set_index)) = (u32::try_from(pool_index), u32::try_from(set_index))
+    {
+        let scope = rustfs_common::mrf_channel::MrfScope { pool_index, set_index };
+        let ingress = rustfs_common::mrf_channel::try_send_mrf_intent_typed(kind, bucket, object, version_uuid, Some(scope));
+        debug!(
+            event = EVENT_SET_DISK_READ,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_SET_DISK,
+            state = "mrf_ingress",
+            mrf_kind = ?kind,
+            bucket,
+            object,
+            version_id,
+            pool_index,
+            set_index,
+            result = ?ingress,
+            "Read-repair MRF ingress result"
+        );
+    }
+
     let Some(dedup_key) = reserve_read_repair_heal(bucket, object, version_id, pool_index, set_index).await else {
         record_read_repair_dedup("duplicate");
         debug!(
@@ -1233,15 +1251,6 @@ pub(in crate::set_disk) async fn submit_read_repair_heal_with_submitter(
         return;
     };
 
-    // Reservation won: this sighting owns the repair records for the object,
-    // including the durable journal intent when the caller asked for one.
-    if let Some((kind, version_uuid)) = mrf_intent
-        && let (Ok(pool_index), Ok(set_index)) = (u32::try_from(pool_index), u32::try_from(set_index))
-    {
-        let scope = rustfs_common::mrf_channel::MrfScope { pool_index, set_index };
-        let _ = rustfs_common::mrf_channel::try_send_mrf_intent_typed(kind, bucket, object, version_uuid, Some(scope));
-    }
-
     let mut request = rustfs_heal_contracts::heal_channel::create_heal_request_with_options(
         bucket.to_string(),
         Some(object.to_string()),
@@ -2481,15 +2490,15 @@ impl SetDisks {
         part_numbers: &[usize],
         read_quorum: usize,
     ) -> disk::error::Result<Vec<ObjectPartInfo>> {
-        let bucket = bucket.to_string();
-        let part_meta_paths = part_meta_paths.to_vec();
+        let bucket: Arc<str> = Arc::from(bucket);
+        let part_meta_paths: Arc<[String]> = Arc::from(part_meta_paths);
 
         let tasks: Vec<_> = disks
             .iter()
             .map(|disk| {
                 let disk = disk.clone();
-                let bucket = bucket.clone();
-                let part_meta_paths = part_meta_paths.clone();
+                let bucket = Arc::clone(&bucket);
+                let part_meta_paths = Arc::clone(&part_meta_paths);
 
                 async move {
                     if let Some(disk) = disk {
@@ -12961,7 +12970,7 @@ mod tests {
 
     #[tokio::test]
     #[serial_test::serial]
-    async fn mrf_intent_is_filed_once_per_read_repair_reservation() {
+    async fn mrf_ingress_is_coalesced_independently_of_read_repair_reservations() {
         // Serial: owns the process-global MRF channel for this test binary
         // (same key as the other channel-owning tests above).
         let bucket = format!("mrf-intent-bucket-{}", Uuid::new_v4());
@@ -12982,8 +12991,13 @@ mod tests {
             }
         }
 
-        // First sighting wins the reservation: the journal intent is filed
-        // synchronously before the admission task is spawned.
+        // An earlier metadata/missing-shard admission has no payload MRF intent.
+        let mut metadata_submission = intent_submission(&bucket, &object);
+        metadata_submission.mrf_intent = None;
+        submit_read_repair_heal_with_submitter(metadata_submission, accepted_read_repair_submitter).await;
+        assert!(receiver.try_recv().is_err());
+
+        // Newly observed corruption must reach MRF despite that reservation.
         submit_read_repair_heal_with_submitter(intent_submission(&bucket, &object), accepted_read_repair_submitter).await;
         let first = receiver.try_recv().expect("first sighting must file exactly one MRF intent");
         assert_eq!(*first.bucket, bucket);
@@ -12993,6 +13007,20 @@ mod tests {
         // second journal record.
         submit_read_repair_heal_with_submitter(intent_submission(&bucket, &object), accepted_read_repair_submitter).await;
         assert!(receiver.try_recv().is_err(), "duplicate sighting must not file another MRF intent");
+
+        let retry_object = format!("retry-object-{}", Uuid::new_v4());
+        rustfs_common::mrf_channel::set_mrf_delivery_enabled(false);
+        submit_read_repair_heal_with_submitter(intent_submission(&bucket, &retry_object), accepted_read_repair_submitter).await;
+        assert!(receiver.try_recv().is_err());
+        rustfs_common::mrf_channel::set_mrf_delivery_enabled(true);
+        submit_read_repair_heal_with_submitter(intent_submission(&bucket, &retry_object), accepted_read_repair_submitter).await;
+        let retried = receiver
+            .try_recv()
+            .expect("a heal reservation cannot suppress a previously rejected MRF intent");
+        assert_eq!(*retried.object, retry_object);
+        rustfs_common::mrf_channel::release_mrf_intent(&first);
+        rustfs_common::mrf_channel::release_mrf_intent(&retried);
+        rustfs_common::mrf_channel::set_mrf_delivery_enabled(false);
     }
 
     #[tokio::test]

crates/ecstore/src/set_disk/mod.rs

@@ -3877,6 +3877,9 @@ pub struct SetDisks {
     pub format: FormatV3,
     #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
     disk_health_cache: Arc<RwLock<Vec<Option<DiskHealthEntry>>>>,
+    /// Coalesce bounded GET probes of missing remote slots. Shared by set clones;
+    /// the timestamp also limits repeated probes while a peer remains absent.
+    read_reconnect: Arc<tokio::sync::Mutex<Option<tokio::time::Instant>>>,
     get_object_metadata_cache: moka::future::Cache<GetObjectMetadataCacheKey, Arc<GetObjectMetadataCacheEntry>>,
     get_object_metadata_cache_hash_builder: std::collections::hash_map::RandomState,
     get_object_metadata_cache_generations: Arc<[AtomicU64]>,
@@ -4743,6 +4746,7 @@ impl SetDisks {
             format,
             set_endpoints,
             disk_health_cache: Arc::new(RwLock::new(Vec::new())),
+            read_reconnect: Arc::new(tokio::sync::Mutex::new(None)),
             get_object_metadata_cache: moka::future::Cache::builder()
                 .max_capacity(get_object_metadata_cache_max_entries() as u64)
                 .time_to_live(GET_OBJECT_METADATA_CACHE_TTL)
@@ -6473,6 +6477,28 @@ async fn verify_inline_part_bitrot(meta: &FileInfo) -> disk::error::Result<()> {
         .map_err(|_| DiskError::FileCorrupt)
 }
 
+fn validate_deep_scan_results(results: &[usize], expected_parts: usize) -> disk::error::Result<()> {
+    if results.len() != expected_parts {
+        return Err(DiskError::other(format!(
+            "incomplete deep scan result: expected {expected_parts} parts, received {}",
+            results.len()
+        )));
+    }
+    for (part, status) in results.iter().enumerate() {
+        match *status {
+            CHECK_PART_SUCCESS | CHECK_PART_FILE_NOT_FOUND | CHECK_PART_FILE_CORRUPT => {}
+            CHECK_PART_DISK_NOT_FOUND => return Err(DiskError::DiskNotFound),
+            crate::disk::CHECK_PART_VOLUME_NOT_FOUND => return Err(DiskError::VolumeNotFound),
+            _ => {
+                return Err(DiskError::other(format!(
+                    "incomplete deep scan result: part {part} has unverified status {status}"
+                )));
+            }
+        }
+    }
+    Ok(())
+}
+
 /// disks_with_all_partsv2 is a corrected version based on Go implementation.
 /// It sets partsMetadata and onlineDisks when xl.meta is inexistant/corrupted or outdated.
 /// It also checks if the status of each part (corrupted, missing, ok) in each drive.
@@ -6693,9 +6719,13 @@ async fn disks_with_all_parts(
             // it needs healing too.
             match disk.verify_file(bucket, object, meta).await {
                 Ok(v) => {
+                    validate_deep_scan_results(&v.results, latest_meta.parts.len())?;
                     verify_resp = v;
                 }
                 Err(err) => {
+                    if !matches!(err, DiskError::FileNotFound | DiskError::FileVersionNotFound | DiskError::FileCorrupt) {
+                        return Err(err);
+                    }
                     debug!(
                         event = EVENT_SET_DISK_HEAL,
                         component = LOG_COMPONENT_ECSTORE,
@@ -6779,7 +6809,10 @@ pub fn should_heal_object_on_disk(
     latest_meta: &FileInfo,
 ) -> (bool, bool, Option<DiskError>) {
     if let Some(err) = err
-        && (err == &DiskError::FileNotFound || err == &DiskError::FileVersionNotFound || err == &DiskError::FileCorrupt)
+        && (err == &DiskError::FileNotFound
+            || err == &DiskError::FileVersionNotFound
+            || err == &DiskError::FileCorrupt
+            || err == &DiskError::VolumeNotFound)
     {
         return (true, true, Some(err.clone()));
     }
@@ -6788,7 +6821,12 @@ pub fn should_heal_object_on_disk(
         return (false, false, err.clone());
     }
 
-    if !meta.equals(latest_meta) {
+    // `FileInfo::equals` intentionally compares the erasure payload shape and
+    // modification time, but it does not compare the selected version or the
+    // delete-marker bit. Heal uses this decision for versioned metadata, so a
+    // stale historical version that is still marked latest must be treated as
+    // outdated even when those storage-level fields happen to match.
+    if !meta.equals(latest_meta) || !heal_metadata_identity_matches(meta, latest_meta) {
         debug!(
             event = EVENT_SET_DISK_HEAL,
             component = LOG_COMPONENT_ECSTORE,
@@ -6811,6 +6849,11 @@ pub fn should_heal_object_on_disk(
     (false, false, None)
 }
 
+fn heal_metadata_identity_matches(meta: &FileInfo, latest_meta: &FileInfo) -> bool {
+    meta.deleted == latest_meta.deleted
+        && meta.version_id.filter(|version| !version.is_nil()) == latest_meta.version_id.filter(|version| !version.is_nil())
+}
+
 /// Probe every drive of the set at once. Each live probe is bounded by the
 /// drive `disk_info` timeout, and the admin peer probe budget only covers one
 /// such timeout; a sequential walk over several stalled drives after a power
@@ -11109,6 +11152,50 @@ mod tests {
         assert_eq!(conv_part_err_to_int(&Some(other_err)), CHECK_PART_UNKNOWN); // Other errors should return UNKNOWN, not SUCCESS
     }
 
+    #[test]
+    fn deep_scan_results_accept_only_complete_verified_or_repairable_parts() {
+        for (results, expected_parts) in [
+            (vec![], 0),
+            (vec![CHECK_PART_SUCCESS], 1),
+            (vec![CHECK_PART_FILE_NOT_FOUND], 1),
+            (vec![CHECK_PART_FILE_CORRUPT], 1),
+            (vec![CHECK_PART_SUCCESS, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_FILE_CORRUPT], 3),
+        ] {
+            validate_deep_scan_results(&results, expected_parts)
+                .expect("complete results must allow healthy or repairable parts");
+        }
+    }
+
+    #[test]
+    fn deep_scan_results_reject_unknown_and_incomplete_observations() {
+        for (results, expected_parts) in [
+            (vec![CHECK_PART_UNKNOWN], 1),
+            (vec![usize::MAX], 1),
+            (vec![CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN], 2),
+            (vec![], 1),
+            (vec![CHECK_PART_SUCCESS], 2),
+            (vec![CHECK_PART_SUCCESS, CHECK_PART_SUCCESS], 1),
+            (vec![CHECK_PART_SUCCESS], 0),
+        ] {
+            let error =
+                validate_deep_scan_results(&results, expected_parts).expect_err("unverified parts must not become healthy");
+            assert!(matches!(error, DiskError::Io(_)), "an incomplete observation is not proven corruption");
+            assert!(error.to_string().contains("incomplete deep scan result"));
+        }
+    }
+
+    #[test]
+    fn deep_scan_results_preserve_disk_and_volume_failures() {
+        for (status, expected_error) in [
+            (CHECK_PART_DISK_NOT_FOUND, DiskError::DiskNotFound),
+            (CHECK_PART_VOLUME_NOT_FOUND, DiskError::VolumeNotFound),
+        ] {
+            let error = validate_deep_scan_results(&[CHECK_PART_SUCCESS, status], 2)
+                .expect_err("an unavailable part cannot certify a healthy disk");
+            assert_eq!(error, expected_error);
+        }
+    }
+
     #[test]
     fn test_has_part_err() {
         // Test checking for part errors
@@ -11189,6 +11276,12 @@ mod tests {
         let (should_heal, _, _) = should_heal_object_on_disk(&err, &[], &meta, &latest_meta);
         assert!(should_heal);
 
+        let err = Some(DiskError::VolumeNotFound);
+        let (should_heal, is_meta, reason) = should_heal_object_on_disk(&err, &[], &meta, &latest_meta);
+        assert!(should_heal);
+        assert!(is_meta);
+        assert_eq!(reason, Some(DiskError::VolumeNotFound));
+
         let err = Some(DiskError::FileCorrupt);
         let (should_heal, is_meta, reason) = should_heal_object_on_disk(&err, &[], &meta, &latest_meta);
         assert!(should_heal);
@@ -11205,6 +11298,29 @@ mod tests {
         assert_eq!(reason, Some(DiskError::FileCorrupt));
     }
 
+    #[test]
+    fn stale_delete_marker_identity_requires_metadata_heal() {
+        let historical_version = Uuid::new_v4();
+        let marker_version = Uuid::new_v4();
+        let mut stale = FileInfo {
+            version_id: Some(historical_version),
+            ..FileInfo::default()
+        };
+        let mut latest = stale.clone();
+        stale.deleted = false;
+        latest.deleted = true;
+        latest.version_id = Some(marker_version);
+
+        // The generic equality helper intentionally considers these records
+        // equal when their erasure shape and mod-time match. Heal must still
+        // repair the version identity and delete-marker state.
+        assert!(stale.equals(&latest));
+        let (should_heal, metadata, reason) = should_heal_object_on_disk(&None, &[], &stale, &latest);
+        assert!(should_heal);
+        assert!(metadata);
+        assert_eq!(reason, Some(DiskError::OutdatedXLMeta));
+    }
+
     #[test]
     fn metadata_io_failures_never_authorize_heal_overwrite() {
         let meta = FileInfo::default();
@@ -13628,6 +13744,7 @@ mod tests {
                 true,
                 false,
                 false,
+                false,
                 GET_OBJECT_PATH_LEGACY_DUPLEX,
                 GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART,
                 metrics_size_bucket,
@@ -13741,6 +13858,7 @@ mod tests {
                 true,
                 false,
                 false,
+                false,
                 GET_OBJECT_PATH_LEGACY_DUPLEX,
                 GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART,
                 metrics_size_bucket,

crates/ecstore/src/set_disk/ops/heal.rs

@@ -502,6 +502,22 @@ fn heal_writer_error_summary(error: &DiskError) -> String {
     }
 }
 
+/// Whether a writer failure can be retried after the target disk or peer
+/// becomes available again.  Keep this decision typed: the aggregate
+/// `all_targets_unavailable` path must not turn permanent errors such as
+/// `DiskFull` into retryable failures.
+fn is_retryable_heal_writer_error(error: &DiskError) -> bool {
+    matches!(
+        error,
+        DiskError::DiskNotFound
+            | DiskError::Timeout
+            | DiskError::SourceStalled
+            | DiskError::FaultyRemoteDisk
+            | DiskError::FaultyDisk
+            | DiskError::RemoteClientUnavailable(_)
+    ) || error.is_retryable_internode_write_failure()
+}
+
 fn warn_heal_writer_failures(
     bucket: &str,
     object: &str,
@@ -854,7 +870,7 @@ impl SetDisks {
         );
 
         let disks = self.get_disks_internal().await;
-        let bucket_heal_scope = crate::store::bucket_heal_scope(bucket);
+        let bucket_heal_scope = crate::store::bucket_heal_scope_for_object(bucket, object);
         if let Some(scope) = &bucket_heal_scope {
             scope.check()?;
         }
@@ -1398,6 +1414,26 @@ impl SetDisks {
                             ));
                         }
 
+                        // `errs` is reported in physical-disk order while the
+                        // reconstruction loop below uses the shard order from
+                        // the selected metadata. Preserve which targets are
+                        // missing their bucket volume across that permutation.
+                        let mut missing_bucket_volumes = vec![false; out_dated_disks.len()];
+                        for (physical_index, error) in errs.iter().enumerate() {
+                            if error.as_ref() != Some(&DiskError::VolumeNotFound) {
+                                continue;
+                            }
+                            let shard_index = latest_meta
+                                .erasure
+                                .distribution
+                                .get(physical_index)
+                                .and_then(|index| index.checked_sub(1))
+                                .unwrap_or(physical_index);
+                            if let Some(target) = missing_bucket_volumes.get_mut(shard_index) {
+                                *target = true;
+                            }
+                        }
+
                         out_dated_disks = Self::shuffle_disks(&out_dated_disks, &latest_meta.erasure.distribution);
                         let mut parts_metadata = Self::shuffle_parts_metadata(&parts_metadata, &latest_meta.erasure.distribution);
                         let mut copy_parts_metadata = vec![None; parts_metadata.len()];
@@ -1424,6 +1460,58 @@ impl SetDisks {
                             }
                         }
 
+                        // A missing bucket volume is a repairable object target,
+                        // but the final rename cannot create its parent volume.
+                        // Recreate only the affected volumes after quorum and
+                        // lifecycle checks have passed. Keep failures visible so
+                        // a partial repair cannot be reported as successful.
+                        let mut volume_creation_error = None;
+                        for (index, outdated_disk) in out_dated_disks.iter_mut().enumerate() {
+                            if !missing_bucket_volumes.get(index).copied().unwrap_or(false) || outdated_disk.is_none() {
+                                continue;
+                            }
+                            if let Some(scope) = &bucket_heal_scope {
+                                scope.check()?;
+                            }
+                            let disk = outdated_disk.as_ref().expect("checked above").clone();
+                            let endpoint = disk.endpoint().to_string();
+                            match disk.make_volume(bucket).await {
+                                Ok(()) | Err(DiskError::VolumeExists) => {}
+                                Err(error) => {
+                                    warn!(
+                                        event = EVENT_SET_DISK_HEAL,
+                                        component = LOG_COMPONENT_ECSTORE,
+                                        subsystem = LOG_SUBSYSTEM_HEAL,
+                                        bucket,
+                                        object,
+                                        version_id,
+                                        disk_index = index,
+                                        endpoint = %endpoint,
+                                        error = %error,
+                                        state = "bucket_volume_recreate_failed",
+                                        "Heal object could not recreate the target bucket volume"
+                                    );
+                                    if volume_creation_error.is_none() {
+                                        volume_creation_error = Some(error.clone());
+                                    }
+                                    *outdated_disk = None;
+                                    disks_to_heal_count = disks_to_heal_count.saturating_sub(1);
+                                    for drive in &mut result.after.drives {
+                                        if drive.endpoint == endpoint {
+                                            drive.state = heal_drive_state_for_error(&error).to_string();
+                                        }
+                                    }
+                                }
+                            }
+                        }
+
+                        if let Some(scope) = &bucket_heal_scope {
+                            scope.check()?;
+                        }
+                        if disks_to_heal_count == 0 {
+                            return Ok((result, volume_creation_error.or(Some(DiskError::ErasureWriteQuorum))));
+                        }
+
                         // We write at temporary location and then rename to final location.
                         let tmp_id = Uuid::new_v4().to_string();
                         // Delete markers and remote (transitioned) objects carry no data_dir and
@@ -1458,6 +1546,8 @@ impl SetDisks {
                             let erasure_info = latest_meta.erasure.clone();
                             let mut writer_failure_count = 0usize;
                             let mut first_writer_failure = None;
+                            let mut first_writer_error = None;
+                            let mut all_writer_failures_retryable = true;
                             let mut writer_failure_warned = false;
 
                             for (part_index, part) in latest_meta.parts.iter().enumerate() {
@@ -1584,9 +1674,11 @@ impl SetDisks {
                                             Ok(writer) => writer,
                                             Err(err) => {
                                                 writer_failure_count += 1;
+                                                all_writer_failures_retryable &= is_retryable_heal_writer_error(&err);
                                                 if first_writer_failure.is_none() {
                                                     first_writer_failure =
                                                         Some((part.number, index, heal_writer_error_summary(&err)));
+                                                    first_writer_error = Some(err);
                                                 }
                                                 writers.push(None);
                                                 continue;
@@ -1714,12 +1806,12 @@ impl SetDisks {
                                     }
                                     // Clean up healed shards written to .rustfs/tmp before bailing (B20).
                                     let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
-                                    return Ok((
-                                        result,
-                                        Some(DiskError::other(format!(
-                                            "all drives had write errors, unable to heal {bucket}/{object}"
-                                        ))),
-                                    ));
+                                    let error = if all_writer_failures_retryable {
+                                        first_writer_error.take().unwrap_or(DiskError::FaultyDisk)
+                                    } else {
+                                        DiskError::other(format!("all drives had write errors, unable to heal {bucket}/{object}"))
+                                    };
+                                    return Ok((result, Some(error)));
                                 }
                             }
 
@@ -1865,6 +1957,10 @@ impl SetDisks {
                             ));
                         }
 
+                        if let Some(error) = volume_creation_error {
+                            return Ok((result, Some(error)));
+                        }
+
                         if result.integrity_verified
                             && let Err(error) =
                                 crate::io_support::shard_integrity::restore_proof_replicas(&latest_meta, &disks, bucket, object)
@@ -3430,7 +3526,7 @@ impl SetDisks {
 mod heal_result_report_tests {
     use super::{
         AbsenceProofRequest, DanglingCheckPartsFailure, DanglingDeleteFailure, DanglingDeleteSafety, ReadRepairCommitFingerprint,
-        ReadRepairPauseScope, SetDisks, heal_writer_error_summary,
+        ReadRepairPauseScope, SetDisks, heal_writer_error_summary, is_retryable_heal_writer_error,
     };
     use super::{HEAL_RENAME_INCOMPLETE, HealRenameFailureScope, HealWriterFailureScope};
     use crate::disk::endpoint::Endpoint;
@@ -3663,6 +3759,24 @@ mod heal_result_report_tests {
         assert!(!summary.contains("sensitive"));
     }
 
+    #[test]
+    fn retryable_heal_writer_error_keeps_permanent_errors_out() {
+        for error in [
+            DiskError::DiskNotFound,
+            DiskError::Timeout,
+            DiskError::SourceStalled,
+            DiskError::FaultyRemoteDisk,
+            DiskError::FaultyDisk,
+            DiskError::RemoteClientUnavailable("peer restarting".to_string()),
+        ] {
+            assert!(is_retryable_heal_writer_error(&error), "{error:?} should be retryable");
+        }
+
+        for error in [DiskError::DiskFull, DiskError::DiskAccessDenied, DiskError::FileCorrupt] {
+            assert!(!is_retryable_heal_writer_error(&error), "{error:?} should remain permanent");
+        }
+    }
+
     #[tokio::test(flavor = "current_thread")]
     #[serial_test::serial]
     async fn heal_writer_failures_emit_one_aggregate_warning_per_object() {
@@ -4223,6 +4337,88 @@ mod heal_result_report_tests {
         );
     }
 
+    #[tokio::test]
+    async fn deep_heal_recreates_missing_bucket_volume_before_rebuilding_shard() {
+        let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
+        let bucket = "deep-heal-missing-bucket-volume";
+        let object = "object.bin";
+        for disk in &disks {
+            disk.make_volume(bucket).await.expect("bucket volume should be created");
+        }
+
+        let mut reader = PutObjReader::from_vec(vec![0x5c; 1024 * 1024]);
+        set.put_object(
+            bucket,
+            object,
+            &mut reader,
+            &ObjectOptions {
+                no_lock: true,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("source object should be written before volume loss");
+        let source = disks[2]
+            .read_version("", bucket, object, "", &ReadOptions::default())
+            .await
+            .expect("source metadata should be readable");
+        let data_dir = source.data_dir.expect("non-inline source should have a data directory");
+
+        disks[1]
+            .delete_volume(bucket, true)
+            .await
+            .expect("target bucket volume should be removable for the regression setup");
+        assert!(matches!(disks[1].stat_volume(bucket).await, Err(DiskError::VolumeNotFound)));
+
+        let (result, error) = set
+            .heal_object(
+                bucket,
+                object,
+                "",
+                &HealOpts {
+                    no_lock: true,
+                    scan_mode: HealScanMode::Deep,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("deep heal should finish after bucket volume loss");
+
+        assert!(error.is_none(), "deep heal should recreate the missing bucket volume: {error:?}");
+        assert!(disks[1].stat_volume(bucket).await.is_ok(), "target bucket volume should be restored");
+        let healed = disks[1]
+            .read_version("", bucket, object, "", &ReadOptions::default())
+            .await
+            .expect("healed target metadata should be readable");
+        assert_eq!(healed.data_dir, Some(data_dir));
+        assert_eq!(result.after.drives[1].state, DriveState::Ok.to_string());
+        assert!(
+            temp_dirs[1]
+                .path()
+                .join(bucket)
+                .join(object)
+                .join(data_dir.to_string())
+                .join("part.1")
+                .exists(),
+            "deep heal should rebuild the object shard after recreating its volume"
+        );
+
+        let (_, retry_error) = set
+            .heal_object(
+                bucket,
+                object,
+                "",
+                &HealOpts {
+                    no_lock: true,
+                    scan_mode: HealScanMode::Deep,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("repeat heal should complete");
+        assert!(retry_error.is_none(), "repeat heal should be idempotent: {retry_error:?}");
+    }
+
     #[tokio::test]
     async fn deep_heal_rebuilds_a_stale_current_delete_marker_metadata_replica() {
         let (_temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
@@ -4378,63 +4574,243 @@ mod heal_result_report_tests {
         );
     }
 
-    #[tokio::test]
-    async fn deep_heal_rebuilds_h2_near_tail_truncated_part() {
-        let (temp_dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(16, 0, 4).await;
-        let bucket = "deep-heal-h2-near-tail-truncation";
-        let object = "object.bin";
-        for disk in &disks {
-            disk.make_volume(bucket).await.expect("bucket volume should be created");
+    #[derive(Debug)]
+    struct FailedIntegrityReadTransport {
+        error: DiskError,
+        reads: std::sync::atomic::AtomicUsize,
+    }
+
+    #[async_trait::async_trait]
+    impl crate::cluster::rpc::internode_data_transport::InternodeDataTransport for FailedIntegrityReadTransport {
+        async fn open_read(
+            &self,
+            _request: crate::cluster::rpc::internode_data_transport::ReadStreamRequest,
+        ) -> crate::disk::error::Result<crate::disk::FileReader> {
+            self.reads.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+            let error = match &self.error {
+                DiskError::FileCorrupt => DiskError::from(rustfs_rio::new_test_remote_file_corrupt_http_io_error()),
+                DiskError::FileNotFound => DiskError::from(rustfs_rio::new_test_remote_file_not_found_http_io_error()),
+                DiskError::VolumeNotFound => DiskError::from(rustfs_rio::new_test_remote_volume_not_found_http_io_error()),
+                error => error.clone(),
+            };
+            Err(error)
         }
 
-        let expected_payload = vec![0x6d; 5 * 1024 * 1024 + 123];
+        async fn open_write(
+            &self,
+            _request: crate::cluster::rpc::internode_data_transport::WriteStreamRequest,
+        ) -> crate::disk::error::Result<crate::disk::FileWriter> {
+            panic!("an integrity scan must not write remote data");
+        }
+
+        async fn open_walk_dir(
+            &self,
+            _request: crate::cluster::rpc::internode_data_transport::WalkDirStreamRequest,
+        ) -> crate::disk::error::Result<crate::disk::FileReader> {
+            panic!("an object integrity scan must not walk remote directories");
+        }
+
+        fn name(&self) -> &'static str {
+            "failed-integrity-read"
+        }
+
+        fn capabilities(&self) -> crate::cluster::rpc::internode_data_transport::InternodeDataTransportCapabilities {
+            crate::cluster::rpc::internode_data_transport::InternodeDataTransportCapabilities::tcp_http()
+        }
+    }
+
+    async fn assert_remote_integrity_read_failure(expected_error: DiskError) {
+        use crate::disk::{CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN};
+        use crate::object_api::ShardIntegrityWriteMode;
+
+        let (dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, 2).await;
+        let bucket = "remote-integrity-read-failure";
+        let object = "object.bin";
+        for disk in &disks {
+            disk.make_volume(bucket).await.expect("create fixture bucket");
+        }
         set.put_object(
             bucket,
             object,
-            &mut PutObjReader::from_vec(expected_payload.clone()),
+            &mut PutObjReader::from_vec(vec![0x6d; 1024 * 1024 + 123]),
             &ObjectOptions {
+                shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Protected),
                 no_lock: true,
                 ..Default::default()
             },
         )
         .await
-        .expect("source object should be written before shard truncation");
-        let source = disks[2]
-            .read_version("", bucket, object, "", &ReadOptions::default())
-            .await
-            .expect("source metadata should be readable");
-        let data_dir = source.data_dir.expect("non-inline source should have a data directory");
-        let truncated_part = temp_dirs[1]
-            .path()
-            .join(bucket)
-            .join(object)
-            .join(data_dir.to_string())
-            .join("part.1");
-        let original_len = tokio::fs::metadata(&truncated_part)
-            .await
-            .expect("target shard should exist")
-            .len();
-        assert!(original_len > 1, "test shard must be large enough to truncate");
-        let file = tokio::fs::OpenOptions::new()
-            .write(true)
-            .open(&truncated_part)
-            .await
-            .expect("target shard should be writable");
-        file.set_len(original_len - 1)
-            .await
-            .expect("target shard should be truncated");
+        .expect("write protected external fixture");
 
-        let mut reader = set
-            .get_object_reader(bucket, object, None, Default::default(), &ObjectOptions::default())
-            .await
-            .expect("GET should remain readable after a one-byte shard truncation");
-        let mut read_back = Vec::new();
-        tokio::io::copy(&mut reader, &mut read_back)
-            .await
-            .expect("GET should reconstruct the truncated shard through EC");
-        assert_eq!(read_back, expected_payload, "EC GET must preserve the object bytes");
+        let mut files = Vec::new();
+        let mut snapshots = Vec::new();
+        for (dir, disk) in dirs.iter().zip(&disks) {
+            let file = disk
+                .read_version("", bucket, object, "", &ReadOptions::default())
+                .await
+                .expect("read fixture metadata");
+            assert!(file.data.is_none(), "the remote test must open an external shard");
+            let object_dir = dir.path().join(bucket).join(object);
+            let part_path = object_dir
+                .join(file.data_dir.expect("external shard directory").to_string())
+                .join("part.1");
+            for path in [part_path, object_dir.join("xl.meta")] {
+                let bytes = tokio::fs::read(&path).await.expect("snapshot fixture before scan");
+                snapshots.push((path, bytes));
+            }
+            files.push(file);
+        }
+        let transport = Arc::new(FailedIntegrityReadTransport {
+            error: expected_error.clone(),
+            reads: std::sync::atomic::AtomicUsize::new(0),
+        });
+        let endpoint = Endpoint {
+            url: url::Url::parse("http://remote-integrity.invalid:9000/data/disk0").expect("fixture endpoint"),
+            is_local: false,
+            pool_idx: 0,
+            set_idx: 0,
+            disk_idx: 0,
+        };
+        let remote = crate::cluster::rpc::RemoteDisk::new(
+            &endpoint,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+            transport.clone(),
+        )
+        .await
+        .expect("construct remote disk with failing data transport");
+        let mut verification_disks: Vec<_> = disks.iter().cloned().map(Some).collect();
+        verification_disks[0] = Some(Arc::new(crate::disk::Disk::Remote(Box::new(remote))));
+        let mut statuses = [(0, vec![CHECK_PART_UNKNOWN; disks.len()])].into_iter().collect();
+        let result = crate::io_support::shard_integrity::verify_deep_parts(
+            &files,
+            &verification_disks,
+            &files[0],
+            bucket,
+            object,
+            &mut statuses,
+        )
+        .await;
+        assert!(transport.reads.load(std::sync::atomic::Ordering::Relaxed) > 0);
+        if matches!(expected_error, DiskError::FileCorrupt | DiskError::FileNotFound) {
+            result.expect("typed remote payload damage must identify a repairable shard");
+            let expected_status = if expected_error == DiskError::FileCorrupt {
+                CHECK_PART_FILE_CORRUPT
+            } else {
+                CHECK_PART_FILE_NOT_FOUND
+            };
+            assert_eq!(statuses[&0][0], expected_status);
+            assert!(statuses[&0][1..].iter().all(|status| *status == CHECK_PART_SUCCESS));
+            let (heal, metadata, reason) = super::should_heal_object_on_disk(&None, &[statuses[&0][0]], &files[0], &files[0]);
+            assert!(heal, "remote payload damage must schedule repair: {expected_error:?}");
+            assert!(!metadata);
+            let expected_reason = if expected_error == DiskError::FileCorrupt {
+                DiskError::FileCorrupt
+            } else {
+                DiskError::PartMissingOrCorrupt
+            };
+            assert_eq!(reason, Some(expected_reason));
+        } else {
+            let error = result.expect_err("unavailable payload storage must abort health verification");
+            if expected_error.is_internode_http_status(500) {
+                assert!(error.is_internode_http_status(500), "preserve the original transport error: {error:?}");
+            } else {
+                assert_eq!(error, expected_error, "preserve the original storage failure");
+            }
+            assert_eq!(statuses[&0][0], CHECK_PART_UNKNOWN, "failed verification cannot certify the shard");
+        }
+        for (path, before) in snapshots {
+            assert_eq!(tokio::fs::read(path).await.expect("read fixture after scan"), before);
+        }
+    }
 
-        let (result, error) = set
+    #[tokio::test]
+    async fn deep_scan_remote_payload_damage_schedules_repair() {
+        for error in [DiskError::FileCorrupt, DiskError::FileNotFound] {
+            assert_remote_integrity_read_failure(error).await;
+        }
+    }
+
+    #[tokio::test]
+    async fn deep_scan_remote_storage_failure_cannot_certify_healthy() {
+        for error in [
+            DiskError::from(rustfs_rio::new_test_internode_http_io_error(
+                rustfs_rio::InternodeHttpErrorKind::HttpStatus(reqwest::StatusCode::INTERNAL_SERVER_ERROR),
+            )),
+            DiskError::DiskNotFound,
+            DiskError::VolumeNotFound,
+            DiskError::FileAccessDenied,
+        ] {
+            assert_remote_integrity_read_failure(error).await;
+        }
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn deep_heal_legacy_verification_error_cannot_certify_healthy() {
+        use crate::object_api::ShardIntegrityWriteMode;
+
+        let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
+        let bucket = "legacy-deep-scan-verification-error";
+        let object = "object.bin";
+        for disk in &disks {
+            disk.make_volume(bucket).await.expect("create legacy fixture bucket");
+        }
+        set.put_object(
+            bucket,
+            object,
+            &mut PutObjReader::from_vec(vec![0x6d; 1024 * 1024 + 123]),
+            &ObjectOptions {
+                shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy),
+                no_lock: true,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("write external legacy fixture");
+
+        let mut files = Vec::new();
+        let mut part_paths = Vec::new();
+        let mut preserved = Vec::new();
+        for (index, (dir, disk)) in dirs.iter().zip(&disks).enumerate() {
+            let file = disk
+                .read_version("", bucket, object, "", &ReadOptions::default())
+                .await
+                .expect("read legacy fixture metadata");
+            assert!(file.data.is_none(), "legacy verification must open the physical shard");
+            assert!(file.parts.iter().all(|part| part.integrity.is_none()));
+            let object_dir = dir.path().join(bucket).join(object);
+            let part = object_dir
+                .join(file.data_dir.expect("external legacy shard directory").to_string())
+                .join("part.1");
+            let metadata = object_dir.join("xl.meta");
+            let bytes = tokio::fs::read(&metadata).await.expect("snapshot legacy metadata");
+            preserved.push((metadata, bytes));
+            if index != 0 {
+                let bytes = tokio::fs::read(&part).await.expect("snapshot healthy legacy shard");
+                preserved.push((part.clone(), bytes));
+            }
+            part_paths.push(part);
+            files.push(file);
+        }
+        let broken_part = &part_paths[0];
+        tokio::fs::remove_file(broken_part)
+            .await
+            .expect("remove fixture shard before symlink fault");
+        std::os::unix::fs::symlink("part.1", broken_part).expect("create self-referential shard symlink");
+        let open_error = tokio::fs::File::open(broken_part)
+            .await
+            .expect_err("self-referential symlink must fail to open");
+        assert_eq!(open_error.raw_os_error(), Some(libc::ELOOP));
+
+        let verification_error = disks[0]
+            .verify_file(bucket, object, &files[0])
+            .await
+            .expect_err("legacy disk verifier must reject the symlink path");
+        assert_eq!(verification_error, DiskError::InvalidPath);
+        let error = set
             .heal_object(
                 bucket,
                 object,
@@ -4446,19 +4822,229 @@ mod heal_result_report_tests {
                 },
             )
             .await
-            .expect("deep heal should finish after a one-byte H2 tail truncation");
-
-        assert!(error.is_none(), "deep heal should recover the H2 truncated shard: {error:?}");
-        assert_eq!(result.drives_healed(), Some(1));
-        assert_eq!(result.before.drives[1].state, DriveState::Corrupt.to_string());
+            .expect_err("an unverified legacy part must fail the complete heal operation");
+        assert_eq!(error, DiskError::InvalidPath, "complete heal must preserve the verifier's path error");
         assert_eq!(
-            tokio::fs::metadata(&truncated_part)
+            tokio::fs::read_link(broken_part)
                 .await
-                .expect("repaired H2 shard should exist")
-                .len(),
-            original_len,
-            "deep heal must restore the complete H2 shard length"
+                .expect("failed heal must preserve the symlink fault"),
+            std::path::Path::new("part.1")
         );
+        for (path, before) in preserved {
+            assert_eq!(tokio::fs::read(path).await.expect("read unchanged legacy fixture"), before);
+        }
+    }
+
+    async fn assert_h2_truncated_shards(
+        mode: crate::object_api::ShardIntegrityWriteMode,
+        coding_indexes: &[usize],
+        retained_len: fn(usize, usize) -> usize,
+    ) {
+        use crate::object_api::ShardIntegrityWriteMode;
+
+        let (temp_dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(16, 0, 4).await;
+        let bucket = "deep-heal-h2-truncation";
+        let object = "object.bin";
+        for disk in &disks {
+            disk.make_volume(bucket).await.expect("create bucket volume");
+        }
+        let expected_payload: Vec<_> = (0..5 * 1024 * 1024 + 123)
+            .map(|index| u8::try_from((index / 4096 + index) % 251).expect("payload byte fits"))
+            .collect();
+        set.put_object(
+            bucket,
+            object,
+            &mut PutObjReader::from_vec(expected_payload.clone()),
+            &ObjectOptions {
+                shard_integrity_write_mode: Some(mode),
+                no_lock: true,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("commit all source shards");
+
+        let mut files = Vec::new();
+        let mut paths = Vec::new();
+        let mut original_shards = Vec::new();
+        let mut original_metadata = Vec::new();
+        let mut damaged = Vec::new();
+        for (slot, disk) in disks.iter().enumerate() {
+            let source = disk
+                .read_version("", bucket, object, "", &ReadOptions::default())
+                .await
+                .expect("read source metadata");
+            assert_eq!((source.erasure.data_blocks, source.erasure.parity_blocks), (12, 4));
+            assert_eq!(source.parts[0].integrity.is_some(), mode == ShardIntegrityWriteMode::Protected);
+            let directory = temp_dirs[slot].path().join(bucket).join(object);
+            let path = directory
+                .join(source.data_dir.expect("external shard data directory").to_string())
+                .join("part.1");
+            let original = tokio::fs::read(&path).await.expect("read original shard");
+            original_metadata.push(tokio::fs::read(directory.join("xl.meta")).await.expect("read xl.meta"));
+            if coding_indexes.contains(&source.erasure.index) {
+                let erasure = crate::erasure::coding::Erasure::try_new_with_options(
+                    12,
+                    4,
+                    source.erasure.block_size,
+                    source.uses_legacy_checksum,
+                )
+                .expect("valid erasure layout");
+                let stripe_size = erasure.shard_size() + source.erasure.get_checksum_info(1).algorithm.size();
+                let keep = retained_len(original.len(), stripe_size);
+                assert!(keep < original.len(), "fixture must truncate the shard");
+                tokio::fs::OpenOptions::new()
+                    .write(true)
+                    .open(&path)
+                    .await
+                    .expect("open target shard")
+                    .set_len(u64::try_from(keep).expect("shard length fits"))
+                    .await
+                    .expect("truncate target shard");
+                damaged.push((slot, keep));
+            }
+            paths.push(path);
+            original_shards.push(original);
+            files.push(source);
+        }
+        assert_eq!(damaged.len(), coding_indexes.len());
+        if mode == ShardIntegrityWriteMode::Protected {
+            let mut statuses = [(0, vec![crate::disk::CHECK_PART_UNKNOWN; disks.len()])]
+                .into_iter()
+                .collect();
+            let online_disks: Vec<_> = disks.iter().cloned().map(Some).collect();
+            crate::io_support::shard_integrity::verify_deep_parts(
+                &files,
+                &online_disks,
+                &files[0],
+                bucket,
+                object,
+                &mut statuses,
+            )
+            .await
+            .expect("verify protected shards directly");
+            for (slot, status) in statuses[&0].iter().enumerate() {
+                let expected = if damaged.iter().any(|(index, _)| *index == slot) {
+                    crate::disk::CHECK_PART_FILE_CORRUPT
+                } else {
+                    crate::disk::CHECK_PART_SUCCESS
+                };
+                assert_eq!(*status, expected, "protected shard {slot}");
+            }
+        }
+
+        let opts = HealOpts {
+            no_lock: true,
+            scan_mode: HealScanMode::Deep,
+            ..Default::default()
+        };
+        let (dry_run, error) = set
+            .heal_object(bucket, object, "", &HealOpts { dry_run: true, ..opts })
+            .await
+            .expect("dry-run should report corruption");
+        assert!(error.is_none(), "dry-run error: {error:?}");
+        assert_eq!(dry_run.drives_healed(), Some(0));
+        for (slot, keep) in &damaged {
+            assert_eq!(dry_run.after.drives[*slot].state, DriveState::Corrupt.to_string());
+            assert_eq!(
+                tokio::fs::read(&paths[*slot]).await.expect("dry-run preserved shard"),
+                original_shards[*slot][..*keep]
+            );
+        }
+
+        let recoverable = damaged.len() <= 4;
+        if recoverable {
+            let mut reader = set
+                .get_object_reader(bucket, object, None, Default::default(), &ObjectOptions::default())
+                .await
+                .expect("degraded GET should remain readable");
+            let mut read_back = Vec::new();
+            tokio::io::copy(&mut reader, &mut read_back)
+                .await
+                .expect("read complete degraded GET");
+            assert_eq!(read_back, expected_payload);
+        }
+        let (result, error) = set
+            .heal_object(bucket, object, "", &opts)
+            .await
+            .expect("heal should report outcome");
+        if recoverable {
+            assert!(error.is_none(), "heal error: {error:?}");
+            assert_eq!(result.drives_healed(), Some(damaged.len()));
+            for (slot, _) in &damaged {
+                assert_eq!(result.before.drives[*slot].state, DriveState::Corrupt.to_string());
+            }
+        } else {
+            assert_eq!(error, Some(DiskError::ErasureReadQuorum));
+            assert_eq!(result.drives_healed(), Some(0));
+        }
+        for (slot, path) in paths.iter().enumerate() {
+            let expected = if !recoverable && let Some((_, keep)) = damaged.iter().find(|(index, _)| *index == slot) {
+                &original_shards[slot][..*keep]
+            } else {
+                original_shards[slot].as_slice()
+            };
+            assert_eq!(
+                tokio::fs::read(path).await.expect("read shard after heal"),
+                expected,
+                "physical shard {slot}"
+            );
+            if !recoverable {
+                let metadata_path = temp_dirs[slot].path().join(bucket).join(object).join("xl.meta");
+                assert_eq!(tokio::fs::read(metadata_path).await.expect("preserved metadata"), original_metadata[slot]);
+            }
+        }
+        if recoverable {
+            let mut reader = set
+                .get_object_reader(bucket, object, None, Default::default(), &ObjectOptions::default())
+                .await
+                .expect("GET after repair");
+            let mut read_back = Vec::new();
+            tokio::io::copy(&mut reader, &mut read_back)
+                .await
+                .expect("read repaired object");
+            assert_eq!(read_back, expected_payload);
+            let (healthy, error) = set.heal_object(bucket, object, "", &opts).await.expect("second deep scan");
+            assert!(error.is_none());
+            assert_eq!(healthy.drives_healed(), Some(0), "repair must converge");
+            assert_eq!(healthy.integrity_verified, mode == ShardIntegrityWriteMode::Protected);
+        }
+    }
+
+    #[tokio::test]
+    async fn deep_heal_rebuilds_h2_near_tail_truncated_part() {
+        use crate::object_api::ShardIntegrityWriteMode::{Legacy, Protected};
+        for mode in [Legacy, Protected] {
+            for coding_index in [1, 13] {
+                assert_h2_truncated_shards(mode, &[coding_index], |length, _| length - 1).await;
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn deep_heal_rebuilds_h2_half_truncated_part() {
+        use crate::object_api::ShardIntegrityWriteMode::{Legacy, Protected};
+        for mode in [Legacy, Protected] {
+            for coding_index in [1, 13] {
+                assert_h2_truncated_shards(mode, &[coding_index], |length, _| length / 2).await;
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn deep_heal_rebuilds_h2_stripe_boundary_truncated_parts_at_read_quorum() {
+        use crate::object_api::ShardIntegrityWriteMode::{Legacy, Protected};
+        for mode in [Legacy, Protected] {
+            assert_h2_truncated_shards(mode, &[1, 2, 13, 16], |_, stripe| stripe).await;
+        }
+    }
+
+    #[tokio::test]
+    async fn deep_heal_rejects_h2_truncated_parts_below_read_quorum() {
+        use crate::object_api::ShardIntegrityWriteMode::{Legacy, Protected};
+        for mode in [Legacy, Protected] {
+            assert_h2_truncated_shards(mode, &[1, 2, 3, 13, 16], |length, _| length / 2).await;
+        }
     }
 
     #[tokio::test]

crates/ecstore/src/set_disk/ops/locking.rs

@@ -341,14 +341,17 @@ impl SetDisks {
     pub async fn renew_disk(&self, ep: &Endpoint) {
         debug!("renew_disk: start {:?}", ep);
 
-        let previous_health = {
+        let previous_disk = {
             let disks = self.disks.read().await;
             disks
                 .iter()
                 .filter_map(|disk| disk.as_ref())
                 .find(|disk| disk.endpoint() == *ep)
-                .and_then(|disk| disk.local_health_tracker_epoch_for_reconnect())
+                .cloned()
         };
+        let previous_health = previous_disk
+            .as_ref()
+            .and_then(|disk| disk.local_health_tracker_epoch_for_reconnect());
 
         let (new_disk, fm) = match Self::connect_endpoint(ep, previous_health).await {
             Ok(res) => res,
@@ -406,6 +409,25 @@ impl SetDisks {
         }
 
         let _ = new_disk.set_disk_id(Some(fm.erasure.this)).await;
+
+        if !new_disk.is_local() {
+            let mut disks = self.disks.write().await;
+            let slot = &mut disks[disk_idx];
+            let unchanged = match (slot.as_ref(), previous_disk.as_ref()) {
+                (None, None) => true,
+                (Some(current), Some(previous)) => Arc::ptr_eq(current, previous),
+                _ => false,
+            };
+            // A GET probe or another reconnect may have filled this slot while
+            // format I/O was pending. Do not replace its handle or start a
+            // second health monitor for an unpublished handle.
+            if unchanged {
+                new_disk.enable_health_check();
+                *slot = Some(new_disk);
+            }
+            return;
+        }
+
         new_disk.enable_health_check();
 
         if new_disk.is_local() {

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -66,6 +66,7 @@ use crate::disk::DiskOption;
 use crate::disk::STORAGE_FORMAT_FILE;
 #[cfg(test)]
 use crate::disk::new_disk;
+use crate::erasure::coding::BitrotWriterWrapper;
 use crate::multipart_listing::paginate_multipart_listing;
 #[cfg(test)]
 use crate::object_api::ObjectLockConfigSnapshot;
@@ -77,7 +78,7 @@ use crate::storage_api_contracts::multipart::MultipartOperations;
 #[cfg(test)]
 use crate::storage_api_contracts::object::HTTPPreconditions;
 use crate::storage_api_contracts::object::ObjectOperations;
-use futures::{StreamExt, stream};
+use futures::{StreamExt, future::join_all, stream};
 #[cfg(test)]
 use http::HeaderMap;
 use rustfs_filemeta::metadata_keys;
@@ -100,6 +101,49 @@ use std::time::Duration;
 use tokio::io::AsyncReadExt;
 use tokio::task::JoinSet;
 
+// The erasure-set width bounds fan-out. Await every opener so errors retain
+// their disk slots and every successful writer remains owned until quorum is checked.
+async fn create_part_writers(
+    disks: &[Option<DiskStore>],
+    path: &str,
+    length: i64,
+    shard_size: usize,
+) -> (Vec<Option<BitrotWriterWrapper>>, Vec<Option<DiskError>>) {
+    join_all(disks.iter().map(|disk| async move {
+        let Some(disk) = disk else {
+            return (None, Some(DiskError::DiskNotFound));
+        };
+        match create_bitrot_writer(
+            false,
+            Some(disk),
+            RUSTFS_META_TMP_BUCKET,
+            path,
+            length,
+            shard_size,
+            HashAlgorithm::HighwayHash256S,
+        )
+        .await
+        {
+            Ok(writer) => (Some(writer), None),
+            Err(err) => {
+                warn!(
+                    event = EVENT_SET_DISK_MULTIPART,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_SET_DISK,
+                    disk = ?disk,
+                    state = "bitrot_writer_skipped",
+                    error = ?err,
+                    "Set disk multipart bitrot writer skipped"
+                );
+                (None, Some(err))
+            }
+        }
+    }))
+    .await
+    .into_iter()
+    .unzip()
+}
+
 const MULTIPART_LIST_IO_CONCURRENCY: usize = 16;
 
 static CAPPED_MULTIPART_STAGING: OnceLock<Mutex<HashMap<String, Arc<tokio::sync::Semaphore>>>> = OnceLock::new();
@@ -1542,45 +1586,13 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
                     .map_err(Error::from)?);
             let writer_setup_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
 
-            let mut writers = Vec::with_capacity(shuffle_disks.len());
-            let mut errors = Vec::with_capacity(shuffle_disks.len());
-            for disk_op in shuffle_disks.iter() {
-                if let Some(disk) = disk_op {
-                    let writer = match create_bitrot_writer(
-                        false,
-                        Some(disk),
-                        RUSTFS_META_TMP_BUCKET,
-                        &tmp_part_path,
-                        erasure.shard_file_size(data.size()),
-                        erasure.shard_size(),
-                        HashAlgorithm::HighwayHash256S,
-                    )
-                    .await
-                    {
-                        Ok(writer) => writer,
-                        Err(err) => {
-                            warn!(
-                                event = EVENT_SET_DISK_MULTIPART,
-                                component = LOG_COMPONENT_ECSTORE,
-                                subsystem = LOG_SUBSYSTEM_SET_DISK,
-                                disk = ?disk,
-                                state = "bitrot_writer_skipped",
-                                error = ?err,
-                                "Set disk multipart bitrot writer skipped"
-                            );
-                            errors.push(Some(err));
-                            writers.push(None);
-                            continue;
-                        }
-                    };
-
-                    writers.push(Some(writer));
-                    errors.push(None);
-                } else {
-                    errors.push(Some(DiskError::DiskNotFound));
-                    writers.push(None);
-                }
-            }
+            let (mut writers, errors) = create_part_writers(
+                &shuffle_disks,
+                &tmp_part_path,
+                erasure.shard_file_size(data.size()),
+                erasure.shard_size(),
+            )
+            .await;
 
             if let Some(stage_start) = writer_setup_stage_start {
                 rustfs_io_metrics::record_put_object_stage_duration(
@@ -3594,6 +3606,82 @@ mod tests {
     use tempfile::TempDir;
     use tokio::sync::{Notify, RwLock};
 
+    #[tokio::test]
+    async fn multipart_writer_setup_opens_disks_concurrently_and_preserves_error_slots() {
+        use crate::cluster::rpc::internode_data_transport::{
+            InternodeDataTransport, InternodeDataTransportCapabilities, ReadStreamRequest, WalkDirStreamRequest,
+            WriteStreamRequest,
+        };
+        use crate::cluster::rpc::remote_disk::RemoteDisk;
+        use crate::disk::{Disk, FileReader, FileWriter};
+
+        #[derive(Debug)]
+        struct BarrierTransport {
+            barrier: Arc<tokio::sync::Barrier>,
+            fail: bool,
+        }
+        #[async_trait::async_trait]
+        impl InternodeDataTransport for BarrierTransport {
+            async fn open_read(&self, _: ReadStreamRequest) -> disk::error::Result<FileReader> {
+                unreachable!("writer setup must not read")
+            }
+            async fn open_walk_dir(&self, _: WalkDirStreamRequest) -> disk::error::Result<FileReader> {
+                unreachable!("writer setup must not list")
+            }
+            async fn open_write(&self, request: WriteStreamRequest) -> disk::error::Result<FileWriter> {
+                assert_eq!(request.volume, RUSTFS_META_TMP_BUCKET);
+                assert_eq!(request.path, "upload/part.1");
+                self.barrier.wait().await;
+                if self.fail {
+                    Err(DiskError::FileAccessDenied)
+                } else {
+                    Ok(Box::new(tokio::io::sink()))
+                }
+            }
+            fn name(&self) -> &'static str {
+                "multipart-writer-test"
+            }
+            fn capabilities(&self) -> InternodeDataTransportCapabilities {
+                InternodeDataTransportCapabilities::tcp_http()
+            }
+        }
+
+        let barrier = Arc::new(tokio::sync::Barrier::new(3));
+        let mut disks = Vec::new();
+        for i in 0..3 {
+            let endpoint = Endpoint {
+                url: url::Url::parse(&format!("http://multipart-test.invalid:9000/disk{i}")).expect("endpoint"),
+                is_local: false,
+                pool_idx: 0,
+                set_idx: 0,
+                disk_idx: i,
+            };
+            let disk = RemoteDisk::new(
+                &endpoint,
+                &DiskOption {
+                    cleanup: false,
+                    health_check: false,
+                },
+                Arc::new(BarrierTransport {
+                    barrier: Arc::clone(&barrier),
+                    fail: i == 1,
+                }),
+            )
+            .await
+            .expect("remote disk");
+            disks.push(Some(Arc::new(Disk::Remote(Box::new(disk)))));
+        }
+        disks.insert(1, None);
+        // A serial opener cannot cross the barrier. The timeout only bounds failures;
+        // the assertion depends on all three independent openers making progress.
+        let (writers, errors) =
+            tokio::time::timeout(Duration::from_secs(10), create_part_writers(&disks, "upload/part.1", 1024, 256))
+                .await
+                .expect("all disk openers must be polled concurrently");
+        assert_eq!(writers.iter().map(Option::is_some).collect::<Vec<_>>(), [true, false, false, true]);
+        assert_eq!(errors, [None, Some(DiskError::DiskNotFound), Some(DiskError::FileAccessDenied), None]);
+    }
+
     #[test]
     fn multipart_bucket_incarnation_metadata_is_consistent_and_non_nil() {
         let incarnation = Uuid::new_v4();

crates/ecstore/src/set_disk/ops/object.rs

@@ -2776,6 +2776,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
                     self.set_index,
                     self.pool_index,
                     opts.skip_verify_bitrot,
+                    opts.suppress_read_repair,
                     true,
                     true,
                     GET_OBJECT_PATH_LEGACY_DUPLEX,
@@ -2805,6 +2806,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
                         self.set_index,
                         self.pool_index,
                         opts.skip_verify_bitrot,
+                        opts.suppress_read_repair,
                         true,
                         false,
                         GET_OBJECT_PATH_LEGACY_DUPLEX,
@@ -2893,6 +2895,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
                 self.set_index,
                 self.pool_index,
                 opts.skip_verify_bitrot,
+                opts.suppress_read_repair,
                 true,
                 false,
                 GET_OBJECT_PATH_DIRECT_MEMORY,
@@ -2949,6 +2952,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
                 self.set_index,
                 self.pool_index,
                 opts.skip_verify_bitrot,
+                opts.suppress_read_repair,
                 object_class.as_str(),
                 size_bucket,
                 false,
@@ -2995,6 +2999,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
                     self.set_index,
                     self.pool_index,
                     opts.skip_verify_bitrot,
+                    opts.suppress_read_repair,
                     object_class.as_str(),
                     size_bucket,
                     codec_streaming_gate.prefer_data_blocks_first_reader_setup,
@@ -3063,6 +3068,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
         let set_index = self.set_index;
         let pool_index = self.pool_index;
         let skip_verify = opts.skip_verify_bitrot;
+        let suppress_read_repair = opts.suppress_read_repair;
         // The producer runs in a separate Tokio task, so carry the caller's
         // read policy across the task boundary explicitly. Tokio task-local
         // values are not inherited by spawned tasks.
@@ -3093,6 +3099,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
                         set_index,
                         pool_index,
                         skip_verify,
+                        suppress_read_repair,
                         false,
                         false,
                         GET_OBJECT_PATH_LEGACY_DUPLEX,
@@ -9493,6 +9500,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         let set_index = self.set_index;
         let pool_index = self.pool_index;
         let skip_verify = opts.skip_verify_bitrot;
+        let suppress_read_repair = opts.suppress_read_repair;
         let metrics_size_bucket = rustfs_io_metrics::get_object_size_bucket(cloned_fi.size);
         let erasure_cache = Arc::clone(&self.erasure_cache);
         let producer = async move {
@@ -9510,6 +9518,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 set_index,
                 pool_index,
                 skip_verify,
+                suppress_read_repair,
                 false,
                 false,
                 GET_OBJECT_PATH_LEGACY_DUPLEX,

crates/ecstore/src/set_disk/shard_source.rs

@@ -18,6 +18,7 @@ use crate::diagnostics::get::{
 use crate::disk::error::Error;
 use crate::layout::disks_layout::MAX_ERASURE_SET_DRIVE_COUNT;
 use smallvec::SmallVec;
+use std::sync::atomic::{AtomicU8, Ordering};
 
 /// Generic codec callers may exceed the production set limit; `SmallVec` then
 /// spills without changing slot semantics.
@@ -25,6 +26,35 @@ pub(crate) const INLINE_SHARD_SLOTS: usize = MAX_ERASURE_SET_DRIVE_COUNT;
 pub(crate) type ShardBuffers = SmallVec<[Option<Vec<u8>>; INLINE_SHARD_SLOTS]>;
 pub(crate) type ShardErrors = SmallVec<[Option<Error>; INLINE_SHARD_SLOTS]>;
 
+/// Repair evidence is independent of reconstruction quorum. Successful shards
+/// must not vote away a minority's observed corruption. Streaming decoders
+/// share this small accumulator with the reader that completes the response.
+#[derive(Debug, Default)]
+pub(crate) struct ShardReadRepair {
+    observed: AtomicU8,
+}
+
+impl ShardReadRepair {
+    pub(crate) fn observe(&self, errors: &[Option<Error>]) {
+        let flags = errors.iter().flatten().fold(0, |flags, error| {
+            flags
+                | match error {
+                    Error::FileCorrupt => 1,
+                    Error::FileNotFound => 2,
+                    _ => 0,
+                }
+        });
+        if flags != 0 {
+            self.observed.fetch_or(flags, Ordering::Relaxed);
+        }
+    }
+
+    pub(crate) fn needed(&self, allow_missing: bool) -> bool {
+        let mask = if allow_missing { 3 } else { 1 };
+        self.observed.load(Ordering::Relaxed) & mask != 0
+    }
+}
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub(crate) enum ShardReadCost {
     Local,
@@ -164,6 +194,19 @@ mod tests {
     use super::*;
     use std::mem::size_of;
 
+    #[test]
+    fn repair_evidence_survives_healthy_stripes_and_excludes_unproven_failures() {
+        let evidence = ShardReadRepair::default();
+        evidence.observe(&[None, Some(Error::Timeout), Some(Error::DiskNotFound)]);
+        assert!(!evidence.needed(true));
+        evidence.observe(&[None, None, Some(Error::FileNotFound)]);
+        assert!(!evidence.needed(false), "unattempted reader slots do not prove a missing shard");
+        assert!(evidence.needed(true));
+        evidence.observe(&[None, None, Some(Error::FileCorrupt)]);
+        evidence.observe(&[None, None, None]);
+        assert!(evidence.needed(false), "a later healthy stripe must not erase observed corruption");
+    }
+
     #[test]
     fn stripe_scratch_capacity_matches_the_production_set_limit() {
         type OversizedShardBuffers = SmallVec<[Option<Vec<u8>>; 32]>;

crates/ecstore/src/store/heal.rs

@@ -58,6 +58,26 @@ pub(crate) fn bucket_heal_scope(bucket: &str) -> Option<Arc<BucketHealScope>> {
         .flatten()
 }
 
+/// Resolve only the two canonical bucket records; other internal paths carry no bucket authority.
+pub(super) fn bucket_metadata_owner(object: &str) -> Option<&str> {
+    use crate::bucket::metadata::{BUCKET_INCARNATION_FILE, BUCKET_METADATA_FILE};
+    let (prefix, rest) = object.split_once('/')?;
+    let (bucket, file) = rest.split_once('/')?;
+    (prefix == disk::BUCKET_META_PREFIX
+        && matches!(file, BUCKET_METADATA_FILE | BUCKET_INCARNATION_FILE)
+        && check_valid_bucket_name_strict(bucket).is_ok())
+    .then_some(bucket)
+}
+
+pub(crate) fn bucket_heal_scope_for_object(bucket: &str, object: &str) -> Option<Arc<BucketHealScope>> {
+    let owner = if bucket == RUSTFS_META_BUCKET {
+        bucket_metadata_owner(object)?
+    } else {
+        bucket
+    };
+    bucket_heal_scope(owner)
+}
+
 /// Storage-owned proof for the exact version and every selected erasure location.
 /// This is an in-process result, never reconstructed from admin drive telemetry.
 #[derive(Debug)]
@@ -422,6 +442,97 @@ impl ECStore {
         Ok(Arc::ptr_eq(&target_set, &pool.get_disks_by_key(POOL_META_NAME)))
     }
 
+    /// Restore the bucket configuration and incarnation before retiring a replacement marker.
+    pub async fn heal_replacement_bucket_metadata(
+        self: &Arc<Self>,
+        bucket: &str,
+        opts: &HealOpts,
+        targets: &[String],
+    ) -> Result<()> {
+        use crate::bucket::metadata::{BUCKET_INCARNATION_FILE, BUCKET_METADATA_FILE};
+        use crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in;
+
+        let (Some(pool_index), Some(set_index)) = (opts.pool, opts.set) else {
+            return Err(Error::PreconditionFailed);
+        };
+        if targets.is_empty() || opts.dry_run || opts.no_lock || check_valid_bucket_name_strict(bucket).is_err() {
+            return Err(Error::PreconditionFailed);
+        }
+        let pool = self
+            .pools
+            .get(pool_index)
+            .ok_or_else(|| invalid_heal_pool_index(pool_index, self.pools.len()))?;
+        let target_set = pool.get_disks_for_heal_object(BUCKET_METADATA_FILE, opts)?;
+        if targets.iter().any(|target| {
+            target_set
+                .set_endpoints
+                .iter()
+                .filter(|endpoint| endpoint.to_string() == *target)
+                .count()
+                != 1
+        }) {
+            return Err(Error::PreconditionFailed);
+        }
+        let objects: Vec<_> = [BUCKET_METADATA_FILE, BUCKET_INCARNATION_FILE]
+            .into_iter()
+            .map(|file| format!("{}/{bucket}/{file}", disk::BUCKET_META_PREFIX))
+            .filter(|object| Arc::ptr_eq(&target_set, &pool.get_disks_by_key(object)))
+            .collect();
+        if objects.is_empty() {
+            return Ok(());
+        }
+
+        // Read the persisted identity without lazy migration: a replacement
+        // must never turn lost configuration into newly fabricated defaults.
+        let incarnation = self.bucket_incarnation_id_from_disk(bucket).await?;
+        let targets = targets.to_vec();
+        self.run_bucket_heal_at_incarnation(bucket, incarnation, opts, move |store, bucket, opts| async move {
+            // Match config writers: lifecycle, metadata transaction, object locks.
+            // Keep the transaction stable through target readback, including when
+            // the caller is cancelled and the storage owner finishes its write.
+            let transaction = acquire_bucket_metadata_transaction_read_lock_in(&store.ctx, &bucket).await?;
+            for object in objects {
+                let scope = bucket_heal_scope(&bucket).ok_or(Error::PreconditionFailed)?;
+                scope.check()?;
+                if transaction.is_lock_lost() {
+                    return Err(Error::PreconditionFailed);
+                }
+                // Config objects occupy one pool, unlike pool.bin. Require a
+                // successful all-pool lookup before accepting another pool's ownership.
+                if !store.single_pool() {
+                    let (owner, _) = store
+                        .get_pool_info_for_delete_marker(RUSTFS_META_BUCKET, &object, &ObjectOptions::default())
+                        .await?;
+                    if owner.index != pool_index {
+                        continue;
+                    }
+                }
+                let metadata_opts = HealOpts { remove: false, ..opts };
+                let (result, error) = store.heal_object(RUSTFS_META_BUCKET, &object, "", &metadata_opts).await?;
+                if let Some(error) = error {
+                    return Err(error);
+                }
+                for target in &targets {
+                    let mut drives = result.after.drives.iter().filter(|drive| &drive.endpoint == target);
+                    if !drives.next().is_some_and(|drive| drive.state == "ok") || drives.next().is_some() {
+                        return Err(Error::PreconditionFailed);
+                    }
+                }
+                if !store
+                    .replacement_targets_have_version(RUSTFS_META_BUCKET, &object, "", pool_index, set_index, &targets)
+                    .await?
+                {
+                    return Err(Error::PreconditionFailed);
+                }
+            }
+            if transaction.is_lock_lost() {
+                return Err(Error::PreconditionFailed);
+            }
+            Ok(())
+        })
+        .await
+    }
+
     /// Return every live erasure set selected by an object-heal scope.
     pub async fn heal_erasure_set_scopes(&self, opts: &HealOpts) -> Result<Vec<(usize, usize)>> {
         let pools = self.get_pools_for_heal_object(opts)?;
@@ -746,11 +857,10 @@ impl ECStore {
         opts: &HealOpts,
         allow_unversioned_absence: bool,
     ) -> Result<HealObjectStorageResult> {
-        if opts.dry_run
-            || opts.no_lock
-            || (!allow_unversioned_absence && version_id.is_empty())
-            || super::utils::is_reserved_or_invalid_bucket(bucket, false)
-        {
+        // Object heal may recreate a missing bucket volume before committing a
+        // shard. Keep the current bucket generation fenced for the whole
+        // operation, including the ordinary unversioned path.
+        if opts.dry_run || opts.no_lock || super::utils::is_reserved_or_invalid_bucket(bucket, false) {
             let (item, error) = self.handle_heal_object(bucket, object, version_id, opts).await?;
             return Ok(HealObjectStorageResult {
                 item,
@@ -1345,6 +1455,16 @@ mod tests {
                 .await
                 .is_err()
         );
+        for file in [".metadata.bin", ".bucket-incarnation"] {
+            let metadata = format!("buckets/{bucket}/{file}");
+            assert!(
+                store
+                    .rename_local_data_at_incarnation(&disk_ref, (&bucket, object), &fi, (RUSTFS_META_BUCKET, &metadata), old)
+                    .await
+                    .is_err(),
+                "stale remote heal must not replace the successor's {file}"
+            );
+        }
         assert!(store.heal_bucket_at_incarnation(&bucket, old, &opts).await.is_err());
         assert!(
             store
@@ -1743,6 +1863,194 @@ mod tests {
         (temp_dir, store, shutdown)
     }
 
+    #[test]
+    fn bucket_metadata_heal_owner_rejects_path_aliases_and_unrelated_records() {
+        for file in [".metadata.bin", ".bucket-incarnation"] {
+            assert_eq!(bucket_metadata_owner(&format!("buckets/example/{file}")), Some("example"));
+        }
+        for path in [
+            "/buckets/example/.metadata.bin",
+            "buckets/../.metadata.bin",
+            "buckets/example/../.metadata.bin",
+            "buckets/example/.metadata.bin/extra",
+            "buckets/example//.metadata.bin",
+            "buckets/example/.metadata.bin.old",
+            "config/iam/example/.metadata.bin",
+            "buckets/example/usage.json",
+        ] {
+            assert_eq!(bucket_metadata_owner(path), None, "unexpected bucket authority for {path}");
+        }
+    }
+
+    #[tokio::test]
+    async fn replacement_bucket_metadata_respects_pool_ownership_and_required_records() {
+        let (root, store, shutdown) = multi_pool_heal_store().await;
+        let bucket = "replacement-bucket-metadata";
+        for invalid_bucket in ["MixedCase", "../example", RUSTFS_META_BUCKET] {
+            assert!(matches!(
+                store
+                    .heal_replacement_bucket_metadata(
+                        invalid_bucket,
+                        &HealOpts {
+                            pool: Some(0),
+                            set: Some(0),
+                            ..Default::default()
+                        },
+                        &[root.path().join("pool0-disk0").to_string_lossy().into_owned()],
+                    )
+                    .await,
+                Err(Error::PreconditionFailed)
+            ));
+        }
+        store
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create bucket and its persisted records");
+        let mut owned = 0;
+        for pool in 0..2 {
+            let target = root.path().join(format!("pool{pool}-disk0"));
+            let records: Vec<_> = [".metadata.bin", ".bucket-incarnation"]
+                .into_iter()
+                .map(|file| {
+                    let path = target.join(RUSTFS_META_BUCKET).join("buckets").join(bucket).join(file);
+                    let exists = path.join("xl.meta").exists();
+                    if exists {
+                        std::fs::remove_dir_all(&path).expect("remove replacement metadata shard");
+                        owned += 1;
+                    }
+                    (path, exists)
+                })
+                .collect();
+            store
+                .heal_replacement_bucket_metadata(
+                    bucket,
+                    &HealOpts {
+                        pool: Some(pool),
+                        set: Some(0),
+                        recreate: true,
+                        ..Default::default()
+                    },
+                    &[target.to_string_lossy().into_owned()],
+                )
+                .await
+                .expect("repair owned records and accept authoritative ownership in the other pool");
+            for (path, existed) in records {
+                assert_eq!(path.join("xl.meta").exists(), existed, "wrong placement for {}", path.display());
+            }
+        }
+        assert_eq!(owned, 2, "each required record must have exactly one owning pool");
+        let object = format!("buckets/{bucket}/.metadata.bin");
+        let owner = store
+            .get_pool_idx_existing_with_opts(RUSTFS_META_BUCKET, &object, &ObjectOptions::default())
+            .await
+            .expect("persisted metadata owner");
+        let disk = store.pools[owner].disk_set[0].disks.read().await[0]
+            .clone()
+            .expect("metadata target disk");
+        let fi = disk
+            .read_version(
+                "",
+                RUSTFS_META_BUCKET,
+                &object,
+                "",
+                &disk::ReadOptions {
+                    read_data: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("inline metadata to repair through target-side admission");
+        assert!(fi.data.is_some(), "fixture must contain an inline metadata shard");
+        let target_path = root.path().join(format!("pool{owner}-disk0"));
+        std::fs::remove_dir_all(target_path.join(RUSTFS_META_BUCKET).join(&object)).expect("remove remote target shard");
+        let incarnation = store
+            .bucket_incarnation_id_from_disk(bucket)
+            .await
+            .expect("current incarnation");
+        store
+            .rename_local_data_at_incarnation(
+                &disk.endpoint().to_string(),
+                (disk::RUSTFS_META_TMP_BUCKET, "metadata-heal-control"),
+                &fi,
+                (RUSTFS_META_BUCKET, &object),
+                incarnation,
+            )
+            .await
+            .expect("the target-side RPC admission must accept current bucket metadata repair");
+        assert!(target_path.join(RUSTFS_META_BUCKET).join(&object).join("xl.meta").exists());
+        let (configuration, _) = read_config_no_lock_preserve_empty_with_metadata(store.clone(), &object)
+            .await
+            .expect("read the current bucket configuration");
+        save_config(store.clone(), &object, configuration)
+            .await
+            .expect("publish a newer config revision in the same bucket incarnation");
+        let committed =
+            std::fs::read(target_path.join(RUSTFS_META_BUCKET).join(&object).join("xl.meta")).expect("new target configuration");
+        assert!(
+            store
+                .rename_local_data_at_incarnation(
+                    &disk.endpoint().to_string(),
+                    (disk::RUSTFS_META_TMP_BUCKET, "delayed-metadata-heal"),
+                    &fi,
+                    (RUSTFS_META_BUCKET, &object),
+                    incarnation,
+                )
+                .await
+                .is_err(),
+            "a delayed repair must not overwrite a newer config in the same incarnation"
+        );
+        assert_eq!(
+            std::fs::read(target_path.join(RUSTFS_META_BUCKET).join(&object).join("xl.meta")).expect("retained configuration"),
+            committed,
+            "rejected repair must leave the newer configuration unchanged"
+        );
+        for pool in 0..2 {
+            for disk in 0..4 {
+                let path = root
+                    .path()
+                    .join(format!("pool{pool}-disk{disk}"))
+                    .join(RUSTFS_META_BUCKET)
+                    .join("buckets")
+                    .join(bucket)
+                    .join(".metadata.bin");
+                if path.exists() {
+                    std::fs::remove_dir_all(path).expect("remove every copy of required bucket configuration");
+                }
+            }
+        }
+        metadata_sys::remove_bucket_metadata_in(&store.ctx, bucket)
+            .await
+            .expect("remove the cached configuration so repair cannot rely on it");
+        assert!(
+            store
+                .heal_replacement_bucket_metadata(
+                    bucket,
+                    &HealOpts {
+                        pool: Some(0),
+                        set: Some(0),
+                        recreate: true,
+                        ..Default::default()
+                    },
+                    &[root.path().join("pool0-disk0").to_string_lossy().into_owned()],
+                )
+                .await
+                .is_err(),
+            "total metadata loss must not be accepted as ownership in another pool"
+        );
+        for pool in 0..2 {
+            assert!(
+                !root
+                    .path()
+                    .join(format!("pool{pool}-disk0"))
+                    .join(RUSTFS_META_BUCKET)
+                    .join(&object)
+                    .exists(),
+                "missing required configuration must not be recreated with defaults"
+            );
+        }
+        shutdown.cancel();
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn absence_proof_requires_every_selected_pool() {

crates/ecstore/src/store/mod.rs

@@ -426,8 +426,8 @@ mod bucket_fence;
 pub(crate) use bucket::await_bucket_namespace_operation;
 pub use bucket_fence::BucketIncarnationFenceGuard;
 mod heal;
-pub(crate) use heal::bucket_heal_scope;
 pub use heal::{HealObjectAbsenceProof, HealObjectStorageResult};
+pub(crate) use heal::{bucket_heal_scope, bucket_heal_scope_for_object};
 mod heal_walk;
 pub use heal_walk::HealWalkVersion;
 mod init;

crates/ecstore/src/store/object.rs

@@ -9142,6 +9142,25 @@ mod tests {
             )
             .await
             .expect("create versioned bucket");
+        let mut history = Vec::new();
+        for value in 1..=3_u8 {
+            let payload = vec![value; 4097];
+            let version = store
+                .put_object(
+                    bucket,
+                    object,
+                    &mut PutObjReader::from_vec(payload.clone()),
+                    &ObjectOptions {
+                        versioned: true,
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("write the historical data versions")
+                .version_id
+                .expect("historical version must have an explicit identity");
+            history.push((version, payload));
+        }
         let marker = store
             .delete_object(
                 bucket,
@@ -9216,6 +9235,55 @@ mod tests {
                 Err(crate::disk::error::DiskError::FileVersionNotFound | crate::disk::error::DiskError::FileNotFound)
             ));
         }
+        let replay = store
+            .heal_object_at_incarnation(
+                bucket,
+                object,
+                &marker.to_string(),
+                current,
+                &rustfs_heal_contracts::heal_channel::HealOpts {
+                    remove: true,
+                    scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Deep,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("replay the exact removed marker identity");
+        assert!(replay.error.is_none(), "exact marker replay: {:?}", replay.error);
+        let absence = replay.absence.expect("exact replay must prove authoritative absence");
+        assert!(!absence.removed, "replay must not claim another physical repair");
+        assert_eq!(absence.bucket_incarnation_id, current);
+        assert_eq!(absence.version_id, marker.to_string());
+        let listed = store
+            .clone()
+            .inner_list_object_versions(bucket, "", None, None, None, 100)
+            .await
+            .expect("list surviving versions after marker cleanup");
+        assert_eq!(listed.objects.len(), history.len(), "an absent marker is not enumerable in a new walk");
+        assert!(listed.objects.iter().all(|info| !info.delete_marker));
+        for (version, expected) in history {
+            assert!(listed.objects.iter().any(|info| info.version_id == Some(version)));
+            let mut reader = store
+                .handle_get_object_reader(
+                    bucket,
+                    object,
+                    None,
+                    HeaderMap::new(),
+                    &ObjectOptions {
+                        version_id: Some(version.to_string()),
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("historical data must remain readable after cleanup and replay");
+            let mut actual = Vec::new();
+            reader
+                .stream
+                .read_to_end(&mut actual)
+                .await
+                .expect("read all historical bytes");
+            assert_eq!(actual, expected);
+        }
     }
 
     #[tokio::test]

crates/ecstore/src/store/peer.rs

@@ -161,14 +161,52 @@ impl ECStore {
                 "incarnation-bound rename requires namespace locking and a non-nil identity",
             ));
         }
+        let bucket = if destination.0 == RUSTFS_META_BUCKET {
+            super::heal::bucket_metadata_owner(destination.1).ok_or(DiskError::FileAccessDenied)?
+        } else {
+            destination.0
+        };
         let guard = Arc::new(
-            self.acquire_bucket_incarnation_fence(destination.0, expected)
+            self.acquire_bucket_incarnation_fence(bucket, expected)
                 .await
                 .map_err(|error| DiskError::other(error.to_string()))?,
         );
         if guard.is_lock_lost() {
             return Err(DiskError::other("bucket heal incarnation fence was lost"));
         }
+        let transaction = if destination.0 == RUSTFS_META_BUCKET {
+            // The receiver must retain the config fence after an RPC timeout.
+            // A delayed request must also reject a newer config in the same
+            // bucket incarnation before publishing its older repair payload.
+            let transaction = crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket)
+                .await
+                .map_err(|error| DiskError::other(error.to_string()))?;
+            let (current, _) = self
+                .get_pool_info_for_delete_marker(
+                    destination.0,
+                    destination.1,
+                    &ObjectOptions {
+                        no_lock: true,
+                        ..Default::default()
+                    },
+                )
+                .await
+                .map_err(|error| DiskError::other(error.to_string()))?;
+            let current = current.object_info;
+            if transaction.is_lock_lost()
+                || guard.is_lock_lost()
+                || fi.mod_time.is_none()
+                || current.mod_time != fi.mod_time
+                || current.data_dir != fi.data_dir
+                || current.size != fi.size
+                || current.etag != fi.get_etag()
+            {
+                return Err(DiskError::FileAccessDenied);
+            }
+            Some(transaction)
+        } else {
+            None
+        };
         rename_local_data_with_ctx(
             &self.ctx,
             disk_ref,
@@ -176,7 +214,7 @@ impl ECStore {
             fi,
             destination,
             RenameDataGuards {
-                external_guard: Some(guard),
+                external_guard: Some(Arc::new((guard, transaction))),
                 ..Default::default()
             },
         )

crates/heal/src/error.rs

@@ -143,6 +143,8 @@ impl Error {
                         err,
                         EcstoreError::DiskNotFound
                             | EcstoreError::VolumeNotFound
+                            | EcstoreError::FaultyDisk
+                            | EcstoreError::FaultyRemoteDisk
                             | EcstoreError::SlowDown
                             | EcstoreError::OperationCanceled
                             | EcstoreError::RemoteClientUnavailable(_)
@@ -348,6 +350,8 @@ mod tests {
         assert!(Error::Disk(DiskError::DiskNotFound).is_recoverable_heal());
         assert!(Error::Storage(EcstoreError::DiskNotFound).is_recoverable_heal());
         assert!(Error::Storage(EcstoreError::VolumeNotFound).is_recoverable_heal());
+        assert!(Error::Storage(EcstoreError::FaultyDisk).is_recoverable_heal());
+        assert!(Error::Storage(EcstoreError::FaultyRemoteDisk).is_recoverable_heal());
     }
 
     #[test]