fix(rpc): recover tier reloads after peer capability revocation (#7920)

The manager step failed with 'the following arguments are required:
--outcome' in the first table run (hidden by continue-on-error): every
other suite workflow passes the test step outcome through.

.agents/skills/security-advisory-lessons/references/advisory-patterns.md

@@ -48,7 +48,7 @@ Update this file only when an advisory adds or changes a reusable lesson, affect
 
 ### S3 object actions, copy, multipart, and upload policy validation
 
-- `GHSA-g8w9-qw9q-fghr`: a valid presigned `PutObject` accepted extra `x-amz-tagging`, website redirect, and storage-class headers omitted from `SignedHeaders`. Lesson: a presigned URL is a bounded capability; reject `x-amz-*` headers that are not cryptographically bound by the signature so unsigned metadata cannot change authorization, lifecycle, redirect, cost, or durability semantics.
+- `GHSA-g8w9-qw9q-fghr` and `GHSA-xm99-m3gq-83g8`: a valid presigned `PutObject` accepted extra `x-amz-*` headers omitted from `SignedHeaders`; `x-amz-copy-source` could turn that upload capability into a cross-bucket read performed as the signer. Lesson: a presigned URL is a bounded capability; inspect all received security-sensitive headers and reject unsigned ones before selecting the S3 operation or reaching storage.
 - `GHSA-3ppv-fx5m-m749`: explicit `versionId` reads and copy sources authorized `s3:GetObject` instead of `s3:GetObjectVersion`. Lesson: version-specific object access must select version-specific actions for direct reads, `CopyObject`, and `UploadPartCopy`, with tests proving the backend is not reached on denial.
 - `GHSA-x298-9x87-fvjq`: anonymous `ListObjectVersions` fell back to `ListBucket` and returned before public-access-block gates. Lesson: compatibility fallbacks must converge on the same post-authorization checks as direct grants, especially `RestrictPublicBuckets` and anonymous data-plane denies.
 - `GHSA-mx42-j6wv-px98`: `UploadPartCopy` missed source authorization and allowed cross-bucket object exfiltration. Lesson: multipart copy must enforce the same source and destination contract as `CopyObject`.
@@ -120,7 +120,7 @@ Use these targeted searches when a diff touches security-sensitive code:
 ```bash
 rg -n "validate_admin_request|check_permissions|AdminAction::|deny_only|is_allowed" rustfs crates
 rg -n "authorize_operation|FtpsDriver|SftpDriver|RETR|MKD|SIZE|MDTM|CreateBucket|GetObject|HeadObject" crates/protocols rustfs
-rg -n "UploadPartCopy|upload_part_copy|CompleteMultipart|PostObject|presign|SignedHeaders|content-length-range|starts-with" rustfs crates
+rg -n "UploadPartCopy|upload_part_copy|CompleteMultipart|PostObject|presign|SignedHeaders|x-amz-copy-source|collect_signed_headers|content-length-range|starts-with" rustfs crates
 rg -n "ListBucketVersions|GetObjectVersion|versionId|VersionId|ExistingObjectTag|ForAllValues|ForAnyValue|POLICY_PLUGIN|opa" rustfs crates
 rg -n "normalize_extract_entry_key|Snowball|auto-extract|PathBuf::join|canonicalize|\\.\\.|x-forwarded-for|x-real-ip|SourceIp" rustfs crates
 rg -n "DEFAULT_SECRET|DEFAULT_ACCESS|TEST_PRIVATE_KEY|rustfs rpc|RUSTFS_RPC_SECRET" rustfs crates
@@ -137,7 +137,7 @@ rg -n "deny_unknown_fields|serde.default|as u32|as usize|as i32" rustfs crates
 - Protocol frontend authz fixes: include denied `RETR`, `SIZE`/`MDTM`, `MKD`, bucket probe, and sibling allowed-operation cases, and assert denied paths do not reach the storage backend.
 - IAM fixes: include import/update/list service-account cases with attacker-controlled parent, claims, access key, secret key, and policy.
 - Copy/upload fixes: include cross-bucket, cross-user, source-denied, destination-denied, copy-source-condition, and multipart completion cases.
-- Presigned upload fixes: include a valid presign with extra unsigned tagging, redirect, and storage-class headers; require rejection before storage access, and verify explicitly signed equivalents still work.
+- Presigned upload fixes: include a valid presign with extra unsigned tagging, redirect, storage-class, and cross-bucket copy-source headers; require rejection before source or destination storage access, and verify explicitly signed equivalents still work.
 - Version-action fixes: include historical UUID, explicit current version, `null`, range, partNumber, presigned, STS/session, service-account, anonymous bucket-policy, copy source, and multipart-copy source cases.
 - Policy-condition fixes: include reserved-key header collisions, missing keys, partially overlapping multi-value sets, plugin mode, and built-in policy mode.
 - Path fixes: include encoded traversal, absolute path, nested traversal, archive entries with `..`, valid object keys that resemble traversal text but should be rejected, and canonical bucket/prefix boundary checks.

.config/e2e-distributed-selection.txt

@@ -1,2 +1,2 @@
-sha256-linux=775825dcb2b4997c4fa24bd9ba9c0546316c503f4d5e369c39abff0678c95e8e
-sha256-darwin=775825dcb2b4997c4fa24bd9ba9c0546316c503f4d5e369c39abff0678c95e8e
+sha256-linux=21ab66fd172018682ce825a53f26aa8f286d57f3f687d8616d1850e56474dcd8
+sha256-darwin=21ab66fd172018682ce825a53f26aa8f286d57f3f687d8616d1850e56474dcd8

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=874c881d7b45f12378a5817c7f42c95c4981960a2ec9ce12dcf4af239ae1f9d5
-sha256-linux=9351e25b45bf7dfce18b951a5e3740225f457cacc53b8bf9f500f6947763ec0e
+sha256-darwin=9e9687ded961aa4e619a15def6e1ee0a5153be05423d009ee7a58126624c83e7
+sha256-linux=364666693c2bf258997643c9d87e07e7b7501e58175f190d201d53c81946eb20

.config/e2e-repl-nightly-selection.txt

@@ -1 +1 @@
-sha256=0e338d305260229e17ccfb2adc48a6212dbdfea36a9ebfb5a4e0d38658e6cc45
+sha256=e4ee3898b840a8374cb02c64342cc15f8d7ecd1becaf4b4a2a4657290f9e7ff1

.config/e2e-smoke-selection.txt

@@ -1 +1 @@
-sha256=6d18f9cce820c51d5589de944e8cc185f73eeca0ea9a9916651943e3759169d0
+sha256=548a58d74f3cb3b5a1ab9dcd4d3f8625b9ddebe847a2e14de1d08c3b2e6292eb

.config/functional-script-revision.txt

@@ -0,0 +1 @@
+27e9584a2d776db9416d0edfd040d524d48b5d4d

.config/make/tests.mak

@@ -49,11 +49,15 @@ script-tests: ## Run shell script tests
 	./scripts/test_python_bin.sh
 	./scripts/check_embedded_secrets.sh --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/ci_gate.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain_health.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_timing_report.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
 	bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test

.config/nextest.toml

@@ -76,6 +76,12 @@ setup = 'ecstore-base-stack'
 filter = 'binary(lifecycle_integration_test) | (package(rustfs) & test(/^app::lifecycle_transition_api_test::/))'
 setup = 'lifecycle-large-stack'
 
+# The pool metadata placement regression composes the full erasure-heal
+# startup future and overflows the default Linux test-thread stack.
+[[profile.default.scripts]]
+filter = 'package(rustfs-heal) & test(=replacement_pool_metadata_follows_real_two_set_placement)'
+setup = 'ecstore-large-stack'
+
 [[profile.default.overrides]]
 filter = 'package(rustfs-ecstore) & (test(concurrent_resend_same_part_commits_one_generation) | test(concurrent_config_writes_from_separate_nodes_do_not_lose_writes) | test(/^store::bucket::tests::bucket_delete_(mark_delete|purge_removes|default_s3_delete)/))'
 test-group = 'ecstore-serial-flaky'
@@ -152,9 +158,18 @@ test-group = 'ecstore-serial-flaky'
 # deterministic commit barriers. Keep the whole init decommission family in one
 # nextest group; serial_test alone cannot isolate separate test processes.
 [[profile.default.overrides]]
-filter = 'package(rustfs-ecstore) & test(/^store::init::tests::(decommission_|suspended_.*decommission)$/)'
+filter = 'package(rustfs-ecstore) & test(/^store::init::tests::(decommission_.*|suspended_.*decommission.*)$/)'
 test-group = 'ecstore-serial-flaky'
 
+# The tier overwrite cleanup-owner matrix restarts a real 4-disk store and
+# drives free-version recovery through lifecycle expiry workers. Reserve the
+# full test budget so competing real-store fixtures cannot starve the walk's
+# filesystem progress deadlines. Keep the production deadlines and no retries.
+[[profile.default.overrides]]
+filter = 'package(rustfs-ecstore) & test(tier_overwrite_put_and_self_copy_recover_persisted_cleanup_owners)'
+test-group = 'ecstore-serial-flaky'
+threads-required = "num-test-threads"
+
 # Serialize the bucket-incarnation / lifecycle-fence tests. They drive
 # init_bucket_metadata_sys and bucket_metadata_sys_of, i.e. process-global
 # OnceLock state that serial_test's #[serial] cannot protect across nextest's
@@ -230,6 +245,12 @@ setup = 'ecstore-base-stack'
 filter = 'binary(lifecycle_integration_test) | (package(rustfs) & test(/^app::lifecycle_transition_api_test::/))'
 setup = 'lifecycle-large-stack'
 
+# The pool metadata placement regression composes the full erasure-heal
+# startup future and overflows the default Linux test-thread stack.
+[[profile.ci.scripts]]
+filter = 'package(rustfs-heal) & test(=replacement_pool_metadata_follows_real_two_set_placement)'
+setup = 'ecstore-large-stack'
+
 # ===========================================================================
 # QUARANTINE — flaky tests granted retries = 2 under the ci profile ONLY.
 #
@@ -323,9 +344,14 @@ filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_reg
 test-group = 'ecstore-serial-flaky'
 
 [[profile.ci.overrides]]
-filter = 'package(rustfs-ecstore) & test(/^store::init::tests::(decommission_|suspended_.*decommission)$/)'
+filter = 'package(rustfs-ecstore) & test(/^store::init::tests::(decommission_.*|suspended_.*decommission.*)$/)'
 test-group = 'ecstore-serial-flaky'
 
+[[profile.ci.overrides]]
+filter = 'package(rustfs-ecstore) & test(tier_overwrite_put_and_self_copy_recover_persisted_cleanup_owners)'
+test-group = 'ecstore-serial-flaky'
+threads-required = "num-test-threads"
+
 # Serialize the bucket-incarnation / lifecycle-fence tests under the ci profile
 # too (see the matching default-profile override near the top). No retries.
 [[profile.ci.overrides]]
@@ -539,6 +565,13 @@ path = "junit.xml"
 filter = 'package(e2e_test)'
 test-group = 'e2e-cluster-nightly'
 
+# The EC8+4 multi-set heal proof deliberately uploads a larger workload so the
+# background root-heal pass can be interrupted after targeting the replacement
+# drive's erasure slots. Keep the extended budget scoped to this proof case.
+[[profile.e2e-nightly.overrides]]
+filter = 'package(e2e_test) & test(=heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_across_multi_set_after_background_target_restart)'
+slow-timeout = { period = "120s", terminate-after = 12, grace-period = "10s" }
+
 # ---------------------------------------------------------------------------
 # e2e-distributed profile — 4-node 4-disk Actions suite
 # ---------------------------------------------------------------------------
@@ -638,8 +671,8 @@ path = "junit.xml"
 # Each e2e test spawns its own single-node rustfs server on a random port with
 # an isolated temp dir (crates/e2e_test/src/common.rs), so the set is
 # parallel-safe — the same property e2e-smoke relies on. The exceptions are the
-# 4-disk reliability / degraded-read fault-injection tests and the fixed-port
-# Vault tests, both serialized below.
+# 4-disk reliability / degraded-read fault-injection tests, multi-node heal
+# clusters, and the fixed-port Vault tests, all serialized below.
 [profile.e2e-full]
 default-filter = """
     package(e2e_test)
@@ -668,6 +701,12 @@ test-group = 'e2e-reliability'
 filter = 'package(e2e_test) & test(/^inline_fast_path_cluster_test::/)'
 test-group = 'e2e-inline-boundaries'
 
+# Heal interruption cases start whole multi-node clusters. Bound their overlap
+# just as the nightly lane does, without changing membership or retry policy.
+[[profile.e2e-full.overrides]]
+filter = 'package(e2e_test) & test(/^heal_erasure_disk_rebuild_test::/)'
+test-group = 'e2e-cluster-nightly'
+
 [[profile.e2e-full.overrides]]
 filter = 'package(e2e_test) & (test(/^kms::kms_vault_test::/) | test(/^kms::kms_rekey_sweep_test::/) | test(/^kms::configured_roundtrip_test::test_configured_vault_kms_admin_and_versioned_cleanup$/))'
 test-group = 'e2e-vault'

.config/security-smoke-floor.txt

@@ -9,4 +9,4 @@
 # if the selected count drops below this number, so a rename or removal that
 # thins the security smoke gate must update this file in the same PR.
 # Adding tests does not require a bump, but bumping keeps the guard tight.
-26
+28

.github/actions/schedule-failure-issue/README.md

@@ -5,20 +5,38 @@ This is the single alerting mechanism for all timed pipelines
 (rustfs/backlog#1149 ci-8, arbitration G3): scheduled workflows must consume
 this action instead of inventing their own notification paths.
 
+Issues are filed in **rustfs/backlog** (override with `target-repository`),
+not the repository the workflow runs in, so scheduled-failure noise stays
+out of the code repository's issue tracker.
+
 ## Behavior
 
 - Issue title: `[scheduled-failure] <workflow name>` — the workflow name is
   the dedupe key.
-- If an **open** issue with that exact title exists, the failure is appended
-  as a comment; otherwise a new issue is created (labeled `infrastructure` by
-  default). Closing the issue resets the cycle: the next failure opens a
-  fresh one.
-- The issue body / comment includes the run URL, run attempt, event, ref and
-  the names of the failed (or timed-out/cancelled) jobs of the current run
-  attempt.
+- If an **open** issue with that exact title exists in the target
+  repository, the failure is appended as a comment; otherwise a new issue is
+  created (labeled `infrastructure` by default; the label must exist in the
+  target repository). Closing the issue resets the cycle: the next failure
+  opens a fresh one.
+- The issue body / comment includes the source repository, run URL, run
+  attempt, event, ref and the names of the failed (or timed-out/cancelled)
+  jobs of the current run attempt.
 - Requires `gh` and `jq` on the runner (both preinstalled on GitHub-hosted
   runners such as `ubuntu-latest`).
 
+## Tokens
+
+Two tokens are involved:
+
+- `github-token` authenticates issue create/comment/search against the
+  target repository. The workflow-scoped `secrets.GITHUB_TOKEN` is scoped to
+  the code repository and **cannot** open issues in rustfs/backlog, so this
+  must be a PAT or GitHub App token with `issues: write` on the target
+  repository, stored as the `BACKLOG_ISSUE_TOKEN` repository secret.
+- `source-token` (defaults to `${{ github.token }}`) reads the failed-job
+  list from the reported run, which lives in the repository running the
+  workflow.
+
 ## Usage
 
 Add a final job to the workflow. `always()` is required — without it the job
@@ -41,7 +59,7 @@ this job only; no other job may gain permissions.
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}
 ```
 
 Notes:

.github/actions/schedule-failure-issue/action.yml

@@ -16,18 +16,31 @@ name: "Schedule Failure Issue"
 description: >-
   Open (or update) a tracking issue when a scheduled workflow run fails or
   does not complete normally.
+  Issues are filed in the target repository (default rustfs/backlog), not
+  the repository the workflow runs in.
   Dedupes by workflow name: if an open issue titled
-  "[scheduled-failure] <workflow name>" already exists, the result is
+  "[scheduled-failure] <workflow name>" already exists there, the result is
   appended as a comment; otherwise a new issue is created. This is the
   single alerting mechanism for all scheduled pipelines (backlog#1149 ci-8).
 
 inputs:
   github-token:
     description: >-
-      Token with issues:write on the repository. Pass secrets.GITHUB_TOKEN
-      from a job that declares `permissions: issues: write` (scope the
-      permission to the alert job only, never workflow-wide).
+      Token with issues:write on the target repository. The workflow-scoped
+      secrets.GITHUB_TOKEN is scoped to the code repository and cannot open
+      issues in rustfs/backlog, so pass a PAT / GitHub App token stored as a
+      repository secret (BACKLOG_ISSUE_TOKEN).
     required: true
+  target-repository:
+    description: "Repository the failure-tracking issue is filed in."
+    required: false
+    default: rustfs/backlog
+  source-token:
+    description: >-
+      Token used to read the failed-job list from the reported run. Defaults
+      to the workflow's own token, which can read runs of its own repository.
+    required: false
+    default: ${{ github.token }}
   workflow-name:
     description: "Workflow name used for the issue title (the dedupe key)."
     required: false
@@ -35,8 +48,8 @@ inputs:
   label:
     description: >-
       Label applied when a new issue is created. Must already exist in the
-      repository; if applying it fails, the issue is created without a label.
-      Set to an empty string to skip labeling.
+      target repository; if applying it fails, the issue is created without a
+      label. Set to an empty string to skip labeling.
     required: false
     default: "infrastructure"
   source-run-id:
@@ -71,6 +84,8 @@ runs:
       shell: bash
       env:
         GH_TOKEN: ${{ inputs.github-token }}
+        SOURCE_TOKEN: ${{ inputs.source-token }}
+        TARGET_REPOSITORY: ${{ inputs.target-repository }}
         WORKFLOW_NAME: ${{ inputs.workflow-name }}
         ISSUE_LABEL: ${{ inputs.label }}
         SOURCE_RUN_ID: ${{ inputs.source-run-id }}
@@ -86,8 +101,10 @@ runs:
         run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}"
 
         # Inspect the reported run attempt. It can be the current in-workflow
-        # failure or a completed run observed by the external watchdog.
-        failed_jobs="$(gh api \
+        # failure or a completed run observed by the external watchdog. The
+        # run lives in the repository running the workflow, so authenticate
+        # with the source token, not the target-repository token.
+        failed_jobs="$(GH_TOKEN="${SOURCE_TOKEN}" gh api \
           "repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}/attempts/${SOURCE_RUN_ATTEMPT}/jobs" \
           --paginate \
           --jq '.jobs[]
@@ -109,6 +126,7 @@ runs:
         body="$(cat <<EOF
         Run of **${WORKFLOW_NAME}** did not complete successfully.
 
+        - Repository: ${GITHUB_REPOSITORY}
         - Run: ${run_url} (attempt ${SOURCE_RUN_ATTEMPT})
         - Event: \`${SOURCE_EVENT}\`
         - Ref: \`${SOURCE_REF_NAME}\` @ \`${SOURCE_SHA}\`
@@ -120,26 +138,26 @@ runs:
         EOF
         )"
 
-        # Dedupe by exact title among OPEN issues (a closed issue means the
-        # earlier breakage was resolved; a new failure gets a fresh issue).
-        # GitHub search strips the [] characters, so search loosely and match
-        # the exact title with jq.
-        existing="$(gh issue list --repo "${GITHUB_REPOSITORY}" --state open --limit 100 \
+        # Dedupe by exact title among OPEN issues in the target repository (a
+        # closed issue means the earlier breakage was resolved; a new failure
+        # gets a fresh issue). GitHub search strips the [] characters, so
+        # search loosely and match the exact title with jq.
+        existing="$(gh issue list --repo "${TARGET_REPOSITORY}" --state open --limit 100 \
           --search "\"scheduled-failure\" in:title" --json number,title \
           | jq -r --arg title "${title}" 'map(select(.title == $title)) | (.[0].number // empty)')"
 
         if [ -n "${existing}" ]; then
-          echo "Appending comment to existing open issue #${existing}"
-          gh issue comment "${existing}" --repo "${GITHUB_REPOSITORY}" --body "${body}"
+          echo "Appending comment to existing open issue rustfs/backlog#${existing}"
+          gh issue comment "${existing}" --repo "${TARGET_REPOSITORY}" --body "${body}"
           exit 0
         fi
 
-        echo "Creating new issue: ${title}"
+        echo "Creating new issue in ${TARGET_REPOSITORY}: ${title}"
         if [ -n "${ISSUE_LABEL}" ]; then
-          if gh issue create --repo "${GITHUB_REPOSITORY}" \
+          if gh issue create --repo "${TARGET_REPOSITORY}" \
                --title "${title}" --body "${body}" --label "${ISSUE_LABEL}"; then
             exit 0
           fi
           echo "::warning::issue creation with label '${ISSUE_LABEL}' failed (missing label?); retrying without a label"
         fi
-        gh issue create --repo "${GITHUB_REPOSITORY}" --title "${title}" --body "${body}"
+        gh issue create --repo "${TARGET_REPOSITORY}" --title "${title}" --body "${body}"

.github/workflows/audit.yml

@@ -207,4 +207,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/build.yml

@@ -207,12 +207,12 @@ jobs:
           fi
 
           all='{"include":[
-            {"target_id":"linux-x86_64-musl","os":"sm-standard-2","target":"x86_64-unknown-linux-musl","cross":false,"platform":"linux","rustflags":""},
-            {"target_id":"linux-aarch64-musl","os":"sm-standard-2","target":"aarch64-unknown-linux-musl","cross":true,"platform":"linux","rustflags":""},
-            {"target_id":"linux-x86_64-gnu","os":"sm-standard-2","target":"x86_64-unknown-linux-gnu","cross":false,"platform":"linux","rustflags":""},
-            {"target_id":"linux-aarch64-gnu","os":"sm-standard-2","target":"aarch64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":""},
-            {"target_id":"macos-aarch64","os":"macos-latest","target":"aarch64-apple-darwin","cross":false,"platform":"macos","rustflags":""},
-            {"target_id":"windows-x86_64","os":"windows-latest","target":"x86_64-pc-windows-msvc","cross":false,"platform":"windows","rustflags":""}
+            {"target_id":"linux-x86_64-musl","os":"sm-standard-2","target":"x86_64-unknown-linux-musl","cross":false,"platform":"linux","rustflags":"","features":""},
+            {"target_id":"linux-aarch64-musl","os":"sm-standard-2","target":"aarch64-unknown-linux-musl","cross":true,"platform":"linux","rustflags":"","features":""},
+            {"target_id":"linux-x86_64-gnu","os":"sm-standard-2","target":"x86_64-unknown-linux-gnu","cross":false,"platform":"linux","rustflags":"","features":"pyroscope"},
+            {"target_id":"linux-aarch64-gnu","os":"sm-standard-2","target":"aarch64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":"","features":"pyroscope"},
+            {"target_id":"macos-aarch64","os":"macos-latest","target":"aarch64-apple-darwin","cross":false,"platform":"macos","rustflags":"","features":"pyroscope"},
+            {"target_id":"windows-x86_64","os":"windows-latest","target":"x86_64-pc-windows-msvc","cross":false,"platform":"windows","rustflags":"","features":""}
           ]}'
 
           if [[ "${selected}" == "all" ]]; then
@@ -406,12 +406,25 @@ jobs:
           # Force rebuild by touching build.rs
           touch rustfs/build.rs
 
-          if [[ "${{ matrix.cross }}" == "true" ]]; then
-            # All cross targets in the matrix are Linux; zigbuild handles them.
-            cargo zigbuild --release --target ${{ matrix.target }} -p rustfs --bin rustfs
-          else
-            cargo build --release --target ${{ matrix.target }} -p rustfs --bin rustfs
+          BINARY_NAMES=(rustfs)
+          if [[ "${{ matrix.platform }}" == "linux" ]]; then
+            BINARY_NAMES+=(rustfs-cli)
           fi
+          FEATURE_ARGS=()
+          if [[ -n "${{ matrix.features }}" ]]; then
+            FEATURE_ARGS=(--features "${{ matrix.features }}")
+          fi
+
+          # Build one binary at a time so release LTO links cannot exhaust a
+          # self-hosted runner by running concurrently.
+          for binary in "${BINARY_NAMES[@]}"; do
+            if [[ "${{ matrix.cross }}" == "true" ]]; then
+              # All cross targets in the matrix are Linux; zigbuild handles them.
+              cargo zigbuild --release --target ${{ matrix.target }} "${FEATURE_ARGS[@]}" -p rustfs --bin "$binary"
+            else
+              cargo build --release --target ${{ matrix.target }} "${FEATURE_ARGS[@]}" -p rustfs --bin "$binary"
+            fi
+          done
 
       - name: Create release package
         id: package
@@ -498,17 +511,24 @@ jobs:
             BINARY_NAME="rustfs"
           fi
 
-          # Verify the binary exists before packaging
-          if [[ ! -f "$BINARY_NAME" ]]; then
-            echo "❌ Binary $BINARY_NAME not found in $(pwd)"
-            if [[ "${{ matrix.platform }}" == "windows" ]]; then
-              dir
-            else
-              ls -la
-            fi
-            exit 1
+          BINARY_FILES=("$BINARY_NAME")
+          if [[ "${{ matrix.platform }}" == "linux" ]]; then
+            BINARY_FILES+=("rustfs-cli")
           fi
 
+          # Verify the binaries exist before packaging
+          for binary in "${BINARY_FILES[@]}"; do
+            if [[ ! -f "$binary" ]]; then
+              echo "❌ Binary $binary not found in $(pwd)"
+              if [[ "${{ matrix.platform }}" == "windows" ]]; then
+                dir
+              else
+                ls -la
+              fi
+              exit 1
+            fi
+          done
+
           # Universal packaging function
           package_zip() {
             local src=$1
@@ -526,8 +546,12 @@ jobs:
           }
 
           # Create the zip package
-          echo "Start packaging: $BINARY_NAME -> ../../../${PACKAGE_NAME}.zip"
-          package_zip "$BINARY_NAME" "../../../${PACKAGE_NAME}.zip"
+          echo "Start packaging: ${BINARY_FILES[*]} -> ../../../${PACKAGE_NAME}.zip"
+          if [[ "${{ matrix.platform }}" == "linux" ]]; then
+            zip "../../../${PACKAGE_NAME}.zip" "${BINARY_FILES[@]}"
+          else
+            package_zip "$BINARY_NAME" "../../../${PACKAGE_NAME}.zip"
+          fi
 
           cd ../../..
 
@@ -601,6 +625,28 @@ jobs:
           echo "🔧 Build type: ${BUILD_TYPE}"
           echo "📊 Version: ${VERSION}"
 
+      - name: Verify packaged Linux CLI
+        if: matrix.platform == 'linux'
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          package_file="${{ steps.package.outputs.package_file }}"
+          archive_members="$(unzip -Z1 "$package_file" | sort)"
+          expected_members=$'rustfs\nrustfs-cli'
+          if [[ "$archive_members" != "$expected_members" ]]; then
+            echo "Linux package must contain exactly rustfs and rustfs-cli" >&2
+            printf '%s\n' "$archive_members" >&2
+            exit 1
+          fi
+
+          if [[ "${{ matrix.target }}" == "x86_64-unknown-linux-gnu" ]]; then
+            cli_dir="$(mktemp -d)"
+            trap 'rm -rf "$cli_dir"' EXIT
+            unzip -q "$package_file" rustfs-cli -d "$cli_dir"
+            "$cli_dir/rustfs-cli" --help >/dev/null
+          fi
+
       - name: Verify packaged console
         if: matrix.target == 'x86_64-unknown-linux-gnu'
         shell: bash
@@ -1100,4 +1146,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/ci.yml

@@ -293,10 +293,10 @@ jobs:
       - name: Run rebalance/decommission migration proofs
         run: ./scripts/check_migration_gate_count.sh
 
-  # The root boundary requires fresh CLI and integration-test builds. Give it
-  # its own time budget instead of sharing the workspace lint/test budget.
-  offline-enrollment-root-boundary:
-    name: Offline Enrollment Root Boundary
+  # This gate builds into a fresh target directory to isolate the E2E root.
+  # Give its cold build a separate budget from workspace tests and migration proofs.
+  offline-enrollment-e2e:
+    name: Offline Enrollment E2E Root Boundary
     if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
     needs: [ quick-checks, classify-changes ]
     runs-on: sm-standard-4
@@ -316,6 +316,7 @@ jobs:
           cache-shared-key: ci-dev
           cache-save-if: 'false'
           install-build-packaging-tools: 'false'
+          install-test-tools: 'false'
 
       - name: Protect Connect test home
         run: chmod go-w "$(realpath "$HOME")"
@@ -362,11 +363,7 @@ jobs:
       # re-enabled by backlog#1304 (restore accepts serialize on a short CAS
       # guard; the copy-back no longer holds the #4877 whole-copy-back lock,
       # so the mid-restore ongoing read and fast 409 rejection it asserts are
-      # the implemented contract). The remaining exclusions each hit a
-      # DIFFERENT, independent issue (all tracked under rustfs/backlog#1148;
-      # they keep #[ignore] with a backlog reference):
-      #   - test_noncurrent_{expiry,transition}_still_works_after_immediate_compensation_transition:
-      #     noncurrent transition/expiry after an immediate compensation transition.
+      # the implemented contract).
       - name: Run ignored ILM integration tests serially
         env:
           # Match the measured Test and Lint link budget. The default exposed
@@ -379,7 +376,7 @@ jobs:
           NEXTEST_HIDE_PROGRESS_BAR=1 timeout --verbose --signal=TERM --kill-after=30s 80m \
             cargo nextest run -j1 --run-ignored ignored-only \
             -p rustfs-scanner -p rustfs \
-            -E '(binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))) and not (test(test_noncurrent_expiry_still_works_after_immediate_compensation_transition) or test(test_noncurrent_transition_still_works_after_immediate_compensation_transition))' \
+            -E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))' \
             --status-level all --final-status-level all \
             2>&1 | tee artifacts/ilm-integration/nextest.log
           status=${PIPESTATUS[0]}
@@ -595,7 +592,7 @@ jobs:
                       digest.update(chunk)
               return digest.hexdigest()
 
-          argv = ["cargo", "build", "-p", "rustfs", "--bins", "--features", "e2e-test-hooks"]
+          argv = ["python3", "scripts/e2e_binary.py", "build", "--bins", "--features", "e2e-test-hooks"]
           commit, tree = git("rev-parse", "HEAD"), git("rev-parse", "HEAD^{tree}")
           clean_before = not git("status", "--porcelain", "--untracked-files=normal")
           if not clean_before:
@@ -629,6 +626,7 @@ jobs:
           name: rustfs-debug-binary
           path: |
             target/debug/rustfs
+            target/debug/rustfs.e2e.json
             target/debug/rustfs.e2e-startup-cas-build.json
           if-no-files-found: error
           retention-days: 1
@@ -661,13 +659,15 @@ jobs:
           install-build-packaging-tools: 'false'
 
       - name: Build debug binary with rio-v2
-        run: cargo build -p rustfs --bins --features rio-v2,e2e-test-hooks
+        run: python3 scripts/e2e_binary.py build --bins --features rio-v2,e2e-test-hooks
 
       - name: Upload debug binary
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
           name: rustfs-debug-binary-rio-v2
-          path: target/debug/rustfs
+          path: |
+            target/debug/rustfs
+            target/debug/rustfs.e2e.json
           if-no-files-found: error
           retention-days: 1
 
@@ -816,7 +816,7 @@ jobs:
           NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
           RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-smoke-logs
         run: |
-          cargo nextest run --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" \
+          python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" \
             --status-level all --final-status-level all --failure-output final
 
       - name: Upload e2e smoke diagnostics
@@ -852,7 +852,7 @@ jobs:
           RUSTFS_TEST_PORT="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()')"
           RUSTFS_TEST_PORT="${RUSTFS_TEST_PORT}" \
           RUSTFS_TEST_LOG="${RUN_ROOT}/rustfs.log" \
-          ./scripts/e2e-run.sh ./target/debug/rustfs "${RUN_ROOT}/data"
+          python3 scripts/e2e_binary.py run --features e2e-test-hooks -- ./scripts/e2e-run.sh ./target/debug/rustfs "${RUN_ROOT}/data"
 
       - name: Upload test logs
         if: failure()
@@ -981,6 +981,7 @@ jobs:
           if manifest["binary_sha256"] != digest.hexdigest() or manifest["commit"] != commit:
               raise SystemExit("downloaded hooks binary identity mismatch")
           shutil.copy2(manifest_path, target / manifest_path.name)
+          shutil.copy2(source.with_name("rustfs.e2e.json"), target / "rustfs.e2e.json")
           binary.chmod(0o755)
           PYINPUT
 
@@ -1001,7 +1002,7 @@ jobs:
           RUSTFS_E2E_STARTUP_CAS_BINARY: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs
           RUSTFS_E2E_STARTUP_CAS_BUILD_MANIFEST: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json
           RUSTFS_E2E_STARTUP_CAS_ARTIFACT_DIR: ${{ runner.temp }}/rustfs-startup-cas-evidence
-        run: cargo nextest run --profile e2e-full -p e2e_test
+        run: python3 scripts/e2e_binary.py run --binary "$RUSTFS_E2E_STARTUP_CAS_BINARY" --features e2e-test-hooks -- cargo nextest run --profile e2e-full -p e2e_test
 
       - name: Upload junit
         if: always()
@@ -1073,7 +1074,7 @@ jobs:
       - name: Run end-to-end tests
         run: |
           s3s-e2e --version
-          ./scripts/e2e-run.sh ./target/debug/rustfs /tmp/rustfs
+          python3 scripts/e2e_binary.py run --features rio-v2,e2e-test-hooks -- ./scripts/e2e-run.sh ./target/debug/rustfs /tmp/rustfs
 
       - name: Upload test logs
         if: failure()
@@ -1116,7 +1117,7 @@ jobs:
           S3_PORT="${S3_PORT}" \
           DATA_ROOT="${RUN_ROOT}" \
           S3TESTS_CONF=artifacts/s3tests-single/s3tests.conf \
-          ./scripts/s3-tests/run.sh
+          python3 scripts/e2e_binary.py run --features e2e-test-hooks -- ./scripts/s3-tests/run.sh
 
       - name: Upload s3 test artifacts
         if: always()
@@ -1198,7 +1199,7 @@ jobs:
           S3_PORT="${S3_PORT}" \
           DATA_ROOT="${RUN_ROOT}" \
           S3TESTS_CONF=artifacts/s3tests-single/s3tests.conf \
-          ./scripts/s3-tests/run.sh
+          python3 scripts/e2e_binary.py run --features e2e-test-hooks -- ./scripts/s3-tests/run.sh
 
       - name: Upload s3 test artifacts
         if: always()
@@ -1217,7 +1218,7 @@ jobs:
       - typos
       - quick-checks
       - test-and-lint
-      - offline-enrollment-root-boundary
+      - offline-enrollment-e2e
       - test-ilm-integration-serial
       - test-and-lint-rio-v2
       - connect-short-credential-boundary
@@ -1251,7 +1252,7 @@ jobs:
       - typos
       - quick-checks
       - test-and-lint
-      - offline-enrollment-root-boundary
+      - offline-enrollment-e2e
       - test-ilm-integration-serial
       - test-and-lint-rio-v2
       - test-and-lint-protocols
@@ -1278,4 +1279,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/connect-performance-drive-acceptance.yml

@@ -0,0 +1,192 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Connect performance.drive service-job acceptance
+
+on:
+  workflow_dispatch:
+    inputs:
+      build_run_id:
+        description: Main-branch Build and Release workflow run ID
+        required: true
+        type: string
+      artifact_id:
+        description: Linux x86_64 GNU artifact ID from that run
+        required: true
+        type: string
+      source_sha:
+        description: Exact 40-character source commit
+        required: true
+        type: string
+      artifact_digest:
+        description: GitHub artifact digest including sha256 prefix
+        required: true
+        type: string
+      binary_sha256:
+        description: Expected rustfs binary SHA-256
+        required: true
+        type: string
+      connect_sha:
+        description: Exact 40-character RustFS Connect harness commit
+        required: true
+        type: string
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  performance-drive:
+    name: Verify native Linux x86_64 performance.drive service job
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - name: Checkout acceptance harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Checkout exact RustFS source
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          path: rustfs-source
+          persist-credentials: false
+          ref: ${{ inputs.source_sha }}
+
+      - name: Checkout exact Connect harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/connect
+          path: connect-harness
+          persist-credentials: false
+          ref: ${{ inputs.connect_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 22.22.2
+          cache: npm
+          cache-dependency-path: connect-harness/web/package-lock.json
+
+      - name: Verify source run and artifact identity
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+        run: |
+          set -euo pipefail
+          [[ "$BUILD_RUN_ID" =~ ^[0-9]+$ ]]
+          [[ "$ARTIFACT_ID" =~ ^[0-9]+$ ]]
+          [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ "$ARTIFACT_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
+          [[ "$CONNECT_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ $(git -C rustfs-source rev-parse HEAD) == "$SOURCE_SHA" ]]
+          [[ $(git -C connect-harness rev-parse HEAD) == "$CONNECT_SHA" ]]
+          [[ $(git -C rustfs-source remote get-url origin) == https://github.com/rustfs/rustfs ]]
+          [[ $(git -C connect-harness remote get-url origin) == https://github.com/rustfs/connect ]]
+
+          run=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}")
+          [[ $(jq -r '.head_sha' <<<"$run") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.head_branch' <<<"$run") == main ]]
+          [[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]]
+          [[ $(jq -r '.name' <<<"$run") == "Build and Release" ]]
+
+          expected_job='Build RustFS (linux-x86_64-gnu, sm-standard-2, x86_64-unknown-linux-gnu, false, linux, pyroscope)'
+          jobs=$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
+          jq -e --arg name "$expected_job" '
+            [.[].jobs[] | select(.name == $name)] as $matches
+            | (($matches | length) == 1 and $matches[0].conclusion == "success")
+          ' <<<"$jobs" >/dev/null
+
+          artifact=$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}")
+          [[ $(jq -r '.workflow_run.id' <<<"$artifact") == "$BUILD_RUN_ID" ]]
+          [[ $(jq -r '.workflow_run.head_sha' <<<"$artifact") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.name' <<<"$artifact") == rustfs-linux-x86_64-gnu-* ]]
+          [[ $(jq -r '.digest' <<<"$artifact") == "$ARTIFACT_DIGEST" ]]
+          [[ $(jq -r '.expired' <<<"$artifact") == false ]]
+
+      - name: Download exact build artifact
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        with:
+          artifact-ids: ${{ inputs.artifact_id }}
+          path: artifact
+          run-id: ${{ inputs.build_run_id }}
+          github-token: ${{ github.token }}
+
+      - name: Extract exact RustFS binary
+        shell: bash
+        env:
+          BINARY_SHA256: ${{ inputs.binary_sha256 }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          short_sha=${SOURCE_SHA:0:7}
+          package=$(find artifact -type f -name "rustfs-linux-x86_64-gnu-dev-${short_sha}.zip" -print -quit)
+          [[ -n "$package" ]]
+          mkdir -p binary
+          unzip -qq "$package" rustfs -d binary
+          chmod +x binary/rustfs
+          printf '%s  %s\n' "$BINARY_SHA256" binary/rustfs | sha256sum --check --strict
+
+      - name: Run signed service job acceptance
+        shell: bash
+        env:
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          printf '%s\n' "$SOURCE_SHA" >connect-harness/tests/e2e/connected/rustfs-ref
+          npm --prefix connect-harness/web ci
+          connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
+          make -C connect-harness e2e-connected-dispatch-check
+          RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
+            RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
+            CONNECT_E2E_PERFORMANCE_DRIVE_EVIDENCE="$GITHUB_WORKSPACE/performance-drive-service-job-evidence.json" \
+            make -C connect-harness e2e-connected E2E_SCENARIO=performance-drive
+
+      - name: Bind workflow and artifact provenance
+        if: ${{ always() && hashFiles('performance-drive-service-job-evidence.json') != '' }}
+        shell: bash
+        env:
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          jq \
+            --arg workflowRunId "$GITHUB_RUN_ID" \
+            --arg buildRunId "$BUILD_RUN_ID" \
+            --arg artifactId "$ARTIFACT_ID" \
+            --arg artifactDigest "$ARTIFACT_DIGEST" \
+            --arg connectSha "$CONNECT_SHA" \
+            --arg sourceSha "$SOURCE_SHA" \
+            '. + {workflowRunId: $workflowRunId, buildRunId: $buildRunId, artifactId: $artifactId, artifactDigest: $artifactDigest, connectSha: $connectSha, sourceSha: $sourceSha}' \
+            performance-drive-service-job-evidence.json >performance-drive-service-job-evidence.bound.json
+          mv performance-drive-service-job-evidence.bound.json performance-drive-service-job-evidence.json
+
+      - name: Upload acceptance evidence
+        if: ${{ always() && hashFiles('performance-drive-service-job-evidence.json') != '' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: connect-performance-drive-evidence-${{ github.run_id }}
+          path: |
+            performance-drive-service-job-evidence.json
+          retention-days: 14
+          if-no-files-found: error

.github/workflows/connect-performance-network-acceptance.yml

@@ -0,0 +1,192 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Connect performance.network service-job acceptance
+
+on:
+  workflow_dispatch:
+    inputs:
+      build_run_id:
+        description: Main-branch Build and Release workflow run ID
+        required: true
+        type: string
+      artifact_id:
+        description: Linux x86_64 GNU artifact ID from that run
+        required: true
+        type: string
+      source_sha:
+        description: Exact 40-character source commit
+        required: true
+        type: string
+      artifact_digest:
+        description: GitHub artifact digest including sha256 prefix
+        required: true
+        type: string
+      binary_sha256:
+        description: Expected rustfs binary SHA-256
+        required: true
+        type: string
+      connect_sha:
+        description: Exact 40-character RustFS Connect harness commit
+        required: true
+        type: string
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  performance-network:
+    name: Verify native Linux x86_64 performance.network service job
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - name: Checkout acceptance harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Checkout exact RustFS source
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          path: rustfs-source
+          persist-credentials: false
+          ref: ${{ inputs.source_sha }}
+
+      - name: Checkout exact Connect harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/connect
+          path: connect-harness
+          persist-credentials: false
+          ref: ${{ inputs.connect_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 22.22.2
+          cache: npm
+          cache-dependency-path: connect-harness/web/package-lock.json
+
+      - name: Verify source run and artifact identity
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+        run: |
+          set -euo pipefail
+          [[ "$BUILD_RUN_ID" =~ ^[0-9]+$ ]]
+          [[ "$ARTIFACT_ID" =~ ^[0-9]+$ ]]
+          [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ "$ARTIFACT_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
+          [[ "$CONNECT_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ $(git -C rustfs-source rev-parse HEAD) == "$SOURCE_SHA" ]]
+          [[ $(git -C connect-harness rev-parse HEAD) == "$CONNECT_SHA" ]]
+          [[ $(git -C rustfs-source remote get-url origin) == https://github.com/rustfs/rustfs ]]
+          [[ $(git -C connect-harness remote get-url origin) == https://github.com/rustfs/connect ]]
+
+          run=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}")
+          [[ $(jq -r '.head_sha' <<<"$run") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.head_branch' <<<"$run") == main ]]
+          [[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]]
+          [[ $(jq -r '.name' <<<"$run") == "Build and Release" ]]
+
+          expected_job='Build RustFS (linux-x86_64-gnu, sm-standard-2, x86_64-unknown-linux-gnu, false, linux, pyroscope)'
+          jobs=$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
+          jq -e --arg name "$expected_job" '
+            [.[].jobs[] | select(.name == $name)] as $matches
+            | (($matches | length) == 1 and $matches[0].conclusion == "success")
+          ' <<<"$jobs" >/dev/null
+
+          artifact=$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}")
+          [[ $(jq -r '.workflow_run.id' <<<"$artifact") == "$BUILD_RUN_ID" ]]
+          [[ $(jq -r '.workflow_run.head_sha' <<<"$artifact") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.name' <<<"$artifact") == rustfs-linux-x86_64-gnu-* ]]
+          [[ $(jq -r '.digest' <<<"$artifact") == "$ARTIFACT_DIGEST" ]]
+          [[ $(jq -r '.expired' <<<"$artifact") == false ]]
+
+      - name: Download exact build artifact
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        with:
+          artifact-ids: ${{ inputs.artifact_id }}
+          path: artifact
+          run-id: ${{ inputs.build_run_id }}
+          github-token: ${{ github.token }}
+
+      - name: Extract exact RustFS binary
+        shell: bash
+        env:
+          BINARY_SHA256: ${{ inputs.binary_sha256 }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          short_sha=${SOURCE_SHA:0:7}
+          package=$(find artifact -type f -name "rustfs-linux-x86_64-gnu-dev-${short_sha}.zip" -print -quit)
+          [[ -n "$package" ]]
+          mkdir -p binary
+          unzip -qq "$package" rustfs -d binary
+          chmod +x binary/rustfs
+          printf '%s  %s\n' "$BINARY_SHA256" binary/rustfs | sha256sum --check --strict
+
+      - name: Run signed service job acceptance
+        shell: bash
+        env:
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          printf '%s\n' "$SOURCE_SHA" >connect-harness/tests/e2e/connected/rustfs-ref
+          npm --prefix connect-harness/web ci
+          connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
+          make -C connect-harness e2e-connected-dispatch-check
+          RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
+            RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
+            CONNECT_E2E_PERFORMANCE_NETWORK_EVIDENCE="$GITHUB_WORKSPACE/performance-network-service-job-evidence.json" \
+            make -C connect-harness e2e-connected E2E_SCENARIO=performance-network
+
+      - name: Bind workflow and artifact provenance
+        if: ${{ always() && hashFiles('performance-network-service-job-evidence.json') != '' }}
+        shell: bash
+        env:
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          jq \
+            --arg workflowRunId "$GITHUB_RUN_ID" \
+            --arg buildRunId "$BUILD_RUN_ID" \
+            --arg artifactId "$ARTIFACT_ID" \
+            --arg artifactDigest "$ARTIFACT_DIGEST" \
+            --arg connectSha "$CONNECT_SHA" \
+            --arg sourceSha "$SOURCE_SHA" \
+            '. + {workflowRunId: $workflowRunId, buildRunId: $buildRunId, artifactId: $artifactId, artifactDigest: $artifactDigest, connectSha: $connectSha, sourceSha: $sourceSha}' \
+            performance-network-service-job-evidence.json >performance-network-service-job-evidence.bound.json
+          mv performance-network-service-job-evidence.bound.json performance-network-service-job-evidence.json
+
+      - name: Upload acceptance evidence
+        if: ${{ always() && hashFiles('performance-network-service-job-evidence.json') != '' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: connect-performance-network-evidence-${{ github.run_id }}
+          path: |
+            performance-network-service-job-evidence.json
+          retention-days: 14
+          if-no-files-found: error

.github/workflows/connect-profile-cpu-acceptance.yml

@@ -0,0 +1,192 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Connect profile.cpu service-job acceptance
+
+on:
+  workflow_dispatch:
+    inputs:
+      build_run_id:
+        description: Main-branch Build and Release workflow run ID
+        required: true
+        type: string
+      artifact_id:
+        description: Linux x86_64 GNU artifact ID from that run
+        required: true
+        type: string
+      source_sha:
+        description: Exact 40-character source commit
+        required: true
+        type: string
+      artifact_digest:
+        description: GitHub artifact digest including sha256 prefix
+        required: true
+        type: string
+      binary_sha256:
+        description: Expected rustfs binary SHA-256
+        required: true
+        type: string
+      connect_sha:
+        description: Exact 40-character RustFS Connect harness commit
+        required: true
+        type: string
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  profile-cpu:
+    name: Verify native Linux x86_64 profile.cpu service job
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - name: Checkout acceptance harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Checkout exact RustFS source
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          path: rustfs-source
+          persist-credentials: false
+          ref: ${{ inputs.source_sha }}
+
+      - name: Checkout exact Connect harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/connect
+          path: connect-harness
+          persist-credentials: false
+          ref: ${{ inputs.connect_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 22.22.2
+          cache: npm
+          cache-dependency-path: connect-harness/web/package-lock.json
+
+      - name: Verify source run and artifact identity
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+        run: |
+          set -euo pipefail
+          [[ "$BUILD_RUN_ID" =~ ^[0-9]+$ ]]
+          [[ "$ARTIFACT_ID" =~ ^[0-9]+$ ]]
+          [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ "$ARTIFACT_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
+          [[ "$CONNECT_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ $(git -C rustfs-source rev-parse HEAD) == "$SOURCE_SHA" ]]
+          [[ $(git -C connect-harness rev-parse HEAD) == "$CONNECT_SHA" ]]
+          [[ $(git -C rustfs-source remote get-url origin) == https://github.com/rustfs/rustfs ]]
+          [[ $(git -C connect-harness remote get-url origin) == https://github.com/rustfs/connect ]]
+
+          run=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}")
+          [[ $(jq -r '.head_sha' <<<"$run") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.head_branch' <<<"$run") == main ]]
+          [[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]]
+          [[ $(jq -r '.name' <<<"$run") == "Build and Release" ]]
+
+          expected_job='Build RustFS (linux-x86_64-gnu, sm-standard-2, x86_64-unknown-linux-gnu, false, linux, pyroscope)'
+          jobs=$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
+          jq -e --arg name "$expected_job" '
+            [.[].jobs[] | select(.name == $name)] as $matches
+            | (($matches | length) == 1 and $matches[0].conclusion == "success")
+          ' <<<"$jobs" >/dev/null
+
+          artifact=$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}")
+          [[ $(jq -r '.workflow_run.id' <<<"$artifact") == "$BUILD_RUN_ID" ]]
+          [[ $(jq -r '.workflow_run.head_sha' <<<"$artifact") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.name' <<<"$artifact") == rustfs-linux-x86_64-gnu-* ]]
+          [[ $(jq -r '.digest' <<<"$artifact") == "$ARTIFACT_DIGEST" ]]
+          [[ $(jq -r '.expired' <<<"$artifact") == false ]]
+
+      - name: Download exact build artifact
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        with:
+          artifact-ids: ${{ inputs.artifact_id }}
+          path: artifact
+          run-id: ${{ inputs.build_run_id }}
+          github-token: ${{ github.token }}
+
+      - name: Extract exact RustFS binary
+        shell: bash
+        env:
+          BINARY_SHA256: ${{ inputs.binary_sha256 }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          short_sha=${SOURCE_SHA:0:7}
+          package=$(find artifact -type f -name "rustfs-linux-x86_64-gnu-dev-${short_sha}.zip" -print -quit)
+          [[ -n "$package" ]]
+          mkdir -p binary
+          unzip -qq "$package" rustfs -d binary
+          chmod +x binary/rustfs
+          printf '%s  %s\n' "$BINARY_SHA256" binary/rustfs | sha256sum --check --strict
+
+      - name: Run signed service job acceptance
+        shell: bash
+        env:
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          printf '%s\n' "$SOURCE_SHA" >connect-harness/tests/e2e/connected/rustfs-ref
+          npm --prefix connect-harness/web ci
+          connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
+          make -C connect-harness e2e-connected-dispatch-check
+          RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
+            RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
+            CONNECT_E2E_PROFILE_EVIDENCE="$GITHUB_WORKSPACE/profile-service-job-evidence.json" \
+            make -C connect-harness e2e-connected E2E_SCENARIO=profile
+
+      - name: Bind workflow and artifact provenance
+        if: ${{ always() && hashFiles('profile-service-job-evidence.json') != '' }}
+        shell: bash
+        env:
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          jq \
+            --arg workflowRunId "$GITHUB_RUN_ID" \
+            --arg buildRunId "$BUILD_RUN_ID" \
+            --arg artifactId "$ARTIFACT_ID" \
+            --arg artifactDigest "$ARTIFACT_DIGEST" \
+            --arg connectSha "$CONNECT_SHA" \
+            --arg sourceSha "$SOURCE_SHA" \
+            '. + {workflowRunId: $workflowRunId, buildRunId: $buildRunId, artifactId: $artifactId, artifactDigest: $artifactDigest, connectSha: $connectSha, sourceSha: $sourceSha}' \
+            profile-service-job-evidence.json >profile-service-job-evidence.bound.json
+          mv profile-service-job-evidence.bound.json profile-service-job-evidence.json
+
+      - name: Upload acceptance evidence
+        if: ${{ always() && hashFiles('profile-service-job-evidence.json') != '' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: connect-profile-cpu-evidence-${{ github.run_id }}
+          path: |
+            profile-service-job-evidence.json
+          retention-days: 14
+          if-no-files-found: error

.github/workflows/connect-profile-threads-acceptance.yml

@@ -0,0 +1,192 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Connect profile.threads service-job acceptance
+
+on:
+  workflow_dispatch:
+    inputs:
+      build_run_id:
+        description: Main-branch Build and Release workflow run ID
+        required: true
+        type: string
+      artifact_id:
+        description: Linux x86_64 GNU artifact ID from that run
+        required: true
+        type: string
+      source_sha:
+        description: Exact 40-character source commit
+        required: true
+        type: string
+      artifact_digest:
+        description: GitHub artifact digest including sha256 prefix
+        required: true
+        type: string
+      binary_sha256:
+        description: Expected rustfs binary SHA-256
+        required: true
+        type: string
+      connect_sha:
+        description: Exact 40-character RustFS Connect harness commit
+        required: true
+        type: string
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  profile-threads:
+    name: Verify native Linux x86_64 profile.threads service job
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - name: Checkout acceptance harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Checkout exact RustFS source
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          path: rustfs-source
+          persist-credentials: false
+          ref: ${{ inputs.source_sha }}
+
+      - name: Checkout exact Connect harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/connect
+          path: connect-harness
+          persist-credentials: false
+          ref: ${{ inputs.connect_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 22.22.2
+          cache: npm
+          cache-dependency-path: connect-harness/web/package-lock.json
+
+      - name: Verify source run and artifact identity
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+        run: |
+          set -euo pipefail
+          [[ "$BUILD_RUN_ID" =~ ^[0-9]+$ ]]
+          [[ "$ARTIFACT_ID" =~ ^[0-9]+$ ]]
+          [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ "$ARTIFACT_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
+          [[ "$CONNECT_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ $(git -C rustfs-source rev-parse HEAD) == "$SOURCE_SHA" ]]
+          [[ $(git -C connect-harness rev-parse HEAD) == "$CONNECT_SHA" ]]
+          [[ $(git -C rustfs-source remote get-url origin) == https://github.com/rustfs/rustfs ]]
+          [[ $(git -C connect-harness remote get-url origin) == https://github.com/rustfs/connect ]]
+
+          run=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}")
+          [[ $(jq -r '.head_sha' <<<"$run") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.head_branch' <<<"$run") == main ]]
+          [[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]]
+          [[ $(jq -r '.name' <<<"$run") == "Build and Release" ]]
+
+          expected_job='Build RustFS (linux-x86_64-gnu, sm-standard-2, x86_64-unknown-linux-gnu, false, linux, pyroscope)'
+          jobs=$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
+          jq -e --arg name "$expected_job" '
+            [.[].jobs[] | select(.name == $name)] as $matches
+            | (($matches | length) == 1 and $matches[0].conclusion == "success")
+          ' <<<"$jobs" >/dev/null
+
+          artifact=$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}")
+          [[ $(jq -r '.workflow_run.id' <<<"$artifact") == "$BUILD_RUN_ID" ]]
+          [[ $(jq -r '.workflow_run.head_sha' <<<"$artifact") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.name' <<<"$artifact") == rustfs-linux-x86_64-gnu-* ]]
+          [[ $(jq -r '.digest' <<<"$artifact") == "$ARTIFACT_DIGEST" ]]
+          [[ $(jq -r '.expired' <<<"$artifact") == false ]]
+
+      - name: Download exact build artifact
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        with:
+          artifact-ids: ${{ inputs.artifact_id }}
+          path: artifact
+          run-id: ${{ inputs.build_run_id }}
+          github-token: ${{ github.token }}
+
+      - name: Extract exact RustFS binary
+        shell: bash
+        env:
+          BINARY_SHA256: ${{ inputs.binary_sha256 }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          short_sha=${SOURCE_SHA:0:7}
+          package=$(find artifact -type f -name "rustfs-linux-x86_64-gnu-dev-${short_sha}.zip" -print -quit)
+          [[ -n "$package" ]]
+          mkdir -p binary
+          unzip -qq "$package" rustfs -d binary
+          chmod +x binary/rustfs
+          printf '%s  %s\n' "$BINARY_SHA256" binary/rustfs | sha256sum --check --strict
+
+      - name: Run signed service job acceptance
+        shell: bash
+        env:
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          printf '%s\n' "$SOURCE_SHA" >connect-harness/tests/e2e/connected/rustfs-ref
+          npm --prefix connect-harness/web ci
+          connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
+          make -C connect-harness e2e-connected-dispatch-check
+          RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
+            RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
+            CONNECT_E2E_PROFILE_THREADS_EVIDENCE="$GITHUB_WORKSPACE/profile-threads-service-job-evidence.json" \
+            make -C connect-harness e2e-connected E2E_SCENARIO=profile-threads
+
+      - name: Bind workflow and artifact provenance
+        if: ${{ always() && hashFiles('profile-threads-service-job-evidence.json') != '' }}
+        shell: bash
+        env:
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          jq \
+            --arg workflowRunId "$GITHUB_RUN_ID" \
+            --arg buildRunId "$BUILD_RUN_ID" \
+            --arg artifactId "$ARTIFACT_ID" \
+            --arg artifactDigest "$ARTIFACT_DIGEST" \
+            --arg connectSha "$CONNECT_SHA" \
+            --arg sourceSha "$SOURCE_SHA" \
+            '. + {workflowRunId: $workflowRunId, buildRunId: $buildRunId, artifactId: $artifactId, artifactDigest: $artifactDigest, connectSha: $connectSha, sourceSha: $sourceSha}' \
+            profile-threads-service-job-evidence.json >profile-threads-service-job-evidence.bound.json
+          mv profile-threads-service-job-evidence.bound.json profile-threads-service-job-evidence.json
+
+      - name: Upload acceptance evidence
+        if: ${{ always() && hashFiles('profile-threads-service-job-evidence.json') != '' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: connect-profile-threads-evidence-${{ github.run_id }}
+          path: |
+            profile-threads-service-job-evidence.json
+          retention-days: 14
+          if-no-files-found: error

.github/workflows/connect-top-api-acceptance.yml

@@ -0,0 +1,192 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Connect top.api service-job acceptance
+
+on:
+  workflow_dispatch:
+    inputs:
+      build_run_id:
+        description: Main-branch Build and Release workflow run ID
+        required: true
+        type: string
+      artifact_id:
+        description: Linux x86_64 GNU artifact ID from that run
+        required: true
+        type: string
+      source_sha:
+        description: Exact 40-character source commit
+        required: true
+        type: string
+      artifact_digest:
+        description: GitHub artifact digest including sha256 prefix
+        required: true
+        type: string
+      binary_sha256:
+        description: Expected rustfs binary SHA-256
+        required: true
+        type: string
+      connect_sha:
+        description: Exact 40-character RustFS Connect harness commit
+        required: true
+        type: string
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  top-api:
+    name: Verify native Linux x86_64 top.api service job
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - name: Checkout acceptance harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Checkout exact RustFS source
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          path: rustfs-source
+          persist-credentials: false
+          ref: ${{ inputs.source_sha }}
+
+      - name: Checkout exact Connect harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/connect
+          path: connect-harness
+          persist-credentials: false
+          ref: ${{ inputs.connect_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 22.22.2
+          cache: npm
+          cache-dependency-path: connect-harness/web/package-lock.json
+
+      - name: Verify source run and artifact identity
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+        run: |
+          set -euo pipefail
+          [[ "$BUILD_RUN_ID" =~ ^[0-9]+$ ]]
+          [[ "$ARTIFACT_ID" =~ ^[0-9]+$ ]]
+          [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ "$ARTIFACT_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
+          [[ "$CONNECT_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ $(git -C rustfs-source rev-parse HEAD) == "$SOURCE_SHA" ]]
+          [[ $(git -C connect-harness rev-parse HEAD) == "$CONNECT_SHA" ]]
+          [[ $(git -C rustfs-source remote get-url origin) == https://github.com/rustfs/rustfs ]]
+          [[ $(git -C connect-harness remote get-url origin) == https://github.com/rustfs/connect ]]
+
+          run=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}")
+          [[ $(jq -r '.head_sha' <<<"$run") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.head_branch' <<<"$run") == main ]]
+          [[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]]
+          [[ $(jq -r '.name' <<<"$run") == "Build and Release" ]]
+
+          expected_job='Build RustFS (linux-x86_64-gnu, sm-standard-2, x86_64-unknown-linux-gnu, false, linux, pyroscope)'
+          jobs=$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
+          jq -e --arg name "$expected_job" '
+            [.[].jobs[] | select(.name == $name)] as $matches
+            | (($matches | length) == 1 and $matches[0].conclusion == "success")
+          ' <<<"$jobs" >/dev/null
+
+          artifact=$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}")
+          [[ $(jq -r '.workflow_run.id' <<<"$artifact") == "$BUILD_RUN_ID" ]]
+          [[ $(jq -r '.workflow_run.head_sha' <<<"$artifact") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.name' <<<"$artifact") == rustfs-linux-x86_64-gnu-* ]]
+          [[ $(jq -r '.digest' <<<"$artifact") == "$ARTIFACT_DIGEST" ]]
+          [[ $(jq -r '.expired' <<<"$artifact") == false ]]
+
+      - name: Download exact build artifact
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        with:
+          artifact-ids: ${{ inputs.artifact_id }}
+          path: artifact
+          run-id: ${{ inputs.build_run_id }}
+          github-token: ${{ github.token }}
+
+      - name: Extract exact RustFS binary
+        shell: bash
+        env:
+          BINARY_SHA256: ${{ inputs.binary_sha256 }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          short_sha=${SOURCE_SHA:0:7}
+          package=$(find artifact -type f -name "rustfs-linux-x86_64-gnu-dev-${short_sha}.zip" -print -quit)
+          [[ -n "$package" ]]
+          mkdir -p binary
+          unzip -qq "$package" rustfs -d binary
+          chmod +x binary/rustfs
+          printf '%s  %s\n' "$BINARY_SHA256" binary/rustfs | sha256sum --check --strict
+
+      - name: Run signed service job acceptance
+        shell: bash
+        env:
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          printf '%s\n' "$SOURCE_SHA" >connect-harness/tests/e2e/connected/rustfs-ref
+          npm --prefix connect-harness/web ci
+          connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
+          make -C connect-harness e2e-connected-dispatch-check
+          RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
+            RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
+            CONNECT_E2E_TOP_API_EVIDENCE="$GITHUB_WORKSPACE/top-api-service-job-evidence.json" \
+            make -C connect-harness e2e-connected E2E_SCENARIO=top-api
+
+      - name: Bind workflow and artifact provenance
+        if: ${{ always() && hashFiles('top-api-service-job-evidence.json') != '' }}
+        shell: bash
+        env:
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          jq \
+            --arg workflowRunId "$GITHUB_RUN_ID" \
+            --arg buildRunId "$BUILD_RUN_ID" \
+            --arg artifactId "$ARTIFACT_ID" \
+            --arg artifactDigest "$ARTIFACT_DIGEST" \
+            --arg connectSha "$CONNECT_SHA" \
+            --arg sourceSha "$SOURCE_SHA" \
+            '. + {workflowRunId: $workflowRunId, buildRunId: $buildRunId, artifactId: $artifactId, artifactDigest: $artifactDigest, connectSha: $connectSha, sourceSha: $sourceSha}' \
+            top-api-service-job-evidence.json >top-api-service-job-evidence.bound.json
+          mv top-api-service-job-evidence.bound.json top-api-service-job-evidence.json
+
+      - name: Upload acceptance evidence
+        if: ${{ always() && hashFiles('top-api-service-job-evidence.json') != '' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: connect-top-api-evidence-${{ github.run_id }}
+          path: |
+            top-api-service-job-evidence.json
+          retention-days: 14
+          if-no-files-found: error

.github/workflows/connect-top-disk-acceptance.yml

@@ -0,0 +1,142 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Connect top.disk artifact acceptance
+
+on:
+  workflow_dispatch:
+    inputs:
+      build_run_id:
+        description: Successful main-branch Build and Release workflow run ID
+        required: true
+        type: string
+      artifact_id:
+        description: Linux x86_64 GNU artifact ID from that run
+        required: true
+        type: string
+      source_sha:
+        description: Exact 40-character source commit
+        required: true
+        type: string
+      artifact_digest:
+        description: GitHub artifact digest including sha256 prefix
+        required: true
+        type: string
+      binary_sha256:
+        description: Expected rustfs binary SHA-256
+        required: true
+        type: string
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  top-disk:
+    name: Verify native Linux x86_64 top.disk artifact
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Checkout acceptance harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Verify source run and artifact identity
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+        run: |
+          set -euo pipefail
+          [[ "$BUILD_RUN_ID" =~ ^[0-9]+$ ]]
+          [[ "$ARTIFACT_ID" =~ ^[0-9]+$ ]]
+          [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ "$ARTIFACT_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
+
+          run=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}")
+          [[ $(jq -r '.conclusion' <<<"$run") == success ]]
+          [[ $(jq -r '.head_sha' <<<"$run") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.head_branch' <<<"$run") == main ]]
+          [[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]]
+          [[ $(jq -r '.name' <<<"$run") == "Build and Release" ]]
+
+          artifact=$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}")
+          [[ $(jq -r '.workflow_run.id' <<<"$artifact") == "$BUILD_RUN_ID" ]]
+          [[ $(jq -r '.workflow_run.head_sha' <<<"$artifact") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.name' <<<"$artifact") == rustfs-linux-x86_64-gnu-* ]]
+          [[ $(jq -r '.digest' <<<"$artifact") == "$ARTIFACT_DIGEST" ]]
+          [[ $(jq -r '.expired' <<<"$artifact") == false ]]
+
+      - name: Download exact build artifact
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        with:
+          artifact-ids: ${{ inputs.artifact_id }}
+          path: artifact
+          run-id: ${{ inputs.build_run_id }}
+          github-token: ${{ github.token }}
+
+      - name: Extract exact RustFS binary
+        shell: bash
+        env:
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          short_sha=${SOURCE_SHA:0:7}
+          package=$(find artifact -type f -name "rustfs-linux-x86_64-gnu-dev-${short_sha}.zip" -print -quit)
+          [[ -n "$package" ]]
+          mkdir -p binary
+          unzip -qq "$package" rustfs -d binary
+          chmod +x binary/rustfs
+
+      - name: Run top.disk acceptance
+        shell: bash
+        env:
+          SOURCE_SHA: ${{ inputs.source_sha }}
+          BINARY_SHA256: ${{ inputs.binary_sha256 }}
+        run: |
+          set -euo pipefail
+          scripts/ci/check_connect_top_disk_artifact.sh \
+            binary/rustfs "$SOURCE_SHA" "$BINARY_SHA256" \
+            top-disk-runtime-evidence.json top-disk.zip
+
+      - name: Bind workflow and artifact provenance
+        shell: bash
+        env:
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+        run: |
+          set -euo pipefail
+          jq \
+            --arg workflowRunId "$GITHUB_RUN_ID" \
+            --arg buildRunId "$BUILD_RUN_ID" \
+            --arg artifactId "$ARTIFACT_ID" \
+            --arg artifactDigest "$ARTIFACT_DIGEST" \
+            '. + {workflowRunId: $workflowRunId, buildRunId: $buildRunId, artifactId: $artifactId, artifactDigest: $artifactDigest}' \
+            top-disk-runtime-evidence.json >top-disk-runtime-evidence.bound.json
+          mv top-disk-runtime-evidence.bound.json top-disk-runtime-evidence.json
+
+      - name: Upload acceptance evidence
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: connect-top-disk-evidence-${{ github.run_id }}
+          path: |
+            top-disk-runtime-evidence.json
+            top-disk.zip
+          retention-days: 14
+          if-no-files-found: error

.github/workflows/connect-top-locks-acceptance.yml

@@ -0,0 +1,192 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Connect top.locks service-job acceptance
+
+on:
+  workflow_dispatch:
+    inputs:
+      build_run_id:
+        description: Main-branch Build and Release workflow run ID
+        required: true
+        type: string
+      artifact_id:
+        description: Linux x86_64 GNU artifact ID from that run
+        required: true
+        type: string
+      source_sha:
+        description: Exact 40-character source commit
+        required: true
+        type: string
+      artifact_digest:
+        description: GitHub artifact digest including sha256 prefix
+        required: true
+        type: string
+      binary_sha256:
+        description: Expected rustfs binary SHA-256
+        required: true
+        type: string
+      connect_sha:
+        description: Exact 40-character RustFS Connect harness commit
+        required: true
+        type: string
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  top-locks:
+    name: Verify native Linux x86_64 top.locks service job
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - name: Checkout acceptance harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Checkout exact RustFS source
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          path: rustfs-source
+          persist-credentials: false
+          ref: ${{ inputs.source_sha }}
+
+      - name: Checkout exact Connect harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/connect
+          path: connect-harness
+          persist-credentials: false
+          ref: ${{ inputs.connect_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 22.22.2
+          cache: npm
+          cache-dependency-path: connect-harness/web/package-lock.json
+
+      - name: Verify source run and artifact identity
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+        run: |
+          set -euo pipefail
+          [[ "$BUILD_RUN_ID" =~ ^[0-9]+$ ]]
+          [[ "$ARTIFACT_ID" =~ ^[0-9]+$ ]]
+          [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ "$ARTIFACT_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
+          [[ "$CONNECT_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ $(git -C rustfs-source rev-parse HEAD) == "$SOURCE_SHA" ]]
+          [[ $(git -C connect-harness rev-parse HEAD) == "$CONNECT_SHA" ]]
+          [[ $(git -C rustfs-source remote get-url origin) == https://github.com/rustfs/rustfs ]]
+          [[ $(git -C connect-harness remote get-url origin) == https://github.com/rustfs/connect ]]
+
+          run=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}")
+          [[ $(jq -r '.head_sha' <<<"$run") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.head_branch' <<<"$run") == main ]]
+          [[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]]
+          [[ $(jq -r '.name' <<<"$run") == "Build and Release" ]]
+
+          expected_job='Build RustFS (linux-x86_64-gnu, sm-standard-2, x86_64-unknown-linux-gnu, false, linux, pyroscope)'
+          jobs=$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
+          jq -e --arg name "$expected_job" '
+            [.[].jobs[] | select(.name == $name)] as $matches
+            | (($matches | length) == 1 and $matches[0].conclusion == "success")
+          ' <<<"$jobs" >/dev/null
+
+          artifact=$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}")
+          [[ $(jq -r '.workflow_run.id' <<<"$artifact") == "$BUILD_RUN_ID" ]]
+          [[ $(jq -r '.workflow_run.head_sha' <<<"$artifact") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.name' <<<"$artifact") == rustfs-linux-x86_64-gnu-* ]]
+          [[ $(jq -r '.digest' <<<"$artifact") == "$ARTIFACT_DIGEST" ]]
+          [[ $(jq -r '.expired' <<<"$artifact") == false ]]
+
+      - name: Download exact build artifact
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        with:
+          artifact-ids: ${{ inputs.artifact_id }}
+          path: artifact
+          run-id: ${{ inputs.build_run_id }}
+          github-token: ${{ github.token }}
+
+      - name: Extract exact RustFS binary
+        shell: bash
+        env:
+          BINARY_SHA256: ${{ inputs.binary_sha256 }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          short_sha=${SOURCE_SHA:0:7}
+          package=$(find artifact -type f -name "rustfs-linux-x86_64-gnu-dev-${short_sha}.zip" -print -quit)
+          [[ -n "$package" ]]
+          mkdir -p binary
+          unzip -qq "$package" rustfs -d binary
+          chmod +x binary/rustfs
+          printf '%s  %s\n' "$BINARY_SHA256" binary/rustfs | sha256sum --check --strict
+
+      - name: Run signed service job acceptance
+        shell: bash
+        env:
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          printf '%s\n' "$SOURCE_SHA" >connect-harness/tests/e2e/connected/rustfs-ref
+          npm --prefix connect-harness/web ci
+          connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
+          make -C connect-harness e2e-connected-dispatch-check
+          RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
+            RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
+            CONNECT_E2E_TOP_LOCKS_EVIDENCE="$GITHUB_WORKSPACE/top-locks-service-job-evidence.json" \
+            make -C connect-harness e2e-connected E2E_SCENARIO=top-locks
+
+      - name: Bind workflow and artifact provenance
+        if: ${{ always() && hashFiles('top-locks-service-job-evidence.json') != '' }}
+        shell: bash
+        env:
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          jq \
+            --arg workflowRunId "$GITHUB_RUN_ID" \
+            --arg buildRunId "$BUILD_RUN_ID" \
+            --arg artifactId "$ARTIFACT_ID" \
+            --arg artifactDigest "$ARTIFACT_DIGEST" \
+            --arg connectSha "$CONNECT_SHA" \
+            --arg sourceSha "$SOURCE_SHA" \
+            '. + {workflowRunId: $workflowRunId, buildRunId: $buildRunId, artifactId: $artifactId, artifactDigest: $artifactDigest, connectSha: $connectSha, sourceSha: $sourceSha}' \
+            top-locks-service-job-evidence.json >top-locks-service-job-evidence.bound.json
+          mv top-locks-service-job-evidence.bound.json top-locks-service-job-evidence.json
+
+      - name: Upload acceptance evidence
+        if: ${{ always() && hashFiles('top-locks-service-job-evidence.json') != '' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: connect-top-locks-evidence-${{ github.run_id }}
+          path: |
+            top-locks-service-job-evidence.json
+          retention-days: 14
+          if-no-files-found: error

.github/workflows/connect-top-rpc-acceptance.yml

@@ -0,0 +1,192 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Connect top.rpc service-job acceptance
+
+on:
+  workflow_dispatch:
+    inputs:
+      build_run_id:
+        description: Main-branch Build and Release workflow run ID
+        required: true
+        type: string
+      artifact_id:
+        description: Linux x86_64 GNU artifact ID from that run
+        required: true
+        type: string
+      source_sha:
+        description: Exact 40-character source commit
+        required: true
+        type: string
+      artifact_digest:
+        description: GitHub artifact digest including sha256 prefix
+        required: true
+        type: string
+      binary_sha256:
+        description: Expected rustfs binary SHA-256
+        required: true
+        type: string
+      connect_sha:
+        description: Exact 40-character RustFS Connect harness commit
+        required: true
+        type: string
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  top-rpc:
+    name: Verify native Linux x86_64 top.rpc service job
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - name: Checkout acceptance harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Checkout exact RustFS source
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          path: rustfs-source
+          persist-credentials: false
+          ref: ${{ inputs.source_sha }}
+
+      - name: Checkout exact Connect harness
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/connect
+          path: connect-harness
+          persist-credentials: false
+          ref: ${{ inputs.connect_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 22.22.2
+          cache: npm
+          cache-dependency-path: connect-harness/web/package-lock.json
+
+      - name: Verify source run and artifact identity
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+        run: |
+          set -euo pipefail
+          [[ "$BUILD_RUN_ID" =~ ^[0-9]+$ ]]
+          [[ "$ARTIFACT_ID" =~ ^[0-9]+$ ]]
+          [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ "$ARTIFACT_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
+          [[ "$CONNECT_SHA" =~ ^[0-9a-f]{40}$ ]]
+          [[ $(git -C rustfs-source rev-parse HEAD) == "$SOURCE_SHA" ]]
+          [[ $(git -C connect-harness rev-parse HEAD) == "$CONNECT_SHA" ]]
+          [[ $(git -C rustfs-source remote get-url origin) == https://github.com/rustfs/rustfs ]]
+          [[ $(git -C connect-harness remote get-url origin) == https://github.com/rustfs/connect ]]
+
+          run=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}")
+          [[ $(jq -r '.head_sha' <<<"$run") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.head_branch' <<<"$run") == main ]]
+          [[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]]
+          [[ $(jq -r '.name' <<<"$run") == "Build and Release" ]]
+
+          expected_job='Build RustFS (linux-x86_64-gnu, sm-standard-2, x86_64-unknown-linux-gnu, false, linux, pyroscope)'
+          jobs=$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
+          jq -e --arg name "$expected_job" '
+            [.[].jobs[] | select(.name == $name)] as $matches
+            | (($matches | length) == 1 and $matches[0].conclusion == "success")
+          ' <<<"$jobs" >/dev/null
+
+          artifact=$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}")
+          [[ $(jq -r '.workflow_run.id' <<<"$artifact") == "$BUILD_RUN_ID" ]]
+          [[ $(jq -r '.workflow_run.head_sha' <<<"$artifact") == "$SOURCE_SHA" ]]
+          [[ $(jq -r '.name' <<<"$artifact") == rustfs-linux-x86_64-gnu-* ]]
+          [[ $(jq -r '.digest' <<<"$artifact") == "$ARTIFACT_DIGEST" ]]
+          [[ $(jq -r '.expired' <<<"$artifact") == false ]]
+
+      - name: Download exact build artifact
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        with:
+          artifact-ids: ${{ inputs.artifact_id }}
+          path: artifact
+          run-id: ${{ inputs.build_run_id }}
+          github-token: ${{ github.token }}
+
+      - name: Extract exact RustFS binary
+        shell: bash
+        env:
+          BINARY_SHA256: ${{ inputs.binary_sha256 }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          short_sha=${SOURCE_SHA:0:7}
+          package=$(find artifact -type f -name "rustfs-linux-x86_64-gnu-dev-${short_sha}.zip" -print -quit)
+          [[ -n "$package" ]]
+          mkdir -p binary
+          unzip -qq "$package" rustfs -d binary
+          chmod +x binary/rustfs
+          printf '%s  %s\n' "$BINARY_SHA256" binary/rustfs | sha256sum --check --strict
+
+      - name: Run signed service job acceptance
+        shell: bash
+        env:
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          printf '%s\n' "$SOURCE_SHA" >connect-harness/tests/e2e/connected/rustfs-ref
+          npm --prefix connect-harness/web ci
+          connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
+          make -C connect-harness e2e-connected-dispatch-check
+          RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
+            RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
+            CONNECT_E2E_TOP_RPC_EVIDENCE="$GITHUB_WORKSPACE/top-rpc-service-job-evidence.json" \
+            make -C connect-harness e2e-connected E2E_SCENARIO=top-rpc
+
+      - name: Bind workflow and artifact provenance
+        if: ${{ always() && hashFiles('top-rpc-service-job-evidence.json') != '' }}
+        shell: bash
+        env:
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          ARTIFACT_ID: ${{ inputs.artifact_id }}
+          ARTIFACT_DIGEST: ${{ inputs.artifact_digest }}
+          CONNECT_SHA: ${{ inputs.connect_sha }}
+          SOURCE_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          jq \
+            --arg workflowRunId "$GITHUB_RUN_ID" \
+            --arg buildRunId "$BUILD_RUN_ID" \
+            --arg artifactId "$ARTIFACT_ID" \
+            --arg artifactDigest "$ARTIFACT_DIGEST" \
+            --arg connectSha "$CONNECT_SHA" \
+            --arg sourceSha "$SOURCE_SHA" \
+            '. + {workflowRunId: $workflowRunId, buildRunId: $buildRunId, artifactId: $artifactId, artifactDigest: $artifactDigest, connectSha: $connectSha, sourceSha: $sourceSha}' \
+            top-rpc-service-job-evidence.json >top-rpc-service-job-evidence.bound.json
+          mv top-rpc-service-job-evidence.bound.json top-rpc-service-job-evidence.json
+
+      - name: Upload acceptance evidence
+        if: ${{ always() && hashFiles('top-rpc-service-job-evidence.json') != '' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: connect-top-rpc-evidence-${{ github.run_id }}
+          path: |
+            top-rpc-service-job-evidence.json
+          retention-days: 14
+          if-no-files-found: error

.github/workflows/coverage.yml

@@ -140,4 +140,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/e2e-distributed.yml

@@ -151,8 +151,7 @@ jobs:
 
       - name: Build rustfs binary
         run: |
-          cargo build -p rustfs --bins
-          : > target/debug/rustfs.features
+          python3 scripts/e2e_binary.py build --bins
 
       - name: Verify distributed e2e membership
         env:
@@ -168,9 +167,9 @@ jobs:
         run: |
           set -euo pipefail
           if [ -n "${FILTER}" ]; then
-            cargo nextest run --profile e2e-distributed -p e2e_test -E "${FILTER}"
+            python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-distributed -p e2e_test -E "${FILTER}"
           else
-            cargo nextest run --profile e2e-distributed -p e2e_test --no-tests=fail
+            python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-distributed -p e2e_test --no-tests=fail
           fi
 
       - name: Upload distributed e2e diagnostics
@@ -213,4 +212,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/e2e-replication-nightly.yml

@@ -89,14 +89,10 @@ jobs:
       - name: Verify awscurl
         run: test -x "$AWSCURL_PATH"
 
-      # Build the rustfs binary once up front. The e2e tests spawn it as a
-      # child process (crates/e2e_test/src/common.rs) and will build it on
-      # demand otherwise, but a single explicit build avoids several parallel
-      # nextest test processes racing to build it at once.
+      # Build once and carry its source/binary identity into the test invocation.
       - name: Build rustfs binary
         run: |
-          cargo build -p rustfs --bins
-          : > target/debug/rustfs.features
+          python3 scripts/e2e_binary.py build --bins
 
       - name: Verify replication e2e membership
         env:
@@ -108,7 +104,7 @@ jobs:
       - name: Run replication e2e nightly suite
         env:
           RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-repl-nightly-logs
-        run: cargo nextest run --profile e2e-repl-nightly -p e2e_test
+        run: python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-repl-nightly -p e2e_test
 
       - name: Upload nextest junit report
         if: always()
@@ -144,8 +140,7 @@ jobs:
 
       - name: Build rustfs binary
         run: |
-          cargo build -p rustfs --bins --features e2e-test-hooks
-          : > target/debug/rustfs.features
+          python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
 
       - name: Verify cluster fault e2e membership
         env:
@@ -158,7 +153,7 @@ jobs:
         env:
           CARGO_BIN_EXE_rustfs: ${{ github.workspace }}/target/debug/rustfs
           RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-nightly-logs
-        run: cargo nextest run --profile e2e-nightly -p e2e_test
+        run: python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-nightly -p e2e_test
 
       - name: Upload cluster fault diagnostics
         if: always()
@@ -199,6 +194,9 @@ jobs:
           sudo apt-get install -y -qq iproute2
           ss -tn state CLOSE-WAIT >/dev/null
 
+      - name: Build protocol server
+        run: python3 scripts/e2e_binary.py build --features "$RUSTFS_BUILD_FEATURES"
+
       # The suite owns fixed protocol ports and serializes its internal cases.
       - name: Verify protocol e2e membership
         env:
@@ -211,7 +209,7 @@ jobs:
         env:
           RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-protocol-e2e-logs
         run: >-
-          cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture
+          python3 scripts/e2e_binary.py run --features "$RUSTFS_BUILD_FEATURES" -- cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture
 
       - name: Upload protocol diagnostics
         if: always()
@@ -244,4 +242,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/e2e-s3tests.yml

@@ -460,4 +460,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/e2e-upgrade.yml

@@ -125,14 +125,13 @@ jobs:
 
       - name: Build current RustFS binary
         run: |
-          cargo build --locked -p rustfs --bin rustfs
-          : > target/debug/rustfs.features
+          python3 scripts/e2e_binary.py build
 
       - name: Run upgrade compatibility test
         env:
           RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR: ${{ runner.temp }}/rustfs-upgrade-g09-evidence/${{ matrix.artifact }}
         run: |
-          cargo test --locked -p e2e_test \
+          python3 scripts/e2e_binary.py run -- cargo test --locked -p e2e_test \
             "upgrade_compatibility_test::${{ matrix.test }}" \
             -- --ignored --exact --nocapture
 

.github/workflows/functional-chain-health.yml

@@ -0,0 +1,31 @@
+name: Functional chain health
+on:
+  schedule:
+    - cron: '23 * * * *'
+  workflow_dispatch:
+permissions:
+  contents: read
+  actions: read
+concurrency:
+  group: functional-chain-health
+  cancel-in-progress: false
+jobs:
+  collect:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Collect and publish verified chain health
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          NIGHTLY_SOURCE_REF: ${{ vars.NIGHTLY_BRANCH || 'main' }}
+        run: python3 scripts/functional_chain_health.py --source-ref "${NIGHTLY_SOURCE_REF}" --publish --output "${RUNNER_TEMP}/chain-health.json"
+      - name: Retain health observation
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-health-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-health.json
+          if-no-files-found: error

.github/workflows/fuzz.yml

@@ -252,4 +252,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/minio-interop.yml

@@ -143,4 +143,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/mint.yml

@@ -270,4 +270,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/nightly-gnu.yml

@@ -42,18 +42,39 @@ env:
   NIGHTLY_BUILD_REF: ${{ github.event_name == 'schedule' && (vars.NIGHTLY_BRANCH || 'main') || (inputs.branch || github.ref_name) }}
 
 jobs:
+  resolve-source:
+    name: Resolve nightly source
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    outputs:
+      source_sha: ${{ steps.source.outputs.sha }}
+      source_ref: ${{ steps.source.outputs.ref }}
+    steps:
+      - name: Checkout selected source
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+          ref: ${{ env.NIGHTLY_BUILD_REF }}
+      - name: Record immutable source
+        id: source
+        run: |
+          echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+          echo "ref=${NIGHTLY_BUILD_REF}" >> "$GITHUB_OUTPUT"
+
   build:
+    needs: resolve-source
     name: Build x86_64 GNU
     runs-on: sm-standard-4
     timeout-minutes: 150
     env:
+      NIGHTLY_BUILD_REF: ${{ needs.resolve-source.outputs.source_ref }}
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: ${{ env.NIGHTLY_BUILD_REF }}
+          ref: ${{ needs.resolve-source.outputs.source_sha }}
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -334,9 +355,10 @@ jobs:
 
           CANDIDATE_FILE="${RUNNER_TEMP}/nightly-candidate-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
           jq -n --arg source_sha "${SOURCE_SHA}" \
+            --arg workflow_sha "${GITHUB_SHA}" --arg source_ref "${NIGHTLY_BUILD_REF}" \
             --argjson build_run_id "${GITHUB_RUN_ID}" --argjson build_run_attempt "${GITHUB_RUN_ATTEMPT}" \
             --arg package_url "${CANDIDATE_URL}" --arg package_sha256 "${DEB_SHA256}" \
-            '{schema: 1, source_sha: $source_sha, build_run_id: $build_run_id, build_run_attempt: $build_run_attempt, package_url: $package_url, package_sha256: $package_sha256}' \
+            '{schema: 2, workflow_sha: $workflow_sha, source_ref: $source_ref, source_sha: $source_sha, build_run_id: $build_run_id, build_run_attempt: $build_run_attempt, package_url: $package_url, package_sha256: $package_sha256}' \
             > "${CANDIDATE_FILE}"
           echo "candidate_file=${CANDIDATE_FILE}" >> "${GITHUB_OUTPUT}"
 
@@ -378,6 +400,7 @@ jobs:
   # self-hosted fleet is heterogeneous — a docker-dependent workflow has been
   # burned by it before (see the banner in e2e-s3tests.yml, rustfs/backlog#1149).
   kms-vault-lane:
+    needs: resolve-source
     name: KMS live Vault lane
     runs-on: ubuntu-latest
     timeout-minutes: 90
@@ -399,7 +422,7 @@ jobs:
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: ${{ env.NIGHTLY_BUILD_REF }}
+          ref: ${{ needs.resolve-source.outputs.source_sha }}
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -477,6 +500,7 @@ jobs:
   # flake cannot mask the main lane's verdict, and vice versa. The script
   # provisions and tears down its own Docker cluster.
   kms-vault-ha-failover:
+    needs: resolve-source
     name: KMS Vault HA failover lane
     runs-on: ubuntu-latest
     timeout-minutes: 60
@@ -488,7 +512,7 @@ jobs:
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
-          ref: ${{ env.NIGHTLY_BUILD_REF }}
+          ref: ${{ needs.resolve-source.outputs.source_sha }}
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -519,4 +543,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/oidc-keycloak.yml

@@ -107,4 +107,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/on-demand-migration-interop.yml

@@ -132,7 +132,7 @@ jobs:
             s3api create-bucket --bucket "${RUSTFS_ODM_INTEROP_BUCKET}"
 
       - name: Build the RustFS binary under test
-        run: cargo build --locked -p rustfs --bins
+        run: python3 scripts/e2e_binary.py build --bins
 
       # The lane selects tests by module, so a rename would quietly shrink it.
       # The committed digest in .config/e2e-odm-interop-selection.txt fails
@@ -143,7 +143,7 @@ jobs:
           python3 ./scripts/check_test_wiring.py --check-profile e2e-odm-interop "${NEXTEST_LISTING}"
 
       - name: Run the interop cases against MinIO
-        run: cargo nextest run --profile e2e-odm-interop -p e2e_test --no-tests=fail
+        run: python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-odm-interop -p e2e_test --no-tests=fail
 
       - name: Build the MinIO interop report
         if: always()
@@ -251,7 +251,7 @@ jobs:
 
       - name: Build the RustFS binary under test
         if: steps.credentials.outputs.present == 'true'
-        run: cargo build --locked -p rustfs --bins
+        run: python3 scripts/e2e_binary.py build --bins
 
       # A filterset that matches nothing is valid, so the count is asserted
       # rather than inferred from a green run.
@@ -272,7 +272,7 @@ jobs:
       - name: Run the three-case minimum
         if: steps.credentials.outputs.present == 'true'
         run: |
-          cargo nextest run --profile e2e-odm-interop -p e2e_test \
+          python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-odm-interop -p e2e_test \
             -E "${CLOUD_CASE_FILTER}" --no-tests=fail
 
       - name: Build the ${{ matrix.provider }} interop report
@@ -312,4 +312,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/performance-ab.yml

@@ -378,4 +378,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/runner-hygiene.yml

@@ -78,4 +78,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/rustfs-fault-tolerance-test.yml

@@ -0,0 +1,327 @@
+# RustFS Fault-Tolerance (degradation) Test
+#
+# Scenario suite for the 2026-09 degradation report: verifies read/write
+# behavior under drive and node loss against the erasure-coding contract and
+# snapshots health-endpoint responses at every tier.
+#
+#   A  single-node 4 drives (SNMD): hide 1/2/3 drives, restore
+#   B  multi-node 4x1 (one drive per node): stop 1/2/3 nodes, restore
+#   C  multi-node 4x4 (16 drives, EC:4): stop 1 node (read-quorum boundary),
+#      stop 2 nodes, restore
+#   C2 multi-node 4x4 with EC:8: 2 nodes down puts 8 drives online -- reads
+#      satisfy the EC read quorum while the lock majority is broken (the
+#      reported divergence window: reads 503 with lock_quorum_unavailable)
+#
+# Expectations come from product source (default_parity_count, erasure set
+# sizing). By default a "reads refused although the read quorum is met"
+# observation is reported as known-divergence without failing the suite; the
+# strict input turns those into failures once the product behavior changes.
+
+name: RustFS Fault-Tolerance Test
+
+on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
+  workflow_dispatch:
+    inputs:
+      package_url:
+        description: 'Direct .deb URL. Required unless the nightly default is wanted.'
+        required: false
+        type: string
+      strict:
+        description: 'Fail the suite when reads are refused despite a met read quorum'
+        type: boolean
+        default: false
+      cleanup_before:
+        description: 'Reset the nodes before the test (DESTROYS existing data/config)'
+        type: boolean
+        default: true
+      cleanup_after:
+        description: 'Reset the nodes after the test (DESTROYS test data/config)'
+        type: boolean
+        default: true
+  repository_dispatch:
+    # Chain handoff: dispatched when the replication suite finishes, ahead of
+    # the performance suite.
+    types: [rustfs-chain-fault-tolerance]
+
+permissions:
+  contents: read
+
+# The suite stops services and hides drive dirs on the shared fleet; only one
+# functional suite may touch the environment at a time.
+concurrency:
+  group: rustfs-shared-functional-tests-v2
+  cancel-in-progress: false
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
+  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
+  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
+  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
+  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
+
+jobs:
+  fault-tolerance-test:
+    runs-on: smoke-testing
+    timeout-minutes: 480
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-ft-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}/evidence"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'EVIDENCE_DIR=%s/evidence\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
+      # Pin the private suite checkout when the chain supplies a candidate.
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
+
+      - name: Show environment
+        run: |
+          uname -a
+          jq --version
+          aws --version
+          df -h /data | tail -1
+
+      - name: Cleanup environment (before)
+        if: ${{ inputs.cleanup_before != 'false' }}
+        run: |
+          ./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}"
+
+      - name: Stage verified candidate package
+        id: chain_package
+        if: ${{ inputs.chain_manifest != '' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/prepare_functional_package.py prepare
+
+      - name: Run fault-tolerance scenarios (A, B, C, C2)
+        id: test
+        # Standalone case failures are reported by the backlog manager.
+        # Chain evidence separately requires complete passing cases.
+        continue-on-error: true
+        run: |
+          ARGS=(--all -y --package-url "${{ steps.chain_package.outputs.package_url || inputs.package_url || env.RUSTFS_NIGHTLY_PACKAGE_URL }}" --log-file "${LOG_FILE}")
+          if [ "${{ inputs.strict }}" = "true" ]; then
+            ARGS+=(--strict)
+          fi
+          ./auto-testing/rustfs-fault-tolerance-test.sh "${ARGS[@]}"
+
+      - name: Generate report
+        id: chain_report
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        run: |
+          set -euo pipefail
+          {
+            echo "# RustFS fault-tolerance test report"
+            echo ""
+            echo "- Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+            echo "- Package: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
+            echo "- Strict mode: ${{ inputs.strict || 'false' }}"
+            echo ""
+            echo "## Per-probe results"
+            echo ""
+            echo '```'
+            grep -E '^FT-(CASE|SUMMARY|REPORT)' "${LOG_FILE}" || echo "(no FT-CASE lines found)"
+            echo '```'
+            echo ""
+            echo "## Health snapshots"
+            echo ""
+            for f in "${FUNCTIONAL_ARTIFACTS_DIR}"/evidence/*.code; do
+              [ -e "${f}" ] || continue
+              printf '%s -> %s\n' "$(basename "${f}" .code)" "$(cat "${f}")"
+            done
+          } > "${REPORT_FILE}"
+          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          # Product gate: failing FT cases keep the run green — the report above and
+          # the backlog issue manager carry the signal. Only harness/environment
+          # breakdowns (no case verdicts at all) turn the workflow red.
+          FT_CASES="$(grep -cE '^FT-CASE:' "${LOG_FILE}" 2>/dev/null || true)"
+          FT_FAILED="$(grep -cE '^FT-CASE: .* verdict=UNEXPECTED' "${LOG_FILE}" 2>/dev/null || true)"
+          FT_PASSED=$(( ${FT_CASES:-0} - ${FT_FAILED:-0} ))
+          HARNESS_OK=0
+          if [ "${{ steps.test.outcome }}" = "success" ]; then
+            HARNESS_OK=1
+          elif [ "${{ steps.test.outcome }}" = "failure" ] && [ "${FT_CASES}" -gt 0 ] && [ "${FT_PASSED}" -ge 1 ]; then
+            HARNESS_OK=1
+          fi
+          [ "${HARNESS_OK}" = "1" ]
+
+      - name: Manage backlog issues (dedup / label / auto-close)
+        # Signal-based lifecycle: dedups against open issues by label
+        # (fault-tolerance + FT case), labels new issues (functional-test,
+        # category, case IDs, env), and closes fixed issues after a fully
+        # green run. Never acts on cancelled runs. Logic lives in
+        # auto-testing/scripts/issue_manager.py, which parses the
+        # FT-CASE verdict lines from the suite log.
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 'fault-tolerance'
+          SUITE_LABEL: 'Fault-Tolerance'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
+            exit 0
+          fi
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
+            exit 0
+          fi
+          PACKAGE_URL='${{ inputs.package_url }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
+          fi
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.test.outcome }}" \
+            --report-file "${REPORT_FILE}" \
+            --log "${LOG_FILE}" \
+            --run-url "${RUN_URL}/attempts/${GITHUB_RUN_ATTEMPT}#summary" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
+
+      - name: Upload test logs & evidence
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: rustfs-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/evidence/
+          if-no-files-found: warn
+
+      - name: Cleanup environment (after)
+        if: ${{ always() && inputs.cleanup_after != 'false' }}
+        run: |
+          ./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}" || true
+
+      - name: "Continue functional chain (next: Performance)"
+        # Only chain-triggered runs forward to the next suite; standalone
+        # workflow_dispatch runs stop after their own cleanup. A failed
+        # handoff retries, then files an alert issue in rustfs/backlog.
+        if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -uo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          DISPATCHED=0
+          for attempt in 1 2 3; do
+            if gh api --method POST repos/rustfs/rustfs/dispatches \
+              -f event_type='rustfs-chain-performance' \
+              -F 'client_payload[from_suite]=fault-tolerance'; then
+              echo "dispatched next suite Performance (attempt ${attempt})"
+              DISPATCHED=1
+              break
+            fi
+            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
+            sleep "${attempt}0"
+          done
+          if [ "${DISPATCHED:-0}" -ne 1 ]; then
+            echo "ERROR: functional chain stalled: could not dispatch Performance after 3 attempts" >&2
+            TITLE="[functional][chain] stalled after fault-tolerance (run ${GITHUB_RUN_ID})"
+            BODY_FILE="$(mktemp)"
+            {
+              echo "The functional chain could not hand off from **fault-tolerance** to **Performance** after 3 attempts."
+              echo ""
+              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+              echo "- Expected next event: 'rustfs-chain-performance'"
+              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+              echo "- Recovery: re-dispatch manually with"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-performance'"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+            } > "${BODY_FILE}"
+            gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test \
+              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
+              || echo "could not file the stall alert issue either; check the token" >&2
+            exit 1
+          fi
+
+      - name: Notify on failure
+        if: failure()
+        run: |
+          echo "RustFS fault-tolerance test failed"
+          echo "Package source: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
+          echo "See the uploaded log artifact and FT-CASE lines for details."
+
+      - name: Remove verified candidate package
+        if: ${{ always() && inputs.chain_manifest != '' && steps.chain.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/prepare_functional_package.py cleanup
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite fault-tolerance
+          --report "${FUNCTIONAL_ARTIFACTS_DIR}/suite.log"
+          --output "${RUNNER_TEMP}/chain-fault-tolerance-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/fault-tolerance.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}/fault-tolerance.json
+          if-no-files-found: error

.github/workflows/rustfs-functional-chain.yml

@@ -12,50 +12,207 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Functional chain driver: runs the ten functional suites in a fixed order
-# (upgrade -> s3 -> kms -> tier -> storage -> heal -> pool -> security ->
-# replication -> performance). Each suite attempts the next handoff even
-# when its tests fail.
-#
-# Each suite workflow can still be dispatched standalone (workflow_dispatch);
-# only chain-triggered runs forward to the next suite via repository_dispatch,
-# so a standalone run never drags the rest of the chain behind it.
-#
-# Why not workflow_run chaining: GitHub does not guarantee delivery of
-# workflow_run events (they are fire-and-forget), and the head-SHA filter made
-# newly added suites (storage) unable to trigger at all. Explicit
-# repository_dispatch handoffs are verifiable and re-drivable.
-
+# Reusable workflows run from this driver's commit in one Actions run. Each
+# suite still runs after an earlier suite fails; the final job requires all twelve.
 name: RustFS Functional Chain
 
 on:
   workflow_dispatch:
+    inputs:
+      build_run_id:
+        description: Successful nightly build run on main
+        type: string
+        required: true
+      build_run_attempt:
+        description: Exact successful build attempt
+        type: string
+        required: true
   workflow_run:
-    # Entry point: start the chain after the nightly build completes. The
-    # build's own conclusion does not gate the chain; each suite reports its
-    # own result to rustfs/backlog and the dashboard.
     workflows: ["Nightly GNU Build"]
     types: [completed]
 
 permissions:
   contents: read
+  actions: read
+
+concurrency:
+  group: rustfs-functional-chain-runs
+  cancel-in-progress: false
 
 jobs:
-  start-chain:
-    name: Start functional chain (upgrade first)
+  prepare:
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.event == 'schedule' }}
     runs-on: ubuntu-latest
     timeout-minutes: 10
-    if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'workflow_run' && github.event.workflow_run.event == 'schedule') }}
+    outputs:
+      manifest: ${{ steps.candidate.outputs.manifest }}
     steps:
-      - name: Dispatch first suite (upgrade)
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Resolve published candidate
+        id: candidate
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
+          CHAIN_OUTPUT: ${{ runner.temp }}/chain-candidate.json
+        run: python3 scripts/resolve_functional_candidate.py
+      - name: Retain candidate identity
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-candidate-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-candidate.json
+          if-no-files-found: error
+
+      - name: Check shared functional runners before scheduling suites
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot start the functional chain" >&2
-            exit 1
-          fi
-          gh api --method POST repos/rustfs/rustfs/dispatches \
-            -f event_type='rustfs-chain-upgrade' \
-            -F 'client_payload[from_suite]=nightly-build'
+        run: python3 scripts/check_functional_runners.py smoke-testing pf-testing
+
+  upgrade:
+    needs: [prepare]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-upgrade-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  s3:
+    needs: [prepare, upgrade]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-s3-compat-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  kms:
+    needs: [prepare, s3]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-kms-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  tier:
+    needs: [prepare, kms]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-tier-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  storage:
+    needs: [prepare, tier]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-storage-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  heal:
+    needs: [prepare, storage]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-heal-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  pool:
+    needs: [prepare, heal]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-pool-expand-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  security:
+    needs: [prepare, pool]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-security-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  replication:
+    needs: [prepare, security]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-replication-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  fault-tolerance:
+    needs: [prepare, replication]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-fault-tolerance-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  table:
+    needs: [prepare, fault-tolerance]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-table-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  performance:
+    needs: [prepare, table]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-performance-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  complete-chain:
+    needs: [prepare, upgrade, s3, kms, tier, storage, heal, pool, security, replication, fault-tolerance, table, performance]
+    if: ${{ always() && needs.prepare.result != 'skipped' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Download suite evidence
+        id: download
+        continue-on-error: true
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        with:
+          pattern: functional-chain-*-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-evidence
+          merge-multiple: true
+      - name: Summarize every suite including failures and missing evidence
+        if: always()
+        env:
+          CHAIN_MANIFEST: ${{ needs.prepare.outputs.manifest }}
+          CHAIN_NEEDS: ${{ toJSON(needs) }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py summarize
+          --directory "${RUNNER_TEMP}/chain-evidence"
+          --output "${RUNNER_TEMP}/chain-report.json"
+      - name: Retain chain report regardless of test verdict
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-report-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            ${{ runner.temp }}/chain-report.json
+            ${{ runner.temp }}/chain-report.md
+          if-no-files-found: error
+      - name: Verify every required suite
+        if: always()
+        env:
+          CHAIN_MANIFEST: ${{ needs.prepare.outputs.manifest }}
+          CHAIN_NEEDS: ${{ toJSON(needs) }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py aggregate
+          --directory "${RUNNER_TEMP}/chain-evidence"
+          --output "${RUNNER_TEMP}/chain-complete.json"
+      - name: Upload complete-chain evidence
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-complete-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-complete.json
+          if-no-files-found: error

.github/workflows/rustfs-heal-test.yml

@@ -1,6 +1,12 @@
 name: RustFS Heal Test
 
 on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       package_url:
@@ -35,7 +41,7 @@ permissions:
 # Only one test at a time: both this and the pool-expansion workflow mutate
 # the same test environment, so they share one concurrency group.
 concurrency:
-  group: rustfs-shared-functional-tests
+  group: rustfs-shared-functional-tests-v2
   cancel-in-progress: false
 
 defaults:
@@ -57,8 +63,13 @@ jobs:
     timeout-minutes: 480
     # Standalone manual run, or one link of the nightly functional chain
     # (storage -> heal -> pool). Pool expansion no longer re-runs heal.
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout chain tooling
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
       - name: Initialize functional evidence
         id: evidence
         run: |
@@ -74,25 +85,21 @@ jobs:
             printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
           } >> "${GITHUB_ENV}"
 
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
 
       - name: Show environment
         run: |
@@ -143,6 +150,9 @@ jobs:
 
       - name: Run heal test (write -> outage -> heal -> verify)
         id: test
+        # Case failures keep the run green: the report and the backlog
+        # issue manager carry the product signal.
+        continue-on-error: true
         run: |
           ./auto-testing/rustfs_heal_test.sh \
             --steps "3,4,5,6,7" -y \
@@ -152,6 +162,7 @@ jobs:
             --log-file "${LOG_FILE}"
 
       - name: Generate report
+        id: chain_report
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
@@ -224,6 +235,20 @@ jobs:
           if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
             RESULT=success
           fi
+
+          # Product gate: failing steps keep the run green — they are reported
+          # below and tracked in rustfs/backlog. Only harness/environment
+          # breakdowns turn the workflow red.
+          STEPS_TOTAL="$(grep -cE '^\[HEAL-STEP\]' "${LOG_FILE}" 2>/dev/null || true)"
+          STEPS_FAIL="$(grep -cE '^\[HEAL-STEP\].* FAIL$' "${LOG_FILE}" 2>/dev/null || true)"
+          STEPS_TOTAL=$(( ${STEPS_TOTAL:-0} + 0 )); STEPS_FAIL=$(( ${STEPS_FAIL:-0} + 0 ))
+          STEPS_PASS=$(( STEPS_TOTAL - STEPS_FAIL ))
+          HARNESS_OK=0
+          if [ '${{ steps.test.outcome }}' = 'success' ]; then
+            HARNESS_OK=1
+          elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${STEPS_TOTAL}" -gt 0 ] && [ "${STEPS_PASS}" -ge 1 ]; then
+            HARNESS_OK=1
+          fi
           {
             echo "# RustFS heal test report"
             echo ""
@@ -235,19 +260,24 @@ jobs:
             echo "- Test Step Outcome: ${RESULT}"
             echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            if [ "${RESULT}" = "success" ]; then
+            if [ -s "${STEPS_TABLE}" ]; then
               cat "${STEPS_TABLE}"
               echo ""
+            fi
+            echo "- Product result: ${STEPS_PASS} passed, ${STEPS_FAIL} failed (failing steps are tracked in rustfs/backlog)"
+            echo ""
+            if [ -s "${STEPS_TABLE}" ]; then
               echo "## Log tail"
               echo '```text'
-              tail -n 200 "${LOG_FILE}"
+              tail -n 200 "${LOG_FILE}" 2>/dev/null || true
               echo '```'
             else
               echo "The suite or evidence validation failed. See this run's artifact for partial step results and suite.log."
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]
+          # Red only for harness/environment breakdowns; step failures stay green.
+          [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
         if: ${{ always() && steps.evidence.outcome == 'success' }}
@@ -280,65 +310,51 @@ jobs:
           fi
           rm -f "${B64_FILE}"
 
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+      - name: Manage backlog issues (dedup / label / auto-close)
+        # Replaces the old per-run failure filing. One entry point that:
+        # - dedups by signal: failing cases are matched against open backlog
+        #   issues by label (category + case ID); covered cases become a
+        #   comment on the existing issue, only uncovered cases file a new one
+        # - labels new issues (functional-test, category, case IDs, env)
+        # - closes fixed issues after a fully green run
+        # - never files or closes on cancelled runs
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'heal'
           SUITE_LABEL: 'Heal'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
             exit 0
           fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          PACKAGE_URL='${{ inputs.package_url }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          echo "filed backlog issue for suite ${SUITE}"
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.test.outcome }}" \
+            --report-file "${REPORT_FILE}" \
+            --log "${LOG_FILE}" \
+            --run-url "${RUN_URL}/attempts/${GITHUB_RUN_ATTEMPT}#summary" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
 
       - name: Upload test logs
         if: ${{ always() && steps.evidence.outcome == 'success' }}
@@ -426,3 +442,24 @@ jobs:
           echo "RustFS heal test failed"
           echo "Package source: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
           echo "See the uploaded log artifact for details."
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite heal
+          --report "${FUNCTIONAL_ARTIFACTS_DIR}/steps.md"
+          --output "${RUNNER_TEMP}/chain-heal-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/heal.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-heal-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-heal-${{ github.run_id }}-${{ github.run_attempt }}/heal.json
+          if-no-files-found: error

.github/workflows/rustfs-kms-test.yml

@@ -1,12 +1,18 @@
 name: RustFS KMS Test
 
 on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       rustfs_version:
-        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
+        description: 'RustFS release tag to test. Leave empty for nightly.'
         required: false
-        default: '1.0.0-rc.4-preview.1'
+        default: ''
       package_url:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false
@@ -31,7 +37,7 @@ permissions:
   contents: read
 
 concurrency:
-  group: rustfs-shared-functional-tests
+  group: rustfs-shared-functional-tests-v2
   cancel-in-progress: false
 
 defaults:
@@ -50,7 +56,7 @@ jobs:
   kms-test:
     runs-on: smoke-testing
     timeout-minutes: 420
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -71,25 +77,21 @@ jobs:
             printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
           } >> "${GITHUB_ENV}"
 
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
 
       - name: Show environment
         run: |
@@ -127,6 +129,9 @@ jobs:
 
       - name: Run KMS suite
         id: test
+        # Case failures keep the run green: the report and the backlog
+        # issue manager carry the product signal.
+        continue-on-error: true
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-kms-test.sh
@@ -156,6 +161,7 @@ jobs:
           ./auto-testing/rustfs-kms-test.sh "${ARGS[@]}"
 
       - name: Generate report
+        id: chain_report
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
@@ -175,6 +181,21 @@ jobs:
           if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
             RESULT=success
           fi
+
+          # Product gate: failing cases keep the run green — they are reported
+          # below and tracked in rustfs/backlog. Only harness/environment
+          # breakdowns (the suite never reached case level, or a wholesale
+          # failure with zero passes) turn the workflow red.
+          CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
+          CASES_PASS=$(( CASES_TOTAL - CASES_FAIL ))
+          HARNESS_OK=0
+          if [ '${{ steps.test.outcome }}' = 'success' ]; then
+            HARNESS_OK=1
+          elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${CASES_TOTAL}" -gt 0 ] && [ "${CASES_PASS}" -ge 1 ]; then
+            HARNESS_OK=1
+          fi
           {
             echo "# RustFS KMS test report"
             echo ""
@@ -186,19 +207,24 @@ jobs:
             echo "- Test Step Outcome: ${RESULT}"
             echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            if [ "${RESULT}" = "success" ]; then
+            if [ -s "${CASE_TABLE}" ]; then
               cat "${CASE_TABLE}"
               echo ""
+            fi
+            echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)"
+            echo ""
+            if [ -s "${CASE_TABLE}" ]; then
               echo "## Log tail"
               echo '```text'
-              tail -n 200 "${LOG_FILE}"
+              tail -n 200 "${LOG_FILE}" 2>/dev/null || true
               echo '```'
             else
               echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]
+          # Red only for harness/environment breakdowns; case failures stay green.
+          [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
         if: ${{ always() && steps.evidence.outcome == 'success' }}
@@ -231,65 +257,55 @@ jobs:
           fi
           rm -f "${B64_FILE}"
 
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+      - name: Manage backlog issues (dedup / label / auto-close)
+        # Replaces the old per-run failure filing. One entry point that:
+        # - dedups by signal: failing cases are matched against open backlog
+        #   issues by label (category + case ID); covered cases become a
+        #   comment on the existing issue, only uncovered cases file a new one
+        # - labels new issues (functional-test, category, case IDs, env)
+        # - closes fixed issues after a fully green run
+        # - never files or closes on cancelled runs
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'kms'
           SUITE_LABEL: 'KMS'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
             exit 0
           fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          echo "filed backlog issue for suite ${SUITE}"
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.test.outcome }}" \
+            --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" \
+            --report-file "${REPORT_FILE}" \
+            --log "${LOG_FILE}" \
+            --run-url "${RUN_URL}/attempts/${GITHUB_RUN_ATTEMPT}#summary" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
 
       - name: Upload report and logs
         if: ${{ always() && steps.evidence.outcome == 'success' }}
@@ -375,3 +391,24 @@ jobs:
         run: |
           echo "RustFS KMS suite failed"
           echo "See the uploaded report and log artifacts for details."
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite kms
+          --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          --output "${RUNNER_TEMP}/chain-kms-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/kms.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-kms-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-kms-${{ github.run_id }}-${{ github.run_attempt }}/kms.json
+          if-no-files-found: error

.github/workflows/rustfs-performance-test.yml

@@ -1,6 +1,12 @@
 name: RustFS Performance Test
 
 on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       package_url:
@@ -58,7 +64,7 @@ permissions:
 # The default performance nodes overlap the other suites' remote VMs, even
 # though the runner differs. Hold the shared lock through cleanup as well.
 concurrency:
-  group: rustfs-shared-functional-tests
+  group: rustfs-shared-functional-tests-v2
   cancel-in-progress: false
 
 defaults:
@@ -79,7 +85,21 @@ env:
   PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
 
 jobs:
+  runner-check:
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Check performance runner before entering its queue
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: python3 scripts/check_functional_runners.py pf-testing
+
   performance-test:
+    needs: runner-check
     runs-on: pf-testing
     timeout-minutes: 900
     env:
@@ -90,8 +110,13 @@ jobs:
       RUSTFS_WARP_CONCURRENCY: ${{ inputs.warp_concurrency || '64' }}
     # Run on manual dispatch, or when the nightly build completed successfully.
     # Skipped when nightly failed.
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout chain tooling
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
       - name: Initialize functional evidence
         id: evidence
         run: |
@@ -108,25 +133,21 @@ jobs:
             printf 'VERSION_FILE=%s/version.txt\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
           } >> "${GITHUB_ENV}"
 
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
 
       - name: Show environment
         run: |
@@ -192,6 +213,7 @@ jobs:
           } > "${VERSION_FILE}"
 
       - name: Upload report to dashboard (reports/YYYY-MM-DD.md)
+        id: chain_report
         if: ${{ steps.benchmark.conclusion == 'success' }}
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
@@ -237,65 +259,48 @@ jobs:
             echo "created ${REPORT_PATH} in rustfs/dashboard"
           fi
 
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.benchmark.outcome == 'failure' || steps.benchmark.outcome == 'cancelled') }}
+      - name: Manage backlog issues (dedup / label / auto-close)
+        # Signal-based lifecycle: dedups against open issues by label,
+        # labels new issues (functional-test, category, case IDs, env),
+        # closes fixed issues after a fully green run. Never acts on
+        # cancelled runs. Logic lives in auto-testing/scripts/.
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'performance'
           SUITE_LABEL: 'Performance'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
             exit 0
           fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          PACKAGE_URL='${{ inputs.package_url }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          echo "filed backlog issue for suite ${SUITE}"
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.benchmark.outcome }}" \
+            --report-file "${REPORT_FILE}" \
+            --log "${LOG_FILE}" \
+            --run-url "${RUN_URL}/attempts/${GITHUB_RUN_ATTEMPT}#summary" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
 
       - name: Upload test logs & results
         if: ${{ always() && steps.evidence.outcome == 'success' }}
@@ -325,3 +330,24 @@ jobs:
           echo "RustFS performance test failed"
           echo "Package source: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
           echo "See the uploaded log artifact for details."
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.benchmark.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite performance
+          --report "${RUSTFS_RESULT_DIR}/summary.tsv"
+          --output "${RUNNER_TEMP}/chain-performance-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/performance.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-performance-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-performance-${{ github.run_id }}-${{ github.run_attempt }}/performance.json
+          if-no-files-found: error

.github/workflows/rustfs-pool-expand-test.yml

@@ -1,6 +1,12 @@
 name: RustFS Pool Expansion Test
 
 on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       rustfs_version:
@@ -52,7 +58,7 @@ permissions:
 # environment (vm000/vm001/vm002), so concurrent runs must not clobber each
 # other.
 concurrency:
-  group: rustfs-shared-functional-tests
+  group: rustfs-shared-functional-tests-v2
   cancel-in-progress: false
 
 defaults:
@@ -63,7 +69,7 @@ env:
   RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
   RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
   RUSTFS_API_ENDPOINT: ${{ secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT }}
-  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
+  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES || 'vm000 vm001 vm002' }}
   RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
   PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
   # Package used by the nightly run (workflow_dispatch inputs are empty for
@@ -77,35 +83,37 @@ jobs:
     name: Pool expansion / decommission test
     runs-on: smoke-testing
     timeout-minutes: 360
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     env:
       RUSTFS_POOL_ADMIN_ENDPOINT: ${{ secrets.RUSTFS_POOL_ADMIN_ENDPOINT || vars.RUSTFS_POOL_ADMIN_ENDPOINT || 'http://rustfs-node1:9000' }}
       RUSTFS_POOL_PROXY_ENDPOINT: http://127.0.0.1:19000
       RUSTFS_POOL_WARP_ENDPOINT: http://127.0.0.1:19000
       RUSTFS_SHARED_PROXY_ENDPOINT: ${{ secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT }}
-      RUSTFS_POOL_NODE_ENDPOINTS: ${{ secrets.RUSTFS_POOL_NODE_ENDPOINTS || vars.RUSTFS_POOL_NODE_ENDPOINTS || 'http://rustfs-node1:9000 http://rustfs-node2:9000 http://rustfs-node3:9000' }}
+      RUSTFS_POOL_NODE_ENDPOINTS: ${{ secrets.RUSTFS_POOL_NODE_ENDPOINTS || vars.RUSTFS_POOL_NODE_ENDPOINTS || 'http://rustfs-node1:9000 http://rustfs-node2:9000 http://rustfs-node3:9000 http://rustfs-node4:9000' }}
     steps:
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
+      - name: Checkout chain tooling
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
 
       - name: Initialize pool test artifacts
+        id: evidence
         run: |
           set -euo pipefail
           ARTIFACT_DIR="${RUNNER_TEMP}/rustfs-pool-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
@@ -120,6 +128,17 @@ jobs:
           warp --version || true
           df -h /data | tail -1
 
+      - name: Validate pool topology before destructive cleanup
+        id: topology
+        run: |
+          set -euo pipefail
+          read -r -a NODES <<< "${RUSTFS_NODES}"
+          read -r -a ENDPOINTS <<< "${RUSTFS_POOL_NODE_ENDPOINTS}"
+          if [ "${#NODES[@]}" -lt 3 ] || [ "${#ENDPOINTS[@]}" -lt "${#NODES[@]}" ]; then
+            echo "Pool tests require at least three nodes and a direct endpoint for every node" >&2
+            exit 1
+          fi
+
       - name: Cleanup environment (before)
         if: ${{ inputs.cleanup_before != 'false' }}
         run: |
@@ -227,6 +246,9 @@ jobs:
 
       - name: Run pool expansion & decommission test
         id: pool_test
+        # Case failures keep the run green: the report and the backlog
+        # issue manager carry the product signal.
+        continue-on-error: true
         run: |
           set -o pipefail
           STEPS="4,5,6"
@@ -363,6 +385,7 @@ jobs:
           fi
 
       - name: Generate report
+        id: chain_report
         if: always()
         run: |
           set -euo pipefail
@@ -471,7 +494,11 @@ jobs:
           }
 
           require_nonempty "${POOL_ARTIFACT_DIR}/pool-test.log"
-          require_nonempty "${POOL_ARTIFACT_DIR}/warp.log"
+          # warp is stopped early (SIGINT/SIGTERM) once the storage threshold
+          # is reached, and it only writes its final report on a clean exit —
+          # an empty warp.log is the expected shape of a healthy run, so only
+          # require that the redirect target exists.
+          require_available "${POOL_ARTIFACT_DIR}/warp.log"
           require_nonempty "${POOL_ARTIFACT_DIR}/pool-report.md"
           require_nonempty "${POOL_ARTIFACT_DIR}/pool-baseline.log"
           require_nonempty "${POOL_ARTIFACT_DIR}/nginx-config-redacted.txt"
@@ -522,6 +549,16 @@ jobs:
             echo "dedicated proxy error log contains an upstream timeout or connection failure" >&2
             failed=1
           fi
+          # A mid-script abort is a product/harness signal: the run step's
+          # continue-on-error would otherwise keep the workflow green, so the
+          # validator must turn it red. Aborts go through die(), which prints
+          # an [ERROR] line; a step that dies never emits its [POOL-STEP]
+          # verdict, so "any FAIL verdict" cannot be the trigger.
+          if grep -q '^\[POOL-STEP\] .* FAIL' "${POOL_ARTIFACT_DIR}/pool-test.log" 2>/dev/null \
+            || grep -q '\[ERROR\]' "${POOL_ARTIFACT_DIR}/pool-test.log" 2>/dev/null; then
+            echo "pool test script reported a failure (step FAIL verdict or die() abort)" >&2
+            failed=1
+          fi
           [ "${failed}" -eq 0 ] || exit 1
 
       - name: Upload functional report to dashboard
@@ -556,64 +593,54 @@ jobs:
           fi
           rm -f "${B64_FILE}"
 
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.pool_test.outcome == 'failure' || steps.pool_test.outcome == 'cancelled') }}
+      - name: Manage backlog issues (dedup / label / auto-close)
+        # Replaces the old per-run failure filing. One entry point that:
+        # - dedups by signal: failing cases are matched against open backlog
+        #   issues by label (category + case ID); covered cases become a
+        #   comment on the existing issue, only uncovered cases file a new one
+        # - labels new issues (functional-test, category, case IDs, env)
+        # - closes fixed issues after a fully green run
+        # - never files or closes on cancelled runs
+        if: ${{ always() }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
           SUITE: 'pool'
           SUITE_LABEL: 'Pool expansion'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '${{ env.POOL_ARTIFACT_DIR }}/pool-report.md'
-          LOG_FILE: '${{ env.POOL_ARTIFACT_DIR }}/pool-test.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
             exit 0
           fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          echo "filed backlog issue for suite ${SUITE}"
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.pool_test.outcome }}" \
+            --report-file "${POOL_ARTIFACT_DIR}/pool-report.md" \
+            --log "${POOL_ARTIFACT_DIR}/pool-test.log" \
+            --run-url "${RUN_URL}/attempts/${GITHUB_RUN_ATTEMPT}#summary" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
 
       - name: Upload test logs
         if: always()
@@ -634,7 +661,7 @@ jobs:
             ./auto-testing/rustfs_pool_nginx_stage.sh cleanup
 
       - name: Cleanup environment (after)
-        if: ${{ always() && inputs.cleanup_after != 'false' }}
+        if: ${{ always() && steps.topology.outcome == 'success' && inputs.cleanup_after != 'false' }}
         run: |
           set -euo pipefail
           read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
@@ -707,3 +734,24 @@ jobs:
           echo "RustFS pool expansion test failed"
           echo "Package source: ${{ inputs.package_url || inputs.rustfs_version || 'nightly (R2 latest)' }}"
           echo "See the uploaded log artifact for details."
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.pool_test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite pool
+          --report "${POOL_ARTIFACT_DIR}/pool-steps.md"
+          --output "${RUNNER_TEMP}/chain-pool-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/pool.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-pool-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-pool-${{ github.run_id }}-${{ github.run_attempt }}/pool.json
+          if-no-files-found: error

.github/workflows/rustfs-replication-test.yml

@@ -15,12 +15,18 @@
 name: RustFS Replication Test
 
 on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       rustfs_version:
-        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
+        description: 'RustFS release tag to test. Leave empty for nightly.'
         required: false
-        default: '1.0.0-rc.4-preview.1'
+        default: ''
       package_url:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false
@@ -43,7 +49,7 @@ permissions:
 # The replication suite uses the same shared VMs as the other functional
 # tests, so it must serialize with them instead of running in parallel.
 concurrency:
-  group: rustfs-shared-functional-tests
+  group: rustfs-shared-functional-tests-v2
   cancel-in-progress: false
 
 defaults:
@@ -62,7 +68,7 @@ jobs:
   replication-test:
     runs-on: smoke-testing
     timeout-minutes: 360
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -83,25 +89,21 @@ jobs:
             printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
           } >> "${GITHUB_ENV}"
 
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
 
       - name: Show environment
         run: |
@@ -131,6 +133,9 @@ jobs:
 
       - name: Run replication suite
         id: test
+        # Case failures keep the run green: the report and the backlog
+        # issue manager carry the product signal.
+        continue-on-error: true
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-replication-test.sh
@@ -153,6 +158,7 @@ jobs:
           ./auto-testing/rustfs-replication-test.sh "${ARGS[@]}"
 
       - name: Generate report
+        id: chain_report
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
@@ -182,6 +188,21 @@ jobs:
           if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
             RESULT=success
           fi
+
+          # Product gate: failing cases keep the run green — they are reported
+          # below and tracked in rustfs/backlog. Only harness/environment
+          # breakdowns (the suite never reached case level, or a wholesale
+          # failure with zero passes) turn the workflow red.
+          CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
+          CASES_PASS=$(( CASES_TOTAL - CASES_FAIL ))
+          HARNESS_OK=0
+          if [ '${{ steps.test.outcome }}' = 'success' ]; then
+            HARNESS_OK=1
+          elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${CASES_TOTAL}" -gt 0 ] && [ "${CASES_PASS}" -ge 1 ]; then
+            HARNESS_OK=1
+          fi
           {
             echo "# RustFS replication test report"
             echo ""
@@ -194,19 +215,24 @@ jobs:
             echo "- Test Step Outcome: ${RESULT}"
             echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            if [ "${RESULT}" = "success" ]; then
+            if [ -s "${CASE_TABLE}" ]; then
               cat "${CASE_TABLE}"
               echo ""
+            fi
+            echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)"
+            echo ""
+            if [ -s "${CASE_TABLE}" ]; then
               echo "## Log tail"
               echo '```text'
-              tail -n 200 "${LOG_FILE}"
+              tail -n 200 "${LOG_FILE}" 2>/dev/null || true
               echo '```'
             else
               echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]
+          # Red only for harness/environment breakdowns; case failures stay green.
+          [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
         if: ${{ always() && steps.evidence.outcome == 'success' }}
@@ -239,65 +265,55 @@ jobs:
           fi
           rm -f "${B64_FILE}"
 
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+      - name: Manage backlog issues (dedup / label / auto-close)
+        # Replaces the old per-run failure filing. One entry point that:
+        # - dedups by signal: failing cases are matched against open backlog
+        #   issues by label (category + case ID); covered cases become a
+        #   comment on the existing issue, only uncovered cases file a new one
+        # - labels new issues (functional-test, category, case IDs, env)
+        # - closes fixed issues after a fully green run
+        # - never files or closes on cancelled runs
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'replication'
           SUITE_LABEL: 'Replication (bucket + site)'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
             exit 0
           fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          echo "filed backlog issue for suite ${SUITE}"
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.test.outcome }}" \
+            --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" \
+            --report-file "${REPORT_FILE}" \
+            --log "${LOG_FILE}" \
+            --run-url "${RUN_URL}/attempts/${GITHUB_RUN_ATTEMPT}#summary" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
 
       - name: Upload report and logs
         if: ${{ always() && steps.evidence.outcome == 'success' }}
@@ -329,7 +345,7 @@ jobs:
             '
           done
 
-      - name: "Continue functional chain (next: Performance)"
+      - name: "Continue functional chain (next: Fault tolerance)"
         if: ${{ always() && github.event_name == 'repository_dispatch' }}
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
@@ -342,9 +358,9 @@ jobs:
           DISPATCHED=0
           for attempt in 1 2 3; do
             if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-performance' \
+              -f event_type='rustfs-chain-fault-tolerance' \
               -F 'client_payload[from_suite]=replication'; then
-              echo "dispatched next suite Performance (attempt ${attempt})"
+              echo "dispatched next suite Fault tolerance (attempt ${attempt})"
               DISPATCHED=1
               break
             fi
@@ -352,19 +368,19 @@ jobs:
             sleep "${attempt}0"
           done
           if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch Performance after 3 attempts" >&2
+            echo "ERROR: functional chain stalled: could not dispatch Fault tolerance after 3 attempts" >&2
             TITLE="[functional][chain] stalled after replication (run ${GITHUB_RUN_ID})"
             BODY_FILE="$(mktemp)"
             trap 'rm -f "${BODY_FILE}"' EXIT
             {
-              echo "The functional chain could not hand off from **replication** to **Performance** after 3 attempts."
+              echo "The functional chain could not hand off from **replication** to **Fault tolerance** after 3 attempts."
               echo ""
               echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-performance'"
+              echo "- Expected next event: 'rustfs-chain-fault-tolerance'"
               echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
               echo "- Recovery: re-dispatch manually with"
               FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-performance'"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-fault-tolerance'"
               FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
             } > "${BODY_FILE}"
             gh issue create -R rustfs/backlog --title "${TITLE}" \
@@ -380,3 +396,24 @@ jobs:
           echo "RustFS replication suite failed"
           echo "Package source: ${{ inputs.package_url || inputs.rustfs_version || 'nightly (R2 latest)' }}"
           echo "See the uploaded report and log artifacts for details."
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite replication
+          --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          --output "${RUNNER_TEMP}/chain-replication-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/replication.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-replication-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-replication-${{ github.run_id }}-${{ github.run_attempt }}/replication.json
+          if-no-files-found: error

.github/workflows/rustfs-s3-compat-test.yml

@@ -1,12 +1,18 @@
 name: RustFS S3 Compatibility Test
 
 on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       rustfs_version:
-        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
+        description: 'RustFS release tag to test. Leave empty for nightly.'
         required: false
-        default: '1.0.0-rc.4-preview.1'
+        default: ''
       package_url:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false
@@ -19,7 +25,7 @@ permissions:
   contents: read
 
 concurrency:
-  group: rustfs-shared-functional-tests
+  group: rustfs-shared-functional-tests-v2
   cancel-in-progress: false
 
 defaults:
@@ -38,7 +44,7 @@ jobs:
   s3-compat-test:
     runs-on: smoke-testing
     timeout-minutes: 360
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -59,25 +65,21 @@ jobs:
             printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
           } >> "${GITHUB_ENV}"
 
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
 
       - name: Show environment
         run: |
@@ -106,6 +108,9 @@ jobs:
 
       - name: Run S3 compatibility suite
         id: test
+        # Case failures keep the run green: the report and the backlog
+        # issue manager carry the product signal.
+        continue-on-error: true
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-s3-compat-test.sh
@@ -122,6 +127,7 @@ jobs:
           ./auto-testing/rustfs-s3-compat-test.sh "${ARGS[@]}"
 
       - name: Generate report
+        id: chain_report
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
@@ -151,6 +157,21 @@ jobs:
           if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
             RESULT=success
           fi
+
+          # Product gate: failing cases keep the run green — they are reported
+          # below and tracked in rustfs/backlog. Only harness/environment
+          # breakdowns (the suite never reached case level, or a wholesale
+          # failure with zero passes) turn the workflow red.
+          CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
+          CASES_PASS=$(( CASES_TOTAL - CASES_FAIL ))
+          HARNESS_OK=0
+          if [ '${{ steps.test.outcome }}' = 'success' ]; then
+            HARNESS_OK=1
+          elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${CASES_TOTAL}" -gt 0 ] && [ "${CASES_PASS}" -ge 1 ]; then
+            HARNESS_OK=1
+          fi
           {
             echo "# RustFS S3 compatibility test report"
             echo ""
@@ -163,19 +184,24 @@ jobs:
             echo "- Test Step Outcome: ${RESULT}"
             echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            if [ "${RESULT}" = "success" ]; then
+            if [ -s "${CASE_TABLE}" ]; then
               cat "${CASE_TABLE}"
               echo ""
+            fi
+            echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)"
+            echo ""
+            if [ -s "${CASE_TABLE}" ]; then
               echo "## Log tail"
               echo '```text'
-              tail -n 200 "${LOG_FILE}"
+              tail -n 200 "${LOG_FILE}" 2>/dev/null || true
               echo '```'
             else
               echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]
+          # Red only for harness/environment breakdowns; case failures stay green.
+          [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
         if: ${{ always() && steps.evidence.outcome == 'success' }}
@@ -208,65 +234,55 @@ jobs:
           fi
           rm -f "${B64_FILE}"
 
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+      - name: Manage backlog issues (dedup / label / auto-close)
+        # Replaces the old per-run failure filing. One entry point that:
+        # - dedups by signal: failing cases are matched against open backlog
+        #   issues by label (category + case ID); covered cases become a
+        #   comment on the existing issue, only uncovered cases file a new one
+        # - labels new issues (functional-test, category, case IDs, env)
+        # - closes fixed issues after a fully green run
+        # - never files or closes on cancelled runs
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 's3'
           SUITE_LABEL: 'S3 compatibility'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
             exit 0
           fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          echo "filed backlog issue for suite ${SUITE}"
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.test.outcome }}" \
+            --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" \
+            --report-file "${REPORT_FILE}" \
+            --log "${LOG_FILE}" \
+            --run-url "${RUN_URL}/attempts/${GITHUB_RUN_ATTEMPT}#summary" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
 
       - name: Upload report and logs
         if: ${{ always() && steps.evidence.outcome == 'success' }}
@@ -352,3 +368,24 @@ jobs:
         run: |
           echo "RustFS S3 compatibility suite failed"
           echo "See the uploaded report and log artifacts for details."
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite s3
+          --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          --output "${RUNNER_TEMP}/chain-s3-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/s3.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-s3-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-s3-${{ github.run_id }}-${{ github.run_attempt }}/s3.json
+          if-no-files-found: error

.github/workflows/rustfs-security-test.yml

@@ -15,12 +15,18 @@
 name: RustFS Security Test
 
 on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       rustfs_version:
-        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
+        description: 'RustFS release tag to test. Leave empty for nightly.'
         required: false
-        default: '1.0.0-rc.4-preview.1'
+        default: ''
       package_url:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false
@@ -56,7 +62,7 @@ permissions:
 # The security suite uses the same shared VMs as the other functional tests,
 # so it must serialize with them instead of running in parallel.
 concurrency:
-  group: rustfs-shared-functional-tests
+  group: rustfs-shared-functional-tests-v2
   cancel-in-progress: false
 
 defaults:
@@ -75,7 +81,7 @@ jobs:
   security-test:
     runs-on: smoke-testing
     timeout-minutes: 360
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       # Checkout the repository into its own subdirectory. Checking out at
       # the workspace root would wipe the auto-testing clone above (that is
@@ -95,25 +101,21 @@ jobs:
           mkdir -- "${SECURITY_ARTIFACTS_DIR}" "${SECURITY_ARTIFACTS_DIR}-scratch"
           printf 'SECURITY_ARTIFACTS_DIR=%s\n' "${SECURITY_ARTIFACTS_DIR}" >> "${GITHUB_ENV}"
 
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
 
       - name: Show environment
         run: |
@@ -145,6 +147,8 @@ jobs:
 
       - name: Run security suite
         id: test
+        # Case failures keep the run green: the report and the backlog
+        # issue manager carry the product signal.
         continue-on-error: true
         env:
           REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
@@ -181,10 +185,31 @@ jobs:
           TEST_OUTCOME: ${{ steps.test.outcome }}
         run: |
           set -euo pipefail
+          LOG_FILE="${SECURITY_ARTIFACTS_DIR}/suite.log"
           RESULT=failure
           if [ "${TEST_OUTCOME}" = "success" ] && [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then
             RESULT=success
           fi
+
+          # Product gate: failing cases keep the run green — they are reported
+          # below and tracked in rustfs/backlog. Only harness/environment
+          # breakdowns turn the workflow red. The suite log lines carry ANSI
+          # color escapes both before and after the verdict tag
+          # (e.g. "\e[1;31m[FAIL]\e[0m STS-105 ..."), so the pattern allows
+          # any number of escapes on each side of it.
+          ESC=$'\033'
+          VERDICTS_TOTAL="$(grep -cE "^(${ESC}\[[0-9;]*m)*\[(PASS|FAIL|UNSUPPORTED)\](${ESC}\[[0-9;]*m)* [A-Z]" "${LOG_FILE}" 2>/dev/null || true)"
+          VERDICTS_FAIL="$(grep -cE "^(${ESC}\[[0-9;]*m)*\[FAIL\](${ESC}\[[0-9;]*m)* [A-Z]" "${LOG_FILE}" 2>/dev/null || true)"
+          VERDICTS_TOTAL=$(( ${VERDICTS_TOTAL:-0} + 0 )); VERDICTS_FAIL=$(( ${VERDICTS_FAIL:-0} + 0 ))
+          VERDICTS_PASS=$(( VERDICTS_TOTAL - VERDICTS_FAIL ))
+          VERDICTS_SKIPPED="$(grep -cE "^(${ESC}\[[0-9;]*m)*\[SKIP\](${ESC}\[[0-9;]*m)* [A-Z]" "${LOG_FILE}" 2>/dev/null || true)"
+          VERDICTS_SKIPPED=$(( ${VERDICTS_SKIPPED:-0} + 0 ))
+          HARNESS_OK=0
+          if [ "${TEST_OUTCOME}" = "success" ]; then
+            HARNESS_OK=1
+          elif [ "${TEST_OUTCOME}" = "failure" ] && [ "${VERDICTS_TOTAL}" -gt 0 ] && [ "${VERDICTS_PASS}" -ge 1 ]; then
+            HARNESS_OK=1
+          fi
           {
             echo "# RustFS security test report"
             echo ""
@@ -197,16 +222,19 @@ jobs:
             echo ""
             # The dashboard prioritizes case rows over the step outcome.
             # Keep partial case results in the artifact when the suite fails.
-            if [ "${RESULT}" = "success" ]; then
+            if [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then
               cat "${SECURITY_ARTIFACTS_DIR}/suite-report.md"
-            elif [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then
-              echo "The suite did not complete successfully. See suite-report.md in this run's artifact for diagnostics."
+              echo ""
+            fi
+            if [ "${VERDICTS_SKIPPED}" -gt 0 ]; then
+              echo "- Product result: ${VERDICTS_PASS} passed, ${VERDICTS_FAIL} failed, ${VERDICTS_SKIPPED} skipped (failing cases are tracked in rustfs/backlog)"
             else
-              echo "The suite did not produce a non-empty report."
+              echo "- Product result: ${VERDICTS_PASS} passed, ${VERDICTS_FAIL} failed (failing cases are tracked in rustfs/backlog)"
             fi
           } > "${SECURITY_ARTIFACTS_DIR}/report.md"
           cat "${SECURITY_ARTIFACTS_DIR}/report.md" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]
+          # Red only for harness/environment breakdowns; case failures stay green.
+          [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
         if: ${{ always() && steps.evidence.outcome == 'success' }}
@@ -240,65 +268,54 @@ jobs:
           fi
           rm -f "${B64_FILE}"
 
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+      - name: Manage backlog issues (dedup / label / auto-close)
+        # Replaces the old per-run failure filing. One entry point that:
+        # - dedups by signal: failing cases are matched against open backlog
+        #   issues by label (category + case ID); covered cases become a
+        #   comment on the existing issue, only uncovered cases file a new one
+        # - labels new issues (functional-test, category, case IDs, env)
+        # - closes fixed issues after a fully green run
+        # - never files or closes on cancelled runs
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
           SUITE: 'security'
           SUITE_LABEL: 'Security'
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/report.md
-          LOG_FILE: ''
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
             exit 0
           fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          echo "filed backlog issue for suite ${SUITE}"
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.test.outcome }}" \
+            --report-file "${SECURITY_ARTIFACTS_DIR}/report.md" \
+            --log "${SECURITY_ARTIFACTS_DIR}/suite.log" \
+            --run-url "${RUN_URL}/attempts/${GITHUB_RUN_ATTEMPT}#summary" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
 
       - name: Upload report and logs
         if: ${{ always() && steps.evidence.outcome == 'success' }}
@@ -355,3 +372,24 @@ jobs:
           echo "RustFS security test failed"
           echo "Package source: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
           echo "See the uploaded report and logs for details."
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite security
+          --report "${SECURITY_ARTIFACTS_DIR}/suite-report.md"
+          --output "${RUNNER_TEMP}/chain-security-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/security.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-security-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-security-${{ github.run_id }}-${{ github.run_attempt }}/security.json
+          if-no-files-found: error

.github/workflows/rustfs-storage-test.yml

@@ -1,12 +1,18 @@
 name: RustFS Storage Engine Test
 
 on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       rustfs_version:
-        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
+        description: 'RustFS release tag to test. Leave empty for nightly.'
         required: false
-        default: '1.0.0-rc.4-preview.1'
+        default: ''
       package_url:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false
@@ -28,7 +34,7 @@ permissions:
   contents: read
 
 concurrency:
-  group: rustfs-shared-functional-tests
+  group: rustfs-shared-functional-tests-v2
   cancel-in-progress: false
 
 defaults:
@@ -47,7 +53,7 @@ jobs:
   storage-test:
     runs-on: smoke-testing
     timeout-minutes: 360
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -68,25 +74,21 @@ jobs:
             printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
           } >> "${GITHUB_ENV}"
 
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
 
       - name: Show environment
         run: |
@@ -115,6 +117,9 @@ jobs:
 
       - name: Run storage engine suite
         id: test
+        # Case failures keep the run green: the report and the backlog
+        # issue manager carry the product signal.
+        continue-on-error: true
         run: |
           set -euo pipefail
           chmod +x auto-testing/rustfs-storage-test.sh
@@ -137,6 +142,7 @@ jobs:
           ./auto-testing/rustfs-storage-test.sh "${ARGS[@]}"
 
       - name: Generate report
+        id: chain_report
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
@@ -166,6 +172,21 @@ jobs:
           if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
             RESULT=success
           fi
+
+          # Product gate: failing cases keep the run green — they are reported
+          # below and tracked in rustfs/backlog. Only harness/environment
+          # breakdowns (the suite never reached case level, or a wholesale
+          # failure with zero passes) turn the workflow red.
+          CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
+          CASES_PASS=$(( CASES_TOTAL - CASES_FAIL ))
+          HARNESS_OK=0
+          if [ '${{ steps.test.outcome }}' = 'success' ]; then
+            HARNESS_OK=1
+          elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${CASES_TOTAL}" -gt 0 ] && [ "${CASES_PASS}" -ge 1 ]; then
+            HARNESS_OK=1
+          fi
           {
             echo "# RustFS storage engine test report"
             echo ""
@@ -178,19 +199,24 @@ jobs:
             echo "- Test Step Outcome: ${RESULT}"
             echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            if [ "${RESULT}" = "success" ]; then
+            if [ -s "${CASE_TABLE}" ]; then
               cat "${CASE_TABLE}"
               echo ""
+            fi
+            echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)"
+            echo ""
+            if [ -s "${CASE_TABLE}" ]; then
               echo "## Log tail"
               echo '```text'
-              tail -n 200 "${LOG_FILE}"
+              tail -n 200 "${LOG_FILE}" 2>/dev/null || true
               echo '```'
             else
               echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]
+          # Red only for harness/environment breakdowns; case failures stay green.
+          [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
         if: ${{ always() && steps.evidence.outcome == 'success' }}
@@ -223,65 +249,55 @@ jobs:
           fi
           rm -f "${B64_FILE}"
 
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+      - name: Manage backlog issues (dedup / label / auto-close)
+        # Replaces the old per-run failure filing. One entry point that:
+        # - dedups by signal: failing cases are matched against open backlog
+        #   issues by label (category + case ID); covered cases become a
+        #   comment on the existing issue, only uncovered cases file a new one
+        # - labels new issues (functional-test, category, case IDs, env)
+        # - closes fixed issues after a fully green run
+        # - never files or closes on cancelled runs
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'storage'
           SUITE_LABEL: 'Storage engine'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
             exit 0
           fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          echo "filed backlog issue for suite ${SUITE}"
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.test.outcome }}" \
+            --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" \
+            --report-file "${REPORT_FILE}" \
+            --log "${LOG_FILE}" \
+            --run-url "${RUN_URL}/attempts/${GITHUB_RUN_ATTEMPT}#summary" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
 
       - name: Upload report and logs
         if: ${{ always() && steps.evidence.outcome == 'success' }}
@@ -367,3 +383,24 @@ jobs:
         run: |
           echo "RustFS storage engine suite failed"
           echo "See the uploaded report and log artifacts for details."
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite storage
+          --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          --output "${RUNNER_TEMP}/chain-storage-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/storage.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-storage-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-storage-${{ github.run_id }}-${{ github.run_attempt }}/storage.json
+          if-no-files-found: error

.github/workflows/rustfs-table-test.yml

@@ -0,0 +1,352 @@
+name: RustFS Table Test
+
+on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
+  workflow_dispatch:
+    inputs:
+      package_url:
+        description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
+        required: false
+        type: string
+      rustfs_version:
+        description: 'RustFS release tag to test (leave empty to use the latest nightly deb)'
+        required: false
+        type: string
+      skip_smoke:
+        description: 'Skip the PyIceberg end-to-end case (TBL-101)'
+        required: false
+        type: boolean
+        default: false
+
+concurrency:
+  group: rustfs-shared-functional-tests-v2
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+env:
+  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES || 'vm000 vm001 vm002 vm003' }}
+  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER || 'azureuser' }}
+  RUSTFS_API_ENDPOINT: ${{ secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || 'http://rustfs-node1:9000' }}
+  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
+  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
+  RUSTFS_NIGHTLY_PACKAGE_URL: https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb
+  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+jobs:
+  table-test:
+    name: table-test
+    runs-on: smoke-testing
+    timeout-minutes: 90
+    steps:
+      - name: Checkout repository (for report parser)
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-table-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
+
+      - name: Show environment
+        run: |
+          set -euo pipefail
+          echo "nodes: ${RUSTFS_NODES}"
+          echo "endpoint: ${RUSTFS_API_ENDPOINT}"
+          aws --version
+          python3 --version
+
+      - name: Cleanup environment (before)
+        continue-on-error: true
+        run: |
+          set -uo pipefail
+          for node in ${RUSTFS_NODES}; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
+            ' || echo "pre-cleanup failed on ${node} (continuing)"
+          done
+
+      - name: Stage verified candidate package
+        id: chain_package
+        if: ${{ inputs.chain_manifest != '' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/prepare_functional_package.py prepare
+
+      - name: Run table suite
+        id: test
+        # Standalone case failures are reported by the backlog manager.
+        # Chain evidence separately requires complete passing cases.
+        continue-on-error: true
+        run: |
+          set -euo pipefail
+          chmod +x auto-testing/rustfs-table-test.sh
+          PACKAGE_URL='${{ steps.chain_package.outputs.package_url || inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          SMOKE_PATH="${GITHUB_WORKSPACE:-$PWD}/scripts/table-catalog/pyiceberg_smoke.py"
+          ARGS=(-y --smoke-script "${SMOKE_PATH}")
+          if [ '${{ inputs.skip_smoke }}' = 'true' ]; then
+            ARGS+=(--skip-smoke)
+          fi
+          if [ -n "${PACKAGE_URL}" ]; then
+            ARGS+=(--package-url "${PACKAGE_URL}")
+          elif [ -n "${RUSTFS_VERSION}" ] && [ "${RUSTFS_VERSION}" != "null" ]; then
+            ARGS+=(--package-url "https://github.com/rustfs/rustfs/releases/download/${RUSTFS_VERSION}/rustfs_${RUSTFS_VERSION}_amd64.deb")
+          else
+            ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
+          fi
+          ./auto-testing/rustfs-table-test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
+
+      - name: Generate report
+        id: chain_report
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        run: |
+          set -euo pipefail
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ] && [ "${RUSTFS_VERSION}" != "null" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
+          fi
+          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          CASE_RESULT=success
+          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
+          RESULT=failure
+          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+            RESULT=success
+          fi
+
+          # Product gate: failing cases keep the run green — they are reported
+          # below and tracked in rustfs/backlog. Only harness/environment
+          # breakdowns (the suite never reached case level, or a wholesale
+          # failure with zero passes) turn the workflow red.
+          CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
+          CASES_PASS=$(( CASES_TOTAL - CASES_FAIL ))
+          HARNESS_OK=0
+          if [ '${{ steps.test.outcome }}' = 'success' ]; then
+            HARNESS_OK=1
+          elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${CASES_TOTAL}" -gt 0 ] && [ "${CASES_PASS}" -ge 1 ]; then
+            HARNESS_OK=1
+          fi
+          {
+            echo "# RustFS S3 Tables (Iceberg REST Catalog) test report"
+            echo ""
+            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+            echo "- Workflow Commit: ${GITHUB_SHA}"
+            echo "- Trigger: ${{ github.event_name }}"
+            echo "- Package: ${PACKAGE_SOURCE}"
+            echo "- Test Step Outcome: ${RESULT}"
+            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
+            echo ""
+            if [ -s "${CASE_TABLE}" ]; then
+              cat "${CASE_TABLE}"
+              echo ""
+            fi
+            echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)"
+            echo ""
+            if [ -s "${CASE_TABLE}" ]; then
+              echo "## Log tail"
+              echo '```text'
+              tail -n 200 "${LOG_FILE}" 2>/dev/null || true
+              echo '```'
+            else
+              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+            fi
+          } | tee "${REPORT_FILE}"
+          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+          # Red only for harness/environment breakdowns; case failures stay green.
+          [ "${HARNESS_OK}" = "1" ]
+
+      - name: Upload functional report to dashboard
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: ${{ env.REPORT_FILE }}
+          SUITE: table
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
+            exit 0
+          fi
+          DATE="$(date -u +%Y-%m-%d)"
+          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
+          B64_FILE="$(mktemp)"
+          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
+          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${SHA}" ]; then
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          else
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+              '{message:$msg, content:($content|rtrimstr("\n"))}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          fi
+          echo "dashboard updated: ${REPORT_PATH}"
+
+      - name: Manage backlog issues (dedup / label / auto-close)
+        # Replaces the old per-run failure filing. One entry point that:
+        # - dedups by signal: failing cases are matched against open backlog
+        #   issues by label (category + case ID); covered cases become a
+        #   comment on the existing issue, only uncovered cases file a new one
+        # - labels new issues (functional-test, category, case IDs, env)
+        # - closes fixed issues after a fully green run
+        # - never files or closes on cancelled runs
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
+            exit 0
+          fi
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found; skipping backlog issue management"
+            exit 0
+          fi
+          PACKAGE_URL='${{ inputs.package_url }}'
+          PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
+          if [ -n "${PACKAGE_URL}" ]; then PACKAGE_SOURCE="${PACKAGE_URL}"; fi
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --outcome "${{ steps.test.outcome }}" \
+            --suite table --category table --suite-label "S3 Tables" \
+            --outcome '${{ steps.test.outcome }}' \
+            --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" \
+            --run-url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}#summary" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${GITHUB_EVENT_NAME}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --report-file "${REPORT_FILE}" \
+            --log "${LOG_FILE}" \
+            --date "$(date -u +%Y-%m-%d)"
+
+      - name: Upload report and logs
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: rustfs-table-test-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+          if-no-files-found: error
+          retention-days: 30
+
+      - name: Cleanup environment (after)
+        if: always()
+        continue-on-error: true
+        run: |
+          set -uo pipefail
+          for node in ${RUSTFS_NODES}; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -f /etc/default/rustfs
+            ' || echo "cleanup failed on ${node} (continuing)"
+          done
+
+      - name: "Continue functional chain (next: Performance)"
+        # Only chain-triggered runs forward to the next suite; standalone
+        # workflow_dispatch runs stop after their own cleanup.
+        if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          echo "Dispatching next functional suite: Performance"
+          gh api --method POST repos/rustfs/rustfs/dispatches \
+            -f event_type='rustfs-chain-performance' \
+            -F 'client_payload[from_suite]=table'
+
+      - name: Notify on failure
+        if: failure()
+        run: echo "RustFS table test workflow failed (harness/environment breakdown); see logs and report artifact."
+
+      - name: Remove verified candidate package
+        if: ${{ always() && inputs.chain_manifest != '' && steps.chain.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/prepare_functional_package.py cleanup
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite table
+          --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          --output "${RUNNER_TEMP}/chain-table-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/table.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-table-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-table-${{ github.run_id }}-${{ github.run_attempt }}/table.json
+          if-no-files-found: error

.github/workflows/rustfs-tier-test.yml

@@ -1,12 +1,18 @@
 name: RustFS Tier Test
 
 on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       rustfs_version:
-        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
+        description: 'RustFS release tag to test. Leave empty for nightly.'
         required: false
-        default: '1.0.0-rc.4-preview.1'
+        default: ''
       package_url:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false
@@ -39,7 +45,7 @@ permissions:
   contents: read
 
 concurrency:
-  group: rustfs-shared-functional-tests
+  group: rustfs-shared-functional-tests-v2
   cancel-in-progress: false
 
 defaults:
@@ -62,8 +68,13 @@ jobs:
   tier-test:
     runs-on: smoke-testing
     timeout-minutes: 420
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout chain tooling
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
       - name: Initialize run evidence directory
         id: evidence
         run: |
@@ -76,25 +87,21 @@ jobs:
           test -d "${TIER_ARTIFACTS_DIR}"
           test ! -L "${TIER_ARTIFACTS_DIR}"
 
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
 
       - name: Prepare pinned RustFS CLI
         id: rc
@@ -258,6 +265,8 @@ jobs:
 
       - name: Run tier suite
         id: test
+        # Case failures keep the run green: the report and the backlog
+        # issue manager carry the product signal.
         continue-on-error: true
         env:
           PACKAGE_URL_INPUT: ${{ inputs.package_url }}
@@ -299,6 +308,7 @@ jobs:
           mv "${TMP_FILE}" "${RESULT_FILE}"
 
       - name: Generate report
+        id: chain_report
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         env:
           PACKAGE_URL_INPUT: ${{ inputs.package_url }}
@@ -343,6 +353,20 @@ jobs:
               echo "Structured report generation failed before producing output (exit ${CASE_GATE_RC})."
             } > "${CASE_TABLE}"
           fi
+          # The tier case table carries a "## Case Summary" bullet list
+          # (Total/PASS/FAIL/...); parse the counts from there. Fall back to
+          # the case-ID table rows if the bullets are absent.
+          CASES_FAIL="$(grep -m1 -E '^- FAIL: [0-9]+' "${CASE_TABLE}" 2>/dev/null | grep -oE '[0-9]+' || true)"
+          if [ -n "${CASES_FAIL}" ]; then
+            CASES_PASS="$(grep -m1 -E '^- PASS: [0-9]+' "${CASE_TABLE}" 2>/dev/null | grep -oE '[0-9]+' || true)"
+            CASES_PASS=$(( ${CASES_PASS:-0} + 0 ))
+          else
+            CASES_TOTAL="$(grep -cE '^\| [A-Za-z0-9_.-]+ \| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
+            CASES_FAIL="$(grep -cE '^\| [A-Za-z0-9_.-]+ \| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
+            CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
+            CASES_PASS=$(( CASES_TOTAL - CASES_FAIL ))
+          fi
+          CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
           {
             echo "# RustFS tier test report"
             echo ""
@@ -355,6 +379,8 @@ jobs:
             echo ""
             cat "${CASE_TABLE}"
             echo ""
+            echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)"
+            echo ""
             echo "## Log tail"
             echo '```text'
             tail -n 200 "${LOG_FILE}" || true
@@ -472,98 +498,70 @@ jobs:
           GATE_RC_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier-gate.rc
         run: |
           set -euo pipefail
+          # Product gate: failing cases keep the run green (the structured gate
+          # rc and the report carry the per-case signal, and the backlog issue
+          # manager tracks failures). Only harness/environment breakdowns —
+          # evidence initialization or a missing structured gate result — turn
+          # the workflow red.
           failed=0
           if [ "${EVIDENCE_OUTCOME}" != "success" ]; then
             echo "tier evidence directory initialization is ${EVIDENCE_OUTCOME}, expected success" >&2
             failed=1
-          fi
-          if [ "${TEST_OUTCOME}" != "success" ]; then
-            echo "tier suite step outcome is ${TEST_OUTCOME}, expected success" >&2
-            failed=1
-          fi
-          if [ "${EVIDENCE_OUTCOME}" != "success" ]; then
-            echo "structured gate result is unavailable because evidence initialization failed" >&2
           elif [ ! -s "${GATE_RC_FILE}" ]; then
             echo "structured gate result is missing" >&2
             failed=1
-          else
-            GATE_RC="$(tr -d '[:space:]' < "${GATE_RC_FILE}")"
-            if ! [[ "${GATE_RC}" =~ ^[0-9]+$ ]] || [ "${GATE_RC}" -ne 0 ]; then
-              echo "structured 56-case gate failed with exit ${GATE_RC:-invalid}" >&2
-              failed=1
-            fi
           fi
           [ "${failed}" -eq 0 ]
 
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled' || steps.evidence_verify.outcome == 'failure' || steps.evidence_verify.outcome == 'cancelled' || steps.gate.outcome == 'failure' || steps.gate.outcome == 'cancelled') }}
+      - name: Manage backlog issues (dedup / label / auto-close)
+        # Replaces the old per-run failure filing. One entry point that:
+        # - dedups by signal: failing cases are matched against open backlog
+        #   issues by label (category + case ID); covered cases become a
+        #   comment on the existing issue, only uncovered cases file a new one
+        # - labels new issues (functional-test, category, case IDs, env)
+        # - closes fixed issues after a fully green run
+        # - never files or closes on cancelled runs
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
           SUITE: 'tier'
           SUITE_LABEL: 'Tier'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          EVIDENCE_DIR: ${{ env.TIER_ARTIFACTS_DIR }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
-          VERIFY_OUTCOME: ${{ steps.evidence_verify.outcome }}
-          GATE_OUTCOME: ${{ steps.gate.outcome }}
-          REPORT_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier-report.md
-          LOG_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier.log
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
             exit 0
           fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo "- Evidence initialization: ${EVIDENCE_OUTCOME}"
-            echo "- Evidence verification: ${VERIFY_OUTCOME}"
-            echo "- Final gate: ${GATE_OUTCOME}"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ "${EVIDENCE_OUTCOME}" != "success" ]; then
-              echo "(the run evidence directory was rejected; its contents were not read)"
-            elif [ ! -d "${EVIDENCE_DIR}" ] || [ -L "${EVIDENCE_DIR}" ]; then
-              echo "(the run evidence directory is missing or unsafe; its contents were not read)"
-            elif [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          echo "filed backlog issue for suite ${SUITE}"
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.test.outcome }}" \
+            --report "${TIER_ARTIFACTS_DIR}/rustfs-tier-cases.md" \
+            --report-file "${TIER_ARTIFACTS_DIR}/rustfs-tier-report.md" \
+            --log "${TIER_ARTIFACTS_DIR}/rustfs-tier.log" \
+            --run-url "${RUN_URL}/attempts/${GITHUB_RUN_ATTEMPT}#summary" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
 
       - name: "Continue functional chain (next: Storage engine)"
         # Only chain-triggered runs forward to the next suite; standalone
@@ -619,3 +617,24 @@ jobs:
         run: |
           echo "RustFS tier suite failed"
           echo "See the uploaded report and log artifacts for details."
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite tier
+          --report "${TIER_ARTIFACTS_DIR}/rustfs-tier-cases.md"
+          --output "${RUNNER_TEMP}/chain-tier-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/tier.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-tier-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-tier-${{ github.run_id }}-${{ github.run_attempt }}/tier.json
+          if-no-files-found: error

.github/workflows/rustfs-upgrade-test.yml

@@ -15,6 +15,12 @@
 name: RustFS Upgrade Test
 
 on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       from_version:
@@ -61,7 +67,7 @@ permissions:
   contents: read
 
 concurrency:
-  group: rustfs-shared-functional-tests
+  group: rustfs-shared-functional-tests-v2
   cancel-in-progress: false
 
 defaults:
@@ -80,7 +86,7 @@ jobs:
   upgrade-test:
     runs-on: smoke-testing
     timeout-minutes: 420
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -101,25 +107,21 @@ jobs:
             printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
           } >> "${GITHUB_ENV}"
 
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
 
       - name: Show environment
         run: |
@@ -160,6 +162,9 @@ jobs:
 
       - name: Run upgrade compatibility suite
         id: test
+        # Case failures keep the run green: the report and the backlog
+        # issue manager carry the product signal.
+        continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
         run: |
@@ -218,6 +223,7 @@ jobs:
           ./auto-testing/rustfs-upgrade-test.sh "${ARGS[@]}"
 
       - name: Generate report
+        id: chain_report
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
@@ -247,6 +253,21 @@ jobs:
           if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
             RESULT=success
           fi
+
+          # Product gate: failing cases keep the run green — they are reported
+          # below and tracked in rustfs/backlog. Only harness/environment
+          # breakdowns (the suite never reached case level, or a wholesale
+          # failure with zero passes) turn the workflow red.
+          CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
+          CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
+          CASES_PASS=$(( CASES_TOTAL - CASES_FAIL ))
+          HARNESS_OK=0
+          if [ '${{ steps.test.outcome }}' = 'success' ]; then
+            HARNESS_OK=1
+          elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${CASES_TOTAL}" -gt 0 ] && [ "${CASES_PASS}" -ge 1 ]; then
+            HARNESS_OK=1
+          fi
           {
             echo "# RustFS upgrade compatibility report"
             echo ""
@@ -259,21 +280,28 @@ jobs:
             echo "- Test Step Outcome: ${RESULT}"
             echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            if [ "${RESULT}" = "success" ]; then
+            if [ -s "${MATRIX_TABLE}" ]; then
               cat "${MATRIX_TABLE}"
               echo ""
+            fi
+            if [ -s "${CASE_TABLE}" ]; then
               cat "${CASE_TABLE}"
               echo ""
+            fi
+            echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)"
+            echo ""
+            if [ -s "${CASE_TABLE}" ]; then
               echo "## Log tail"
               echo '```text'
-              tail -n 200 "${LOG_FILE}"
+              tail -n 200 "${LOG_FILE}" 2>/dev/null || true
               echo '```'
             else
               echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]
+          # Red only for harness/environment breakdowns; case failures stay green.
+          [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
         if: ${{ always() && steps.evidence.outcome == 'success' }}
@@ -306,65 +334,62 @@ jobs:
           fi
           rm -f "${B64_FILE}"
 
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+      - name: Manage backlog issues (dedup / label / auto-close)
+        # Replaces the old per-run failure filing. One entry point that:
+        # - dedups by signal: failing cases are matched against open backlog
+        #   issues by label (category + case ID); covered cases become a
+        #   comment on the existing issue, only uncovered cases file a new one
+        # - labels new issues (functional-test, category, case IDs, env)
+        # - closes fixed issues after a fully green run
+        # - never files or closes on cancelled runs
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'upgrade'
           SUITE_LABEL: 'Upgrade compatibility'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
             exit 0
           fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          FROM_URL='${{ inputs.from_url }}'
+          FROM_VERSION='${{ inputs.from_version }}'
+          TO_URL='${{ inputs.to_url }}'
+          TO_VERSION='${{ inputs.to_version }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${TO_URL}" ]; then
+            PACKAGE_SOURCE="to ${TO_URL}"
+          elif [ -n "${TO_VERSION}" ]; then
+            PACKAGE_SOURCE="to version ${TO_VERSION}"
+          else
+            PACKAGE_SOURCE="to ${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          echo "filed backlog issue for suite ${SUITE}"
+          if [ -n "${FROM_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_SOURCE}; from ${FROM_URL}"
+          elif [ -n "${FROM_VERSION}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_SOURCE}; from version ${FROM_VERSION}"
+          fi
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.test.outcome }}" \
+            --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" \
+            --report-file "${REPORT_FILE}" \
+            --log "${LOG_FILE}" \
+            --run-url "${RUN_URL}/attempts/${GITHUB_RUN_ATTEMPT}#summary" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
 
       - name: Upload report and logs
         if: ${{ always() && steps.evidence.outcome == 'success' }}
@@ -454,3 +479,24 @@ jobs:
           echo "From: ${{ inputs.from_url || inputs.from_version || 'release (default)' }}"
           echo "To: ${{ inputs.to_url || inputs.to_version || 'nightly (R2 latest)' }}"
           echo "See the uploaded report and logs for details."
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite upgrade
+          --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          --output "${RUNNER_TEMP}/chain-upgrade-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/upgrade.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-upgrade-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-upgrade-${{ github.run_id }}-${{ github.run_attempt }}/upgrade.json
+          if-no-files-found: error

.github/workflows/schedule-failure-alert-drill.yml

@@ -19,7 +19,9 @@
 # 'failure')` wiring used by the real consumers (e2e-s3tests, mint, fuzz,
 # performance-ab). Dispatch it twice to verify both the issue-creation and
 # the dedupe-comment paths, then close the resulting
-# "[scheduled-failure] Schedule Failure Alert Drill" issue.
+# "[scheduled-failure] Schedule Failure Alert Drill" issue in rustfs/backlog
+# (the default target repository). Requires the BACKLOG_ISSUE_TOKEN secret
+# (a token with issues:write on rustfs/backlog).
 #
 # The run itself is expected to end red (the forced failure); only the
 # alert-on-failure job result matters.
@@ -68,4 +70,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

.github/workflows/scheduled-validation-freshness.yml

@@ -54,5 +54,5 @@ jobs:
         if: failure()
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}
           details-file: ${{ runner.temp }}/scheduled-validation-freshness.md

.github/workflows/scheduled-validation-watchdog.yml

@@ -55,7 +55,7 @@ jobs:
       - name: Open or update incomplete-run issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}
           workflow-name: ${{ github.event.workflow_run.name }}
           source-run-id: ${{ github.event.workflow_run.id }}
           source-run-attempt: ${{ github.event.workflow_run.run_attempt }}

.github/workflows/targets-integration.yml

@@ -189,4 +189,4 @@ jobs:
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Security
+
+- **Header-signed SigV4 requests honour only signed headers** (GHSA-xm99-m3gq-83g8): a request authenticated with a SigV4 `Authorization` header that carries an `x-amz-*` request header not listed in its `SignedHeaders` is now rejected with `403 AccessDenied` ("There were headers present in the request which were not signed"), matching AWS S3 and the presigned rule from GHSA-g8w9-qw9q-fghr. Previously anyone holding one header-signed `PutObject` request could add an unsigned `x-amz-copy-source` and turn it into a `CopyObject` that ran with the signer's permissions, copying any object the signer could read into the target. An `Authorization` header whose algorithm token is not `AWS4-HMAC-SHA256` is now rejected instead of being verified as SigV4. The request-envelope headers `x-amz-content-sha256`, `x-amz-decoded-content-length`, `x-amz-trailer` and `x-amz-checksum-algorithm` (the same set the upstream `s3s` fix exempts) and `x-amz-cf-id` (CloudFront) remain tolerated unsigned; AWS SDKs and RustFS's own signers already sign every other `x-amz-*` header. SigV2, JWT and anonymous requests are unchanged.
+
 ### Replication
 
 - Object Lock replication PUTs now carry a required integrity header, fixing target rejection introduced by the plain-payload default ([#7097](https://github.com/rustfs/rustfs/pull/7097)). This changes the default outbound request for locked objects but adds no persisted format.
@@ -21,9 +25,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - **Presigned URLs honour only signed headers** (GHSA-g8w9-qw9q-fghr): a SigV4 presigned request that carries an `x-amz-*` request header not listed in `X-Amz-SignedHeaders` is now rejected with `403 AccessDenied` ("There were headers present in the request which were not signed"), matching AWS S3. Previously the holder of a presigned `PutObject` URL could add unsigned `x-amz-tagging`, `x-amz-storage-class`, `x-amz-website-redirect-location`, ACL, metadata, Object Lock or SSE headers and have them applied. Presigners that intend a property must set it before signing so the SDK lists the header in `SignedHeaders`; `x-amz-cf-id` (CloudFront) remains tolerated unsigned. Header-signed SigV4 and SigV2 requests are unchanged.
 
 ### Fixed
+- **Vault backends accepted key names that escape the key prefix**: only the Local backend refused a key identifier containing a path separator, so on Vault KV2 `POST /rustfs/admin/v3/kms/keys` with a name such as `bad/name` succeeded and created a nested KV2 path that the key listing then reported as a directory rather than a key, and an identifier containing `..` addressed a record outside the configured key prefix once the HTTP client normalised the URL. Both Vault backends now apply the Local backend's containment rule at the point where the identifier becomes a path or a Transit key name: an empty identifier, one containing `/`, `\\` or NUL, or the dot segments `.` and `..` is refused with `400` (`InvalidKey`) before any request reaches Vault. The AWS backend is unaffected, since it addresses keys by ARN and alias. A Vault deployment that created such a key on an earlier build must re-create it under a plain name; objects encrypted under the old name stay readable only until that key is refused, so migrate them first. Refs rustfs/backlog#2474.
 - **Fresh multi-pool bootstrap with distinct format creators**: a new deployment whose pools have their first endpoint on different nodes (for example two single-node pools) could never publish its initial `pool.bin`: each node held fresh-bootstrap proof only for the pool it formatted, the deployment-wide proof collapsed to none, and every node died with `pool metadata recovery required: no durable bootstrap identity or pool.bin replica is available` after the startup retry budget. The first pool's creator now mints the pending cluster identity on its own pool, every other creator copies that nonce-bound identity onto the pool it formatted first-hand, and the elected writer publishes `pool.bin` once every pool replica carries the same pending identity. Corrupt or disagreeing replicas, pools that merely have a format, expansion pools joining an initialized deployment, and restarts without first-hand proof still fail closed. Non-elected nodes that start before `pool.bin` exists, and the elected writer while it waits for the other creators, no longer latch their pool-metadata write gate for the life of the process. Refs rustfs/backlog#2338, rustfs/backlog#2375.
 - **Lock RPC timeout storms** (#7363): the remote lock client no longer evicts and re-dials the shared internode HTTP/2 channel on every request deadline. A timeout evicts only when the peer has not completed any lock RPC for two deadlines, evictions and transport-failure re-dials are rate limited per peer (`RUSTFS_OBJECT_LOCK_RPC_EVICTION_COOLDOWN_MS`, default 5 s), and a timed-out request is left running instead of being reset (bounded per peer by `RUSTFS_OBJECT_LOCK_RPC_DETACHED_LIMIT`, default 256), so a slow lock endpoint can no longer drive the `RST_STREAM`/`GOAWAY too_many_resets`/reconnect loop. A lock granted after its caller timed out is released immediately, and unlocks that fail the quick retries continue on a deferred 1/2/4/8/16 s schedule before the server lease reclaims them. New `rustfs_remote_lock_*` metrics cover timeouts, evictions, suppressed evictions, detached streams, late completions and late releases per peer. Operator guide at `docs/operations/lock-rpc-storm-protection.md`.
 - **KMS failures on the S3 data path carry an actionable status**: only "key not found" and a backend outage were classified; every other KMS failure — a disabled or pending-deletion key, a denied KMS grant, an encryption-context mismatch, an unsupported algorithm, a credential or timeout failure, a capability the backend does not have — collapsed onto `500 InternalError`. SDKs therefore applied exponential backoff to configuration errors that no retry can fix, and monitoring filed every one of them as a server fault. Unusable-key and request-side failures now return `400`, a denied grant `403`, transient backend failures `503` — including a key store the backend could not read, so an outage stays distinguishable from a missing key all the way to the client — and a missing backend capability `501`. Damaged or unreadable key material still returns `500`, which is what it is.
+- **Bare SSE-KMS writes on a node without a running KMS**: `x-amz-server-side-encryption: aws:kms` without a key id (and no bucket default key) returned `500 InternalError` while KMS was stopped or never configured, because the "no key available" branch exited before the availability classification that the keyed form already received. `PutObject` and `CreateMultipartUpload` now return `503 ServiceUnavailable` while a configured KMS is stopped, `400 InvalidRequest` when KMS was never configured, and `400 InvalidRequest` naming the missing key id when a running KMS has no default key.
+- **Reading an object whose KMS key is gone returned `500`**: `GetObject`, `CopyObject` and `UploadPartCopy` on an SSE-KMS object whose key had been deleted reported `500 InternalError` ("KMS key not found"), while the same condition on `PutObject` already returned `400 KMS.NotFoundException`. The read path carried only four classifications across the storage boundary and folded a missing key, a denied KMS grant and a missing backend capability onto "decryption failed". Those reads now return `400 KMS.NotFoundException`, `403 AccessDenied` and `501 NotImplemented` respectively; `HeadObject` is unaffected because it never unwraps the data key. An envelope the configured backend cannot unwrap (the key was re-created under the same name, or the backend was switched) stays `500` but now says so instead of the generic internal-error text.
+- **KMS key-management routes answered `500` for client-side failures**: `POST /rustfs/admin/v3/kms/keys` and the legacy `create-key` alias reported every backend refusal as `500`, including a blank key name (each backend failed differently, the Local backend by writing a key file with an empty stem) and a name that already exists; `POST /rustfs/admin/v3/kms/generate-data-key` did the same for an unknown or disabled key, although `describe` and `delete` already classified those. A blank or whitespace key name is now refused before it reaches any backend (`400`), a taken name is `409`, an unknown key is `404` (`KMS.NotFoundException`), a disabled key `400`, and a capability the backend lacks `501`; damaged key material stays `500`. The read-only Static backend now reports create, delete and cancel-deletion as missing capabilities (`501`), matching its rotate and enable/disable answers, instead of `400`/`500`.
+- **SSE-S3 responses named the internal wrapping key**: `PutObject`, `CopyObject`, `CreateMultipartUpload` and `GetObject` for an `AES256` object returned `x-amz-server-side-encryption-aws-kms-key-id` carrying the KMS key that wraps the SSE-S3 data key (the service default key, or the literal `default` on a node without KMS), although the header is defined for `aws:kms` objects only and `CompleteMultipartUpload` and `HeadObject` already omitted it. Those responses now advertise a key id only for `aws:kms` objects.
+- **PutBucketEncryption accepted algorithms the server cannot honour**: a default-encryption rule naming an unknown `SSEAlgorithm` (for example `AES128`), a rule without `ApplyServerSideEncryptionByDefault`, an empty rule list, or a `KMSMasterKeyID` on an `AES256` rule was stored as written. `GetBucketEncryption` then reported that configuration while every header-less write was encrypted under the `AES256` fallback, so the bucket's advertised and actual schemes disagreed. Those configurations are now refused with `400` (`MalformedXML` for a malformed rule, `InvalidArgument` for a key id on a non-KMS rule) and nothing is stored.
 - **SSE-C on buckets with default encryption**: a `PutObject` carrying a valid SSE-C header triple on a bucket that has default encryption configured no longer fails with `400 InvalidArgument` ("The SSE-C and managed server-side encryption headers cannot be used together"). PUT and the POST-object/extract path resolved the bucket default with a hard-coded "no explicit SSE-C" flag, so the default was layered onto the request and then tripped the request's own mutual-exclusion check; an SSE-C request now suppresses the bucket default on all three write paths, matching COPY and AWS S3. Every bucket with default encryption previously refused SSE-C single PUTs outright, while `CreateMultipartUpload` on the same bucket succeeded.
 - **Explicit SSE-S3 on SSE-KMS-default buckets**: `x-amz-server-side-encryption: AES256` against a bucket whose default is `aws:kms` no longer fails with `400 InvalidArgument`. The bucket default's KMS key id was inherited independently of the effective algorithm, producing a self-contradictory `AES256` + key-id pair; the key id is now inherited only when the effective algorithm is `aws:kms`. `PutBucketEncryption` fills in a default key id automatically, so this affected nearly every SSE-KMS-default bucket.
 - **Restore of encrypted or compressed multipart objects (silent data corruption)**: restoring a multipart object from a remote tier addressed the tier in *plaintext* coordinates while the copy-back reads the *stored* representation. Every part received a misaligned slice of the remote object whose length still satisfied the range, the hash reader and the completion size check, so the restore reported success and replaced the object's bytes. Restore now accumulates stored part sizes, passes the stored length to the hash reader alongside the plaintext length, and validates against the stored size. Objects restored by an affected release must be re-restored from the tier or recovered from a backup — this release does not detect or repair them retroactively.

Cargo.toml

@@ -72,8 +72,8 @@ resolver = "3"
 edition = "2024"
 license = "Apache-2.0"
 repository = "https://github.com/rustfs/rustfs"
-rust-version = "1.98.0"
-version = "1.0.0-rc.6"
+rust-version = "1.98.1"
+version = "1.0.0"
 homepage = "https://rustfs.com"
 description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
 keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
@@ -90,62 +90,62 @@ redundant_clone = "warn"
 
 [workspace.dependencies]
 # RustFS Internal Crates
-rustfs = { path = "./rustfs", version = "1.0.0-rc.6" }
-rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.6" }
-rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.0-rc.6" }
-rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.0-rc.6" }
-rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.6" }
-rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.6" }
-rustfs-common = { path = "crates/common", version = "1.0.0-rc.6" }
-rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.6" }
-rustfs-config = { path = "./crates/config", version = "1.0.0-rc.6" }
-rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.6" }
-rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.6" }
-rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.6" }
-rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.6" }
-rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.6" }
-rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.6" }
-rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.6" }
-rustfs-license = { path = "crates/license", version = "1.0.0-rc.6" }
-rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.6" }
-rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.6" }
-rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.6" }
-rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.6" }
-rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.6" }
-rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.6" }
-rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.6" }
-rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.6" }
-rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.6", default-features = false }
-rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.6" }
-rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.6" }
-rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.6" }
-rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.6" }
-rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.6" }
-rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.6" }
-rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.6" }
-rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.6" }
-rustfs-s3-client = { path = "crates/s3-client", version = "1.0.0-rc.6" }
-rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.6" }
-rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.6" }
-rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.6" }
-rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.6" }
-rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.6" }
-rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.6" }
-rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.6" }
-rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.6" }
-rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.6" }
-rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.6" }
-rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.6" }
-rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.6" }
-rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.6" }
-rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.6" }
-rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.6" }
+rustfs = { path = "./rustfs", version = "1.0.0" }
+rustfs-heal = { path = "crates/heal", version = "1.0.0" }
+rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.0" }
+rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.0" }
+rustfs-audit = { path = "crates/audit", version = "1.0.0" }
+rustfs-checksums = { path = "crates/checksums", version = "1.0.0" }
+rustfs-common = { path = "crates/common", version = "1.0.0" }
+rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0" }
+rustfs-config = { path = "./crates/config", version = "1.0.0" }
+rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0" }
+rustfs-credentials = { path = "crates/credentials", version = "1.0.0" }
+rustfs-crypto = { path = "crates/crypto", version = "1.0.0" }
+rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0" }
+rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0" }
+rustfs-iam = { path = "crates/iam", version = "1.0.0" }
+rustfs-keystone = { path = "crates/keystone", version = "1.0.0" }
+rustfs-license = { path = "crates/license", version = "1.0.0" }
+rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0" }
+rustfs-kms = { path = "crates/kms", version = "1.0.0" }
+rustfs-lock = { path = "crates/lock", version = "1.0.0" }
+rustfs-madmin = { path = "crates/madmin", version = "1.0.0" }
+rustfs-notify = { path = "crates/notify", version = "1.0.0" }
+rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0" }
+rustfs-io-core = { path = "crates/io-core", version = "1.0.0" }
+rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0" }
+rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0", default-features = false }
+rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0" }
+rustfs-obs = { path = "crates/obs", version = "1.0.0" }
+rustfs-policy = { path = "crates/policy", version = "1.0.0" }
+rustfs-protos = { path = "crates/protos", version = "1.0.0" }
+rustfs-protocols = { path = "crates/protocols", version = "1.0.0" }
+rustfs-replication = { path = "crates/replication", version = "1.0.0" }
+rustfs-rio = { path = "crates/rio", version = "1.0.0" }
+rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0" }
+rustfs-s3-client = { path = "crates/s3-client", version = "1.0.0" }
+rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0" }
+rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0" }
+rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0" }
+rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0" }
+rustfs-scanner = { path = "crates/scanner", version = "1.0.0" }
+rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0" }
+rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0" }
+rustfs-signer = { path = "crates/signer", version = "1.0.0" }
+rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0" }
+rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0" }
+rustfs-targets = { path = "crates/targets", version = "1.0.0" }
+rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0" }
+rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0" }
+rustfs-utils = { path = "crates/utils", version = "1.0.0" }
+rustfs-zip = { path = "./crates/zip", version = "1.0.0" }
 
 # Async Runtime and Networking
 async-channel = "2.5.0"
 async_zip = { default-features = false, version = "0.0.19" }
 mysql_async = { default-features = false, version = "0.37.1" }
-async-compression = { version = "0.4.46" }
+async-compression = { version = "0.4.47" }
 async-recursion = "1.1.1"
 async-trait = "0.1.92"
 async-nats = { version = "0.50.0", default-features = false }
@@ -211,7 +211,7 @@ openidconnect = { default-features = false, version = "4.0" }
 pbkdf2 = "0.13.0"
 p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
 rsa = { version = "=0.10.0-rc.18" }
-rustls = { default-features = false, version = "0.23.44" }
+rustls = { default-features = false, version = "0.23.45" }
 rustls-native-certs = "0.8"
 rustls-pki-types = "1.15.1"
 x509-parser = "0.18.1"
@@ -224,7 +224,7 @@ proptest = "1"
 # Time and Date
 chrono = { version = "0.4.45" }
 humantime = "2.4.0"
-jiff = { version = "0.2.35" }
+jiff = { version = "0.2.37" }
 time = { version = "0.3.55" }
 
 # Database
@@ -235,8 +235,7 @@ tokio-postgres-rustls = "0.14.0"
 # Utilities and Tools
 anyhow = "1.0.104"
 arc-swap = "1.9.2"
-# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin while Snowball and Swift still depend on it. Remove after Snowball uses a released tar-codec/tar-framing API that exposes precedence-resolved MinIO vendor records, RustFS preserves cancellation-safe ownership of large streamed members, footerless minio-go input is accepted only at an authenticated complete request boundary, the existing resource-limit, cancellation, and error-fuse regressions pass, and Swift no longer needs this fork.
-astral-tokio-tar = { git = "https://github.com/cxymds/tokio-tar.git", rev = "603756478b7668436e464519c77ccac22a99ba96" }
+rustfs-tokio-tar = { version = "0.7.1" }
 # Candidate Snowball parser versions exercised by rustfs-zip compatibility fixtures.
 tar-codec = "0.0.14"
 tar-framing = "0.0.14"
@@ -245,7 +244,7 @@ atomic_enum = "0.3.0"
 aws-config = { version = "1.12.0" }
 aws-credential-types = { version = "1.3.0" }
 aws-sdk-kms = { default-features = false, version = "1.118.0" }
-aws-sdk-s3 = { default-features = false, version = "1.146.0" }
+aws-sdk-s3 = { default-features = false, version = "1.146.1" }
 aws-sdk-sts = { default-features = false, version = "1.114.0" }
 aws-smithy-async = { version = "1.3.0" }
 aws-smithy-http-client = { default-features = false, version = "1.4.0" }
@@ -253,7 +252,7 @@ aws-smithy-runtime-api = { version = "1.16.0" }
 aws-smithy-types = { version = "1.6.3" }
 base64-simd = "0.8.0"
 brotli = "9.0.0"
-clap = { version = "4.6.6" }
+clap = { version = "4.6.7" }
 const-str = { version = "1.1.0" }
 convert_case = "0.12.0"
 criterion = { version = "0.8" }
@@ -261,8 +260,9 @@ crossbeam-queue = "0.3.14"
 crossbeam-channel = "0.5.17"
 crossbeam-deque = "0.8.8"
 crossbeam-utils = "0.8.23"
-datafusion = { default-features = false, version = "55.0.0" }
+datafusion = { default-features = false, version = "55.1.0" }
 derive_builder = "0.20.2"
+ed25519-dalek = "3.0.0"
 enumset = "1.1.14"
 faster-hex = "0.10.0"
 flate2 = "1.1.10"
@@ -312,11 +312,12 @@ rustify = { version = "0.7", default-features = false }
 rustix = { version = "1.1.4" }
 rust-embed = { version = "8.12.0" }
 rustc-hash = { version = "2.1.3" }
-s3s = { git = "https://github.com/rustfs/s3s.git", rev = "bdcb6259339c41369f9f1c60e3a42b5ab8da607b", version = "0.15.0", features = ["minio"] }
+s3s = { version = "0.16.0", features = ["minio"] }
+s3s-sigv4 = { version = "0.16.0" }
 serial_test = "4.0.1"
 shadow-rs = { default-features = false, version = "2.0.0" }
 siphasher = "1.0.3"
-smallvec = { version = "1.16.0" }
+smallvec = { version = "1.16.1" }
 compact_str = "0.10.0"
 snap = "1.1.2"
 starshard = { version = "2.3.0" }
@@ -387,6 +388,14 @@ ignored = ["hotpath", "rustfs"]
 #   CARGO_PROFILE_DEV_DEBUG=full cargo build
 debug = "line-tables-only"
 
+# To further speed up the process, completely disable debug information for dependencies.
+[profile.dev.package."*"]
+debug = false
+# When you really need to debug, you can enable full debugging information using the `--profile debugging` option.
+[profile.debugging]
+inherits = "dev"
+debug = true
+
 [profile.release]
 opt-level = 3
 lto = "thin"

README.md

@@ -141,7 +141,7 @@ chown -R 10001:10001 data logs
 docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
 
 # Using specific version
-docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.6
+docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0
 ```
 
 If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command

README_ZH.md

@@ -138,7 +138,7 @@ chown -R 10001:10001 data logs
 docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
 
 # 使用指定版本运行
-docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.6
+docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0
 ```
 
 如果您通过绑定挂载启用 TLS 证书目录，也请用同样方式准备该目录：

crates/common/Cargo.toml

@@ -41,6 +41,7 @@ uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
 metrics = { workspace = true }
 smallvec = { workspace = true }
 tracing = { workspace = true }
+thiserror.workspace = true
 
 [lib]
 doctest = false

crates/common/src/mrf_channel.rs

@@ -16,12 +16,15 @@
 //!
 //! Producers on error paths (read decode failure, scanner metadata
 //! corruption, partial-write recovery) hand a lightweight [`MrfIntent`] to the
-//! heal crate through a global bounded channel. Delivery is strictly
+//! heal crate through a global bounded channel. Hint delivery is strictly
 //! non-blocking: `try_send_mrf_intent` never awaits and drops the intent
 //! (counting it) when the channel is full or uninitialized — losing one heal
 //! hint is always preferred over stalling an IO path. Durable replay of
 //! unconsumed intents is the consumer's job (see `rustfs-heal`
-//! `heal::mrf_queue`), mirroring MinIO's `.heal/mrf/list.bin`.
+//! `heal::mrf_queue`), mirroring MinIO's `.heal/mrf/list.bin`. Committed partial
+//! writes and committed partial delete-marker purges use a separate bounded
+//! channel and await a checkpoint receipt; ordinary complete writes do not use
+//! it.
 
 use std::collections::HashMap;
 use std::collections::hash_map::RandomState;
@@ -33,7 +36,7 @@ use std::sync::{
     atomic::{AtomicBool, Ordering},
 };
 use std::time::{Duration, Instant};
-use tokio::sync::mpsc;
+use tokio::sync::{mpsc, oneshot};
 use uuid::Uuid;
 
 /// Bounded capacity of the global MRF channel. Backpressure is resolved by
@@ -44,10 +47,10 @@ const MRF_COALESCER_MAX_KEYS: usize = 8192;
 const MRF_COALESCER_MAX_BYTES: usize = 16 * 1024 * 1024;
 const MRF_COALESCER_TTL: Duration = Duration::from_secs(60);
 const MRF_MAX_IDENTITY_COMPONENT: usize = 1024;
+pub const MRF_MAX_DELETE_MARKER_BYTES: usize = 1024 * 1024;
 
 /// Why an intent was produced. Drives the heal priority mapping on the
-/// consumer side (DecodeFailure -> Urgent, MetadataCorruption -> High,
-/// PartialWrite -> Normal).
+/// consumer side.
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
 pub enum MrfKind {
     /// Erasure decode failed while serving a read (read path).
@@ -56,6 +59,8 @@ pub enum MrfKind {
     MetadataCorruption,
     /// A write left the object with fewer committed shards than the set size.
     PartialWrite,
+    /// A committed explicit delete-marker purge missed at least one member.
+    DeleteMarkerPurge,
 }
 
 impl MrfKind {
@@ -64,8 +69,60 @@ impl MrfKind {
             MrfKind::DecodeFailure => "decode-failure",
             MrfKind::MetadataCorruption => "metadata-corruption",
             MrfKind::PartialWrite => "partial-write",
+            MrfKind::DeleteMarkerPurge => "delete-marker-purge",
         }
     }
+
+    pub const fn is_durable(self) -> bool {
+        matches!(self, Self::PartialWrite | Self::DeleteMarkerPurge)
+    }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub struct MrfDeleteMarkerPurgeIdentity {
+    pub bucket_incarnation_id: Uuid,
+    pub marker_incarnation_id: Uuid,
+    pub marker_identity: [u8; 32],
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct MrfDeleteMarkerPurge {
+    pub bucket_incarnation_id: Uuid,
+    pub marker_incarnation_id: Uuid,
+    pub marker_identity: [u8; 32],
+    pub marker: Arc<[u8]>,
+}
+
+impl MrfDeleteMarkerPurge {
+    pub fn new(
+        bucket_incarnation_id: Uuid,
+        marker_incarnation_id: Uuid,
+        marker_identity: [u8; 32],
+        marker: Vec<u8>,
+    ) -> Option<Self> {
+        let payload = Self {
+            bucket_incarnation_id,
+            marker_incarnation_id,
+            marker_identity,
+            marker: Arc::from(marker),
+        };
+        payload.is_valid().then_some(payload)
+    }
+
+    pub const fn identity(&self) -> MrfDeleteMarkerPurgeIdentity {
+        MrfDeleteMarkerPurgeIdentity {
+            bucket_incarnation_id: self.bucket_incarnation_id,
+            marker_incarnation_id: self.marker_incarnation_id,
+            marker_identity: self.marker_identity,
+        }
+    }
+
+    pub fn is_valid(&self) -> bool {
+        !self.bucket_incarnation_id.is_nil()
+            && self.marker_incarnation_id == self.bucket_incarnation_id
+            && !self.marker.is_empty()
+            && self.marker.len() <= MRF_MAX_DELETE_MARKER_BYTES
+    }
 }
 
 /// One repair intent. Kept deliberately small so the in-memory queue and the
@@ -78,6 +135,10 @@ pub struct MrfIntent {
     /// Version the intent targets, as raw UUID bytes.
     pub version_id: Option<[u8; 16]>,
     pub kind: MrfKind,
+    /// Present only for [`MrfKind::DeleteMarkerPurge`]. The encoded marker is a
+    /// destructive-operation precondition, while `marker_identity` is its
+    /// deterministic queue and proof identity.
+    pub delete_marker_purge: Option<MrfDeleteMarkerPurge>,
     /// Stable erasure-set scope when the producer has it. Kept optional so
     /// metadata corruption and legacy producers do not invent a scope.
     pub scope: Option<MrfScope>,
@@ -90,13 +151,14 @@ pub struct MrfIntent {
     pub attempts: u8,
 }
 
-#[derive(Clone, Debug, PartialEq, Eq)]
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
 pub struct MrfDurableRepairAnchor {
     pub kind: MrfKind,
     pub bucket: Arc<str>,
     pub object: Arc<str>,
     pub version_id: Option<[u8; 16]>,
     pub scope: Option<MrfScope>,
+    pub delete_marker_purge: Option<MrfDeleteMarkerPurgeIdentity>,
     pub lease: MrfIngressLease,
     pub bucket_incarnation_id: Uuid,
 }
@@ -106,17 +168,22 @@ impl MrfDurableRepairAnchor {
     /// incarnation and the original ingress lease. Legacy replay records lack
     /// both pieces and therefore remain fail-closed.
     pub fn from_intent(intent: &MrfIntent, bucket_incarnation_id: Uuid) -> Option<Self> {
+        let lease = intent.lease?;
+        let (version_id, scope) = canonical_identity(intent.kind, intent.version_id, intent.scope);
+        let delete_marker_purge = canonical_delete_marker_purge_identity(intent)?;
+        let bucket_incarnation_id = delete_marker_purge
+            .map(|identity| identity.bucket_incarnation_id)
+            .unwrap_or(bucket_incarnation_id);
         if bucket_incarnation_id.is_nil() {
             return None;
         }
-        let lease = intent.lease?;
-        let (version_id, scope) = canonical_identity(intent.kind, intent.version_id, intent.scope);
         Some(Self {
             kind: intent.kind,
             bucket: intent.bucket.clone(),
             object: intent.object.clone(),
             version_id,
             scope,
+            delete_marker_purge,
             lease,
             bucket_incarnation_id,
         })
@@ -131,6 +198,7 @@ impl MrfDurableRepairAnchor {
             && self.object == event.object
             && self.version_id == event.version_id
             && self.scope == event.scope
+            && self.delete_marker_purge == event.delete_marker_purge
             && self.lease == lease
             && self.bucket_incarnation_id == event.bucket_incarnation_id
     }
@@ -201,6 +269,7 @@ pub enum MrfDropReason {
     Uninitialized,
     Full,
     OversizedIdentity,
+    InvalidPayload,
     CoalescerFull,
 }
 
@@ -219,12 +288,133 @@ impl MrfIntent {
     pub fn estimated_bytes(&self) -> usize {
         // Struct + strings + version bytes; buckets and objects are usually
         // far below this bound, so rounding up keeps the budget conservative.
-        64 + self.bucket.len() + self.object.len()
+        64usize
+            .saturating_add(self.bucket.len())
+            .saturating_add(self.object.len())
+            .saturating_add(
+                self.delete_marker_purge
+                    .as_ref()
+                    .map_or(0, |purge| std::mem::size_of::<MrfDeleteMarkerPurge>().saturating_add(purge.marker.len())),
+            )
     }
 }
 
 static GLOBAL_MRF_SENDER: OnceLock<mpsc::Sender<MrfIntent>> = OnceLock::new();
 
+static GLOBAL_DURABLE_MRF_SENDER: OnceLock<mpsc::Sender<MrfDurableSubmission>> = OnceLock::new();
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)]
+pub enum MrfDurableAdmissionError {
+    #[error("MRF delivery is disabled")]
+    Disabled,
+    #[error("MRF durable consumer is unavailable")]
+    Unavailable,
+    #[error("MRF responsibility capacity is exhausted")]
+    Full,
+    #[error("MRF responsibility identity is invalid or oversized")]
+    InvalidIdentity,
+    #[error("MRF responsibility checkpoint could not be persisted")]
+    Persistence,
+    #[error("MRF durable admission timed out; responsibility may still be retained")]
+    Timeout,
+}
+
+/// A committed storage responsibility awaiting a durable checkpoint, separate
+/// from the best-effort read/scanner channel. Dropping the reply does not
+/// cancel repair.
+pub struct MrfDurableSubmission {
+    pub intent: MrfIntent,
+    pub response: oneshot::Sender<Result<(), MrfDurableAdmissionError>>,
+}
+
+pub fn init_durable_mrf_channel() -> Result<mpsc::Receiver<MrfDurableSubmission>, &'static str> {
+    let (sender, receiver) = mpsc::channel(MRF_CHANNEL_CAPACITY);
+    GLOBAL_DURABLE_MRF_SENDER
+        .set(sender)
+        .map_err(|_| "MRF durable channel sender already initialized")?;
+    Ok(receiver)
+}
+
+/// Wait for checkpoint publication on a committed partial-write path. Every
+/// write gets a fresh lease: a previous repair of the same unversioned key
+/// cannot discharge a later overwrite. An error is not durable admission.
+pub async fn persist_partial_write_intent(
+    bucket: &str,
+    object: &str,
+    version_id: Option<Uuid>,
+    scope: MrfScope,
+) -> Result<(), MrfDurableAdmissionError> {
+    let intent = MrfIntent {
+        bucket: Arc::from(bucket),
+        object: Arc::from(object),
+        version_id: canonical_version(version_id),
+        kind: MrfKind::PartialWrite,
+        delete_marker_purge: None,
+        scope: Some(scope),
+        lease: None,
+        enqueued_at_ms: unix_now_ms(),
+        attempts: 0,
+    };
+    persist_durable_intent(intent).await
+}
+
+pub async fn persist_delete_marker_purge_intent(
+    bucket: &str,
+    object: &str,
+    version_id: Uuid,
+    scope: MrfScope,
+    delete_marker_purge: MrfDeleteMarkerPurge,
+) -> Result<(), MrfDurableAdmissionError> {
+    if version_id.is_nil() {
+        return Err(MrfDurableAdmissionError::InvalidIdentity);
+    }
+    let intent = MrfIntent {
+        bucket: Arc::from(bucket),
+        object: Arc::from(object),
+        version_id: canonical_version(Some(version_id)),
+        kind: MrfKind::DeleteMarkerPurge,
+        delete_marker_purge: Some(delete_marker_purge),
+        scope: Some(scope),
+        lease: None,
+        enqueued_at_ms: unix_now_ms(),
+        attempts: 0,
+    };
+    persist_durable_intent_unconditionally(intent).await
+}
+
+async fn persist_durable_intent(intent: MrfIntent) -> Result<(), MrfDurableAdmissionError> {
+    if !mrf_delivery_enabled() {
+        return Err(MrfDurableAdmissionError::Disabled);
+    }
+    persist_durable_intent_unconditionally(intent).await
+}
+
+async fn persist_durable_intent_unconditionally(mut intent: MrfIntent) -> Result<(), MrfDurableAdmissionError> {
+    if intent.bucket.is_empty()
+        || intent.object.is_empty()
+        || intent.bucket.len() > MRF_MAX_IDENTITY_COMPONENT
+        || intent.object.len() > MRF_MAX_IDENTITY_COMPONENT
+        || !valid_intent_payload(&intent)
+    {
+        return Err(MrfDurableAdmissionError::InvalidIdentity);
+    }
+    let sender = GLOBAL_DURABLE_MRF_SENDER.get().ok_or(MrfDurableAdmissionError::Unavailable)?;
+    if try_rearm_mrf_replay_intent(&mut intent) != MrfIngressResult::Enqueued {
+        return Err(MrfDurableAdmissionError::InvalidIdentity);
+    }
+    let (response, receipt) = oneshot::channel();
+    sender
+        .try_send(MrfDurableSubmission { intent, response })
+        .map_err(|err| match err {
+            mpsc::error::TrySendError::Full(_) => MrfDurableAdmissionError::Full,
+            mpsc::error::TrySendError::Closed(_) => MrfDurableAdmissionError::Unavailable,
+        })?;
+    tokio::time::timeout(Duration::from_secs(10), receipt)
+        .await
+        .map_err(|_| MrfDurableAdmissionError::Timeout)?
+        .map_err(|_| MrfDurableAdmissionError::Unavailable)?
+}
+
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
 struct MrfIdentityKey {
     kind: MrfKind,
@@ -278,10 +468,27 @@ fn canonical_identity(
     let version_id = version_id.filter(|bytes| *bytes != [0; 16]);
     match kind {
         MrfKind::MetadataCorruption => (None, None),
-        MrfKind::DecodeFailure | MrfKind::PartialWrite => (version_id, scope),
+        MrfKind::DecodeFailure | MrfKind::PartialWrite | MrfKind::DeleteMarkerPurge => (version_id, scope),
     }
 }
 
+fn canonical_delete_marker_purge_identity(intent: &MrfIntent) -> Option<Option<MrfDeleteMarkerPurgeIdentity>> {
+    match intent.kind {
+        MrfKind::DeleteMarkerPurge => intent
+            .delete_marker_purge
+            .as_ref()
+            .filter(|payload| payload.is_valid())
+            .map(|payload| Some(payload.identity())),
+        _ if intent.delete_marker_purge.is_none() => Some(None),
+        _ => None,
+    }
+}
+
+fn valid_intent_payload(intent: &MrfIntent) -> bool {
+    canonical_delete_marker_purge_identity(intent).is_some()
+        && (intent.kind != MrfKind::DeleteMarkerPurge || (intent.version_id.is_some() && intent.scope.is_some()))
+}
+
 fn identity_estimated_bytes(key: &MrfIdentityKey) -> usize {
     64usize
         .saturating_add(key.bucket.len())
@@ -431,6 +638,9 @@ pub fn try_send_mrf_intent_typed(
     if !mrf_delivery_enabled() {
         return MrfIngressResult::Dropped(MrfDropReason::Disabled);
     }
+    if kind == MrfKind::DeleteMarkerPurge {
+        return MrfIngressResult::Dropped(MrfDropReason::InvalidPayload);
+    }
     let Some(sender) = GLOBAL_MRF_SENDER.get() else {
         return MrfIngressResult::Dropped(MrfDropReason::Uninitialized);
     };
@@ -454,6 +664,7 @@ pub fn try_send_mrf_intent_typed(
         object: key.object.clone(),
         version_id: key.version_id,
         kind,
+        delete_marker_purge: None,
         scope,
         lease: Some(lease),
         enqueued_at_ms: unix_now_ms(),
@@ -481,15 +692,18 @@ pub fn try_send_mrf_intent_typed(
 /// producer coalescer key: the replay queue first deduplicates legacy records,
 /// then manager admission owns task-level deduplication with live producers.
 pub fn try_rearm_mrf_replay_intent(intent: &mut MrfIntent) -> MrfIngressResult {
-    if intent.lease.is_some() {
-        return MrfIngressResult::Enqueued;
-    }
     if intent.bucket.len() > MRF_MAX_IDENTITY_COMPONENT || intent.object.len() > MRF_MAX_IDENTITY_COMPONENT {
         return MrfIngressResult::Dropped(MrfDropReason::OversizedIdentity);
     }
     let (version_id, scope) = canonical_identity(intent.kind, intent.version_id, intent.scope);
     intent.version_id = version_id;
     intent.scope = scope;
+    if !valid_intent_payload(intent) {
+        return MrfIngressResult::Dropped(MrfDropReason::InvalidPayload);
+    }
+    if intent.lease.is_some() {
+        return MrfIngressResult::Enqueued;
+    }
     intent.lease = Some(MrfIngressLease::new(NEXT_MRF_LEASE.fetch_add(1, Ordering::Relaxed)));
     MrfIngressResult::Enqueued
 }
@@ -554,6 +768,7 @@ pub struct MrfVerifiedRepairEvent {
     pub object: Arc<str>,
     pub version_id: Option<[u8; 16]>,
     pub scope: Option<MrfScope>,
+    pub delete_marker_purge: Option<MrfDeleteMarkerPurgeIdentity>,
     pub lease: Option<MrfIngressLease>,
     pub bucket_incarnation_id: Uuid,
     pub disposition: MrfVerifiedRepairDisposition,
@@ -655,6 +870,7 @@ mod tests {
             object: Arc::from("object"),
             version_id: Some([0u8; 16]),
             kind: MrfKind::DecodeFailure,
+            delete_marker_purge: None,
             scope: None,
             lease: None,
             enqueued_at_ms: 0,
@@ -719,6 +935,7 @@ mod tests {
             object: Arc::from("object"),
             version_id: Some([0; 16]),
             kind: MrfKind::PartialWrite,
+            delete_marker_purge: None,
             scope: Some(MrfScope {
                 pool_index: 1,
                 set_index: 2,
@@ -758,6 +975,7 @@ mod tests {
             object: Arc::from("object"),
             version_id: Some([0; 16]),
             kind: MrfKind::PartialWrite,
+            delete_marker_purge: None,
             scope: Some(MrfScope {
                 pool_index: 2,
                 set_index: 3,
@@ -792,6 +1010,7 @@ mod tests {
                 pool_index: 4,
                 set_index: 5,
             }),
+            delete_marker_purge: None,
             lease,
             bucket_incarnation_id: incarnation,
         };
@@ -801,6 +1020,7 @@ mod tests {
             object,
             version_id: anchor.version_id,
             scope: anchor.scope,
+            delete_marker_purge: None,
             lease: Some(lease),
             bucket_incarnation_id: incarnation,
             disposition: MrfVerifiedRepairDisposition::Repaired,
@@ -868,6 +1088,7 @@ mod tests {
                 pool_index: 1,
                 set_index: 2,
             }),
+            delete_marker_purge: None,
             lease,
             bucket_incarnation_id: incarnation,
         };
@@ -882,6 +1103,7 @@ mod tests {
             object: retained_anchor.object.clone(),
             version_id: retained_anchor.version_id,
             scope: retained_anchor.scope,
+            delete_marker_purge: None,
             lease: Some(retained_anchor.lease),
             bucket_incarnation_id: retained_anchor.bucket_incarnation_id,
             disposition: MrfVerifiedRepairDisposition::Repaired,
@@ -995,6 +1217,7 @@ mod tests {
                 pool_index: 2,
                 set_index: 3,
             }),
+            delete_marker_purge: None,
             lease: Some(MrfIngressLease::new(42)),
             bucket_incarnation_id,
             disposition: MrfVerifiedRepairDisposition::Repaired,
@@ -1011,3 +1234,70 @@ mod tests {
         assert_eq!(take_mrf_verified_repair_events_for("verified-bucket-b").len(), 1);
     }
 }
+#[tokio::test]
+async fn partial_write_durable_admission_waits_for_persistence_and_renews_lease() {
+    let mut receiver = init_durable_mrf_channel().expect("isolated test should initialize durable channel");
+    let scope = MrfScope {
+        pool_index: 2,
+        set_index: 3,
+    };
+    let first = tokio::spawn(async move { persist_partial_write_intent("bucket", "object", None, scope).await });
+    let submission = receiver.recv().await.expect("first submission should arrive");
+    let first_lease = submission.intent.lease;
+    assert!(!first.is_finished(), "channel delivery alone must not acknowledge durable ownership");
+    submission
+        .response
+        .send(Err(MrfDurableAdmissionError::Persistence))
+        .expect("producer should await receipt");
+    assert_eq!(first.await.expect("producer should complete"), Err(MrfDurableAdmissionError::Persistence));
+    let second = tokio::spawn(async move { persist_partial_write_intent("bucket", "object", None, scope).await });
+    let submission = receiver.recv().await.expect("replacement submission should arrive");
+    assert_ne!(
+        first_lease, submission.intent.lease,
+        "an overwrite must not reuse the previous proof identity"
+    );
+    assert_eq!(submission.intent.scope, Some(scope));
+    submission
+        .response
+        .send(Ok(()))
+        .expect("second producer should await receipt");
+    assert_eq!(second.await.expect("second producer should complete"), Ok(()));
+
+    let bucket_incarnation_id = Uuid::new_v4();
+    let marker_identity = [7; 32];
+    let purge = MrfDeleteMarkerPurge::new(bucket_incarnation_id, bucket_incarnation_id, marker_identity, vec![1, 2, 3])
+        .expect("valid purge payload");
+    let version_id = Uuid::new_v4();
+    let purge_submit =
+        tokio::spawn(async move { persist_delete_marker_purge_intent("bucket", "marker", version_id, scope, purge).await });
+    let submission = receiver.recv().await.expect("purge submission should arrive");
+    assert_eq!(submission.intent.kind, MrfKind::DeleteMarkerPurge);
+    assert_eq!(submission.intent.version_id, Some(*version_id.as_bytes()));
+    assert_eq!(submission.intent.scope, Some(scope));
+    assert_eq!(
+        submission
+            .intent
+            .delete_marker_purge
+            .as_ref()
+            .map(MrfDeleteMarkerPurge::identity),
+        Some(MrfDeleteMarkerPurgeIdentity {
+            bucket_incarnation_id,
+            marker_incarnation_id: bucket_incarnation_id,
+            marker_identity,
+        })
+    );
+    assert!(submission.intent.lease.is_some());
+    submission.response.send(Ok(())).expect("purge producer should await receipt");
+    assert_eq!(purge_submit.await.expect("purge producer should complete"), Ok(()));
+
+    let invalid = MrfDeleteMarkerPurge {
+        bucket_incarnation_id,
+        marker_incarnation_id: Uuid::new_v4(),
+        marker_identity,
+        marker: Arc::from([1u8]),
+    };
+    assert_eq!(
+        persist_delete_marker_purge_intent("bucket", "marker", version_id, scope, invalid).await,
+        Err(MrfDurableAdmissionError::InvalidIdentity)
+    );
+}

crates/common/src/trace_bus.rs

@@ -127,6 +127,44 @@ pub struct TraceEvent {
     pub attrs: SmallVec<[TraceAttr; TRACE_ATTR_INLINE_CAPACITY]>,
 }
 
+/// A telemetry operation whose published shape cannot retain request data.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum TelemetryTraceOperation {
+    GetObject,
+    PutObject,
+    HeadObject,
+    ListObjects,
+    InternalRpc,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum TelemetryTraceStatus {
+    Ok,
+    Error,
+}
+
+/// A pre-classified S3 or internode RPC observation.
+///
+/// This event deliberately has no request path, headers, object identity,
+/// payload, or free-form attributes. Producers must classify the operation
+/// and outcome before publishing it.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct TelemetryTraceEvent {
+    pub operation: TelemetryTraceOperation,
+    pub duration: Duration,
+    pub status: TelemetryTraceStatus,
+}
+
+impl TelemetryTraceEvent {
+    pub const fn new(operation: TelemetryTraceOperation, duration: Duration, status: TelemetryTraceStatus) -> Self {
+        Self {
+            operation,
+            duration,
+            status,
+        }
+    }
+}
+
 impl TraceEvent {
     pub fn new(kind: TraceKind, func: TraceFunc) -> Self {
         Self {
@@ -174,15 +212,20 @@ impl TraceEvent {
 pub struct TraceBus {
     sender: broadcast::Sender<Arc<TraceEvent>>,
     subscriber_count: Arc<AtomicUsize>,
+    telemetry_sender: broadcast::Sender<Arc<TelemetryTraceEvent>>,
+    telemetry_subscriber_count: Arc<AtomicUsize>,
 }
 
 impl TraceBus {
     pub fn new(capacity: usize) -> Self {
         let capacity = capacity.max(1);
         let (sender, _receiver) = broadcast::channel(capacity);
+        let (telemetry_sender, _telemetry_receiver) = broadcast::channel(capacity);
         Self {
             sender,
             subscriber_count: Arc::new(AtomicUsize::new(0)),
+            telemetry_sender,
+            telemetry_subscriber_count: Arc::new(AtomicUsize::new(0)),
         }
     }
 
@@ -206,6 +249,27 @@ impl TraceBus {
 
         self.sender.send(Arc::new(build())).is_ok()
     }
+
+    pub fn telemetry_subscriber_count(&self) -> usize {
+        self.telemetry_subscriber_count.load(Ordering::Acquire)
+    }
+
+    pub fn subscribe_telemetry(&self) -> TelemetryTraceSubscription {
+        let receiver = self.telemetry_sender.subscribe();
+        self.telemetry_subscriber_count.fetch_add(1, Ordering::AcqRel);
+        TelemetryTraceSubscription {
+            receiver,
+            subscriber_count: Arc::clone(&self.telemetry_subscriber_count),
+        }
+    }
+
+    pub fn emit_telemetry(&self, build: impl FnOnce() -> TelemetryTraceEvent) -> bool {
+        if self.telemetry_subscriber_count() == 0 {
+            return false;
+        }
+
+        self.telemetry_sender.send(Arc::new(build())).is_ok()
+    }
 }
 
 impl Default for TraceBus {
@@ -236,6 +300,28 @@ impl Drop for TraceSubscription {
     }
 }
 
+#[derive(Debug)]
+pub struct TelemetryTraceSubscription {
+    receiver: broadcast::Receiver<Arc<TelemetryTraceEvent>>,
+    subscriber_count: Arc<AtomicUsize>,
+}
+
+impl TelemetryTraceSubscription {
+    pub async fn recv(&mut self) -> Result<Arc<TelemetryTraceEvent>, broadcast::error::RecvError> {
+        self.receiver.recv().await
+    }
+
+    pub fn try_recv(&mut self) -> Result<Arc<TelemetryTraceEvent>, broadcast::error::TryRecvError> {
+        self.receiver.try_recv()
+    }
+}
+
+impl Drop for TelemetryTraceSubscription {
+    fn drop(&mut self) {
+        self.subscriber_count.fetch_sub(1, Ordering::AcqRel);
+    }
+}
+
 pub fn global_trace_bus() -> &'static TraceBus {
     GLOBAL_TRACE_BUS.get_or_init(TraceBus::default)
 }
@@ -252,6 +338,18 @@ pub fn trace_subscriber_count() -> usize {
     global_trace_bus().subscriber_count()
 }
 
+pub fn subscribe_telemetry_trace_events() -> TelemetryTraceSubscription {
+    global_trace_bus().subscribe_telemetry()
+}
+
+pub fn telemetry_trace_emit(build: impl FnOnce() -> TelemetryTraceEvent) -> bool {
+    global_trace_bus().emit_telemetry(build)
+}
+
+pub fn telemetry_trace_subscriber_count() -> usize {
+    global_trace_bus().telemetry_subscriber_count()
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -330,4 +428,30 @@ mod tests {
             .expect_err("receiver should observe lag instead of blocking publishers");
         assert!(matches!(err, broadcast::error::RecvError::Lagged(_)));
     }
+
+    #[tokio::test]
+    async fn telemetry_subscription_exposes_only_classified_fields() {
+        let bus = TraceBus::new(4);
+        let _unrelated_subscription = bus.subscribe();
+        let built = AtomicUsize::new(0);
+        assert!(!bus.emit_telemetry(|| {
+            built.fetch_add(1, Ordering::Relaxed);
+            TelemetryTraceEvent::new(TelemetryTraceOperation::GetObject, Duration::ZERO, TelemetryTraceStatus::Ok)
+        }));
+        assert_eq!(built.load(Ordering::Relaxed), 0);
+
+        let mut subscription = bus.subscribe_telemetry();
+        assert_eq!(bus.telemetry_subscriber_count(), 1);
+
+        assert!(bus.emit_telemetry(|| {
+            TelemetryTraceEvent::new(TelemetryTraceOperation::GetObject, Duration::from_micros(7), TelemetryTraceStatus::Ok)
+        }));
+
+        let event = subscription.recv().await.expect("classified telemetry event");
+        assert_eq!(event.operation, TelemetryTraceOperation::GetObject);
+        assert_eq!(event.duration, Duration::from_micros(7));
+        assert_eq!(event.status, TelemetryTraceStatus::Ok);
+        drop(subscription);
+        assert_eq!(bus.telemetry_subscriber_count(), 0);
+    }
 }

crates/config/src/constants/body_limits.rs

@@ -64,6 +64,14 @@ pub const MAX_HEAL_REQUEST_SIZE: usize = 1024 * 1024; // 1 MB
 /// memory exhaustion from malicious or misconfigured remote services.
 pub const MAX_S3_CLIENT_RESPONSE_SIZE: usize = 10 * 1024 * 1024; // 10 MB
 
+/// Maximum body size accepted by a single `PutObject` or `UploadPart` request (5 GiB).
+/// Used for: the s3s streaming-body limit and the request-header admission check.
+/// Rationale: matches the AWS S3 single-PUT / single-part ceiling. Larger objects
+/// must use multipart upload. The header check rejects an oversize
+/// `Content-Length` before any body byte is read so the client gets
+/// `EntityTooLarge` immediately instead of streaming 5 GiB into a mid-stream failure.
+pub const MAX_SINGLE_PUT_OBJECT_SIZE: u64 = 5 * 1024 * 1024 * 1024; // 5 GiB
+
 /// Maximum size for OIDC provider response bodies (1 MB)
 /// Used for: discovery documents, JWKS documents and token endpoint responses
 /// Rationale: a hostile or compromised identity provider must not be able to exhaust

crates/config/src/constants/heal.rs

@@ -208,9 +208,9 @@ pub const DEFAULT_HEAL_MRF_REPLAY_BATCH: usize = 256;
 
 /// Environment variable selecting how admin heal starts behave when the
 /// requested path overlaps an already running or queued heal: `merge`
-/// (default, keep today's dedup/merge semantics) or `minio_error` (return a
-/// typed already-running / overlapping-paths rejection like madmin).
+/// (default, merge equivalent starts and reject other overlapping admin
+/// scopes) or `minio_error` (also reject equivalent starts like madmin).
 pub const ENV_HEAL_OVERLAP_POLICY: &str = "RUSTFS_HEAL_OVERLAP_POLICY";
 
-/// Default overlap policy: merge duplicate/overlapping requests.
+/// Default overlap policy: merge equivalent requests, reject overlapping scopes.
 pub const DEFAULT_HEAL_OVERLAP_POLICY: &str = "merge";

crates/config/src/constants/object.rs

@@ -116,6 +116,30 @@ pub const ENV_OBJECT_GET_SKIP_BITROT_VERIFY: &str = "RUSTFS_OBJECT_GET_SKIP_BITR
 /// Default: bitrot verification is enabled on GetObject reads (do not skip).
 pub const DEFAULT_OBJECT_GET_SKIP_BITROT_VERIFY: bool = false;
 
+/// Create independent shard commitments for new writes after the fleet is upgraded.
+/// Existing protected objects and multipart uploads retain their protection.
+pub const ENV_SHARD_INTEGRITY_WRITE: &str = "RUSTFS_SHARD_INTEGRITY_WRITE";
+pub const DEFAULT_SHARD_INTEGRITY_WRITE: bool = false;
+
+/// Operator confirmation that every reader, writer and background coordinator
+/// understands independent shard commitments. This is not capability discovery
+/// or a fence against an old binary rejoining the fleet.
+pub const ENV_SHARD_INTEGRITY_FLEET_CONFIRMED: &str = "RUSTFS_SHARD_INTEGRITY_FLEET_CONFIRMED";
+pub const DEFAULT_SHARD_INTEGRITY_FLEET_CONFIRMED: bool = false;
+
+const _: () = assert!(!DEFAULT_SHARD_INTEGRITY_WRITE);
+const _: () = assert!(!DEFAULT_SHARD_INTEGRITY_FLEET_CONFIRMED);
+
+/// How object writes treat a bucket whose stored versioning configuration
+/// cannot be parsed: `permissive` writes as if unversioned (the historical
+/// behavior, recorded by metrics and an error log) and `strict` refuses the
+/// write with 503. Paths that already refuse an unreadable configuration do
+/// so in both modes. Any other value fails startup.
+pub const ENV_BUCKET_CONFIG_PARSE_MODE: &str = "RUSTFS_BUCKET_CONFIG_PARSE_MODE";
+
+/// Default bucket config parse mode.
+pub const DEFAULT_BUCKET_CONFIG_PARSE_MODE: &str = "permissive";
+
 /// Request writing the complete remote-tier version state into object metadata.
 ///
 /// This remains ineffective until

crates/e2e_test/AGENTS.md

@@ -27,4 +27,4 @@ follow.
 
 ## Suggested Validation
 
-- `cargo test --package e2e_test`
+- `python3 scripts/e2e_binary.py build --features e2e-test-hooks`, then `python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo test --package e2e_test` (running `cargo test -p e2e_test` directly fails with a missing E2E run receipt; see [`README.md`](README.md#how-to-run))

crates/e2e_test/Cargo.toml

@@ -131,7 +131,7 @@ md-5 = { workspace = true }
 opentelemetry-proto = { workspace = true }
 prost.workspace = true
 sha2 = { workspace = true }
-astral-tokio-tar = { workspace = true }
+rustfs-tokio-tar = { workspace = true }
 s3s = { workspace = true, features = ["minio"] }
 zstd.workspace = true
 time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }

crates/e2e_test/README.md

@@ -1,7 +1,7 @@
 # e2e_test
 
 End-to-end test suite for RustFS. Each test spawns a **real `rustfs` binary**
-(built on demand from the workspace) and drives it over the network with the
+(built and identified before the test invocation) and drives it over the network with the
 AWS SDK (`aws-sdk-s3`), raw HTTP (`reqwest` / `awscurl`), or a protocol client
 (FTPS / WebDAV / SFTP). This is the black-box integration layer: exhaustive
 end-to-end behavior lives here, unit behavior stays in the source crates
@@ -34,39 +34,35 @@ The external-tool `storage_metric_ownership_test` validates the OTLP/Collector/P
 
 ## How to run
 
-All commands assume repo root. `cargo test` triggers an on-demand build of the
-`rustfs` binary from [`src/common.rs`](src/common.rs) (`rustfs_binary_path`) on
-first use — the first invocation is slow, later ones reuse the binary.
-
-Root-heal interruption scenarios use a test-only commit barrier. Prebuild with `e2e-test-hooks` and pin that binary so concurrent cases do not replace it through on-demand builds:
+All commands assume repo root and Python 3.9 or newer on Linux or macOS. Build the server once through the provenance entry point, then run the test command through the same script:
 
 ```bash
-cargo build -p rustfs --bin rustfs --features e2e-test-hooks
-CARGO_BIN_EXE_rustfs="$PWD/target/debug/rustfs" cargo nextest run -p e2e_test -E 'test(heal_erasure_disk_rebuild_test)'
-```
+python3 scripts/e2e_binary.py build --features e2e-test-hooks
 
-```bash
-# Whole crate (default = ignored tests skipped)
-cargo nextest run -p e2e_test
+# Whole crate (ignored tests remain skipped)
+python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run -p e2e_test
 
 # One module
-cargo nextest run -p e2e_test -E 'test(list_objects_v2_pagination_test)'
-
-# PR smoke subset (see "CI smoke subset" below)
-cargo nextest run --profile e2e-smoke -p e2e_test
-
-# ILM serial lane — ignored lifecycle tests, single-threaded (mirrors CI)
-cargo nextest run -j1 --run-ignored ignored-only -p rustfs-scanner -p rustfs \
-  -E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))'
+python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run -p e2e_test -E 'test(list_objects_v2_pagination_test)'
 
+# PR smoke subset
+python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke -p e2e_test
 ```
 
-The protocols suite has its own contract (fixed bind ports 9022–9301,
-single-worker execution, feature-gated scheduling) documented in
-[`src/protocols/README.md`](src/protocols/README.md). `RUSTFS_BUILD_FEATURES`
-selects which features the spawned binary is built with; leave it unset to run
-every protocol entry. Use the exact profile command under
-[Troubleshooting](#troubleshooting) for CI-equivalent execution.
+Root-heal interruption scenarios use a test-only commit barrier, so build and run them with `e2e-test-hooks`:
+
+```bash
+python3 scripts/e2e_binary.py build --features e2e-test-hooks
+python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run -p e2e_test -E 'test(heal_erasure_disk_rebuild_test)'
+```
+
+`build` records the source contents, HEAD, resolved Cargo features, profile, toolchain, and binary SHA-256 beside the executable in `rustfs.e2e.json`. `run` validates that identity before and after the command, preserves command failures, and removes its temporary run receipt on completion. The Rust harness checks that receipt before starting each server; it never compiles a server inside a test process. Source or binary changes during a run invalidate the result, even when the test command succeeds. Use an isolated worktree and keep it unchanged until the command finishes.
+
+The additional `--features` arguments must match between `build` and `run`; Cargo defaults remain enabled. The wrapper supplies `RUSTFS_BUILD_FEATURES` from Cargo's resolved feature list, including features enabled by `full`. Protocol helpers require a subset of that list. `CARGO_TARGET_DIR` and `--profile release` are supported. An in-workspace target directory must be Git-ignored; tracked files are always included in the source identity. `build --bins` preserves CI lanes that compile all RustFS binary targets. For a downloaded artifact, copy both the executable and its sidecar, then use `run`; do not generate a new identity for an arbitrary prebuilt binary. `CARGO_BIN_EXE_rustfs` cannot override the verified executable.
+
+Each build/run holds an exclusive `rustfs.e2e.lock` marker beside the binary; concurrent wrappers fail immediately. Use a private target directory and do not run ordinary Cargo builds against it while tests are active: Cargo does not honor this marker. Interrupted runs fail and terminate their command group. After an uncatchable kill, inspect the PID recorded in a leftover marker and remove it only after confirming its owner has stopped. Embedded file symlinks are hashed through their target; embedded directory symlinks are rejected because their contents cannot be enumerated safely by this entry point.
+
+The protocols suite has its own fixed-port and single-worker contract in [`src/protocols/README.md`](src/protocols/README.md). Use its command under [Troubleshooting](#troubleshooting).
 
 ### `#[ignore]` semantics
 
@@ -132,7 +128,7 @@ via `create_s3_client(idx)` / `create_all_clients()`. See
 | `wait_for_server_ready` | Poll readiness before issuing requests |
 | `create_s3_client` / `create_test_bucket` / `delete_test_bucket` | aws-sdk-s3 client + bucket lifecycle |
 | `find_available_port` | Random free port (isolation primitive) |
-| `rustfs_binary_path` / `_with_features` | Locate/build the binary; honors `RUSTFS_BUILD_FEATURES` |
+| `rustfs_binary_path` / `_with_features` | Verify this run's binary receipt and required feature subset |
 | `requested_rustfs_build_features` / `rustfs_build_feature_enabled` | Feature-gate a test to what the binary was built with |
 | `execute_awscurl` / `awscurl_post` / `_get` / `_put` / `_delete` / `awscurl_post_sts_form_urlencoded` | Admin/STS API calls via `awscurl`; missing binaries are test failures |
 | `replication_fast_env` | Env vars that shrink replication timers (from repl-4); pass to `start_rustfs_server_with_env` |
@@ -198,35 +194,33 @@ the wiring source of truth. Committed test-ID digests under
 **Reproduce a CI failure locally** — run the exact profile/lane:
 
 ```bash
-# Smoke (e2e-tests job) — includes the 20 fast replication tests
-cargo nextest run --profile e2e-smoke -p e2e_test
-# Full single-node merge/main lane
-cargo nextest run --profile e2e-full -p e2e_test
-# Cluster fault nightly lane
-cargo nextest run --profile e2e-nightly -p e2e_test
-# 4-node 4-disk distributed lane (S3 / lock / versioning / replication / decommission / chaos / upgrade)
-# Upgrade cases need RUSTFS_UPGRADE_SOURCE_BINARY; without it they fail closed.
-cargo nextest run --profile e2e-distributed -p e2e_test
-# Replication nightly lane; awscurl is required for STS paths
-cargo nextest run --profile e2e-repl-nightly -p e2e_test
-# Fixed-port protocol nightly lane
-RUSTFS_BUILD_FEATURES=ftps,webdav,sftp \
-  cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture
-# ILM serial lane
+# Smoke, full, and cluster lanes share a server with fault-test hooks.
+python3 scripts/e2e_binary.py build --features e2e-test-hooks
+python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke -p e2e_test
+python3 scripts/e2e_binary.py run --binary "$RUSTFS_E2E_STARTUP_CAS_BINARY" --features e2e-test-hooks -- cargo nextest run --profile e2e-full -p e2e_test
+python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-nightly -p e2e_test
+
+# Distributed 4-node 4-disk lane uses the default server.
+# Upgrade cases require RUSTFS_UPGRADE_SOURCE_BINARY and fail closed without it.
+python3 scripts/e2e_binary.py build
+python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-distributed -p e2e_test
+
+# Replication nightly uses the default server; awscurl is required for STS.
+python3 scripts/e2e_binary.py build
+python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-repl-nightly -p e2e_test
+
+# Protocol nightly owns fixed ports.
+python3 scripts/e2e_binary.py build --features ftps,webdav,sftp
+python3 scripts/e2e_binary.py run --features ftps,webdav,sftp -- cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture
+
+# The ILM serial lane does not use this server harness.
 cargo nextest run -j1 --run-ignored ignored-only -p rustfs-scanner -p rustfs \
   -E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))'
-# s3s-e2e black box
-./scripts/e2e-run.sh ./target/debug/rustfs /tmp/rustfs-e2e-data
 ```
 
-**Stale binary.** Tests build the `rustfs` binary once and reuse it. To avoid
-rebuilding while iterating on tests, `common.rs` reuses an existing binary when
-running *inside* the e2e test process even if sources changed
-(`can_reuse_inside_e2e`, [`src/common.rs`](src/common.rs) line 98). Downside: if
-you changed **server** code, force a rebuild with
-`cargo build -p rustfs` (or `touch` a source file outside the reuse window)
-before re-running, or CI's freshly built artifact will diverge from your local
-one.
+The full lane also requires the startup-CAS build manifest generated by the `Build debug binary` step in `.github/workflows/ci.yml`. Preserve that binary and both sidecars as its `Preserve startup CAS binary input` step does, and use the same `RUSTFS_E2E_STARTUP_CAS_*` environment as `Run e2e full suite`. A generic local build alone does not supply that fixture evidence.
+
+**Stale or unverified binary.** Re-run the matching `build` command after changing source or features, then invoke tests through `run`. A missing receipt, copied old executable, or mismatched build identity is a prerequisite failure. Bare Cargo invocations that start a server deliberately fail; unit tests that do not start a server can still run directly.
 
 **Port already in use / orphan processes.** A hard-killed run can leak a
 `rustfs` child holding its port. Find and kill it:
@@ -258,7 +252,8 @@ spawn error. Install the pinned CI version before running their profiles.
 A subset of this crate runs on every PR via the `e2e-tests` job:
 
 ```bash
-cargo nextest run --profile e2e-smoke -p e2e_test
+python3 scripts/e2e_binary.py build --features e2e-test-hooks
+python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke -p e2e_test
 ```
 
 The selection lives in `.config/nextest.toml` under `[profile.e2e-smoke]`

crates/e2e_test/src/chaos.rs

@@ -61,6 +61,7 @@ pub(crate) struct VersionShardCensus {
     pub has_xl_meta: bool,
     pub data_dir: Option<String>,
     pub erasure_index: Option<usize>,
+    pub erasure_distribution: Option<Vec<usize>>,
     pub data_blocks: Option<usize>,
     pub parity_blocks: Option<usize>,
     pub expected_part_numbers: BTreeSet<usize>,
@@ -90,6 +91,7 @@ impl VersionShardCensus {
             && manifest.is_complete()
             && self.data_dir == manifest.data_dir
             && self.erasure_index == manifest.erasure_index
+            && self.erasure_distribution == manifest.erasure_distribution
             && self.data_blocks == manifest.data_blocks
             && self.parity_blocks == manifest.parity_blocks
             && self.expected_part_numbers == manifest.expected_part_numbers
@@ -317,6 +319,7 @@ pub(crate) fn census_object_version_on_disk(
             has_xl_meta: false,
             data_dir: None,
             erasure_index: None,
+            erasure_distribution: None,
             data_blocks: None,
             parity_blocks: None,
             expected_part_numbers: BTreeSet::new(),
@@ -334,6 +337,7 @@ pub(crate) fn census_object_version_on_disk(
     };
     let data_dir = file_info.data_dir.map(|id| id.to_string());
     let erasure_index = Some(file_info.erasure.index);
+    let erasure_distribution = Some(file_info.erasure.distribution.clone());
     let inline_data_fingerprint = file_info.data.as_deref().map(shard_fingerprint).transpose()?;
     let part_dir = data_dir.as_ref().map_or_else(|| object_dir.clone(), |id| object_dir.join(id));
     let present_part_fingerprints = match std::fs::read_dir(&part_dir) {
@@ -366,6 +370,7 @@ pub(crate) fn census_object_version_on_disk(
         has_xl_meta: true,
         data_dir,
         erasure_index,
+        erasure_distribution,
         data_blocks: Some(file_info.erasure.data_blocks),
         parity_blocks: Some(file_info.erasure.parity_blocks),
         expected_part_numbers,
@@ -374,6 +379,30 @@ pub(crate) fn census_object_version_on_disk(
     })
 }
 
+pub(crate) fn is_cluster_heal_coordination_unavailable(error: &(dyn std::error::Error + Send + Sync)) -> bool {
+    let message = error.to_string();
+    message.contains("500 Internal Server Error") && message.contains("cluster heal coordination unavailable")
+}
+
+/// Wait for the restarted cluster to admit the first root heal request.
+pub(crate) async fn start_root_heal_when_control_ready(
+    heal_url: &str,
+    heal_body: &str,
+    access_key: &str,
+    secret_key: &str,
+) -> ChaosResult<()> {
+    let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(45);
+    loop {
+        match signed_admin_post(heal_url, Some(heal_body), access_key, secret_key).await {
+            Ok(_) => return Ok(()),
+            Err(error) if is_cluster_heal_coordination_unavailable(error.as_ref()) && tokio::time::Instant::now() < deadline => {
+                tokio::time::sleep(std::time::Duration::from_millis(250)).await;
+            }
+            Err(error) => return Err(error),
+        }
+    }
+}
+
 /// `POST` a signed (SigV4, service `s3`) admin request without relying on the
 /// external `awscurl` binary. Mirrors the admin heal calls used by the heal
 /// regression suite.
@@ -421,6 +450,7 @@ mod tests {
             has_xl_meta: true,
             data_dir: Some("data-dir".to_string()),
             erasure_index: Some(3),
+            erasure_distribution: Some(vec![1, 2, 3, 4]),
             data_blocks: Some(2),
             parity_blocks: Some(2),
             expected_part_numbers: BTreeSet::from([1]),

crates/e2e_test/src/common.rs

@@ -31,7 +31,6 @@ use rustfs_signer::constants::UNSIGNED_PAYLOAD;
 use rustfs_signer::sign_v4;
 use s3s::Body;
 use serde_json;
-use std::ffi::OsStr;
 use std::fs as stdfs;
 use std::io::ErrorKind;
 use std::net::SocketAddr;
@@ -44,7 +43,6 @@ use tokio::net::TcpStream;
 use tokio::time::sleep;
 use tracing::{error, info, warn};
 use uuid::Uuid;
-use walkdir::WalkDir;
 
 // Common constants for all E2E tests
 pub const DEFAULT_ACCESS_KEY: &str = "rustfsadmin";
@@ -437,59 +435,75 @@ fn resolve_rustfs_binary_path(workspace: &Path, configured_target_dir: Option<&P
     path
 }
 
-/// Resolve the RustFS binary relative to the workspace, optionally requesting build features.
+/// Resolve the server verified by `scripts/e2e_binary.py run` for this test invocation.
+/// Requested features are a required subset of the server's resolved Cargo features.
 pub fn rustfs_binary_path_with_features(requested_features: Option<&str>) -> PathBuf {
-    if let Some(path) = std::env::var_os("CARGO_BIN_EXE_rustfs") {
-        return PathBuf::from(path);
-    }
-    let requested_features = requested_features.and_then(normalize_rustfs_build_features);
-
     let workspace = workspace_root();
     let configured_target_dir = std::env::var_os("CARGO_TARGET_DIR").map(PathBuf::from);
-    let binary_path = resolve_rustfs_binary_path(&workspace, configured_target_dir.as_deref());
+    let binary_path = std::env::var_os("CARGO_BIN_EXE_rustfs")
+        .map(PathBuf::from)
+        .unwrap_or_else(|| resolve_rustfs_binary_path(&workspace, configured_target_dir.as_deref()));
+    let receipt_path = std::env::var_os("RUSTFS_E2E_BINARY_RECEIPT").map(PathBuf::from);
+    receipt_path
+        .ok_or_else(|| std::io::Error::new(ErrorKind::NotFound, "missing E2E run receipt"))
+        .and_then(|receipt| verify_e2e_binary_receipt(&receipt, &workspace, &binary_path, requested_features))
+        .unwrap_or_else(|error| {
+            panic!(
+                "E2E server prerequisite failed: {error}. Build with `python3 scripts/e2e_binary.py build --features <features>` and run tests with `python3 scripts/e2e_binary.py run --features <features> -- cargo nextest run ...`"
+            )
+        })
+}
 
-    let features_match = binary_features_match(&binary_path, requested_features.as_deref());
-    let source_is_newer = workspace_sources_newer_than_binary(&binary_path);
-    let can_reuse_inside_e2e = running_inside_e2e_test_binary() && requested_features.is_none() && features_match;
-    if binary_path.is_file() && features_match && (!source_is_newer || can_reuse_inside_e2e) {
-        if source_is_newer {
-            warn!(
-                "RustFS binary at {:?} appears older than workspace sources; reusing it inside cargo test to avoid nested builds",
-                binary_path
-            );
-        }
-        info!("Using existing RustFS binary at {:?}", binary_path);
-        return binary_path;
+#[derive(serde::Deserialize)]
+#[serde(deny_unknown_fields)]
+struct E2eBinaryReceipt {
+    schema: u32,
+    workspace: PathBuf,
+    binary: PathBuf,
+    size: u64,
+    modified_ns: u128,
+    features: Vec<String>,
+}
+
+fn verify_e2e_binary_receipt(
+    receipt_path: &Path,
+    workspace: &Path,
+    binary_path: &Path,
+    requested_features: Option<&str>,
+) -> std::io::Result<PathBuf> {
+    let receipt: E2eBinaryReceipt = serde_json::from_slice(&stdfs::read(receipt_path)?)?;
+    let binary = binary_path.canonicalize()?;
+    let metadata = binary.metadata()?;
+    let modified_ns = metadata
+        .modified()?
+        .duration_since(std::time::UNIX_EPOCH)
+        .map_err(std::io::Error::other)?
+        .as_nanos();
+    // The runner hashes source and binary before/after the entire suite. Each
+    // nextest process checks only this invocation's path, features, and file stat.
+    if receipt.schema != 1
+        || receipt.workspace != workspace.canonicalize()?
+        || receipt.binary != binary
+        || !metadata.is_file()
+        || receipt.size != metadata.len()
+        || receipt.modified_ns != modified_ns
+    {
+        return Err(std::io::Error::new(
+            ErrorKind::InvalidData,
+            "E2E server differs from this run's verified binary",
+        ));
     }
-
-    info!("Building RustFS binary to ensure it's up to date...");
-    build_rustfs_binary(requested_features.as_deref(), &binary_path);
-
-    info!("Using RustFS binary at {:?}", binary_path);
-    binary_path
-}
-
-fn workspace_sources_newer_than_binary(binary_path: &PathBuf) -> bool {
-    let Ok(binary_meta) = std::fs::metadata(binary_path) else {
-        return true;
-    };
-    let Ok(binary_modified) = binary_meta.modified() else {
-        return true;
-    };
-
-    let workspace = workspace_root();
-    let watch_roots = [
-        workspace.join("Cargo.toml"),
-        workspace.join("Cargo.lock"),
-        workspace.join("rustfs"),
-        workspace.join("crates"),
-    ];
-
-    watch_roots.iter().any(|path| path_is_newer_than(binary_modified, path))
-}
-
-fn running_inside_e2e_test_binary() -> bool {
-    std::env::var("CARGO_PKG_NAME").is_ok_and(|value| value == "e2e_test")
+    if let Some(requested) = requested_features.and_then(normalize_rustfs_build_features)
+        && requested
+            .split(',')
+            .any(|feature| !receipt.features.iter().any(|actual| actual == feature))
+    {
+        return Err(std::io::Error::new(
+            ErrorKind::InvalidInput,
+            "E2E server is missing a requested build feature",
+        ));
+    }
+    Ok(binary)
 }
 
 pub fn requested_rustfs_build_features() -> Option<String> {
@@ -519,96 +533,6 @@ pub fn rustfs_build_feature_enabled(requested_features: Option<&str>, required_f
         .any(|feature| feature.eq_ignore_ascii_case(RUSTFS_FULL_FEATURE) || feature.eq_ignore_ascii_case(required_feature))
 }
 
-fn rustfs_binary_features_stamp_path(binary_path: &Path) -> PathBuf {
-    binary_path.with_extension("features")
-}
-
-fn binary_features_match(binary_path: &Path, requested_features: Option<&str>) -> bool {
-    let stamp_path = rustfs_binary_features_stamp_path(binary_path);
-    let recorded = stdfs::read_to_string(stamp_path)
-        .ok()
-        .and_then(|value| normalize_rustfs_build_features(&value));
-    let requested = requested_features.and_then(normalize_rustfs_build_features);
-
-    match requested.as_deref() {
-        Some(features) => recorded.as_deref() == Some(features),
-        None => recorded.is_none(),
-    }
-}
-
-fn path_is_newer_than(binary_modified: std::time::SystemTime, path: &Path) -> bool {
-    if path.is_file() {
-        return std::fs::metadata(path)
-            .and_then(|meta| meta.modified())
-            .map(|modified| modified > binary_modified)
-            .unwrap_or(false);
-    }
-
-    if !path.is_dir() {
-        return false;
-    }
-
-    WalkDir::new(path)
-        .into_iter()
-        .filter_entry(|entry| {
-            let name = entry.file_name();
-            name != OsStr::new("target") && name != OsStr::new(".git")
-        })
-        .filter_map(Result::ok)
-        .filter(|entry| entry.file_type().is_file())
-        .any(|entry| {
-            std::fs::metadata(entry.path())
-                .and_then(|meta| meta.modified())
-                .map(|modified| modified > binary_modified)
-                .unwrap_or(false)
-        })
-}
-
-/// Build the RustFS binary using cargo
-fn build_rustfs_binary(requested_features: Option<&str>, binary_path: &Path) {
-    let workspace = workspace_root();
-    info!("Building RustFS binary from workspace: {:?}", workspace);
-
-    let _profile = if cfg!(debug_assertions) {
-        info!("Building in debug mode");
-        "dev"
-    } else {
-        info!("Building in release mode");
-        "release"
-    };
-
-    let mut cmd = Command::new("cargo");
-    cmd.current_dir(&workspace).args(["build", "--bin", "rustfs"]);
-
-    if let Some(features) = requested_features {
-        cmd.arg("--features").arg(features);
-        info!("Building with features: {}", features);
-    }
-
-    if !cfg!(debug_assertions) {
-        cmd.arg("--release");
-    }
-
-    info!(
-        "Executing: cargo build --bin rustfs {}",
-        if cfg!(debug_assertions) { "" } else { "--release" }
-    );
-
-    let output = cmd.output().expect("Failed to execute cargo build command");
-
-    if !output.status.success() {
-        let stderr = String::from_utf8_lossy(&output.stderr);
-        panic!("Failed to build RustFS binary. Error: {stderr}");
-    }
-
-    let stamp_path = rustfs_binary_features_stamp_path(binary_path);
-    if let Err(err) = stdfs::write(&stamp_path, requested_features.unwrap_or_default()) {
-        warn!("Failed to write RustFS feature stamp {:?}: {}", stamp_path, err);
-    }
-
-    info!("✅ RustFS binary built successfully");
-}
-
 fn awscurl_binary_path() -> PathBuf {
     std::env::var_os("AWSCURL_PATH")
         .map(PathBuf::from)
@@ -2255,16 +2179,66 @@ mod tests {
     }
 
     #[test]
-    fn binary_feature_stamp_matching_uses_normalized_features() {
-        let binary_path = std::env::temp_dir().join(format!("rustfs-feature-stamp-test-{}", Uuid::new_v4()));
-        let stamp_path = rustfs_binary_features_stamp_path(&binary_path);
+    fn explicit_binary_without_run_receipt_is_rejected() {
+        const CHILD_ENV: &str = "RUSTFS_E2E_RECEIPT_TEST_CHILD";
+        if std::env::var_os(CHILD_ENV).is_some() {
+            rustfs_binary_path_with_features(None);
+            return;
+        }
+        let executable = std::env::current_exe().expect("locate isolated test process");
+        let output = Command::new(&executable)
+            .args([
+                "--exact",
+                "common::tests::explicit_binary_without_run_receipt_is_rejected",
+                "--nocapture",
+            ])
+            .env(CHILD_ENV, "1")
+            .env("CARGO_BIN_EXE_rustfs", &executable)
+            .env_remove("RUSTFS_E2E_BINARY_RECEIPT")
+            .output()
+            .expect("run the missing-receipt scenario with isolated environment variables");
+        assert!(!output.status.success(), "an explicit binary must not bypass run verification");
+        assert!(String::from_utf8_lossy(&output.stderr).contains("missing E2E run receipt"));
+    }
 
-        stdfs::write(&stamp_path, " SFTP, ftps ").expect("write feature stamp");
-        assert!(binary_features_match(&binary_path, Some("sftp,ftps")));
-        assert!(binary_features_match(&binary_path, Some(" SFTP, FTPS ")));
-        assert!(!binary_features_match(&binary_path, Some("sftp")));
-
-        stdfs::remove_file(stamp_path).ok();
+    #[test]
+    fn e2e_run_receipt_rejects_replaced_binary_and_missing_features() {
+        let directory = std::env::temp_dir().join(format!("rustfs-e2e-receipt-test-{}", Uuid::new_v4()));
+        stdfs::create_dir(&directory).expect("create receipt fixture");
+        let binary = directory.join("rustfs");
+        let receipt = directory.join("receipt.json");
+        stdfs::write(&binary, "server").expect("write fixture binary");
+        let metadata = binary.metadata().expect("stat fixture binary");
+        let record = serde_json::json!({
+            "schema": 1,
+            "workspace": directory.canonicalize().expect("canonical workspace"),
+            "binary": binary.canonicalize().expect("canonical binary"),
+            "size": metadata.len(),
+            "modified_ns": metadata.modified().expect("modified time").duration_since(std::time::UNIX_EPOCH).expect("positive timestamp").as_nanos(),
+            "features": ["default", "full", "ftps", "webdav", "sftp"]
+        });
+        stdfs::write(&receipt, serde_json::to_vec(&record).expect("serialize receipt")).expect("write receipt");
+        verify_e2e_binary_receipt(&receipt, &directory, &binary, Some("sftp,webdav")).expect("resolved feature subset");
+        verify_e2e_binary_receipt(&receipt, &directory, &binary, Some("full")).expect("full was actually requested");
+        assert_eq!(
+            verify_e2e_binary_receipt(&receipt, &directory, &binary, Some("rio-v2"))
+                .expect_err("full does not enable rio-v2")
+                .kind(),
+            ErrorKind::InvalidInput
+        );
+        let other = directory.join("old-server");
+        stdfs::write(&other, "server").expect("write alternate binary");
+        assert!(verify_e2e_binary_receipt(&receipt, &directory, &other, None).is_err());
+        stdfs::write(&binary, "different server").expect("replace fixture binary");
+        assert!(verify_e2e_binary_receipt(&receipt, &directory, &binary, None).is_err());
+        stdfs::remove_file(&receipt).expect("remove expired receipt");
+        assert_eq!(
+            verify_e2e_binary_receipt(&receipt, &directory, &binary, None)
+                .expect_err("expired receipt")
+                .kind(),
+            ErrorKind::NotFound
+        );
+        stdfs::remove_dir_all(directory).expect("remove receipt fixture");
     }
 
     /// Build a cluster environment struct in-memory (no ports, no processes) so

crates/e2e_test/src/copy_object_metadata_test.rs

@@ -18,10 +18,14 @@
 mod tests {
     use crate::common::{RustFSTestEnvironment, init_logging};
     use aws_sdk_s3::error::ProvideErrorMetadata;
+    use aws_sdk_s3::presigning::PresigningConfig;
     use aws_sdk_s3::primitives::{ByteStream, DateTime, DateTimeFormat};
     use aws_sdk_s3::types::{
         BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, MetadataDirective, StorageClass, VersioningConfiguration,
     };
+    use flate2::{Compression, read::GzDecoder, write::GzEncoder};
+    use std::io::{Read, Write};
+    use std::time::Duration;
     use tracing::info;
 
     #[tokio::test]
@@ -108,6 +112,24 @@ mod tests {
             copied_head.metadata().and_then(|metadata| metadata.get("stale")),
             Some(&"must-be-removed".to_string())
         );
+        let copied_get = client
+            .get_object()
+            .bucket(bucket)
+            .key(copied_key)
+            .send()
+            .await
+            .expect("GET failed after default copy");
+        assert_eq!(copied_get.content_language(), Some("en-US"));
+        assert_eq!(
+            copied_get
+                .body
+                .collect()
+                .await
+                .expect("Failed to collect copied body")
+                .into_bytes()
+                .as_ref(),
+            content,
+        );
 
         client
             .copy_object()
@@ -241,6 +263,7 @@ mod tests {
             .send()
             .await
             .expect("GET failed after self-copy");
+        assert_eq!(get_resp.content_language(), Some("fr-FR"));
         let body = get_resp
             .body
             .collect()
@@ -286,6 +309,7 @@ mod tests {
             .send()
             .await
             .expect("GET failed after empty metadata replacement");
+        assert_eq!(empty_get_resp.content_language(), None);
         let empty_body = empty_get_resp
             .body
             .collect()
@@ -522,9 +546,143 @@ mod tests {
             multipart_body
         );
 
+        assert_versioned_content_language_round_trip(&client, bucket).await;
         env.stop_server();
     }
 
+    async fn assert_versioned_content_language_round_trip(client: &aws_sdk_s3::Client, bucket: &str) {
+        let http = reqwest::Client::builder()
+            .no_proxy()
+            .no_gzip()
+            .build()
+            .expect("Failed to build HTTP client without gzip decoding");
+        let original = b"Content-Language must follow the requested object version.";
+        for encoding in [None, Some("gzip")] {
+            let key = format!("language-{}.bin", encoding.unwrap_or("plain"));
+            let payload = if encoding.is_some() {
+                let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
+                encoder.write_all(original).expect("Failed to encode gzip fixture");
+                encoder.finish().expect("Failed to finish gzip fixture")
+            } else {
+                original.to_vec()
+            };
+            let mut versions = Vec::new();
+            for language in [Some("zh-CN"), None, Some("en-US")] {
+                let put = client
+                    .put_object()
+                    .bucket(bucket)
+                    .key(&key)
+                    .content_type("application/octet-stream")
+                    .set_content_encoding(encoding.map(str::to_string))
+                    .set_content_language(language.map(str::to_string))
+                    .metadata("content-language", "user-language")
+                    .body(ByteStream::from(payload.clone()))
+                    .send()
+                    .await
+                    .expect("Failed to write versioned language fixture");
+                versions.push((put.version_id().expect("PUT must return a version ID").to_string(), language));
+            }
+
+            let current_version = &versions.last().expect("Fixture has three versions").0;
+            let reads = std::iter::once((None, Some("en-US")))
+                .chain(versions.iter().map(|(version, language)| (Some(version.as_str()), *language)));
+            for (version, language) in reads {
+                let expected_version = version.unwrap_or(current_version);
+                let head = client
+                    .head_object()
+                    .bucket(bucket)
+                    .key(&key)
+                    .set_version_id(version.map(str::to_string))
+                    .send()
+                    .await
+                    .expect("HEAD failed for language fixture");
+                assert_eq!(head.content_language(), language);
+                assert_eq!(head.version_id(), Some(expected_version));
+
+                for range in [None, Some("bytes=1-7")] {
+                    let expected_body = if range.is_some() { &payload[1..8] } else { payload.as_slice() };
+                    let content_range = range.map(|_| format!("bytes 1-7/{}", payload.len()));
+                    let request = client
+                        .get_object()
+                        .bucket(bucket)
+                        .key(&key)
+                        .set_version_id(version.map(str::to_string))
+                        .set_range(range.map(str::to_string));
+                    let get = request.clone().send().await.expect("GET failed for language fixture");
+                    assert_eq!(get.content_language(), language);
+                    assert_eq!(get.content_type(), Some("application/octet-stream"));
+                    assert_eq!(get.content_encoding(), encoding);
+                    assert_eq!(
+                        get.content_length(),
+                        Some(i64::try_from(expected_body.len()).expect("Fixture length fits i64"))
+                    );
+                    assert_eq!(get.content_range(), content_range.as_deref());
+                    assert_eq!(get.version_id(), Some(expected_version));
+                    assert_eq!(get.e_tag(), head.e_tag());
+                    assert_eq!(
+                        get.metadata()
+                            .and_then(|metadata| metadata.get("content-language"))
+                            .map(String::as_str),
+                        Some("user-language")
+                    );
+                    assert_eq!(
+                        get.body
+                            .collect()
+                            .await
+                            .expect("Failed to collect GET body")
+                            .into_bytes()
+                            .as_ref(),
+                        expected_body
+                    );
+
+                    for override_language in [None, Some("fr-FR")] {
+                        let presigned = request
+                            .clone()
+                            .set_response_content_language(override_language.map(str::to_string))
+                            .presigned(PresigningConfig::expires_in(Duration::from_secs(300)).expect("Valid signature lifetime"))
+                            .await
+                            .expect("Failed to presign language GET");
+                        let mut raw_request = http.get(presigned.uri());
+                        for (name, value) in presigned.headers() {
+                            raw_request = raw_request.header(name, value);
+                        }
+                        let response = raw_request.send().await.expect("Presigned GET failed");
+                        let status = if range.is_some() {
+                            reqwest::StatusCode::PARTIAL_CONTENT
+                        } else {
+                            reqwest::StatusCode::OK
+                        };
+                        assert_eq!(response.status(), status);
+                        let header = |name: &str| {
+                            response
+                                .headers()
+                                .get(name)
+                                .map(|value| value.to_str().expect("ASCII response header"))
+                        };
+                        assert_eq!(header("content-language"), override_language.or(language));
+                        assert_eq!(header("x-amz-meta-content-language"), Some("user-language"));
+                        assert_eq!(header("content-encoding"), encoding);
+                        assert_eq!(header("content-range"), content_range.as_deref());
+                        assert_eq!(header("x-amz-version-id"), Some(expected_version));
+                        assert_eq!(
+                            response.content_length(),
+                            Some(u64::try_from(expected_body.len()).expect("Fixture length fits u64"))
+                        );
+                        let body = response.bytes().await.expect("Failed to read wire body");
+                        assert_eq!(body.as_ref(), expected_body);
+                        if encoding.is_some() && range.is_none() {
+                            let mut decoded = Vec::new();
+                            GzDecoder::new(body.as_ref())
+                                .read_to_end(&mut decoded)
+                                .expect("GET body must contain actual gzip bytes");
+                            assert_eq!(decoded, original);
+                        }
+                    }
+                }
+            }
+        }
+    }
+
     #[tokio::test]
     async fn invalid_replacement_metadata_does_not_mutate_destination() {
         init_logging();

crates/e2e_test/src/delete_authorization_test.rs

@@ -0,0 +1,656 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Standard S3 deletion permissions and the explicit recursive-delete extension.
+
+use crate::common::{
+    AdminTransport, RustFSTestEnvironment, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user,
+    init_logging,
+};
+use aws_sdk_s3::Client;
+use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
+use aws_sdk_s3::operation::delete_object::{DeleteObjectError, DeleteObjectOutput};
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{BucketVersioningStatus, Delete, ObjectIdentifier, VersioningConfiguration};
+use futures::{StreamExt, TryStreamExt, stream};
+use serde_json::{Value, json};
+use std::collections::BTreeSet;
+use std::error::Error;
+use uuid::Uuid;
+
+type TestResult<T = ()> = Result<T, Box<dyn Error + Send + Sync>>;
+type VersionSnapshot = BTreeSet<(String, String, bool)>;
+
+async fn set_policy(env: &RustFSTestEnvironment, name: &str, policy: &Value) -> TestResult {
+    admin_add_canned_policy_via(
+        AdminTransport::Signed,
+        &env.url,
+        &env.access_key,
+        &env.secret_key,
+        name,
+        &policy.to_string(),
+    )
+    .await
+}
+
+async fn policy_user(env: &RustFSTestEnvironment, policy_name: &str, policy: Option<Value>) -> TestResult<Client> {
+    let username = Uuid::new_v4().simple().to_string();
+    let secret = Uuid::new_v4().simple().to_string();
+    admin_create_user(env, &username, &secret).await?;
+    if let Some(policy) = policy {
+        set_policy(env, policy_name, &policy).await?;
+    }
+    admin_attach_user_policy_via(AdminTransport::Signed, &env.url, &env.access_key, &env.secret_key, policy_name, &username)
+        .await?;
+    Ok(env.create_s3_client_with_credentials(&username, &secret))
+}
+
+async fn versioning(client: &Client, bucket: &str, status: BucketVersioningStatus) -> TestResult {
+    client
+        .put_bucket_versioning()
+        .bucket(bucket)
+        .versioning_configuration(VersioningConfiguration::builder().status(status).build())
+        .send()
+        .await?;
+    Ok(())
+}
+
+async fn put(client: &Client, bucket: &str, key: &str) -> TestResult<String> {
+    let result = client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"delete authorization fixture"))
+        .send()
+        .await?;
+    Ok(result.version_id().unwrap_or("null").to_string())
+}
+
+async fn versions(client: &Client, bucket: &str, prefix: &str) -> TestResult<VersionSnapshot> {
+    let mut result = BTreeSet::new();
+    let mut markers = (None, None);
+    loop {
+        let page = client
+            .list_object_versions()
+            .bucket(bucket)
+            .prefix(prefix)
+            .set_key_marker(markers.0.clone())
+            .set_version_id_marker(markers.1.clone())
+            .send()
+            .await?;
+        for version in page.versions() {
+            result.insert((
+                version.key().ok_or("listed version missing key")?.to_string(),
+                version.version_id().ok_or("listed version missing ID")?.to_string(),
+                false,
+            ));
+        }
+        for marker in page.delete_markers() {
+            result.insert((
+                marker.key().ok_or("listed delete marker missing key")?.to_string(),
+                marker.version_id().ok_or("listed delete marker missing ID")?.to_string(),
+                true,
+            ));
+        }
+        if page.is_truncated() != Some(true) {
+            return Ok(result);
+        }
+        let next = (
+            Some(
+                page.next_key_marker()
+                    .ok_or("truncated versions page missing next key marker")?
+                    .to_string(),
+            ),
+            page.next_version_id_marker().map(str::to_string),
+        );
+        assert_ne!(markers, next, "ListObjectVersions pagination must advance");
+        markers = next;
+    }
+}
+
+async fn force_delete(
+    client: &Client,
+    bucket: &str,
+    prefix: &str,
+) -> Result<DeleteObjectOutput, Box<SdkError<DeleteObjectError>>> {
+    client
+        .delete_object()
+        .bucket(bucket)
+        .key(prefix)
+        .customize()
+        .mutate_request(|request| {
+            request.headers_mut().insert("x-rustfs-force-delete", "true");
+        })
+        .send()
+        .await
+        .map_err(Box::new)
+}
+
+async fn replica_force_delete(
+    client: &Client,
+    bucket: &str,
+    prefix: &str,
+) -> Result<DeleteObjectOutput, Box<SdkError<DeleteObjectError>>> {
+    client
+        .delete_object()
+        .bucket(bucket)
+        .key(prefix)
+        .customize()
+        .mutate_request(|request| {
+            request.headers_mut().insert("x-rustfs-force-delete", "true");
+            request.headers_mut().insert("x-amz-replication-status", "REPLICA");
+        })
+        .send()
+        .await
+        .map_err(Box::new)
+}
+
+fn assert_denied<T, E>(result: Result<T, SdkError<E>>)
+where
+    T: std::fmt::Debug,
+    E: ProvideErrorMetadata + std::fmt::Debug,
+{
+    let error = result.expect_err("request must be denied by its S3 permission");
+    assert_eq!(
+        error.as_service_error().and_then(ProvideErrorMetadata::code),
+        Some("AccessDenied"),
+        "expected an S3 authorization denial, got {error:?}"
+    );
+}
+
+fn assert_boxed_denied<T, E>(result: Result<T, Box<SdkError<E>>>)
+where
+    T: std::fmt::Debug,
+    E: ProvideErrorMetadata + std::fmt::Debug,
+{
+    assert_denied(result.map_err(|error| *error));
+}
+
+#[tokio::test]
+async fn sdk_version_deletion_requires_only_delete_object_version() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    let root = env.create_s3_client();
+    let bucket = "delete-version-permissions";
+    root.create_bucket().bucket(bucket).send().await?;
+    put(&root, bucket, "single-null.txt").await?;
+    put(&root, bucket, "batch-null.txt").await?;
+    versioning(&root, bucket, BucketVersioningStatus::Enabled).await?;
+    let old = put(&root, bucket, "single.txt").await?;
+    let current = put(&root, bucket, "single.txt").await?;
+    let batch_version = put(&root, bucket, "batch.txt").await?;
+    let ordinary_version = put(&root, bucket, "ordinary.txt").await?;
+    let user = policy_user(
+        &env,
+        "version-deleter",
+        Some(json!({"Version":"2012-10-17","Statement":[
+            {"Effect":"Allow","Action":"s3:DeleteObjectVersion","Resource":format!("arn:aws:s3:::{bucket}/*")},
+            {"Effect":"Deny","Action":"s3:DeleteObject","Resource":format!("arn:aws:s3:::{bucket}/*")}
+        ]})),
+    )
+    .await?;
+
+    user.delete_object()
+        .bucket(bucket)
+        .key("single.txt")
+        .version_id(&old)
+        .send()
+        .await?;
+    user.delete_object()
+        .bucket(bucket)
+        .key("single-null.txt")
+        .version_id("null")
+        .send()
+        .await?;
+    assert_denied(user.delete_object().bucket(bucket).key("ordinary.txt").send().await);
+
+    let batch = user
+        .delete_objects()
+        .bucket(bucket)
+        .delete(
+            Delete::builder()
+                .objects(
+                    ObjectIdentifier::builder()
+                        .key("batch.txt")
+                        .version_id(&batch_version)
+                        .build()?,
+                )
+                .objects(ObjectIdentifier::builder().key("batch-null.txt").version_id("null").build()?)
+                .objects(ObjectIdentifier::builder().key("ordinary.txt").build()?)
+                .build()?,
+        )
+        .send()
+        .await?;
+    assert_eq!(batch.deleted().len(), 2, "both explicit version items must succeed");
+    assert_eq!(batch.errors().len(), 1, "only the unversioned item must be denied");
+    assert_eq!(batch.errors()[0].key(), Some("ordinary.txt"));
+    assert_eq!(batch.errors()[0].code(), Some("AccessDenied"));
+    assert_eq!(
+        versions(&root, bucket, "").await?,
+        BTreeSet::from([
+            ("single.txt".into(), current, false),
+            ("ordinary.txt".into(), ordinary_version, false)
+        ]),
+        "version-only deletion must preserve the current single-object version and denied object"
+    );
+    Ok(())
+}
+
+#[tokio::test]
+async fn sdk_list_bucket_and_list_bucket_versions_permissions_are_independent() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    let root = env.create_s3_client();
+    let bucket = "list-version-permissions";
+    root.create_bucket().bucket(bucket).send().await?;
+    versioning(&root, bucket, BucketVersioningStatus::Enabled).await?;
+    put(&root, bucket, "visible.txt").await?;
+    for (action, name) in [
+        ("s3:ListBucket", "object-lister"),
+        ("s3:ListBucketVersions", "version-lister"),
+    ] {
+        let user = policy_user(
+            &env,
+            name,
+            Some(json!({"Version":"2012-10-17","Statement":[
+                {"Effect":"Allow","Action":action,"Resource":format!("arn:aws:s3:::{bucket}")}
+            ]})),
+        )
+        .await?;
+        if action == "s3:ListBucket" {
+            assert_eq!(user.list_objects_v2().bucket(bucket).send().await?.contents().len(), 1);
+            assert_denied(user.list_object_versions().bucket(bucket).send().await);
+        } else {
+            assert_eq!(user.list_object_versions().bucket(bucket).send().await?.versions().len(), 1);
+            assert_denied(user.list_objects_v2().bucket(bucket).send().await);
+        }
+    }
+    Ok(())
+}
+
+#[tokio::test]
+async fn console_admin_force_delete_removes_prefix_versions_and_delete_markers() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    let root = env.create_s3_client();
+    let bucket = "force-console-admin";
+    root.create_bucket().bucket(bucket).send().await?;
+    put(&root, bucket, "folder/null.txt").await?;
+    versioning(&root, bucket, BucketVersioningStatus::Enabled).await?;
+    for key in ["folder/a.txt", "folder/deep/b.txt", "single.txt"] {
+        put(&root, bucket, key).await?;
+        put(&root, bucket, key).await?;
+        root.delete_object().bucket(bucket).key(key).send().await?;
+    }
+    put(&root, bucket, "keep.txt").await?;
+    put(&root, bucket, "folder-sibling/keep.txt").await?;
+    let keep = versions(&root, bucket, "keep.txt").await?;
+    let sibling = versions(&root, bucket, "folder-sibling/").await?;
+    let user = policy_user(&env, "consoleAdmin", None).await?;
+
+    force_delete(&user, bucket, "folder/").await?;
+    assert!(
+        versions(&root, bucket, "folder/").await?.is_empty(),
+        "force prefix deletion must remove null versions and markers"
+    );
+    assert_eq!(versions(&root, bucket, "folder-sibling/").await?, sibling);
+    assert_eq!(
+        versions(&root, bucket, "single.txt").await?.len(),
+        3,
+        "the separate key must survive folder deletion"
+    );
+    force_delete(&user, bucket, "single.txt").await?;
+    assert!(
+        versions(&root, bucket, "single.txt").await?.is_empty(),
+        "explicit force deletion must remove every version of the selected key"
+    );
+    assert_eq!(versions(&root, bucket, "keep.txt").await?, keep);
+    Ok(())
+}
+
+#[tokio::test]
+async fn force_delete_authorizes_only_its_path_scope_without_list_permissions() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    let root = env.create_s3_client();
+    let bucket = "force-delete-only";
+    root.create_bucket().bucket(bucket).send().await?;
+    versioning(&root, bucket, BucketVersioningStatus::Enabled).await?;
+    put(&root, bucket, "selected.txt").await?;
+    put(&root, bucket, "selected.txt/child.txt").await?;
+    put(&root, bucket, "selected.txt-sibling").await?;
+    let sibling = versions(&root, bucket, "selected.txt-sibling").await?;
+    let user = policy_user(
+        &env,
+        "delete-only",
+        Some(json!({"Version":"2012-10-17","Statement":[
+            {"Effect":"Allow","Action":["s3:DeleteObject","s3:DeleteObjectVersion"],"Resource":[
+                format!("arn:aws:s3:::{bucket}/selected.txt"), format!("arn:aws:s3:::{bucket}/selected.txt/*")
+            ]},
+            {"Effect":"Deny","Action":["s3:DeleteObject","s3:DeleteObjectVersion"],"Resource":format!("arn:aws:s3:::{bucket}/selected.txt-sibling")}
+        ]})),
+    )
+    .await?;
+    assert_denied(user.list_objects_v2().bucket(bucket).send().await);
+    assert_denied(user.list_object_versions().bucket(bucket).send().await);
+    force_delete(&user, bucket, "selected.txt").await?;
+    assert_eq!(
+        versions(&root, bucket, "selected.txt").await?,
+        sibling,
+        "force deletion must remove the selected path and descendants without authorizing or deleting its similarly prefixed sibling"
+    );
+    Ok(())
+}
+
+#[tokio::test]
+async fn force_directory_delete_cannot_remove_an_unauthorized_colliding_parent() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    let root = env.create_s3_client();
+    let bucket = "force-directory-collision";
+    root.create_bucket().bucket(bucket).send().await?;
+    versioning(&root, bucket, BucketVersioningStatus::Enabled).await?;
+    let protected_parent_version = put(&root, bucket, "collision.txt").await?;
+    for key in ["collision.txt/child", "collision.txt-sibling"] {
+        put(&root, bucket, key).await?;
+    }
+    put(&root, bucket, "collision.txt").await?;
+    root.delete_object().bucket(bucket).key("collision.txt").send().await?;
+    let user = policy_user(
+        &env,
+        "parent-denier",
+        Some(json!({"Version":"2012-10-17","Statement":[
+            {"Effect":"Allow","Action":["s3:DeleteObject","s3:DeleteObjectVersion"],"Resource":format!("arn:aws:s3:::{bucket}/*")},
+            {"Effect":"Deny","Action":"s3:DeleteObjectVersion","Resource":format!("arn:aws:s3:::{bucket}/collision.txt"),
+             "Condition":{"StringEquals":{"s3:VersionId":protected_parent_version}}}
+        ]})),
+    )
+    .await?;
+    let mut expected = versions(&root, bucket, "").await?;
+    expected.retain(|(key, _, _)| key != "collision.txt/child");
+    force_delete(&user, bucket, "collision.txt/").await?;
+    assert_eq!(
+        versions(&root, bucket, "").await?,
+        expected,
+        "folder deletion must preserve the denied parent's historical versions and delete marker, plus its sibling"
+    );
+    Ok(())
+}
+
+#[tokio::test]
+async fn force_unversioned_directory_requires_only_delete_object() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    let root = env.create_s3_client();
+    let bucket = "force-unversioned-permissions";
+    root.create_bucket().bucket(bucket).send().await?;
+    for key in ["folder/", "folder/child.txt", "outside.txt"] {
+        put(&root, bucket, key).await?;
+    }
+    let outside = versions(&root, bucket, "outside.txt").await?;
+    let user = policy_user(
+        &env,
+        "unversioned-deleter",
+        Some(json!({"Version":"2012-10-17","Statement":[
+            {"Effect":"Allow","Action":"s3:DeleteObject","Resource":format!("arn:aws:s3:::{bucket}/*")},
+            {"Effect":"Deny","Action":"s3:DeleteObjectVersion","Resource":format!("arn:aws:s3:::{bucket}/*")}
+        ]})),
+    )
+    .await?;
+    force_delete(&user, bucket, "folder/").await?;
+    assert_eq!(
+        versions(&root, bucket, "").await?,
+        outside,
+        "unversioned force deletion, including a synthetic nil directory marker, must use DeleteObject permission"
+    );
+    Ok(())
+}
+
+#[tokio::test]
+async fn force_delete_denied_child_preserves_every_object_despite_bucket_allow() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    let root = env.create_s3_client();
+    let bucket = "force-child-denial";
+    root.create_bucket().bucket(bucket).send().await?;
+    for key in ["folder/a-allowed.txt", "folder/z-denied.txt"] {
+        put(&root, bucket, key).await?;
+    }
+    let user = policy_user(
+        &env,
+        "child-denier",
+        Some(json!({"Version":"2012-10-17","Statement":[
+            {"Effect":"Allow","Action":["s3:DeleteObject","s3:DeleteObjectVersion","s3:ReplicateDelete"],"Resource":format!("arn:aws:s3:::{bucket}/*")},
+            {"Effect":"Deny","Action":["s3:DeleteObject","s3:DeleteObjectVersion","s3:ReplicateDelete"],"Resource":format!("arn:aws:s3:::{bucket}/folder/z-denied.txt")}
+        ]})),
+    )
+    .await?;
+    root.put_bucket_policy()
+        .bucket(bucket)
+        .policy(json!({"Version":"2012-10-17","Statement":[
+            {"Effect":"Allow","Principal":"*","Action":["s3:DeleteObject","s3:DeleteObjectVersion","s3:ReplicateDelete"],"Resource":format!("arn:aws:s3:::{bucket}/*")}
+        ]}).to_string())
+        .send().await?;
+    let before = versions(&root, bucket, "folder/").await?;
+    assert_boxed_denied(force_delete(&user, bucket, "folder/").await);
+    assert_eq!(
+        versions(&root, bucket, "folder/").await?,
+        before,
+        "a denied descendant must prevent every mutation in the force scope"
+    );
+    assert_boxed_denied(replica_force_delete(&user, bucket, "folder/").await);
+    assert_eq!(
+        versions(&root, bucket, "folder/").await?,
+        before,
+        "the REPLICA header must not bypass a descendant's ReplicateDelete denial"
+    );
+
+    root.delete_bucket_policy().bucket(bucket).send().await?;
+    let replica_user = policy_user(
+        &env,
+        "replica-deleter",
+        Some(json!({"Version":"2012-10-17","Statement":[
+            {"Effect":"Allow","Action":"s3:DeleteObject","Resource":format!("arn:aws:s3:::{bucket}/*")},
+            {"Effect":"Allow","Action":"s3:ReplicateDelete","Resource":format!("arn:aws:s3:::{bucket}/*")}
+        ]})),
+    )
+    .await?;
+    replica_force_delete(&replica_user, bucket, "folder/").await?;
+    assert!(
+        versions(&root, bucket, "folder/").await?.is_empty(),
+        "an authorized replica force request must check ReplicateDelete for its descendants"
+    );
+    Ok(())
+}
+
+#[tokio::test]
+async fn force_delete_denied_historical_version_preserves_versions_and_markers() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    let root = env.create_s3_client();
+    let bucket = "force-version-denial";
+    root.create_bucket().bucket(bucket).send().await?;
+    put(&root, bucket, "folder/null.txt").await?;
+    versioning(&root, bucket, BucketVersioningStatus::Enabled).await?;
+    let protected_version = put(&root, bucket, "folder/versioned.txt").await?;
+    put(&root, bucket, "folder/versioned.txt").await?;
+    let marker = root.delete_object().bucket(bucket).key("folder/versioned.txt").send().await?;
+    let marker_version = marker
+        .version_id()
+        .ok_or("versioned delete must return a marker version ID")?;
+    put(&root, bucket, "folder/a-allowed.txt").await?;
+    let policy = |version: &str| {
+        json!({"Version":"2012-10-17","Statement":[
+            {"Effect":"Allow","Action":["s3:DeleteObject","s3:DeleteObjectVersion"],"Resource":format!("arn:aws:s3:::{bucket}/*")},
+            {"Effect":"Deny","Action":"s3:DeleteObjectVersion","Resource":format!("arn:aws:s3:::{bucket}/folder/*"),
+             "Condition":{"StringEquals":{"s3:VersionId":version}}}
+        ]})
+    };
+    let user = policy_user(&env, "version-denier", Some(policy(&protected_version))).await?;
+    let before = versions(&root, bucket, "folder/").await?;
+    for version in [protected_version.as_str(), "null", marker_version] {
+        set_policy(&env, "version-denier", &policy(version)).await?;
+        assert_boxed_denied(force_delete(&user, bucket, "folder/").await);
+        assert_eq!(
+            versions(&root, bucket, "folder/").await?,
+            before,
+            "denial of a historical, null, or delete-marker version must prevent recursive deletion"
+        );
+    }
+    Ok(())
+}
+
+#[tokio::test]
+async fn sdk_ordinary_deletion_preserves_versions_and_directory_children() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    let root = env.create_s3_client();
+    let user = policy_user(
+        &env,
+        "ordinary-deleter",
+        Some(json!({"Version":"2012-10-17","Statement":[
+            {"Effect":"Allow","Action":"s3:DeleteObject","Resource":"arn:aws:s3:::*/*"},
+            {"Effect":"Deny","Action":"s3:DeleteObjectVersion","Resource":"arn:aws:s3:::*/*"}
+        ]})),
+    )
+    .await?;
+    for state in ["unversioned", "enabled", "suspended"] {
+        let bucket = format!("ordinary-directory-{state}");
+        root.create_bucket().bucket(&bucket).send().await?;
+        if state != "unversioned" {
+            versioning(&root, &bucket, BucketVersioningStatus::Enabled).await?;
+        }
+        let historical = put(&root, &bucket, "object.txt").await?;
+        if state == "suspended" {
+            versioning(&root, &bucket, BucketVersioningStatus::Suspended).await?;
+            put(&root, &bucket, "object.txt").await?;
+        }
+        put(&root, &bucket, "folder/").await?;
+        put(&root, &bucket, "folder/child.txt").await?;
+        let child_before = versions(&root, &bucket, "folder/child.txt").await?;
+        user.delete_object().bucket(&bucket).key("folder/").send().await?;
+        assert_eq!(
+            versions(&root, &bucket, "folder/").await?,
+            child_before,
+            "ordinary {state} directory-key deletion must remove only its synthetic marker and preserve children"
+        );
+        let deleted = user.delete_object().bucket(&bucket).key("object.txt").send().await?;
+        let object_versions = versions(&root, &bucket, "object.txt").await?;
+        if state == "unversioned" {
+            assert!(object_versions.is_empty());
+        } else {
+            assert_eq!(deleted.delete_marker(), Some(true));
+            assert_eq!(
+                object_versions.len(),
+                2,
+                "ordinary {state} deletion must retain its historical data version"
+            );
+            assert!(object_versions.contains(&("object.txt".into(), historical, false)));
+            if state == "suspended" {
+                assert!(object_versions.contains(&("object.txt".into(), "null".into(), true)));
+            }
+        }
+    }
+    Ok(())
+}
+
+#[tokio::test]
+async fn sdk_delete_objects_force_header_keeps_explicit_item_scope() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    let root = env.create_s3_client();
+    let bucket = "batch-force-explicit-scope";
+    root.create_bucket().bucket(bucket).send().await?;
+    put(&root, bucket, "folder/").await?;
+    put(&root, bucket, "folder/child.txt").await?;
+    let child = versions(&root, bucket, "folder/child.txt").await?;
+    let user = policy_user(&env, "consoleAdmin", None).await?;
+    let result = user
+        .delete_objects()
+        .bucket(bucket)
+        .delete(
+            Delete::builder()
+                .objects(ObjectIdentifier::builder().key("folder/").build()?)
+                .build()?,
+        )
+        .customize()
+        .mutate_request(|request| {
+            request.headers_mut().insert("x-rustfs-force-delete", "true");
+        })
+        .send()
+        .await?;
+    assert!(result.errors().is_empty());
+    assert_eq!(result.deleted().len(), 1);
+    assert_eq!(
+        versions(&root, bucket, "folder/").await?,
+        child,
+        "batch deletion must remove only the explicit directory marker even with the force header"
+    );
+    Ok(())
+}
+
+#[tokio::test]
+async fn force_delete_checks_every_version_page_before_mutation() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    let root = env.create_s3_client();
+    let bucket = "force-delete-pagination";
+    root.create_bucket().bucket(bucket).send().await?;
+    versioning(&root, bucket, BucketVersioningStatus::Enabled).await?;
+    stream::iter(0..1000)
+        .map(|index| {
+            let root = &root;
+            async move { put(root, bucket, &format!("folder/{index:04}.txt")).await.map(|_| ()) }
+        })
+        .buffer_unordered(16)
+        .try_collect::<Vec<_>>()
+        .await?;
+    put(&root, bucket, "folder/z-denied.txt").await?;
+    let allow = json!({"Effect":"Allow","Action":["s3:DeleteObject","s3:DeleteObjectVersion"],"Resource":format!("arn:aws:s3:::{bucket}/*")});
+    let user = policy_user(
+        &env,
+        "paged-deleter",
+        Some(json!({"Version":"2012-10-17","Statement":[allow.clone(),
+            {"Effect":"Deny","Action":"s3:DeleteObjectVersion","Resource":format!("arn:aws:s3:::{bucket}/folder/z-denied.txt")}
+        ]})),
+    )
+    .await?;
+    let before = versions(&root, bucket, "folder/").await?;
+    assert_eq!(before.len(), 1001, "the denied key must be beyond one default versions page");
+    assert_boxed_denied(force_delete(&user, bucket, "folder/").await);
+    assert_eq!(
+        versions(&root, bucket, "folder/").await?,
+        before,
+        "a denial on the second page must preserve the first page too"
+    );
+
+    set_policy(&env, "paged-deleter", &json!({"Version":"2012-10-17","Statement":[allow]})).await?;
+    force_delete(&user, bucket, "folder/").await?;
+    assert!(
+        versions(&root, bucket, "folder/").await?.is_empty(),
+        "authorized recursive deletion must cover all pages"
+    );
+    Ok(())
+}

crates/e2e_test/src/distributed/heal_test.rs

@@ -16,7 +16,7 @@ use super::harness::{
     DistCluster, DistLayout, TestResult, assert_inventory, get_object_bytes, payload_for, put_object, sha256_hex, unique_bucket,
     wait_until,
 };
-use crate::chaos::{VersionShardCensus, census_object_version_on_disk, signed_admin_post};
+use crate::chaos::{VersionShardCensus, census_object_version_on_disk, start_root_heal_when_control_ready};
 use crate::common::init_logging;
 use aws_sdk_s3::Client;
 use aws_sdk_s3::primitives::ByteStream;
@@ -26,16 +26,14 @@ use std::collections::{BTreeMap, HashSet};
 use std::io::{Read, Write};
 use std::path::{Path, PathBuf};
 use std::time::Duration;
-use tokio::time::{Instant, sleep};
 
 const EC84_NODE_COUNT: usize = 3;
 const EC84_DRIVES_PER_NODE: usize = 4;
 const EC84_DATA_BLOCKS: usize = 8;
 const EC84_PARITY_BLOCKS: usize = 4;
+const EC84_ERASURE_SET_DRIVE_COUNT: usize = EC84_DATA_BLOCKS + EC84_PARITY_BLOCKS;
 const EC84_TARGET_DRIVE_RESTART_CASE: &str = "ec84-target-drive-restart";
 const EC84_TARGET_DRIVE_RESTART_ORACLE: &str = "ec84-target-drive-restart.json";
-const EC84_HEAL_CONTROL_READY_TIMEOUT: Duration = Duration::from_secs(45);
-const EC84_HEAL_CONTROL_RETRY_DELAY: Duration = Duration::from_millis(250);
 
 #[derive(Clone)]
 struct ExpectedShard {
@@ -185,6 +183,7 @@ async fn write_scanner_heal_evidence(context: ScannerHealEvidenceContext, payloa
         "binary_sha256": string_field(&context.run, "binary.sha256")?,
         "test_binary_sha256": string_field(&context.run, "test_binary.sha256")?,
         "topology": {"nodes": EC84_NODE_COUNT, "drives_per_node": EC84_DRIVES_PER_NODE},
+        "erasure_set_drive_count": EC84_ERASURE_SET_DRIVE_COUNT,
         "pid_before": payload.pid_before,
         "pid_after": payload.pid_after,
         "unclean_shutdown_marker": false,
@@ -214,29 +213,6 @@ fn assert_replaced_drive_empty(drive: &Path, bucket: &str, keys: &[String]) -> T
     Ok(())
 }
 
-fn is_cluster_heal_coordination_unavailable(error: &(dyn std::error::Error + Send + Sync)) -> bool {
-    let message = error.to_string();
-    message.contains("500 Internal Server Error") && message.contains("cluster heal coordination unavailable")
-}
-
-async fn start_ec84_root_heal_when_control_ready(
-    heal_url: &str,
-    heal_body: &str,
-    access_key: &str,
-    secret_key: &str,
-) -> TestResult {
-    let deadline = Instant::now() + EC84_HEAL_CONTROL_READY_TIMEOUT;
-    loop {
-        match signed_admin_post(heal_url, Some(heal_body), access_key, secret_key).await {
-            Ok(_) => return Ok(()),
-            Err(error) if is_cluster_heal_coordination_unavailable(error.as_ref()) && Instant::now() < deadline => {
-                sleep(EC84_HEAL_CONTROL_RETRY_DELAY).await;
-            }
-            Err(error) => return Err(error),
-        }
-    }
-}
-
 async fn put_large_inventory(client: &Client, bucket: &str) -> TestResult<Vec<ExpectedShard>> {
     let mut expected = Vec::new();
     for index in 0..4 {
@@ -251,6 +227,7 @@ async fn put_large_inventory(client: &Client, bucket: &str) -> TestResult<Vec<Ex
                 has_xl_meta: false,
                 data_dir: None,
                 erasure_index: None,
+                erasure_distribution: None,
                 data_blocks: None,
                 parity_blocks: None,
                 expected_part_numbers: Default::default(),
@@ -330,7 +307,7 @@ async fn three_node_four_drive_ec8_4_root_heal_rebuilds_replaced_drive_after_res
     let heal_body =
         r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
     let heal_url = format!("{}/rustfs/admin/v3/heal/{bucket}?forceStart=true", dist.cluster.nodes[0].url);
-    start_ec84_root_heal_when_control_ready(&heal_url, heal_body, &dist.cluster.access_key, &dist.cluster.secret_key).await?;
+    start_root_heal_when_control_ready(&heal_url, heal_body, &dist.cluster.access_key, &dist.cluster.secret_key).await?;
 
     wait_until(
         Duration::from_secs(120),
@@ -395,6 +372,7 @@ async fn three_node_four_drive_ec8_4_root_heal_rebuilds_replaced_drive_after_res
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::chaos::is_cluster_heal_coordination_unavailable;
 
     #[test]
     fn cluster_heal_coordination_retry_is_exact() {
@@ -410,4 +388,16 @@ mod tests {
             "admin POST failed: 503 Service Unavailable cluster heal coordination unavailable".into();
         assert!(!is_cluster_heal_coordination_unavailable(wrong_status.as_ref()));
     }
+
+    #[test]
+    fn ec84_drive_restart_evidence_shape_matches_registry() {
+        assert_eq!(EC84_ERASURE_SET_DRIVE_COUNT, EC84_NODE_COUNT * EC84_DRIVES_PER_NODE);
+
+        let registry: Value = serde_json::from_str(include_str!("../../../../.config/scanner-heal-required-tests.json"))
+            .expect("scanner/heal registry is valid JSON");
+        let case = &registry["cases"][EC84_TARGET_DRIVE_RESTART_CASE];
+        assert_eq!(case["erasure_set_drive_count"], EC84_ERASURE_SET_DRIVE_COUNT);
+        assert_eq!(case["topology"]["nodes"], EC84_NODE_COUNT);
+        assert_eq!(case["topology"]["drives_per_node"], EC84_DRIVES_PER_NODE);
+    }
 }

crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs

@@ -16,7 +16,9 @@
 
 #[cfg(test)]
 mod tests {
-    use crate::chaos::{VersionShardCensus, census_object_version_on_disk, sha256_hex, signed_admin_post};
+    use crate::chaos::{
+        VersionShardCensus, census_object_version_on_disk, sha256_hex, signed_admin_post, start_root_heal_when_control_ready,
+    };
     use crate::common::{
         ClusterTopology, FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request,
         init_logging, rustfs_binary_path,
@@ -24,7 +26,7 @@ mod tests {
     use crate::storage_api::RUSTFS_META_BUCKET;
     use aws_sdk_s3::{
         error::{ProvideErrorMetadata, SdkError},
-        operation::put_object::PutObjectError,
+        operation::{delete_object::DeleteObjectError, get_object::GetObjectError, put_object::PutObjectError},
         primitives::ByteStream,
     };
     use http::Method;
@@ -45,6 +47,7 @@ mod tests {
 
     struct ReplacementDriveSelection {
         replaced_disk: PathBuf,
+        drive_index: usize,
         replacement_format_path: PathBuf,
         replacement_format: Vec<u8>,
         expected_pool_metadata: Option<VersionShardCensus>,
@@ -461,6 +464,12 @@ mod tests {
         shard_census: VersionShardCensus,
     }
 
+    #[derive(Debug, Default)]
+    struct OutagePeerManifest {
+        erasure_indices: HashSet<usize>,
+        erasure_distribution: Option<Vec<usize>>,
+    }
+
     fn deterministic_object_body(len: usize, seed: u8) -> Vec<u8> {
         let mut value = seed;
         std::iter::repeat_with(|| {
@@ -486,6 +495,81 @@ mod tests {
         Ok(matching)
     }
 
+    fn collect_outage_peer_manifest(
+        cluster: &RustFSTestClusterEnvironment,
+        offline_node_index: usize,
+        bucket: &str,
+        key: &str,
+        erasure_set_drive_count: usize,
+    ) -> Result<OutagePeerManifest, Box<dyn Error + Send + Sync>> {
+        let mut manifest = OutagePeerManifest::default();
+        for (node_index, node) in cluster.nodes.iter().enumerate() {
+            if node_index == offline_node_index {
+                continue;
+            }
+            for (drive_index, drive) in node.data_dirs.iter().enumerate() {
+                let census = census_object_version_on_disk(Path::new(drive), bucket, key, None)?;
+                if !census.has_xl_meta {
+                    continue;
+                }
+                assert!(
+                    census.is_complete(),
+                    "online node {node_index} drive {drive_index} must hold a complete outage-object shard: {census:?}"
+                );
+                let erasure_index = census.erasure_index.ok_or_else(|| {
+                    format!("online node {node_index} drive {drive_index} outage-object shard has no erasure index: {census:?}")
+                })?;
+                assert!(
+                    (1..=erasure_set_drive_count).contains(&erasure_index),
+                    "online node {node_index} drive {drive_index} outage-object erasure index is out of range: {census:?}"
+                );
+                let distribution = census.erasure_distribution.as_ref().ok_or_else(|| {
+                    format!(
+                        "online node {node_index} drive {drive_index} outage-object shard has no erasure distribution: {census:?}"
+                    )
+                })?;
+                assert_eq!(
+                    distribution.len(),
+                    erasure_set_drive_count,
+                    "online node {node_index} drive {drive_index} outage-object distribution must match the erasure set: {census:?}"
+                );
+                match &manifest.erasure_distribution {
+                    Some(existing) => {
+                        assert_eq!(existing, distribution, "outage-object shards must agree on one erasure distribution")
+                    }
+                    None => manifest.erasure_distribution = Some(distribution.clone()),
+                }
+                assert!(
+                    manifest.erasure_indices.insert(erasure_index),
+                    "outage-object erasure index {erasure_index} is duplicated across online drives"
+                );
+            }
+        }
+        Ok(manifest)
+    }
+
+    fn outage_candidate_replacement_erasure_index(
+        peer_manifest: &OutagePeerManifest,
+        erasure_set_drive_count: usize,
+        replacement_set_slot: usize,
+    ) -> Option<usize> {
+        let distribution = peer_manifest.erasure_distribution.as_ref()?;
+        distribution.get(replacement_set_slot).copied().filter(|replacement_index| {
+            (1..=erasure_set_drive_count).contains(replacement_index)
+                && !peer_manifest.erasure_indices.contains(replacement_index)
+        })
+    }
+
+    fn outage_candidate_targets_replacement(
+        peer_manifest: &OutagePeerManifest,
+        erasure_set_drive_count: usize,
+        replacement_set_slot: usize,
+    ) -> bool {
+        let min_online_data_shards = erasure_set_drive_count.saturating_sub(4);
+        peer_manifest.erasure_indices.len() >= min_online_data_shards
+            && outage_candidate_replacement_erasure_index(peer_manifest, erasure_set_drive_count, replacement_set_slot).is_some()
+    }
+
     fn metadata_count(disk: &Path, bucket: &str, expected_manifests: &[PhysicalObjectManifest]) -> usize {
         expected_manifests
             .iter()
@@ -519,7 +603,9 @@ mod tests {
                         .set_continuation_token(continuation_token.clone())
                         .send(),
                 )
-                .await??;
+                .await
+                .map_err(|error| format!("node {node_index} recovery listing timed out for bucket {bucket}: {error}"))?
+                .map_err(|error| format!("node {node_index} recovery listing failed for bucket {bucket}: {error}"))?;
                 listed_keys.extend(
                     response
                         .contents()
@@ -586,8 +672,8 @@ mod tests {
             && operations["retryingTasks"].as_u64() == Some(0)
     }
 
-    // Queued low-priority repairs cannot execute while the single admin slot is
-    // occupied; ownership is determined by active and retrying tasks only.
+    // Admission slots are per node: remote read repairs can briefly overlap
+    // the coordinator's admin task. Require them to drain before interruption.
     fn only_admin_heal_is_active(status: &serde_json::Value) -> bool {
         let operations = &status["healOperations"];
         status["clusterStatusComplete"] == serde_json::Value::Bool(true)
@@ -597,10 +683,49 @@ mod tests {
             && operations["activeBySource"]["admin"].as_u64() == Some(1)
     }
 
-    fn is_service_unavailable_put(error: &SdkError<PutObjectError>) -> bool {
+    fn is_retryable_outage_put(error: &SdkError<PutObjectError>) -> bool {
+        matches!(
+            error.as_service_error().and_then(ProvideErrorMetadata::code),
+            Some("SlowDownRead" | "ServiceUnavailable")
+        )
+    }
+
+    fn is_service_unavailable_delete(error: &SdkError<DeleteObjectError>) -> bool {
         error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("ServiceUnavailable")
     }
 
+    fn is_retryable_recovery_get(error: &SdkError<GetObjectError>) -> bool {
+        matches!(
+            error.as_service_error().and_then(ProvideErrorMetadata::code),
+            Some("SlowDownRead" | "ServiceUnavailable")
+        )
+    }
+
+    async fn get_object_after_recovery(
+        client: &aws_sdk_s3::Client,
+        bucket: &str,
+        key: &str,
+        timeout_secs: u64,
+    ) -> Result<bytes::Bytes, Box<dyn Error + Send + Sync>> {
+        let deadline = Instant::now() + Duration::from_secs(timeout_secs);
+        loop {
+            match timeout(Duration::from_secs(30), client.get_object().bucket(bucket).key(key).send()).await {
+                Ok(Ok(response)) => return Ok(response.body.collect().await?.into_bytes()),
+                Ok(Err(error)) if is_retryable_recovery_get(&error) && Instant::now() < deadline => {
+                    sleep(Duration::from_millis(250)).await;
+                }
+                Ok(Err(error)) => {
+                    return Err(
+                        format!("recovery GET failed for {bucket}/{key} after waiting up to {timeout_secs}s: {error}").into(),
+                    );
+                }
+                Err(error) => {
+                    return Err(format!("recovery GET timed out for {bucket}/{key} after 30s attempt: {error}").into());
+                }
+            }
+        }
+    }
+
     fn select_replacement_drive(
         cluster: &RustFSTestClusterEnvironment,
         node_index: usize,
@@ -612,7 +737,7 @@ mod tests {
             .ok_or_else(|| format!("replacement node {node_index} is absent"))?;
         let mut incomplete_pool_metadata = Vec::new();
 
-        for drive in &node.data_dirs {
+        for (drive_index, drive) in node.data_dirs.iter().enumerate() {
             let replaced_disk = PathBuf::from(drive);
             let replacement_format_path = replaced_disk.join(".rustfs.sys").join("format.json");
             let replacement_format = std::fs::read(&replacement_format_path).map_err(|err| {
@@ -621,6 +746,7 @@ mod tests {
             if !require_pool_metadata {
                 return Ok(ReplacementDriveSelection {
                     replaced_disk,
+                    drive_index,
                     replacement_format_path,
                     replacement_format,
                     expected_pool_metadata: None,
@@ -631,6 +757,7 @@ mod tests {
             if census.is_complete() {
                 return Ok(ReplacementDriveSelection {
                     replaced_disk,
+                    drive_index,
                     replacement_format_path,
                     replacement_format,
                     expected_pool_metadata: Some(census),
@@ -960,19 +1087,26 @@ mod tests {
 
         let online_key = "cluster/online-before-replacement.bin";
         let online_body = b"object written while all cluster nodes are online".to_vec();
-        clients[0]
-            .put_object()
-            .bucket(bucket)
-            .key(online_key)
-            .body(ByteStream::from(online_body.clone()))
-            .send()
-            .await?;
-
         let replaced_disk = PathBuf::from(&cluster.nodes[1].data_dir);
-        assert!(
-            object_metadata_exists_on_disk(&replaced_disk, bucket, online_key),
-            "node 1 should contain metadata before disk replacement"
-        );
+        // A quorum write need not include the disk this fixture will replace.
+        // Establish that disk's baseline before testing its reconstruction.
+        timeout(Duration::from_secs(30), async {
+            loop {
+                clients[0]
+                    .put_object()
+                    .bucket(bucket)
+                    .key(online_key)
+                    .body(ByteStream::from(online_body.clone()))
+                    .send()
+                    .await?;
+                if object_metadata_exists_on_disk(&replaced_disk, bucket, online_key) {
+                    return Ok::<_, Box<dyn Error + Send + Sync>>(());
+                }
+                sleep(Duration::from_millis(100)).await;
+            }
+        })
+        .await
+        .map_err(|_| "node 1 did not store the baseline object before disk replacement")??;
 
         cluster.stop_node(1)?;
         std::fs::remove_dir_all(&replaced_disk)?;
@@ -1017,7 +1151,7 @@ mod tests {
 
         let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
         let heal_url = format!("{}/rustfs/admin/v3/heal/?forceStart=true", cluster.nodes[0].url);
-        signed_admin_post(&heal_url, Some(heal_body), &cluster.access_key, &cluster.secret_key).await?;
+        start_root_heal_when_control_ready(&heal_url, heal_body, &cluster.access_key, &cluster.secret_key).await?;
 
         let expected_objects = [(online_key, online_body.as_slice()), (outage_key, outage_body.as_slice())];
         let mut remaining_rebuild_keys: HashSet<&str> = expected_objects.iter().map(|(key, _)| *key).collect();
@@ -1180,7 +1314,7 @@ mod tests {
     async fn test_cluster_root_heal_recovers_ec84_shards_across_multi_set_after_background_target_restart()
     -> Result<(), Box<dyn Error + Send + Sync>> {
         timeout(
-            Duration::from_secs(600),
+            Duration::from_secs(900),
             run_cluster_root_heal_interruption(InterruptionScenario::BackgroundTargetRestartEc84MultiSet),
         )
         .await?
@@ -1332,10 +1466,13 @@ mod tests {
 
         let ReplacementDriveSelection {
             replaced_disk,
+            drive_index: replacement_drive_index,
             replacement_format_path,
             replacement_format,
             expected_pool_metadata,
         } = select_replacement_drive(&cluster, 1, background_enabled)?;
+        let replacement_global_drive_index = topology.drives_per_node + replacement_drive_index;
+        let replacement_set_slot = replacement_global_drive_index % erasure_set_drive_count;
         let default_online_object_count = if !outage_target_manifest_required { 64 } else { 24 };
         let online_object_count = std::env::var("RUSTFS_HEAL_CHAOS_OBJECT_COUNT")
             .ok()
@@ -1391,6 +1528,23 @@ mod tests {
             expected_manifests.push(PhysicalObjectManifest { key, shard_census });
             attempt_count += 1;
         }
+        for manifest in &expected_manifests {
+            let distribution = manifest
+                .shard_census
+                .erasure_distribution
+                .as_ref()
+                .ok_or_else(|| format!("replacement baseline manifest has no erasure distribution: {manifest:?}"))?;
+            assert_eq!(
+                distribution.len(),
+                erasure_set_drive_count,
+                "replacement baseline distribution must match the erasure set: {manifest:?}"
+            );
+            assert_eq!(
+                manifest.shard_census.erasure_index,
+                distribution.get(replacement_set_slot).copied(),
+                "replacement baseline shard must match the selected drive's erasure-set slot"
+            );
+        }
 
         if background_enabled {
             wait_for_scanner_cycle_after(&cluster, 0).await?;
@@ -1409,12 +1563,30 @@ mod tests {
             "replacement target must retain only its preformatted topology identity"
         );
 
+        // A multi-set pool can route a successful outage PUT away from the
+        // replacement drive. Identify its set through a witnessed baseline.
+        let target_set_peer_drives = cluster
+            .nodes
+            .iter()
+            .enumerate()
+            .filter(|(node_index, _)| *node_index != 1)
+            .flat_map(|(_, node)| node.data_dirs.iter().map(PathBuf::from))
+            .filter(|drive| object_metadata_exists_on_disk(drive, bucket, &expected_manifests[0].key))
+            .collect::<Vec<_>>();
+        assert!(
+            !outage_target_manifest_required || !target_set_peer_drives.is_empty(),
+            "the replacement erasure set must retain an online baseline shard"
+        );
+
         let outage_payload_seed = 0xf1;
         let max_outage_write_attempts = topology.total_drives().max(1);
         let mut outage_key = None;
         let mut outage_write_deferred_until_rejoin = false;
         let mut service_unavailable_outage_writes = 0usize;
         let mut last_service_unavailable = None;
+        let mut outage_peer_manifest = OutagePeerManifest::default();
+        let mut replacement_outage_erasure_index = None;
+        let mut rejected_outage_keys = Vec::new();
         for attempt in 0..max_outage_write_attempts {
             let candidate_key = format!("cluster/written-while-node-down-{attempt:04}.bin");
             let put_result = timeout(
@@ -1429,15 +1601,43 @@ mod tests {
             .await;
             match put_result {
                 Ok(Ok(_)) => {
+                    if outage_target_manifest_required {
+                        let candidate_peer_manifest =
+                            collect_outage_peer_manifest(&cluster, 1, bucket, &candidate_key, erasure_set_drive_count)?;
+                        if !outage_candidate_targets_replacement(
+                            &candidate_peer_manifest,
+                            erasure_set_drive_count,
+                            replacement_set_slot,
+                        ) || !target_set_peer_drives
+                            .iter()
+                            .any(|drive| object_metadata_exists_on_disk(drive, bucket, &candidate_key))
+                        {
+                            rejected_outage_keys.push(candidate_key);
+                            continue;
+                        }
+                        replacement_outage_erasure_index = outage_candidate_replacement_erasure_index(
+                            &candidate_peer_manifest,
+                            erasure_set_drive_count,
+                            replacement_set_slot,
+                        );
+                        outage_peer_manifest = candidate_peer_manifest;
+                    }
                     outage_key = Some(candidate_key);
                     break;
                 }
-                Ok(Err(error)) if is_service_unavailable_put(&error) => {
+                Ok(Err(error)) if is_retryable_outage_put(&error) => {
                     service_unavailable_outage_writes += 1;
                     last_service_unavailable = Some(format!("{error:?}"));
                 }
-                Ok(Err(error)) => return Err(error.into()),
-                Err(error) => return Err(error.into()),
+                Ok(Err(error)) => {
+                    return Err(format!("outage PUT candidate {candidate_key} failed before target rejoin: {error}").into());
+                }
+                Err(error) => {
+                    return Err(format!(
+                        "outage PUT candidate {candidate_key} timed out before target rejoin after 30s: {error}"
+                    )
+                    .into());
+                }
             }
         }
         let outage_key = match outage_key {
@@ -1456,57 +1656,45 @@ mod tests {
             }
         };
 
-        let mut outage_peer_erasure_indices = HashSet::new();
-        if !outage_write_deferred_until_rejoin {
-            for (node_index, node) in cluster.nodes.iter().enumerate() {
-                if node_index == 1 {
-                    continue;
-                }
-                for (drive_index, drive) in node.data_dirs.iter().enumerate() {
-                    let census = census_object_version_on_disk(Path::new(drive), bucket, &outage_key, None)?;
-                    if !census.has_xl_meta {
-                        continue;
-                    }
-                    assert!(
-                        census.is_complete(),
-                        "online node {node_index} drive {drive_index} must hold a complete outage-object shard: {census:?}"
-                    );
-                    let erasure_index = census.erasure_index.ok_or_else(|| {
-                        format!(
-                            "online node {node_index} drive {drive_index} outage-object shard has no erasure index: {census:?}"
-                        )
-                    })?;
-                    assert!(
-                        (1..=erasure_set_drive_count).contains(&erasure_index),
-                        "online node {node_index} drive {drive_index} outage-object erasure index is out of range: {census:?}"
-                    );
-                    assert!(
-                        outage_peer_erasure_indices.insert(erasure_index),
-                        "outage-object erasure index {erasure_index} is duplicated across online drives"
-                    );
-                }
-            }
+        assert!(
+            !outage_target_manifest_required
+                || target_set_peer_drives
+                    .iter()
+                    .any(|drive| object_metadata_exists_on_disk(drive, bucket, &outage_key)),
+            "outage object {outage_key} belongs to a different erasure set than the replacement drive"
+        );
+
+        if !outage_write_deferred_until_rejoin && outage_peer_manifest.erasure_indices.is_empty() {
+            outage_peer_manifest = collect_outage_peer_manifest(&cluster, 1, bucket, &outage_key, erasure_set_drive_count)?;
+            replacement_outage_erasure_index =
+                outage_candidate_replacement_erasure_index(&outage_peer_manifest, erasure_set_drive_count, replacement_set_slot);
         }
         assert!(
             outage_write_deferred_until_rejoin
-                || (!outage_peer_erasure_indices.is_empty() && outage_peer_erasure_indices.len() <= erasure_set_drive_count),
+                || (!outage_peer_manifest.erasure_indices.is_empty()
+                    && outage_peer_manifest.erasure_indices.len() <= erasure_set_drive_count),
             "outage-object must occupy one non-empty erasure set"
         );
         if outage_target_manifest_required {
             let min_online_data_shards = erasure_set_drive_count.saturating_sub(4);
             assert!(
-                outage_peer_erasure_indices.len() >= min_online_data_shards,
+                outage_peer_manifest.erasure_indices.len() >= min_online_data_shards,
                 "online drives in the selected erasure set must retain at least the EC data quorum"
             );
         }
         let missing_outage_erasure_indices = (1..=erasure_set_drive_count)
-            .filter(|index| !outage_peer_erasure_indices.contains(index))
+            .filter(|index| !outage_peer_manifest.erasure_indices.contains(index))
             .collect::<HashSet<_>>();
         if outage_target_manifest_required {
             assert!(
                 !missing_outage_erasure_indices.is_empty(),
                 "the stopped target must account for at least one missing outage-object erasure index"
             );
+            assert_eq!(
+                replacement_outage_erasure_index.filter(|index| missing_outage_erasure_indices.contains(index)),
+                replacement_outage_erasure_index,
+                "the outage object must target the selected replacement drive's erasure-set slot"
+            );
         }
 
         let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
@@ -1541,6 +1729,24 @@ mod tests {
             None
         };
         cluster.start_node_from_binary(1, &server_binary).await?;
+        for rejected_key in rejected_outage_keys {
+            let delete_deadline = Instant::now() + Duration::from_secs(60);
+            loop {
+                let delete_result = timeout(
+                    Duration::from_secs(30),
+                    clients[0].delete_object().bucket(bucket).key(&rejected_key).send(),
+                )
+                .await;
+                match delete_result {
+                    Ok(Ok(_)) => break,
+                    Ok(Err(error)) if is_service_unavailable_delete(&error) && Instant::now() < delete_deadline => {
+                        sleep(Duration::from_secs(1)).await;
+                    }
+                    Ok(Err(error)) => return Err(error.into()),
+                    Err(error) => return Err(error.into()),
+                }
+            }
+        }
 
         let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url);
         let recovery_deadline = Instant::now() + Duration::from_secs(60);
@@ -1679,9 +1885,22 @@ mod tests {
                 "interruption tests require a server built with e2e-test-hooks"
             );
         }
-        let pre_interrupt_status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
-        let pre_interrupt_status: serde_json::Value = serde_json::from_str(&pre_interrupt_status_body)
-            .map_err(|err| format!("pre-interrupt background heal status is not JSON ({err}): {pre_interrupt_status_body}"))?;
+        let pre_interrupt_status = loop {
+            let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
+            let status: serde_json::Value = serde_json::from_str(&status_body)
+                .map_err(|err| format!("pre-interrupt background heal status is not JSON ({err}): {status_body}"))?;
+            if background_enabled || only_admin_heal_is_active(&status) {
+                break status;
+            }
+            if Instant::now() >= partial_deadline {
+                return Err(format!("non-admin heal did not drain before the interruption deadline: {status}").into());
+            }
+            sleep(Duration::from_millis(50)).await;
+        };
+        assert!(
+            metadata_count(&replaced_disk, bucket, &expected_manifests) < expected_manifests.len(),
+            "root heal must still be partially complete at the interruption checkpoint"
+        );
         let pre_interrupt_replacement = replacement_recovery_status(&cluster).await?;
         if background_rejoin_heal_evidence {
             let target_log = std::fs::read_to_string(format!("{log_dir}/node1.log"))?;
@@ -2014,9 +2233,9 @@ mod tests {
             assert_eq!(
                 outage_census
                     .erasure_index
-                    .filter(|index| missing_outage_erasure_indices.contains(index)),
+                    .filter(|index| replacement_outage_erasure_index == Some(*index)),
                 outage_census.erasure_index,
-                "the outage object must be rebuilt into one of the stopped node's missing erasure slots"
+                "the outage object must be rebuilt into the selected replacement drive's erasure-set slot"
             );
         }
 
@@ -2039,11 +2258,21 @@ mod tests {
                 .await;
                 match put_result {
                     Ok(Ok(_)) => break,
-                    Ok(Err(error)) if is_service_unavailable_put(&error) && Instant::now() < deferred_deadline => {
+                    Ok(Err(error)) if is_retryable_outage_put(&error) && Instant::now() < deferred_deadline => {
                         sleep(Duration::from_secs(1)).await;
                     }
-                    Ok(Err(error)) => return Err(error.into()),
-                    Err(error) => return Err(error.into()),
+                    Ok(Err(error)) => {
+                        return Err(format!(
+                            "deferred outage PUT failed for {bucket}/{outage_key} after target rejoin and up to 60s wait: {error}"
+                        )
+                        .into());
+                    }
+                    Err(error) => {
+                        return Err(format!(
+                            "deferred outage PUT timed out for {bucket}/{outage_key} after 30s attempt: {error}"
+                        )
+                        .into());
+                    }
                 }
             }
             info!(
@@ -2064,9 +2293,13 @@ mod tests {
         let node_listings = assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?;
 
         let target_client = cluster.create_s3_client(1)?;
+        let readback_timeout_secs = std::env::var("RUSTFS_HEAL_READBACK_TIMEOUT_SECS")
+            .ok()
+            .and_then(|value| value.parse::<u64>().ok())
+            .unwrap_or(60)
+            .clamp(5, 180);
         for (key, payload_seed) in &created_online_objects {
-            let response = target_client.get_object().bucket(bucket).key(key).send().await?;
-            let actual = response.body.collect().await?.into_bytes();
+            let actual = get_object_after_recovery(&target_client, bucket, key, readback_timeout_secs).await?;
             let expected_body = deterministic_object_body(object_size_bytes, *payload_seed);
             assert_eq!(actual.as_ref(), expected_body.as_slice(), "object body changed for {key}");
             if evidence_run.is_some()
@@ -2084,8 +2317,7 @@ mod tests {
                 }));
             }
         }
-        let response = target_client.get_object().bucket(bucket).key(&outage_key).send().await?;
-        let actual = response.body.collect().await?.into_bytes();
+        let actual = get_object_after_recovery(&target_client, bucket, &outage_key, readback_timeout_secs).await?;
         let expected_outage_body = deterministic_object_body(object_size_bytes, outage_payload_seed);
         assert_eq!(actual.as_ref(), expected_outage_body.as_slice(), "object body changed for {outage_key}");
         if evidence_run.is_some() {
@@ -2296,4 +2528,28 @@ mod tests {
 
         Ok(())
     }
+
+    #[test]
+    fn outage_candidate_must_target_replacement_erasure_index() {
+        let distribution = vec![4, 7, 10, 1, 5, 8, 11, 2, 6, 9, 12, 3];
+        let replacement_set_slot = 8;
+        let replacement_index = distribution[replacement_set_slot];
+        let peers_missing_replacement = OutagePeerManifest {
+            erasure_indices: (1..=12).filter(|index| *index != replacement_index).collect(),
+            erasure_distribution: Some(distribution.clone()),
+        };
+        assert!(outage_candidate_targets_replacement(&peers_missing_replacement, 12, replacement_set_slot));
+
+        let peers_missing_other_slot = OutagePeerManifest {
+            erasure_indices: (1..=12).filter(|index| *index != 9).collect(),
+            erasure_distribution: Some(distribution),
+        };
+        assert!(!outage_candidate_targets_replacement(&peers_missing_other_slot, 12, replacement_set_slot));
+
+        let insufficient_peer_shards = OutagePeerManifest {
+            erasure_indices: [1, 3, 4, 5, 6, 7, 8].into_iter().collect(),
+            erasure_distribution: Some(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]),
+        };
+        assert!(!outage_candidate_targets_replacement(&insufficient_peer_shards, 12, replacement_set_slot));
+    }
 }

crates/e2e_test/src/inline_fast_path_cluster_test.rs

@@ -929,7 +929,13 @@ async fn assert_case(
         case.label
     );
     assert_storage_layout(cluster, bucket, &key, version_id.as_deref(), case.stored_inline)?;
-    Ok((key, body, put.e_tag().map(str::to_owned), version_id))
+    // A suspended PUT omits the response version, but reads identify the
+    // stored null version explicitly.
+    let read_version_id = match state {
+        VersionState::Suspended => Some("null".to_owned()),
+        _ => version_id,
+    };
+    Ok((key, body, put.e_tag().map(str::to_owned), read_version_id))
 }
 
 async fn get_and_assert(

crates/e2e_test/src/internode_rpc_signature_e2e_test.rs

@@ -58,10 +58,11 @@
 //!
 //! Every covered handler checks the digest before touching storage, and
 //! `MakeVolume` resolves its disk *after* that check. Aiming at a disk that
-//! cannot exist gives three cleanly separable outcomes with zero side effects
+//! cannot exist gives four cleanly separable outcomes with zero side effects
 //! on the server's real data:
 //!
 //! - `Err(Unauthenticated)` — rejected by `check_auth` (signature layer).
+//! - `Err(Unavailable)` — a valid signature with a stale boot epoch, rejected before execution.
 //! - `Err(PermissionDenied)` — rejected by the handler's body-digest gate.
 //! - `Ok(success: false)` — **authentication passed**; the request reached
 //!   handler logic and only then failed on the bogus disk.
@@ -468,8 +469,8 @@ async fn replay_scope_rejects_replay_path_transplant_and_stale_epoch_e2e() -> Te
     rustfs_protos::evict_failed_connection(&url).await;
     assert_rejected(
         call_make_volume(&url, request.clone(), stale_epoch).await,
-        Code::Unauthenticated,
-        None,
+        Code::Unavailable,
+        Some("RPC boot epoch changed"),
         "a replay-scoped signature captured before the receiving process restart",
     );
 

crates/e2e_test/src/kms/README.md

@@ -57,50 +57,46 @@ Broad integration tests that exercise:
    pip install awscurl
    ```
 
-2. **Build RustFS**
+2. **Build RustFS** (from the repository root)
    ```bash
-   cargo build
+   python3 scripts/e2e_binary.py build
    ```
 
 ### Run individual suites
 
+Run every command below from the repository root through `scripts/e2e_binary.py run`; plain `cargo test -p e2e_test` fails with a missing E2E run receipt.
+
 #### Local backend
 ```bash
-cd crates/e2e_test
-cargo test test_local_kms_end_to_end -- --nocapture
+python3 scripts/e2e_binary.py run -- cargo test -p e2e_test test_local_kms_end_to_end -- --nocapture
 ```
 
 #### Vault backend
 ```bash
-cd crates/e2e_test
-cargo test test_vault_kms_end_to_end -- --nocapture
+python3 scripts/e2e_binary.py run -- cargo test -p e2e_test test_vault_kms_end_to_end -- --nocapture
 ```
 
 #### High availability
 ```bash
-cd crates/e2e_test
-cargo test test_vault_kms_high_availability -- --nocapture
+python3 scripts/e2e_binary.py run -- cargo test -p e2e_test test_vault_kms_high_availability -- --nocapture
 ```
 
 #### Comprehensive features (disabled)
 ```bash
-cd crates/e2e_test
 # Disabled due to AWS SDK compatibility gaps
-# cargo test test_comprehensive_kms_functionality -- --nocapture
-# cargo test test_sse_modes_compatibility -- --nocapture
-# cargo test test_kms_api_comprehensive -- --nocapture
+# python3 scripts/e2e_binary.py run -- cargo test -p e2e_test test_comprehensive_kms_functionality -- --nocapture
+# python3 scripts/e2e_binary.py run -- cargo test -p e2e_test test_sse_modes_compatibility -- --nocapture
+# python3 scripts/e2e_binary.py run -- cargo test -p e2e_test test_kms_api_comprehensive -- --nocapture
 ```
 
 ### Run all KMS suites
 ```bash
-cd crates/e2e_test
-cargo test kms -- --nocapture
+python3 scripts/e2e_binary.py run -- cargo test -p e2e_test kms -- --nocapture
 ```
 
 ### Run serially (avoid port conflicts)
 ```bash
-cd crates/e2e_test
-cargo test kms -- --nocapture --test-threads=1
+python3 scripts/e2e_binary.py run -- cargo test -p e2e_test kms -- --nocapture --test-threads=1
 ```
 
 ## 🔧 Configuration
@@ -120,7 +116,7 @@ export RUST_LOG=debug
 ### Required binaries
 
 Tests look for:
-- `../../target/debug/rustfs` – RustFS server
+- RustFS server – the binary verified by `scripts/e2e_binary.py` (default `target/debug/rustfs`)
 - `vault` – Vault CLI (must be on PATH)
 - `/Users/dandan/Library/Python/3.9/bin/awscurl` – AWS SigV4 helper
 
@@ -174,14 +170,14 @@ which awscurl  # Update the path in tests accordingly
 
 **Q: Tests time out**
 ```bash
-RUST_LOG=debug cargo test test_local_kms_end_to_end -- --nocapture
+RUST_LOG=debug python3 scripts/e2e_binary.py run -- cargo test -p e2e_test test_local_kms_end_to_end -- --nocapture
 ```
 
 ### Debug tips
 
 1. **Enable verbose logs**
    ```bash
-   RUST_LOG=rustfs_kms=debug,rustfs=info cargo test -- --nocapture
+   RUST_LOG=rustfs_kms=debug,rustfs=info python3 scripts/e2e_binary.py run -- cargo test -p e2e_test kms -- --nocapture
    ```
 
 2. **Keep temporary files** – comment out cleanup logic to inspect generated configs
@@ -237,9 +233,8 @@ Designed to run inside CI/CD pipelines:
     sudo apt-get install -y vault
     pip install awscurl
 
-    cargo build
-    cd crates/e2e_test
-    cargo test kms -- --nocapture --test-threads=1
+    python3 scripts/e2e_binary.py build
+    python3 scripts/e2e_binary.py run -- cargo test -p e2e_test kms -- --nocapture --test-threads=1
 ```
 
 ## 📚 References

crates/e2e_test/src/kms/bucket_default_encryption_test.rs

@@ -21,6 +21,7 @@
 
 use super::common::LocalKMSTestEnvironment;
 use crate::common::{TEST_BUCKET, init_logging};
+use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{
     ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart, ServerSideEncryption,
@@ -631,3 +632,92 @@ async fn test_sse_kms_without_key_id_populates_default() -> Result<(), Box<dyn s
     info!("Test passed: SSE-KMS without key ID correctly populates default key '{}'", default_key_id);
     Ok(())
 }
+
+/// A default-encryption configuration the write path cannot honour as written
+/// must be refused rather than stored: the write path falls back to AES256 for
+/// any algorithm it does not know, so storing `AES128` would make
+/// GetBucketEncryption report a scheme no object is encrypted under.
+#[tokio::test]
+async fn test_put_bucket_encryption_rejects_unknown_algorithm_and_misplaced_key_id()
+-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+
+    let mut kms_env = LocalKMSTestEnvironment::new().await?;
+    let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
+    kms_env.wait_for_kms_ready().await?;
+
+    let s3_client = kms_env.base_env.create_s3_client();
+    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
+
+    let config_with = |algorithm: ServerSideEncryption, key_id: Option<&str>| {
+        let mut by_default = ServerSideEncryptionByDefault::builder().sse_algorithm(algorithm);
+        if let Some(key_id) = key_id {
+            by_default = by_default.kms_master_key_id(key_id);
+        }
+        ServerSideEncryptionConfiguration::builder()
+            .rules(
+                ServerSideEncryptionRule::builder()
+                    .apply_server_side_encryption_by_default(by_default.build().unwrap())
+                    .build(),
+            )
+            .build()
+            .unwrap()
+    };
+
+    // Baseline the bucket on AES256 so a refused update has something to leave untouched.
+    s3_client
+        .put_bucket_encryption()
+        .bucket(TEST_BUCKET)
+        .server_side_encryption_configuration(config_with(ServerSideEncryption::Aes256, None))
+        .send()
+        .await?;
+
+    let unknown = s3_client
+        .put_bucket_encryption()
+        .bucket(TEST_BUCKET)
+        .server_side_encryption_configuration(config_with(ServerSideEncryption::from("AES128"), None))
+        .send()
+        .await
+        .expect_err("an unknown SSEAlgorithm must be refused");
+    assert_eq!(unknown.raw_response().map(|response| response.status().as_u16()), Some(400));
+    assert_eq!(
+        unknown.as_service_error().and_then(ProvideErrorMetadata::code),
+        Some("MalformedXML"),
+        "unknown algorithm error was {unknown:?}"
+    );
+
+    let misplaced_key = s3_client
+        .put_bucket_encryption()
+        .bucket(TEST_BUCKET)
+        .server_side_encryption_configuration(config_with(ServerSideEncryption::Aes256, Some(&default_key_id)))
+        .send()
+        .await
+        .expect_err("KMSMasterKeyID with AES256 must be refused");
+    assert_eq!(misplaced_key.raw_response().map(|response| response.status().as_u16()), Some(400));
+    assert_eq!(
+        misplaced_key.as_service_error().and_then(ProvideErrorMetadata::code),
+        Some("InvalidArgument"),
+        "misplaced key id error was {misplaced_key:?}"
+    );
+
+    // The refused updates left the baseline in place.
+    let stored = s3_client.get_bucket_encryption().bucket(TEST_BUCKET).send().await?;
+    let by_default = stored
+        .server_side_encryption_configuration()
+        .and_then(|config| config.rules().first())
+        .and_then(|rule| rule.apply_server_side_encryption_by_default())
+        .expect("baseline configuration must still be present");
+    assert_eq!(by_default.sse_algorithm(), &ServerSideEncryption::Aes256);
+    assert_eq!(by_default.kms_master_key_id(), None);
+
+    // aws:kms with a key id stays accepted.
+    s3_client
+        .put_bucket_encryption()
+        .bucket(TEST_BUCKET)
+        .server_side_encryption_configuration(config_with(ServerSideEncryption::AwsKms, Some(&default_key_id)))
+        .send()
+        .await?;
+
+    kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
+    Ok(())
+}

crates/e2e_test/src/kms/common.rs

@@ -176,6 +176,30 @@ pub async fn start_kms(
     Ok(())
 }
 
+/// Stop the running KMS service via admin API, keeping its configuration so
+/// `start_kms` can bring it back.
+pub async fn stop_kms(
+    base_url: &str,
+    access_key: &str,
+    secret_key: &str,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let response = kms_admin_request(
+        base_url,
+        http::Method::POST,
+        "/rustfs/admin/v3/kms/stop",
+        Some("{}"),
+        access_key,
+        secret_key,
+    )
+    .await?;
+    let response: serde_json::Value = serde_json::from_str(&response)?;
+    if response["success"] != true {
+        return Err(format!("KMS stop failed: {}", response["message"].as_str().unwrap_or("unknown error")).into());
+    }
+    info!("KMS stopped successfully");
+    Ok(())
+}
+
 /// Get KMS status via admin API
 pub async fn get_kms_status(
     base_url: &str,
@@ -325,7 +349,7 @@ pub async fn create_default_key(
     secret_key: &str,
 ) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
     let create_key_body = serde_json::json!({
-        "key_usage": "ENCRYPT_DECRYPT",
+        "key_usage": "EncryptDecrypt",
         "description": "Default key for e2e testing"
     })
     .to_string();

crates/e2e_test/src/kms/encryption_metadata_test.rs

@@ -357,3 +357,97 @@ async fn test_multipart_upload_writes_encrypted_data() -> Result<(), Box<dyn std
 
     Ok(())
 }
+
+/// `x-amz-server-side-encryption-aws-kms-key-id` is defined for `aws:kms`
+/// objects only. SSE-S3 wraps its data key under the service default key too,
+/// but that key is internal: PutObject, CopyObject and CreateMultipartUpload
+/// responses for an `AES256` object must not name it.
+#[tokio::test]
+async fn test_sse_s3_write_responses_carry_no_kms_key_id() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+
+    let mut kms_env = LocalKMSTestEnvironment::new().await?;
+    let default_key = kms_env.start_rustfs_for_local_kms().await?;
+    kms_env.wait_for_kms_ready().await?;
+
+    let s3_client = kms_env.base_env.create_s3_client();
+    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
+
+    let put = s3_client
+        .put_object()
+        .bucket(TEST_BUCKET)
+        .key("sse-s3-put")
+        .body(ByteStream::from_static(b"sse-s3 payload"))
+        .server_side_encryption(ServerSideEncryption::Aes256)
+        .send()
+        .await?;
+    assert_eq!(put.server_side_encryption(), Some(&ServerSideEncryption::Aes256));
+    assert_eq!(put.ssekms_key_id(), None, "PutObject AES256 must not advertise the wrapping key");
+
+    let get = s3_client.get_object().bucket(TEST_BUCKET).key("sse-s3-put").send().await?;
+    assert_eq!(get.server_side_encryption(), Some(&ServerSideEncryption::Aes256));
+    assert_eq!(get.ssekms_key_id(), None, "GetObject AES256 must not advertise the wrapping key");
+    let head = s3_client.head_object().bucket(TEST_BUCKET).key("sse-s3-put").send().await?;
+    assert_eq!(head.ssekms_key_id(), None, "HeadObject AES256 must not advertise the wrapping key");
+
+    let copy = s3_client
+        .copy_object()
+        .bucket(TEST_BUCKET)
+        .key("sse-s3-copy")
+        .copy_source(format!("{TEST_BUCKET}/sse-s3-put"))
+        .server_side_encryption(ServerSideEncryption::Aes256)
+        .send()
+        .await?;
+    assert_eq!(copy.server_side_encryption(), Some(&ServerSideEncryption::Aes256));
+    assert_eq!(copy.ssekms_key_id(), None, "CopyObject AES256 must not advertise the wrapping key");
+
+    let multipart = s3_client
+        .create_multipart_upload()
+        .bucket(TEST_BUCKET)
+        .key("sse-s3-multipart")
+        .server_side_encryption(ServerSideEncryption::Aes256)
+        .send()
+        .await?;
+    assert_eq!(multipart.server_side_encryption(), Some(&ServerSideEncryption::Aes256));
+    assert_eq!(
+        multipart.ssekms_key_id(),
+        None,
+        "CreateMultipartUpload AES256 must not advertise the wrapping key"
+    );
+    s3_client
+        .abort_multipart_upload()
+        .bucket(TEST_BUCKET)
+        .key("sse-s3-multipart")
+        .upload_id(multipart.upload_id().expect("upload id"))
+        .send()
+        .await?;
+
+    // Control: the same responses keep naming the key for an aws:kms object.
+    let kms_put = s3_client
+        .put_object()
+        .bucket(TEST_BUCKET)
+        .key("sse-kms-put")
+        .body(ByteStream::from_static(b"sse-kms payload"))
+        .server_side_encryption(ServerSideEncryption::AwsKms)
+        .send()
+        .await?;
+    assert_eq!(kms_put.ssekms_key_id(), Some(default_key.as_str()));
+    let kms_multipart = s3_client
+        .create_multipart_upload()
+        .bucket(TEST_BUCKET)
+        .key("sse-kms-multipart")
+        .server_side_encryption(ServerSideEncryption::AwsKms)
+        .send()
+        .await?;
+    assert_eq!(kms_multipart.ssekms_key_id(), Some(default_key.as_str()));
+    s3_client
+        .abort_multipart_upload()
+        .bucket(TEST_BUCKET)
+        .key("sse-kms-multipart")
+        .upload_id(kms_multipart.upload_id().expect("upload id"))
+        .send()
+        .await?;
+
+    kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
+    Ok(())
+}

crates/e2e_test/src/kms/kms_fault_recovery_test.rs

@@ -21,7 +21,7 @@
 //! - Corrupted key files
 //! - Recovery from transient failures
 
-use super::common::LocalKMSTestEnvironment;
+use super::common::{LocalKMSTestEnvironment, create_default_key, create_key_with_specific_id, kms_admin_request};
 use crate::common::{TEST_BUCKET, init_logging};
 use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::types::ServerSideEncryption;
@@ -534,3 +534,129 @@ async fn test_kms_concurrent_encryption_requests() -> Result<(), Box<dyn std::er
     kms_env.base_env.delete_test_bucket(TEST_BUCKET).await?;
     Ok(())
 }
+
+/// Once the key an object was wrapped under is deleted, the object cannot be
+/// read until the key is restored. That is the same `400 KMS.NotFoundException`
+/// a write under a missing key returns, not a `500`; `HeadObject` never unwraps
+/// the data key and keeps answering `200`.
+#[tokio::test]
+async fn test_reads_under_a_deleted_kms_key_report_key_not_found() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+
+    let mut kms_env = LocalKMSTestEnvironment::new().await?;
+    let default_key_id = "rustfs-e2e-test-default-key";
+    create_key_with_specific_id(&kms_env.kms_keys_dir, default_key_id).await?;
+    let key_dir = kms_env.kms_keys_dir.clone();
+    kms_env
+        .base_env
+        .start_rustfs_server_with_env(
+            vec![
+                "--kms-enable",
+                "--kms-backend",
+                "local",
+                "--kms-key-dir",
+                &key_dir,
+                "--kms-default-key-id",
+                default_key_id,
+            ],
+            &[
+                ("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true"),
+                // Immediate deletion is refused on a default server; the test
+                // needs the key gone now rather than after the waiting window.
+                ("RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION", "true"),
+            ],
+        )
+        .await?;
+    kms_env.wait_for_kms_ready().await?;
+    let base = &kms_env.base_env;
+    let s3_client = base.create_s3_client();
+    base.create_test_bucket(TEST_BUCKET).await?;
+
+    let doomed_key_id = create_default_key(&base.url, &base.access_key, &base.secret_key).await?;
+    let object_key = "wrapped-under-doomed-key";
+    let payload = b"readable only while the key exists".to_vec();
+    let put = s3_client
+        .put_object()
+        .bucket(TEST_BUCKET)
+        .key(object_key)
+        .body(aws_sdk_s3::primitives::ByteStream::from(payload.clone()))
+        .server_side_encryption(ServerSideEncryption::AwsKms)
+        .ssekms_key_id(&doomed_key_id)
+        .send()
+        .await?;
+    assert_eq!(put.ssekms_key_id(), Some(doomed_key_id.as_str()));
+
+    info!("🗑️ deleting {doomed_key_id} immediately");
+    kms_admin_request(
+        &base.url,
+        http::Method::DELETE,
+        "/rustfs/admin/v3/kms/keys/delete",
+        Some(
+            &serde_json::json!({
+                "key_id": doomed_key_id,
+                "force_immediate": true,
+                "confirm_key_id": doomed_key_id,
+            })
+            .to_string(),
+        ),
+        &base.access_key,
+        &base.secret_key,
+    )
+    .await?;
+    kms_admin_request(
+        &base.url,
+        http::Method::POST,
+        "/rustfs/admin/v3/kms/clear-cache",
+        Some("{}"),
+        &base.access_key,
+        &base.secret_key,
+    )
+    .await?;
+
+    let head = s3_client.head_object().bucket(TEST_BUCKET).key(object_key).send().await?;
+    assert_eq!(head.ssekms_key_id(), Some(doomed_key_id.as_str()));
+
+    let get_error = s3_client
+        .get_object()
+        .bucket(TEST_BUCKET)
+        .key(object_key)
+        .send()
+        .await
+        .expect_err("an object wrapped under a deleted key must not be readable");
+    assert_eq!(get_error.raw_response().map(|response| response.status().as_u16()), Some(400));
+    assert_eq!(
+        get_error.as_service_error().and_then(ProvideErrorMetadata::code),
+        Some("KMS.NotFoundException"),
+        "GetObject error was {get_error:?}"
+    );
+
+    let copy_error = s3_client
+        .copy_object()
+        .bucket(TEST_BUCKET)
+        .key("copied-from-doomed-source")
+        .copy_source(format!("{TEST_BUCKET}/{object_key}"))
+        .server_side_encryption(ServerSideEncryption::AwsKms)
+        .send()
+        .await
+        .expect_err("copying from an object wrapped under a deleted key must fail the same way");
+    assert_eq!(copy_error.raw_response().map(|response| response.status().as_u16()), Some(400));
+    assert_eq!(
+        copy_error.as_service_error().and_then(ProvideErrorMetadata::code),
+        Some("KMS.NotFoundException"),
+        "CopyObject error was {copy_error:?}"
+    );
+
+    // The default key is untouched, so the node keeps serving other objects.
+    let unaffected = s3_client
+        .put_object()
+        .bucket(TEST_BUCKET)
+        .key("wrapped-under-default-key")
+        .body(aws_sdk_s3::primitives::ByteStream::from_static(b"still fine"))
+        .server_side_encryption(ServerSideEncryption::AwsKms)
+        .send()
+        .await?;
+    assert_eq!(unaffected.ssekms_key_id(), Some(default_key_id));
+
+    base.delete_test_bucket(TEST_BUCKET).await?;
+    Ok(())
+}

crates/e2e_test/src/kms/kms_service_stop_test.rs

@@ -0,0 +1,231 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! SSE-KMS writes against a node whose KMS is stopped, was never configured,
+//! or runs without a default key.
+//!
+//! `docs/operations/kms-backend-security.md` promises `503` for a configured
+//! KMS that is not running and `400 InvalidRequest` when KMS was never
+//! configured. The bare `aws:kms` form (no key id, no bucket default) used to
+//! miss both branches and surface as `500`, so every scenario here is
+//! exercised with and without a key id.
+
+use super::common::{
+    LocalKMSTestEnvironment, assert_s3_error, create_key_with_specific_id, start_kms, stop_kms, wait_for_kms_ready,
+};
+use crate::common::{RustFSTestEnvironment, TEST_BUCKET, init_logging};
+use aws_sdk_s3::Client;
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::ServerSideEncryption;
+use tracing::info;
+
+const SERVICE_UNAVAILABLE_MESSAGE: &str = "The service is unavailable. Please retry.";
+const KMS_NOT_CONFIGURED_MESSAGE: &str = "SSE-KMS requires a configured and running KMS service";
+const KMS_NO_DEFAULT_KEY_MESSAGE: &str =
+    "SSE-KMS requires a KMS key id: the request named none and the KMS service has no default key";
+
+/// Issue an SSE-KMS PutObject and a CreateMultipartUpload, each with and
+/// without a key id, and require every one of them to fail with `status`/`code`/`message`.
+async fn assert_sse_kms_writes_refused(client: &Client, key_prefix: &str, status: u16, code: &str, message: &str) {
+    for (label, key_id) in [("bare", None), ("keyed", Some("rustfs-e2e-refused-key"))] {
+        let object_key = format!("{key_prefix}-{label}");
+        let mut put = client
+            .put_object()
+            .bucket(TEST_BUCKET)
+            .key(&object_key)
+            .body(ByteStream::from_static(b"must not be published"))
+            .server_side_encryption(ServerSideEncryption::AwsKms);
+        if let Some(key_id) = key_id {
+            put = put.ssekms_key_id(key_id);
+        }
+        assert_s3_error(put.send().await, status, code, message, &format!("{label} SSE-KMS PutObject"));
+
+        let mut create = client
+            .create_multipart_upload()
+            .bucket(TEST_BUCKET)
+            .key(&object_key)
+            .server_side_encryption(ServerSideEncryption::AwsKms);
+        if let Some(key_id) = key_id {
+            create = create.ssekms_key_id(key_id);
+        }
+        assert_s3_error(
+            create.send().await,
+            status,
+            code,
+            message,
+            &format!("{label} SSE-KMS CreateMultipartUpload"),
+        );
+
+        let absence = client
+            .get_object()
+            .bucket(TEST_BUCKET)
+            .key(&object_key)
+            .send()
+            .await
+            .expect_err("a refused SSE-KMS write must not publish an object");
+        assert_eq!(absence.raw_response().map(|response| response.status().as_u16()), Some(404));
+        assert_eq!(absence.as_service_error().and_then(ProvideErrorMetadata::code), Some("NoSuchKey"));
+    }
+}
+
+/// A configured KMS that an operator stopped is a transient outage: every
+/// SSE-KMS write is refused with `503` until `kms/start`, after which the bare
+/// form resolves the service default key again and earlier objects read back.
+#[tokio::test]
+async fn test_sse_kms_writes_are_refused_with_503_while_kms_is_stopped() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+    let mut kms_env = LocalKMSTestEnvironment::new().await?;
+    let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
+    kms_env.wait_for_kms_ready().await?;
+    let base = &kms_env.base_env;
+    let client = base.create_s3_client();
+    base.create_test_bucket(TEST_BUCKET).await?;
+
+    let object_key = "written-before-stop";
+    let payload = b"encrypted under the service default key".to_vec();
+    let put = client
+        .put_object()
+        .bucket(TEST_BUCKET)
+        .key(object_key)
+        .body(ByteStream::from(payload.clone()))
+        .server_side_encryption(ServerSideEncryption::AwsKms)
+        .send()
+        .await?;
+    assert_eq!(put.server_side_encryption(), Some(&ServerSideEncryption::AwsKms));
+    assert_eq!(put.ssekms_key_id(), Some(default_key_id.as_str()));
+
+    info!("stopping KMS through the admin API");
+    stop_kms(&base.url, &base.access_key, &base.secret_key).await?;
+    assert_sse_kms_writes_refused(&client, "while-stopped", 503, "ServiceUnavailable", SERVICE_UNAVAILABLE_MESSAGE).await;
+
+    info!("starting KMS again");
+    start_kms(&base.url, &base.access_key, &base.secret_key).await?;
+    wait_for_kms_ready(&base.url, &base.access_key, &base.secret_key).await?;
+
+    let restored = client
+        .put_object()
+        .bucket(TEST_BUCKET)
+        .key("written-after-start")
+        .body(ByteStream::from_static(b"service is back"))
+        .server_side_encryption(ServerSideEncryption::AwsKms)
+        .send()
+        .await?;
+    assert_eq!(restored.ssekms_key_id(), Some(default_key_id.as_str()));
+
+    let read_back = client.get_object().bucket(TEST_BUCKET).key(object_key).send().await?;
+    assert_eq!(read_back.body.collect().await?.into_bytes().as_ref(), payload.as_slice());
+
+    base.delete_test_bucket(TEST_BUCKET).await?;
+    Ok(())
+}
+
+/// A node that only carries `RUSTFS_SSE_S3_MASTER_KEY` serves SSE-S3 but has
+/// no KMS to name: SSE-KMS is a client configuration error (`400`), and it
+/// must never be downgraded onto the local master key.
+#[tokio::test]
+async fn test_sse_kms_writes_are_refused_with_400_when_kms_was_never_configured()
+-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    // base64 of 32 zero bytes: a valid master key shape for the SSE-S3 fallback.
+    env.start_rustfs_server_with_env(vec![], &[("RUSTFS_SSE_S3_MASTER_KEY", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")])
+        .await?;
+    let client = env.create_s3_client();
+    env.create_test_bucket(TEST_BUCKET).await?;
+
+    let sse_s3 = client
+        .put_object()
+        .bucket(TEST_BUCKET)
+        .key("sse-s3-fallback")
+        .body(ByteStream::from_static(b"local master key still serves AES256"))
+        .server_side_encryption(ServerSideEncryption::Aes256)
+        .send()
+        .await?;
+    assert_eq!(sse_s3.server_side_encryption(), Some(&ServerSideEncryption::Aes256));
+
+    assert_sse_kms_writes_refused(&client, "no-kms", 400, "InvalidRequest", KMS_NOT_CONFIGURED_MESSAGE).await;
+
+    env.delete_test_bucket(TEST_BUCKET).await?;
+    Ok(())
+}
+
+/// A running KMS without a default key can serve a keyed request but has
+/// nothing to resolve a bare `aws:kms` request to; that is the caller's
+/// omission, not a server fault.
+#[tokio::test]
+async fn test_bare_sse_kms_write_is_refused_with_400_when_kms_has_no_default_key()
+-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+    let mut kms_env = LocalKMSTestEnvironment::new().await?;
+    let named_key_id = "rustfs-e2e-named-key";
+    create_key_with_specific_id(&kms_env.kms_keys_dir, named_key_id).await?;
+    let key_dir = kms_env.kms_keys_dir.clone();
+    kms_env
+        .base_env
+        .start_rustfs_server_with_env(
+            vec!["--kms-enable", "--kms-backend", "local", "--kms-key-dir", &key_dir],
+            &[("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true")],
+        )
+        .await?;
+    kms_env.wait_for_kms_ready().await?;
+    let base = &kms_env.base_env;
+    let client = base.create_s3_client();
+    base.create_test_bucket(TEST_BUCKET).await?;
+
+    let keyed = client
+        .put_object()
+        .bucket(TEST_BUCKET)
+        .key("keyed-without-default")
+        .body(ByteStream::from_static(b"a named key needs no default"))
+        .server_side_encryption(ServerSideEncryption::AwsKms)
+        .ssekms_key_id(named_key_id)
+        .send()
+        .await?;
+    assert_eq!(keyed.ssekms_key_id(), Some(named_key_id));
+
+    let bare = client
+        .put_object()
+        .bucket(TEST_BUCKET)
+        .key("bare-without-default")
+        .body(ByteStream::from_static(b"must not be published"))
+        .server_side_encryption(ServerSideEncryption::AwsKms)
+        .send()
+        .await;
+    assert_s3_error(
+        bare,
+        400,
+        "InvalidRequest",
+        KMS_NO_DEFAULT_KEY_MESSAGE,
+        "bare SSE-KMS PutObject without default key",
+    );
+
+    let bare_multipart = client
+        .create_multipart_upload()
+        .bucket(TEST_BUCKET)
+        .key("bare-without-default")
+        .server_side_encryption(ServerSideEncryption::AwsKms)
+        .send()
+        .await;
+    assert_s3_error(
+        bare_multipart,
+        400,
+        "InvalidRequest",
+        KMS_NO_DEFAULT_KEY_MESSAGE,
+        "bare SSE-KMS CreateMultipartUpload without default key",
+    );
+
+    base.delete_test_bucket(TEST_BUCKET).await?;
+    Ok(())
+}

crates/e2e_test/src/kms/mod.rs

@@ -39,6 +39,9 @@ mod kms_edge_cases_test;
 #[cfg(test)]
 mod kms_fault_recovery_test;
 
+#[cfg(test)]
+mod kms_service_stop_test;
+
 #[cfg(test)]
 mod bucket_default_encryption_test;
 

crates/e2e_test/src/lib.rs

@@ -179,6 +179,9 @@ mod compression_test;
 #[cfg(test)]
 mod delete_objects_versioning_test;
 
+#[cfg(test)]
+mod delete_authorization_test;
+
 // Regression test for signed DELETE Object?versionId requests without Content-Length.
 #[cfg(test)]
 mod delete_object_no_content_length_test;
@@ -254,6 +257,9 @@ mod copy_object_version_restore_test;
 #[cfg(test)]
 mod copy_object_checksum_test;
 
+#[cfg(test)]
+mod multipart_copy_readiness_test;
+
 #[cfg(test)]
 mod ssec_copy_test;
 

crates/e2e_test/src/multipart_copy_readiness_test.rs

@@ -0,0 +1,260 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Harbor / Docker Distribution multipart staging to CopyObject regressions.
+
+use crate::common::{RustFSTestEnvironment, init_logging};
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart, StorageClass};
+use std::error::Error;
+use std::sync::Arc;
+use tokio::sync::Barrier;
+use tracing::info;
+
+const BUCKET: &str = "multipart-copy-readiness";
+const SOURCE_KEY: &str = "docker/registry/v2/repositories/example/_uploads/upload-id/data";
+const TARGET_KEY: &str = "docker/registry/v2/blobs/sha256/c0/digest/data";
+const UNFINISHED_SOURCE_KEY: &str = "docker/registry/v2/repositories/example/_uploads/upload-id-unfinished/data";
+const UNFINISHED_TARGET_KEY: &str = "docker/registry/v2/blobs/sha256/c1/digest/data";
+const OVERLAP_SOURCE_KEY: &str = "docker/registry/v2/repositories/example/_uploads/upload-id-overlap/data";
+const OVERLAP_TARGET_KEY: &str = "docker/registry/v2/blobs/sha256/c2/digest/data";
+const OVERLAP_RETRY_TARGET_KEY: &str = "docker/registry/v2/blobs/sha256/c3/digest/data";
+
+fn list_contains_key(output: &aws_sdk_s3::operation::list_objects_v2::ListObjectsV2Output, key: &str) -> bool {
+    output
+        .contents()
+        .iter()
+        .any(|object| object.key().is_some_and(|candidate| candidate == key))
+}
+
+async fn upload_one_part_mpu(
+    client: &aws_sdk_s3::Client,
+    bucket: &str,
+    key: &str,
+    payload: Vec<u8>,
+) -> Result<(String, CompletedMultipartUpload), Box<dyn Error + Send + Sync>> {
+    let create = client.create_multipart_upload().bucket(bucket).key(key).send().await?;
+    let upload_id = create.upload_id().ok_or("missing upload id")?.to_string();
+    let part = client
+        .upload_part()
+        .bucket(bucket)
+        .key(key)
+        .upload_id(&upload_id)
+        .part_number(1)
+        .body(ByteStream::from(payload))
+        .send()
+        .await?;
+    let completed = CompletedMultipartUpload::builder()
+        .parts(
+            CompletedPart::builder()
+                .part_number(1)
+                .set_e_tag(part.e_tag().map(str::to_string))
+                .build(),
+        )
+        .build();
+
+    Ok((upload_id, completed))
+}
+
+#[tokio::test]
+async fn harbor_style_multipart_staging_copy_object_boundaries() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+    info!("backlog#2185: Harbor-style one-part MPU staging followed by CopyObject");
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let client = env.create_s3_client();
+    env.create_test_bucket(BUCKET).await?;
+
+    let payload = vec![0xAB; 273];
+    let (upload_id, completed) = upload_one_part_mpu(&client, BUCKET, SOURCE_KEY, payload.clone()).await?;
+    client
+        .complete_multipart_upload()
+        .bucket(BUCKET)
+        .key(SOURCE_KEY)
+        .upload_id(&upload_id)
+        .multipart_upload(completed)
+        .send()
+        .await?;
+
+    let completed_list = client
+        .list_objects_v2()
+        .bucket(BUCKET)
+        .prefix(SOURCE_KEY)
+        .max_keys(1)
+        .send()
+        .await?;
+    assert!(
+        list_contains_key(&completed_list, SOURCE_KEY),
+        "completed multipart staging object must be immediately list-visible"
+    );
+
+    client
+        .copy_object()
+        .bucket(BUCKET)
+        .key(TARGET_KEY)
+        .copy_source(format!("/{BUCKET}/{SOURCE_KEY}"))
+        .storage_class(StorageClass::Standard)
+        .send()
+        .await?;
+
+    let copied = client.get_object().bucket(BUCKET).key(TARGET_KEY).send().await?;
+    let copied_body = copied.body.collect().await?.into_bytes();
+    assert_eq!(copied_body.as_ref(), payload.as_slice());
+
+    let overlap_payload = vec![0xBC; 15_796];
+    let (overlap_upload_id, overlap_completed) =
+        upload_one_part_mpu(&client, BUCKET, OVERLAP_SOURCE_KEY, overlap_payload.clone()).await?;
+
+    let barrier = Arc::new(Barrier::new(2));
+    let complete_client = client.clone();
+    let complete_barrier = Arc::clone(&barrier);
+    let complete_upload_id = overlap_upload_id.clone();
+    let complete_task = tokio::spawn(async move {
+        complete_barrier.wait().await;
+        complete_client
+            .complete_multipart_upload()
+            .bucket(BUCKET)
+            .key(OVERLAP_SOURCE_KEY)
+            .upload_id(complete_upload_id)
+            .multipart_upload(overlap_completed)
+            .send()
+            .await
+    });
+
+    let copy_client = client.clone();
+    let copy_barrier = Arc::clone(&barrier);
+    let copy_task = tokio::spawn(async move {
+        copy_barrier.wait().await;
+        copy_client
+            .copy_object()
+            .bucket(BUCKET)
+            .key(OVERLAP_TARGET_KEY)
+            .copy_source(format!("/{BUCKET}/{OVERLAP_SOURCE_KEY}"))
+            .storage_class(StorageClass::Standard)
+            .send()
+            .await
+    });
+
+    complete_task.await??;
+    match copy_task.await? {
+        Ok(_) => {}
+        Err(copy_err) => {
+            assert_eq!(
+                copy_err.raw_response().map(|response| response.status().as_u16()),
+                Some(404),
+                "overlapped CopyObject may race before publication, but must not leak a 5xx response: {copy_err:?}"
+            );
+            assert_eq!(
+                copy_err.as_service_error().and_then(ProvideErrorMetadata::code),
+                Some("NoSuchKey"),
+                "overlapped CopyObject that wins before Complete must look like an unpublished ordinary object: {copy_err:?}"
+            );
+        }
+    }
+
+    client
+        .copy_object()
+        .bucket(BUCKET)
+        .key(OVERLAP_RETRY_TARGET_KEY)
+        .copy_source(format!("/{BUCKET}/{OVERLAP_SOURCE_KEY}"))
+        .storage_class(StorageClass::Standard)
+        .send()
+        .await?;
+    let overlap_copied = client
+        .get_object()
+        .bucket(BUCKET)
+        .key(OVERLAP_RETRY_TARGET_KEY)
+        .send()
+        .await?;
+    let overlap_copied_body = overlap_copied.body.collect().await?.into_bytes();
+    assert_eq!(
+        overlap_copied_body.as_ref(),
+        overlap_payload.as_slice(),
+        "a completed staging object must not become permanently unreadable after an overlapped copy attempt"
+    );
+
+    let unfinished = client
+        .create_multipart_upload()
+        .bucket(BUCKET)
+        .key(UNFINISHED_SOURCE_KEY)
+        .send()
+        .await?;
+    let unfinished_upload_id = unfinished.upload_id().ok_or("missing unfinished upload id")?.to_string();
+    client
+        .upload_part()
+        .bucket(BUCKET)
+        .key(UNFINISHED_SOURCE_KEY)
+        .upload_id(&unfinished_upload_id)
+        .part_number(1)
+        .body(ByteStream::from_static(b"not-yet-committed"))
+        .send()
+        .await?;
+
+    let unfinished_list = client
+        .list_objects_v2()
+        .bucket(BUCKET)
+        .prefix(UNFINISHED_SOURCE_KEY)
+        .max_keys(1)
+        .send()
+        .await?;
+    assert!(
+        !list_contains_key(&unfinished_list, UNFINISHED_SOURCE_KEY),
+        "UploadPart alone must not publish the staging object namespace entry"
+    );
+
+    let copy_err = client
+        .copy_object()
+        .bucket(BUCKET)
+        .key(UNFINISHED_TARGET_KEY)
+        .copy_source(format!("/{BUCKET}/{UNFINISHED_SOURCE_KEY}"))
+        .storage_class(StorageClass::Standard)
+        .send()
+        .await
+        .expect_err("copying an uncompleted multipart upload source must be rejected");
+    assert_eq!(
+        copy_err.raw_response().map(|response| response.status().as_u16()),
+        Some(404),
+        "unfinished MPU source must not leak a 5xx response: {copy_err:?}"
+    );
+    assert_eq!(
+        copy_err.as_service_error().and_then(ProvideErrorMetadata::code),
+        Some("NoSuchKey"),
+        "unfinished MPU source must be reported as a missing ordinary object: {copy_err:?}"
+    );
+
+    client
+        .abort_multipart_upload()
+        .bucket(BUCKET)
+        .key(UNFINISHED_SOURCE_KEY)
+        .upload_id(unfinished_upload_id)
+        .send()
+        .await?;
+    client
+        .delete_object()
+        .bucket(BUCKET)
+        .key(OVERLAP_RETRY_TARGET_KEY)
+        .send()
+        .await?;
+    let _ = client.delete_object().bucket(BUCKET).key(OVERLAP_TARGET_KEY).send().await;
+    client.delete_object().bucket(BUCKET).key(OVERLAP_SOURCE_KEY).send().await?;
+    client.delete_object().bucket(BUCKET).key(TARGET_KEY).send().await?;
+    client.delete_object().bucket(BUCKET).key(SOURCE_KEY).send().await?;
+    env.delete_test_bucket(BUCKET).await?;
+    env.stop_server();
+
+    Ok(())
+}

crates/e2e_test/src/negative_sigv4_test.rs

@@ -261,6 +261,118 @@ async fn valid_header_sigv4_request_succeeds() -> Result<(), Box<dyn std::error:
     Ok(())
 }
 
+const XM99_SOURCE_BUCKET: &str = "xm99-private-source";
+const XM99_SOURCE_BODY: &[u8] = b"private source object";
+const XM99_TARGET_BODY: &[u8] = b"original target object";
+
+/// Seed a private source object and upload the target with a plain
+/// header-signed PutObject, returning the target path and the copy source.
+async fn xm99_seed_target(
+    env: &RustFSTestEnvironment,
+    signer: &SigV4,
+) -> Result<(String, String), Box<dyn std::error::Error + Send + Sync>> {
+    env.create_test_bucket(XM99_SOURCE_BUCKET).await?;
+    env.create_s3_client()
+        .put_object()
+        .bucket(XM99_SOURCE_BUCKET)
+        .key("secret")
+        .body(ByteStream::from_static(XM99_SOURCE_BODY))
+        .send()
+        .await?;
+
+    let path = format!("/{BUCKET}/xm99-target");
+    let signed = signer.sign("PUT", &path, "", UNSIGNED_PAYLOAD);
+    let resp = send_signed(env, reqwest::Method::PUT, &path, &signed, Some(XM99_TARGET_BODY.to_vec())).await?;
+    assert_eq!(resp.status().as_u16(), 200, "plain header-signed upload must succeed");
+    Ok((path, format!("/{XM99_SOURCE_BUCKET}/secret")))
+}
+
+async fn xm99_target_body(env: &RustFSTestEnvironment) -> Result<Vec<u8>, Box<dyn std::error::Error + Send + Sync>> {
+    let object = env
+        .create_s3_client()
+        .get_object()
+        .bucket(BUCKET)
+        .key("xm99-target")
+        .send()
+        .await?;
+    Ok(object.body.collect().await?.into_bytes().to_vec())
+}
+
+/// GHSA-xm99-m3gq-83g8: replaying a header-signed PutObject with an unsigned
+/// `x-amz-copy-source` must not become a CopyObject that reads another bucket
+/// with the signer's permissions, and swapping the algorithm token must not
+/// route the request around the check.
+#[tokio::test]
+async fn ghsa_xm99_header_sigv4_rejects_unsigned_copy_source() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    setup(&mut env).await?;
+    let signer = SigV4::new(&env);
+    let (path, copy_source) = xm99_seed_target(&env, &signer).await?;
+
+    // The PutObject signature covers host, payload hash and date only.
+    let signed = signer.sign("PUT", &path, "", UNSIGNED_PAYLOAD);
+    let variants = [
+        (
+            signed.authorization.clone(),
+            "There were headers present in the request which were not signed",
+        ),
+        (
+            signed.authorization.replacen(SIGN_V4_ALGORITHM, "OTHER", 1),
+            "Unsupported SigV4 authorization algorithm",
+        ),
+    ];
+    for (authorization, message) in variants {
+        let resp = local_http_client()
+            .put(format!("{}{path}", env.url))
+            .header("authorization", &authorization)
+            .header("x-amz-date", &signed.amz_date)
+            .header("x-amz-content-sha256", &signed.content_sha256)
+            .header("x-amz-copy-source", &copy_source)
+            .send()
+            .await?;
+        let status = resp.status().as_u16();
+        let body = resp.text().await?;
+        assert_eq!(status, 403, "unsigned x-amz-copy-source must be denied, got body:\n{body}");
+        assert_error_code(&body, "AccessDenied");
+        assert!(body.contains(message), "expected {message:?} in response body, got:\n{body}");
+    }
+
+    assert_eq!(
+        xm99_target_body(&env).await?,
+        XM99_TARGET_BODY,
+        "a rejected copy must leave the destination object unchanged"
+    );
+    Ok(())
+}
+
+/// Positive control for GHSA-xm99-m3gq-83g8: the same CopyObject succeeds when
+/// the credential holder signs `x-amz-copy-source`.
+#[tokio::test]
+async fn ghsa_xm99_header_sigv4_accepts_signed_copy_source() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    setup(&mut env).await?;
+    let signer = SigV4::new(&env);
+    let (path, copy_source) = xm99_seed_target(&env, &signer).await?;
+
+    let signed = signer.sign_with_extra_headers("PUT", &path, "", UNSIGNED_PAYLOAD, &[("x-amz-copy-source", &copy_source)]);
+    let resp = local_http_client()
+        .put(format!("{}{path}", env.url))
+        .header("authorization", &signed.authorization)
+        .header("x-amz-date", &signed.amz_date)
+        .header("x-amz-content-sha256", &signed.content_sha256)
+        .header("x-amz-copy-source", &copy_source)
+        .send()
+        .await?;
+    let status = resp.status().as_u16();
+    let body = resp.text().await?;
+    assert_eq!(status, 200, "signed copy must succeed, got body:\n{body}");
+    assert!(body.contains("CopyObjectResult"), "expected CopyObjectResult, got:\n{body}");
+    assert_eq!(xm99_target_body(&env).await?, XM99_SOURCE_BODY, "signed copy must replace the target");
+    Ok(())
+}
+
 /// (a) Tampering the `Signature=` component must be rejected with
 /// SignatureDoesNotMatch / 403.
 #[tokio::test]

crates/e2e_test/src/policy/README.md

@@ -24,7 +24,8 @@ The tests cover the following AWS policy variable scenarios:
 
 ```bash
 # From the project root directory
-cargo test -p e2e_test policy:: -- --nocapture
+python3 scripts/e2e_binary.py build
+python3 scripts/e2e_binary.py run -- cargo test -p e2e_test policy:: -- --nocapture
 ```
 
 Each test starts an isolated RustFS server on a dynamically allocated local port and cleans it up afterward.

crates/e2e_test/src/presigned_negative_test.rs

@@ -527,7 +527,16 @@ async fn ghsa_g8w9_presigned_put_rejects_unsigned_copy_source() -> Result<(), Bo
         .presigned(valid_config())
         .await?;
 
-    let copy_source = format!("/{BUCKET}/{CANONICAL_KEY}");
+    let source_bucket = "presigned-copy-source";
+    env.create_test_bucket(source_bucket).await?;
+    env.create_s3_client()
+        .put_object()
+        .bucket(source_bucket)
+        .key(CANONICAL_KEY)
+        .body(ByteStream::from_static(CANONICAL_BODY))
+        .send()
+        .await?;
+    let copy_source = format!("/{source_bucket}/{CANONICAL_KEY}");
     let unsigned: Vec<(&str, &str)> = vec![("x-amz-copy-source", copy_source.as_str())];
     let headers = pr.headers().chain(unsigned.iter().copied());
     let resp = send_raw(pr.method(), pr.uri(), headers, None).await?;

crates/e2e_test/src/protocols/README.md

@@ -17,15 +17,13 @@ Use the canonical CI-equivalent protocol command in the parent
 For targeted debugging of the core suite only:
 
 ```bash
-RUSTFS_BUILD_FEATURES=ftps,webdav,sftp cargo test --package e2e_test test_protocol_core_suite -- --test-threads=1 --nocapture
+python3 scripts/e2e_binary.py build --features ftps,webdav,sftp
+python3 scripts/e2e_binary.py run --features ftps,webdav,sftp -- cargo test --package e2e_test test_protocol_core_suite -- --test-threads=1 --nocapture
 ```
 
 This targeted command does not cover the full `e2e-protocols` profile.
 
-`RUSTFS_BUILD_FEATURES` controls which features the test rustfs binary is
-built with. When this variable is set, the protocol test runner schedules
-only entries whose protocol is present in the requested feature list. Leave
-it unset to run every protocol entry.
+`e2e_binary.py` supplies `RUSTFS_BUILD_FEATURES` from the verified server's resolved Cargo features. The protocol runner schedules only entries present in that feature list; helpers check that their required features are available without rebuilding the server.
 `--test-threads=1` is required because every entry spawns a rustfs server
 on fixed bind ports.
 

crates/e2e_test/src/reliant/grpc_lock_server.rs

@@ -360,6 +360,13 @@ impl NodeService for MinimalLockNodeService {
         Err(Status::unimplemented("lock-only test server"))
     }
 
+    async fn heal_bucket_at_incarnation(
+        &self,
+        _request: Request<rustfs_protos::proto_gen::node_service::HealBucketRequest>,
+    ) -> Result<Response<rustfs_protos::proto_gen::node_service::HealBucketResponse>, Status> {
+        Err(Status::unimplemented("lock-only test server"))
+    }
+
     async fn list_bucket(
         &self,
         _request: Request<rustfs_protos::proto_gen::node_service::ListBucketRequest>,
@@ -402,6 +409,13 @@ impl NodeService for MinimalLockNodeService {
         Err(Status::unimplemented("lock-only test server"))
     }
 
+    async fn compare_and_update_file(
+        &self,
+        _request: Request<rustfs_protos::proto_gen::node_service::CompareAndUpdateFileRequest>,
+    ) -> Result<Response<rustfs_protos::proto_gen::node_service::CompareAndUpdateFileResponse>, Status> {
+        Err(Status::unimplemented("lock-only test server"))
+    }
+
     async fn delete(
         &self,
         _request: Request<rustfs_protos::proto_gen::node_service::DeleteRequest>,
@@ -409,6 +423,13 @@ impl NodeService for MinimalLockNodeService {
         Err(Status::unimplemented("lock-only test server"))
     }
 
+    async fn delete_at_incarnation(
+        &self,
+        _request: Request<rustfs_protos::proto_gen::node_service::DeleteRequest>,
+    ) -> Result<Response<rustfs_protos::proto_gen::node_service::DeleteResponse>, Status> {
+        Err(Status::unimplemented("lock-only test server"))
+    }
+
     async fn verify_file(
         &self,
         _request: Request<rustfs_protos::proto_gen::node_service::VerifyFileRequest>,
@@ -506,6 +527,13 @@ impl NodeService for MinimalLockNodeService {
         Err(Status::unimplemented("lock-only test server"))
     }
 
+    async fn rename_data_at_incarnation(
+        &self,
+        _request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,
+    ) -> Result<Response<rustfs_protos::proto_gen::node_service::RenameDataResponse>, Status> {
+        Err(Status::unimplemented("lock-only test server"))
+    }
+
     async fn make_volumes(
         &self,
         _request: Request<rustfs_protos::proto_gen::node_service::MakeVolumesRequest>,
@@ -562,6 +590,13 @@ impl NodeService for MinimalLockNodeService {
         Err(Status::unimplemented("lock-only test server"))
     }
 
+    async fn write_metadata_at_incarnation(
+        &self,
+        _request: Request<rustfs_protos::proto_gen::node_service::WriteMetadataRequest>,
+    ) -> Result<Response<rustfs_protos::proto_gen::node_service::WriteMetadataResponse>, Status> {
+        Err(Status::unimplemented("lock-only test server"))
+    }
+
     async fn read_version(
         &self,
         _request: Request<rustfs_protos::proto_gen::node_service::ReadVersionRequest>,
@@ -583,6 +618,20 @@ impl NodeService for MinimalLockNodeService {
         Err(Status::unimplemented("lock-only test server"))
     }
 
+    async fn delete_version_at_incarnation(
+        &self,
+        _request: Request<rustfs_protos::proto_gen::node_service::DeleteVersionRequest>,
+    ) -> Result<Response<rustfs_protos::proto_gen::node_service::DeleteVersionResponse>, Status> {
+        Err(Status::unimplemented("lock-only test server"))
+    }
+
+    async fn delete_retired_marker(
+        &self,
+        _request: Request<rustfs_protos::proto_gen::node_service::DeleteVersionRequest>,
+    ) -> Result<Response<rustfs_protos::proto_gen::node_service::DeleteVersionResponse>, Status> {
+        Err(Status::unimplemented("lock-only test server"))
+    }
+
     async fn delete_versions(
         &self,
         _request: Request<rustfs_protos::proto_gen::node_service::DeleteVersionsRequest>,

crates/e2e_test/src/reliant/tiering.rs

@@ -52,8 +52,8 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{
     BucketLifecycleConfiguration, BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, ExpirationStatus,
-    LifecycleRule, LifecycleRuleFilter, NoncurrentVersionTransition, RestoreRequest, Transition, TransitionStorageClass,
-    VersioningConfiguration,
+    LifecycleRule, LifecycleRuleFilter, MetadataDirective, NoncurrentVersionTransition, RestoreRequest, Transition,
+    TransitionStorageClass, VersioningConfiguration,
 };
 use http::Method;
 use serde::Deserialize;
@@ -963,6 +963,81 @@ async fn test_hermetic_transition_main_path() -> TestResult {
     Ok(())
 }
 
+/// PUT and materialized self-copy must retain cleanup ownership of a replaced
+/// transitioned null version while publishing the new bytes and metadata.
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn test_hermetic_transition_overwrite_and_self_copy() -> TestResult {
+    let mut cold = RustFSTestEnvironment::new().await?;
+    cold.access_key = "coldtieradmin".to_string();
+    cold.secret_key = "coldtiersecret".to_string();
+    cold.start_rustfs_server_without_cleanup(vec![]).await?;
+    let cold_client = cold.create_s3_client();
+    cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
+
+    let mut hot = RustFSTestEnvironment::new().await?;
+    start_tier_source(&mut hot, crate::common::FAST_DATA_USAGE_SCANNER_ENV).await?;
+    let hot_client = hot.create_s3_client();
+    add_rustfs_tier(&hot, &cold).await?;
+    hot_client.create_bucket().bucket(SOURCE_BUCKET).send().await?;
+
+    let data = payload();
+    for self_copy in [false, true] {
+        hot_client
+            .put_bucket_lifecycle_configuration()
+            .bucket(SOURCE_BUCKET)
+            .lifecycle_configuration(BucketLifecycleConfiguration::builder().rules(transition_rule()?).build()?)
+            .send()
+            .await?;
+        put_multipart_object(&hot_client, SOURCE_BUCKET, OBJECT_KEY, &data).await?;
+        wait_for_transition(&hot_client, SOURCE_BUCKET, OBJECT_KEY, StdDuration::from_secs(90)).await?;
+        assert_eq!(cold_tier_object_count(&cold_client).await?, 1);
+        // Keep the replacement local so disappearance of the old remote
+        // object cannot be confused with another automatic transition.
+        hot_client.delete_bucket_lifecycle().bucket(SOURCE_BUCKET).send().await?;
+
+        let expected = if self_copy { data.clone() } else { vec![0x73; 513] };
+        if self_copy {
+            hot_client
+                .copy_object()
+                .bucket(SOURCE_BUCKET)
+                .key(OBJECT_KEY)
+                .copy_source(format!("{SOURCE_BUCKET}/{}", urlencoding::encode(OBJECT_KEY)))
+                .metadata_directive(MetadataDirective::Replace)
+                .content_type("text/plain")
+                .metadata("replacement", "kept")
+                .send()
+                .await?;
+        } else {
+            hot_client
+                .put_object()
+                .bucket(SOURCE_BUCKET)
+                .key(OBJECT_KEY)
+                .body(ByteStream::from(expected.clone()))
+                .content_type("text/plain")
+                .metadata("replacement", "kept")
+                .send()
+                .await?;
+        }
+        wait_for_cold_tier_empty(&cold_client, StdDuration::from_secs(90)).await?;
+        let current = hot_client.get_object().bucket(SOURCE_BUCKET).key(OBJECT_KEY).send().await?;
+        assert_eq!(current.content_type(), Some("text/plain"));
+        assert_eq!(current.metadata().and_then(|m| m.get("replacement")).map(String::as_str), Some("kept"));
+        assert!(
+            current
+                .metadata()
+                .is_none_or(|metadata| !metadata.contains_key(USER_META_KEY))
+        );
+        assert_eq!(current.body.collect().await?.into_bytes().as_ref(), expected.as_slice());
+        hot_client
+            .delete_object()
+            .bucket(SOURCE_BUCKET)
+            .key(OBJECT_KEY)
+            .send()
+            .await?;
+    }
+    Ok(())
+}
+
 /// Restore a transitioned object through a real RustFS remote tier.
 ///
 /// The test covers the externally visible copy-back contract that a mock tier

crates/e2e_test/src/replication_extension_test.rs

@@ -3875,11 +3875,10 @@ async fn test_bucket_replication_replicates_directory_marker_in_versioned_bucket
         .body(ByteStream::from_static(body))
         .send()
         .await?;
-    assert!(
-        put.version_id()
-            .is_none_or(|id| id == "null" || id == uuid::Uuid::nil().to_string()),
-        "a directory marker is the null version even in a versioned bucket: {:?}",
-        put.version_id()
+    assert_eq!(
+        put.version_id(),
+        Some("null"),
+        "a directory marker must expose its null version without leaking the internal nil UUID"
     );
 
     wait_for_source_replication_status(&source_client, source_bucket, marker_key, "COMPLETED", false).await?;
@@ -6134,6 +6133,327 @@ async fn test_site_replication_allows_private_ca_https_with_ca_cert_pem_real_dua
     Ok(())
 }
 
+/// rustfs/backlog#2479: a site resync must count a replicated delete marker
+/// as converged. The peer answers `HEAD ?versionId=<marker>` with a bodiless
+/// 405, which the SDK surfaces without an error code; the resync worker used
+/// to record that as `target service error` and fail the whole bucket.
+#[tokio::test]
+async fn test_site_replication_resync_replicates_delete_marker() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+    let process_env = [
+        ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", "true"),
+        ("RUSTFS_REPL_RESYNC_POLL_MAX_MS", "100"),
+        ("RUST_LOG", "error"),
+    ];
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server_with_env(vec![], &process_env).await?;
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_with_env(vec![], &process_env).await?;
+
+    let bucket = "site-repl-resync-marker";
+    let live_key = "live.bin";
+    let gone_key = "gone.txt";
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+
+    source_client.create_bucket().bucket(bucket).send().await?;
+    enable_bucket_versioning(&source_env, bucket).await?;
+
+    let add_status = site_replication_add(
+        &source_env,
+        &[
+            PeerSite {
+                name: "source-site".to_string(),
+                endpoint: source_env.url.clone(),
+                access_key: source_env.access_key.clone(),
+                secret_key: source_env.secret_key.clone(),
+                ..Default::default()
+            },
+            PeerSite {
+                name: "target-site".to_string(),
+                endpoint: target_env.url.clone(),
+                access_key: target_env.access_key.clone(),
+                secret_key: target_env.secret_key.clone(),
+                ..Default::default()
+            },
+        ],
+    )
+    .await?;
+    assert!(add_status.success, "unexpected site add result: {:?}", add_status);
+
+    let source_info = wait_for_site_replication_enabled(&source_env, 2).await?;
+    wait_for_site_replication_enabled(&target_env, 2).await?;
+    let remote_peer = source_info
+        .sites
+        .into_iter()
+        .find(|peer| peer.endpoint == target_env.url)
+        .ok_or("target peer missing from source site replication info")?;
+    wait_for_bucket_on_target(&target_client, bucket).await?;
+    wait_for_remote_target_arn(&source_env, bucket).await?;
+
+    // One live object and one key whose latest version is a delete marker,
+    // both converged to the peer through live replication first so the
+    // resync re-drives objects the peer already holds.
+    source_client
+        .put_object()
+        .bucket(bucket)
+        .key(live_key)
+        .body(ByteStream::from(vec![b'l'; 4096]))
+        .send()
+        .await?;
+    source_client
+        .put_object()
+        .bucket(bucket)
+        .key(gone_key)
+        .body(ByteStream::from(vec![b'g'; 128]))
+        .send()
+        .await?;
+    let delete = source_client.delete_object().bucket(bucket).key(gone_key).send().await?;
+    assert_eq!(delete.delete_marker(), Some(true), "a versioned delete must create a delete marker");
+    wait_for_object_on_target(&target_client, bucket, live_key).await?;
+    wait_for_target_delete_marker(&target_client, bucket, gone_key).await?;
+
+    let started = site_replication_resync_op(&source_env, "start", &remote_peer).await?;
+    assert_eq!(started.status, "success", "unexpected start result: {:?}", started);
+
+    let deadline = tokio::time::Instant::now() + Duration::from_secs(60);
+    let finished = loop {
+        let status = site_replication_resync_op(&source_env, "status", &remote_peer).await?;
+        match status.state.as_str() {
+            "completed" | "failed" => break status,
+            _ if tokio::time::Instant::now() < deadline => sleep(Duration::from_millis(250)).await,
+            _ => return Err(format!("site resync did not reach a terminal state in time: {status:?}").into()),
+        }
+    };
+    let entry = finished
+        .buckets
+        .iter()
+        .find(|entry| entry.bucket == bucket)
+        .ok_or_else(|| format!("resync status lost the bucket: {finished:?}"))?;
+    assert_eq!(
+        (finished.state.as_str(), entry.status.as_str(), entry.failed_objects),
+        ("completed", "completed", 0),
+        "the delete marker must verify as replicated, not fail the bucket: {finished:?}"
+    );
+    assert!(
+        entry.replicated_objects >= 2,
+        "the live object and the delete marker both count as replicated: {entry:?}"
+    );
+    assert!(entry.err_detail.is_empty(), "unexpected bucket error: {entry:?}");
+
+    Ok(())
+}
+
+/// rustfs/backlog#2489: an operator's bucket-level replication to a site that
+/// later becomes a peer keeps working through the site's add, resync and
+/// removal. Site replication wires its own same-name target next to the
+/// operator's and never takes the operator's target over; a bucket-level
+/// `replication-reset` run before the join must not make the site resync
+/// report the bucket as owned by another resync.
+#[tokio::test]
+async fn test_site_replication_keeps_operator_bucket_target_to_peer() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+    let process_env = [
+        ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", "true"),
+        ("RUSTFS_REPL_RESYNC_POLL_MAX_MS", "100"),
+        ("RUST_LOG", "error"),
+    ];
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server_with_env(vec![], &process_env).await?;
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_with_env(vec![], &process_env).await?;
+
+    let source_bucket = "site-repl-operator-src";
+    let operator_target_bucket = "site-repl-operator-dst";
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+
+    source_client.create_bucket().bucket(source_bucket).send().await?;
+    enable_bucket_versioning(&source_env, source_bucket).await?;
+    target_client.create_bucket().bucket(operator_target_bucket).send().await?;
+    enable_bucket_versioning(&target_env, operator_target_bucket).await?;
+
+    let operator_arn = set_replication_target(&source_env, source_bucket, &target_env, operator_target_bucket).await?;
+    put_bucket_replication(&source_env, source_bucket, &operator_arn).await?;
+    let put_and_wait = |key: &'static str, fill: u8, on: Vec<&'static str>| {
+        let source_client = source_client.clone();
+        let target_client = target_client.clone();
+        async move {
+            source_client
+                .put_object()
+                .bucket(source_bucket)
+                .key(key)
+                .body(ByteStream::from(vec![fill; 4096]))
+                .send()
+                .await?;
+            for bucket in on {
+                let body = wait_for_object_on_target(&target_client, bucket, key).await?;
+                if body != vec![fill; 4096] {
+                    return Err(format!("{key} arrived on {bucket} with the wrong body").into());
+                }
+            }
+            Ok::<(), Box<dyn Error + Send + Sync>>(())
+        }
+    };
+    put_and_wait("before-add.bin", b'a', vec![operator_target_bucket]).await?;
+    let (reset_arn, reset_id) = start_bucket_replication_reset(&source_env, source_bucket).await?;
+    assert_eq!(reset_arn, operator_arn, "the reset must target the operator ARN");
+    wait_for_replication_reset_target(&source_env, source_bucket, &operator_arn, |target| {
+        target.reset_id == reset_id && matches!(target.status.as_str(), "Completed" | "Failed")
+    })
+    .await?;
+
+    let add_status = site_replication_add(
+        &source_env,
+        &[
+            PeerSite {
+                name: "source-site".to_string(),
+                endpoint: source_env.url.clone(),
+                access_key: source_env.access_key.clone(),
+                secret_key: source_env.secret_key.clone(),
+                ..Default::default()
+            },
+            PeerSite {
+                name: "target-site".to_string(),
+                endpoint: target_env.url.clone(),
+                access_key: target_env.access_key.clone(),
+                secret_key: target_env.secret_key.clone(),
+                ..Default::default()
+            },
+        ],
+    )
+    .await?;
+    assert!(add_status.success, "unexpected site add result: {:?}", add_status);
+    let source_info = wait_for_site_replication_enabled(&source_env, 2).await?;
+    wait_for_site_replication_enabled(&target_env, 2).await?;
+    let remote_peer = source_info
+        .sites
+        .into_iter()
+        .find(|peer| peer.endpoint == target_env.url)
+        .ok_or("target peer missing from source site replication info")?;
+    wait_for_bucket_on_target(&target_client, source_bucket).await?;
+
+    // Both targets: the operator's (untouched) and the site's same-name one.
+    let list_targets = || async {
+        let response = list_replication_targets_request(&source_env, Some(source_bucket)).await?;
+        let targets: Vec<serde_json::Value> = if response.status() == StatusCode::OK {
+            response.json().await?
+        } else {
+            Vec::new()
+        };
+        Ok::<Vec<(String, String)>, Box<dyn Error + Send + Sync>>(
+            targets
+                .iter()
+                .map(|target| {
+                    (
+                        target["arn"].as_str().unwrap_or_default().to_string(),
+                        target["targetbucket"].as_str().unwrap_or_default().to_string(),
+                    )
+                })
+                .collect(),
+        )
+    };
+    let deadline = tokio::time::Instant::now() + Duration::from_secs(30);
+    let site_arn = loop {
+        let targets = list_targets().await?;
+        let operator_kept = targets
+            .iter()
+            .any(|(arn, target_bucket)| arn == &operator_arn && target_bucket == operator_target_bucket);
+        let site = targets
+            .iter()
+            .find(|(arn, target_bucket)| arn.contains(&remote_peer.deployment_id) && target_bucket == source_bucket)
+            .map(|(arn, _)| arn.clone());
+        assert!(operator_kept, "site replication must not take the operator target over: {targets:?}");
+        if let Some(arn) = site {
+            break arn;
+        }
+        if tokio::time::Instant::now() >= deadline {
+            return Err(format!("site replication never added its own target next to the operator's: {targets:?}").into());
+        }
+        sleep(Duration::from_millis(250)).await;
+    };
+    assert_ne!(site_arn, operator_arn);
+
+    // The operator's path and the site's path both deliver.
+    put_and_wait("after-add.bin", b'b', vec![operator_target_bucket, source_bucket]).await?;
+
+    let started = site_replication_resync_op(&source_env, "start", &remote_peer).await?;
+    let entry = started
+        .buckets
+        .iter()
+        .find(|entry| entry.bucket == source_bucket)
+        .ok_or_else(|| format!("start response lost the bucket: {started:?}"))?;
+    assert_ne!(
+        entry.status, "conflict",
+        "a finished bucket-level resync must not block the site resync: {entry:?}"
+    );
+    assert_eq!(
+        entry.target_arn, site_arn,
+        "the site resync must drive the site target, not the operator's"
+    );
+    assert_eq!(started.status, "success", "unexpected start result: {:?}", started);
+    let deadline = tokio::time::Instant::now() + Duration::from_secs(60);
+    let finished = loop {
+        let status = site_replication_resync_op(&source_env, "status", &remote_peer).await?;
+        match status.state.as_str() {
+            "completed" | "failed" => break status,
+            _ if tokio::time::Instant::now() < deadline => sleep(Duration::from_millis(250)).await,
+            _ => return Err(format!("site resync did not reach a terminal state in time: {status:?}").into()),
+        }
+    };
+    let entry = finished
+        .buckets
+        .iter()
+        .find(|entry| entry.bucket == source_bucket)
+        .ok_or_else(|| format!("resync status lost the bucket: {finished:?}"))?;
+    assert_eq!(
+        (finished.state.as_str(), entry.status.as_str(), entry.failed_objects),
+        ("completed", "completed", 0),
+        "{finished:?}"
+    );
+
+    // Leaving site replication removes only the site's own target and rule.
+    let removed = site_replication_remove(
+        &source_env,
+        &SRRemoveReq {
+            remove_all: true,
+            ..Default::default()
+        },
+    )
+    .await?;
+    assert!(removed.err_detail.is_empty(), "unexpected remove result: {removed:?}");
+    let targets = list_targets().await?;
+    assert!(
+        targets
+            .iter()
+            .any(|(arn, target_bucket)| arn == &operator_arn && target_bucket == operator_target_bucket),
+        "peer removal must keep the operator target: {targets:?}"
+    );
+    assert!(
+        !targets.iter().any(|(arn, _)| arn == &site_arn),
+        "peer removal must drop the site target: {targets:?}"
+    );
+    let rules = source_client
+        .get_bucket_replication()
+        .bucket(source_bucket)
+        .send()
+        .await?
+        .replication_configuration
+        .map(|config| config.rules)
+        .unwrap_or_default();
+    assert!(
+        rules
+            .iter()
+            .any(|rule| rule.destination.as_ref().map(|d| d.bucket.as_str()) == Some(operator_arn.as_str())),
+        "peer removal must keep the operator rule: {rules:?}"
+    );
+    put_and_wait("after-remove.bin", b'c', vec![operator_target_bucket]).await?;
+
+    Ok(())
+}
+
 #[tokio::test]
 async fn test_site_replication_resync_lifecycle_survives_real_server_restart() -> Result<(), Box<dyn Error + Send + Sync>> {
     init_logging();
@@ -6383,6 +6703,18 @@ async fn test_site_replication_edit_and_status_peer_state_real_three_node() -> R
     let relayed_key = "after-edit-from-relay.txt";
     let relayed_payload = b"site replication after endpoint edit from relay".to_vec();
 
+    // The first joining receiver owns data before the third site has the
+    // shared account. Initial probes and backfill must wait for every join.
+    target_client.create_bucket().bucket(bucket).send().await?;
+    enable_bucket_versioning(&target_env, bucket).await?;
+    target_client
+        .put_object()
+        .bucket(bucket)
+        .key(baseline_key)
+        .body(ByteStream::from(baseline_payload.clone()))
+        .send()
+        .await?;
+
     let add_status = site_replication_add(
         &source_env,
         &[
@@ -6410,7 +6742,10 @@ async fn test_site_replication_edit_and_status_peer_state_real_three_node() -> R
         ],
     )
     .await?;
-    assert!(add_status.success, "unexpected site add result: {:?}", add_status);
+    assert!(
+        add_status.success && add_status.err_detail.is_empty() && add_status.initial_sync_error_message.is_empty(),
+        "unexpected site add result: {add_status:?}"
+    );
 
     let source_info = wait_for_site_replication_enabled(&source_env, 3).await?;
     let _target_info = wait_for_site_replication_enabled(&target_env, 3).await?;
@@ -6421,19 +6756,11 @@ async fn test_site_replication_edit_and_status_peer_state_real_three_node() -> R
         .find(|peer| peer.endpoint == target_env.url)
         .ok_or("target peer missing from source site replication info")?;
 
-    source_client.create_bucket().bucket(bucket).send().await?;
-    enable_bucket_versioning(&source_env, bucket).await?;
-    wait_for_bucket_on_target(&target_client, bucket).await?;
-    wait_for_bucket_on_target(&relay_client, bucket).await?;
-    source_client
-        .put_object()
-        .bucket(bucket)
-        .key(baseline_key)
-        .body(ByteStream::from(baseline_payload.clone()))
-        .send()
-        .await?;
-    let replicated_baseline = wait_for_object_on_target(&target_client, bucket, baseline_key).await?;
-    assert_eq!(replicated_baseline, baseline_payload);
+    for client in [&source_client, &relay_client] {
+        wait_for_bucket_on_target(client, bucket).await?;
+        let backfilled = wait_for_object_on_target(client, bucket, baseline_key).await?;
+        assert_eq!(backfilled, baseline_payload);
+    }
 
     let old_target_address = target_env.address.clone();
     let new_target_port = RustFSTestEnvironment::find_available_port().await?;
@@ -10178,13 +10505,14 @@ async fn test_get_object_tagging_proxies_unreplicated_object_to_replication_targ
     let target_bucket = "proxy-tag-dst";
     let (target, source_env, source_client, target_client) = start_read_proxy_lab(source_bucket, target_bucket).await?;
 
-    target_client
+    let tagged = target_client
         .put_object()
         .bucket(target_bucket)
         .key("proxy-tagged")
         .body(ByteStream::from_static(b"tagged payload"))
         .send()
         .await?;
+    let tagged_version = tagged.version_id().ok_or("versioned target PUT omitted its identity")?;
     target_client
         .put_object_tagging()
         .bucket(target_bucket)
@@ -10208,6 +10536,11 @@ async fn test_get_object_tagging_proxies_unreplicated_object_to_replication_targ
     assert_eq!(tags.tag_set.len(), 1, "proxied tagging read must return the target's tags");
     assert_eq!(tags.tag_set[0].key.as_str(), "team");
     assert_eq!(tags.tag_set[0].value.as_str(), "storage");
+    assert_eq!(
+        tags.version_id(),
+        Some(tagged_version),
+        "proxy must preserve the resolved remote identity"
+    );
 
     let record = target
         .requests()
@@ -10221,6 +10554,33 @@ async fn test_get_object_tagging_proxies_unreplicated_object_to_replication_targ
     );
     assert!(record.proxy_headers.replication_check.is_none());
 
+    let empty = target_client
+        .put_object()
+        .bucket(target_bucket)
+        .key("proxy-tagged")
+        .body(ByteStream::from_static(b"new version without tags"))
+        .send()
+        .await?;
+    let empty_version = empty.version_id().ok_or("empty-tag version omitted its identity")?;
+    for selector in [None, Some(empty_version), Some(tagged_version)] {
+        let tags = source_client
+            .get_object_tagging()
+            .bucket(source_bucket)
+            .key("proxy-tagged")
+            .set_version_id(selector.map(str::to_owned))
+            .send()
+            .await?;
+        let expected_version = selector.unwrap_or(empty_version);
+        assert_eq!(tags.version_id(), Some(expected_version));
+        if expected_version == tagged_version {
+            assert_eq!(tags.tag_set().len(), 1);
+            assert_eq!(tags.tag_set()[0].key(), "team");
+            assert_eq!(tags.tag_set()[0].value(), "storage");
+        } else {
+            assert!(tags.tag_set().is_empty());
+        }
+    }
+
     drop(source_env);
     target.shutdown().await;
     Ok(())

crates/e2e_test/src/version_id_regression_test.rs

@@ -27,7 +27,9 @@ mod tests {
     use aws_sdk_s3::Client;
     use aws_sdk_s3::error::ProvideErrorMetadata;
     use aws_sdk_s3::primitives::ByteStream;
-    use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
+    use aws_sdk_s3::types::{
+        BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, VersioningConfiguration,
+    };
     use tracing::info;
 
     fn create_s3_client(env: &RustFSTestEnvironment) -> Client {
@@ -83,6 +85,156 @@ mod tests {
         Ok(())
     }
 
+    async fn assert_version_tags(client: &Client, bucket: &str, key: &str, version: Option<&str>, value: Option<&str>) {
+        let tags = client
+            .get_object_tagging()
+            .bucket(bucket)
+            .key(key)
+            .set_version_id(version.map(str::to_owned))
+            .send()
+            .await
+            .expect("GetObjectTagging must accept the exact version selector");
+        let expected = value
+            .map(|value| vec![Tag::builder().key("generation").value(value).build().expect("valid tag")])
+            .unwrap_or_default();
+        assert_eq!(tags.tag_set(), expected, "version selector: {version:?}");
+    }
+
+    async fn assert_null_tagging_across_versioning_changes(client: &Client, bucket: &str, key: &str) {
+        assert_version_tags(client, bucket, key, None, None).await;
+        assert_version_tags(client, bucket, key, Some("null"), None).await;
+        client
+            .put_object_tagging()
+            .bucket(bucket)
+            .key(key)
+            .version_id("null")
+            .tagging(
+                Tagging::builder()
+                    .tag_set(
+                        Tag::builder()
+                            .key("generation")
+                            .value("original-null")
+                            .build()
+                            .expect("valid tag"),
+                    )
+                    .build()
+                    .expect("valid tagging"),
+            )
+            .send()
+            .await
+            .expect("tag the original null version");
+
+        enable_versioning(client, bucket)
+            .await
+            .expect("enable versioning over a null version");
+        let versioned = client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .tagging("generation=versioned")
+            .body(ByteStream::from_static(b"new version"))
+            .send()
+            .await
+            .expect("write a newer UUID version");
+        let version = versioned.version_id().expect("versioned PUT must return a UUID");
+        assert_ne!(version, "null");
+        assert_version_tags(client, bucket, key, None, Some("versioned")).await;
+        assert_version_tags(client, bucket, key, Some(version), Some("versioned")).await;
+        assert_version_tags(client, bucket, key, Some("null"), Some("original-null")).await;
+
+        client
+            .put_object_tagging()
+            .bucket(bucket)
+            .key(key)
+            .version_id("null")
+            .tagging(
+                Tagging::builder()
+                    .tag_set(
+                        Tag::builder()
+                            .key("generation")
+                            .value("updated-null")
+                            .build()
+                            .expect("valid tag"),
+                    )
+                    .build()
+                    .expect("valid tagging"),
+            )
+            .send()
+            .await
+            .expect("update tags on the noncurrent null version");
+        assert_version_tags(client, bucket, key, Some("null"), Some("updated-null")).await;
+        assert_version_tags(client, bucket, key, None, Some("versioned")).await;
+        client
+            .delete_object_tagging()
+            .bucket(bucket)
+            .key(key)
+            .version_id("null")
+            .send()
+            .await
+            .expect("delete only the noncurrent null version tags");
+        assert_version_tags(client, bucket, key, Some("null"), None).await;
+        assert_version_tags(client, bucket, key, Some(version), Some("versioned")).await;
+
+        suspend_versioning(client, bucket).await.expect("suspend versioning");
+        client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .tagging("generation=suspended-null")
+            .body(ByteStream::from_static(b"replacement null version"))
+            .send()
+            .await
+            .expect("replace the null version while suspended");
+        assert_version_tags(client, bucket, key, None, Some("suspended-null")).await;
+        assert_version_tags(client, bucket, key, Some("null"), Some("suspended-null")).await;
+        assert_version_tags(client, bucket, key, Some(version), Some("versioned")).await;
+
+        enable_versioning(client, bucket).await.expect("re-enable versioning");
+        client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .tagging("generation=latest")
+            .body(ByteStream::from_static(b"latest version"))
+            .send()
+            .await
+            .expect("write a new latest version");
+        assert_version_tags(client, bucket, key, Some("null"), Some("suspended-null")).await;
+        assert_version_tags(client, bucket, key, None, Some("latest")).await;
+        client
+            .delete_object()
+            .bucket(bucket)
+            .key(key)
+            .version_id("null")
+            .send()
+            .await
+            .expect("remove only the null version");
+        assert_version_tags(client, bucket, key, None, Some("latest")).await;
+        let absent_version = uuid::Uuid::new_v4().to_string();
+        for (missing_key, selector, expected_code) in [
+            (key, Some("null"), "NoSuchVersion"),
+            (key, Some(absent_version.as_str()), "NoSuchVersion"),
+            ("never-created", None, "NoSuchKey"),
+            ("never-created", Some("null"), "NoSuchVersion"),
+            ("never-created", Some(absent_version.as_str()), "NoSuchVersion"),
+        ] {
+            let missing = client
+                .get_object_tagging()
+                .bucket(bucket)
+                .key(missing_key)
+                .set_version_id(selector.map(str::to_owned))
+                .send()
+                .await
+                .expect_err("a missing version must not fall back to latest");
+            assert_eq!(
+                missing.as_service_error().and_then(ProvideErrorMetadata::code),
+                Some(expected_code),
+                "key: {missing_key}, version selector: {selector:?}"
+            );
+        }
+        assert_version_tags(client, bucket, key, None, Some("latest")).await;
+    }
+
     /// Test 1: PutObject should return version_id when versioning is enabled
     /// This directly addresses the Veeam issue from #1066
     #[tokio::test]
@@ -262,7 +414,9 @@ mod tests {
         info!("🧪 TEST: PutObject behavior without versioning (no regression)");
 
         let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
-        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+        env.start_rustfs_server_without_cleanup(vec![])
+            .await
+            .expect("Failed to start isolated RustFS");
 
         let client = create_s3_client(&env);
         let bucket = "test-no-versioning";
@@ -290,6 +444,9 @@ mod tests {
             output.version_id().is_none() || output.version_id() == Some("null"),
             "non-versioned PUT must omit version ID or return the S3 null version"
         );
+        // Reuse this unversioned fixture to prove explicit null never becomes
+        // an implicit latest-version read after enable/suspend transitions.
+        assert_null_tagging_across_versioning_changes(&client, bucket, key).await;
         info!("✅ PASSED: PutObject works correctly without versioning");
     }
 

crates/ecstore/Cargo.toml

@@ -214,7 +214,6 @@ aws-smithy-types = { workspace = true }
 aws-smithy-runtime-api = { workspace = true, features = ["http-1x"] }
 parking_lot = { workspace = true }
 base64-simd.workspace = true
-serde_urlencoded.workspace = true
 google-cloud-storage = { workspace = true, optional = true }
 google-cloud-auth = { workspace = true, optional = true }
 faster-hex = { workspace = true }

crates/ecstore/src/api/mod.rs

@@ -168,12 +168,19 @@ pub mod bucket {
             BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, BUCKET_QUOTA_CONFIG_FILE,
             BUCKET_REPLICATION_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG, BUCKET_SSECONFIG, BUCKET_TABLE_CATALOG_META_PREFIX,
             BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX, BUCKET_TABLE_CONFIG, BUCKET_TABLE_RESERVED_PREFIX, BUCKET_TAGGING_CONFIG,
-            BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, BUCKET_WEBSITE_CONFIG, BucketMetadata, OBJECT_LOCK_CONFIG,
-            load_bucket_metadata, table_catalog_path_hash,
+            BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, BUCKET_WEBSITE_CONFIG, BucketMetadata, ConfigState,
+            OBJECT_LOCK_CONFIG, UnreadableBucketConfig, is_unreadable_config_error, load_bucket_metadata,
+            table_catalog_path_hash, unreadable_config_refusal,
         };
         pub use crate::bucket::metadata::{BUCKET_DURABILITY_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
     }
 
+    pub mod config_parse_mode {
+        pub use crate::bucket::config_parse_mode::{
+            BucketConfigParseMode, bucket_config_parse_mode, validate_bucket_config_parse_mode_env,
+        };
+    }
+
     pub mod durability {
         pub use crate::bucket::durability::{
             BUCKET_DURABILITY_MODE_NONE, BUCKET_DURABILITY_MODE_RELAXED, BUCKET_DURABILITY_MODE_STRICT, BucketDurabilityConfig,
@@ -202,7 +209,9 @@ pub mod bucket {
     }
 
     pub mod migration {
-        pub use crate::bucket::migration::{LegacyBlobDecryptFn, try_migrate_bucket_metadata, try_migrate_iam_config};
+        pub use crate::bucket::migration::{
+            LegacyBlobDecryptFn, migration_startup_error, try_migrate_bucket_metadata, try_migrate_iam_config,
+        };
     }
 
     pub mod object_lock {
@@ -402,7 +411,7 @@ pub mod data_usage {
 
 pub mod disk {
     pub use crate::disk::disk_store::get_object_disk_read_timeout;
-    pub use crate::disk::local::ScanGuard;
+    pub use crate::disk::local::{ReplacementExecutionLease, ScanGuard};
     #[cfg(all(feature = "test-util", not(windows)))]
     pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage};
     pub use crate::disk::{
@@ -501,9 +510,9 @@ pub mod object {
         ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, QuotaAdmission,
         RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest,
         SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, ScannerPublicationCommitScope, ScannerPublicationCommitStartError,
-        ScannerPublicationCommitState, StreamConsumer, WriteCompletion, get_object_body_cache_plaintext_len,
-        lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
-        unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
+        ScannerPublicationCommitState, ShardIntegrityWriteMode, StreamConsumer, WriteCompletion,
+        get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
+        register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
     };
     pub use crate::store::{
         PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
@@ -539,20 +548,21 @@ pub mod rio {
 
 pub mod rpc {
     pub use crate::cluster::rpc::{
-        AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
-        PeerRestClient, PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
-        ScannerBucketListing, ScannerDirtyUsageAcknowledgement, ScannerPeerActivity, ScannerPeerDirtyUsageBucket,
-        ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease, ScannerScopedDirtyUsageAckEntry, TONIC_RPC_PREFIX,
-        TonicInterceptor, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, decode_heal_bucket_rpc_options,
-        encode_heal_bucket_rpc_options, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
-        gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
-        normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, sign_ns_scanner_capability,
-        sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
-        tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
-        verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer,
-        verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
-        verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
-        verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
+        AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, MAX_NETWORK_PROBE_BYTES, MAX_NETWORK_PROBE_DURATION,
+        NetworkPeerProbeClient, NetworkPeerProbeError, NetworkPeerProbeMeasurement, NetworkPeerTarget, PEER_RESTDRY_RUN,
+        PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG,
+        SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing, ScannerDirtyUsageAcknowledgement, ScannerPeerActivity,
+        ScannerPeerDirtyUsageBucket, ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease, ScannerScopedDirtyUsageAckEntry,
+        TONIC_RPC_PREFIX, TonicInterceptor, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce,
+        decode_heal_bucket_rpc_options, encode_heal_bucket_rpc_options, gen_signature_headers, gen_tonic_replay_scope_headers,
+        gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
+        node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
+        sign_ns_scanner_capability, sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability,
+        sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
+        tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation,
+        verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response,
+        verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned,
+        verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
     };
 }
 
@@ -589,6 +599,7 @@ pub mod storage {
         all_local_disk_path, find_local_disk_by_ref, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients,
         prewarm_local_disk_id_map, prewarm_local_disk_id_map_with_instance_ctx,
     };
+    pub use crate::store::{HealObjectAbsenceProof, HealObjectStorageResult};
 }
 
 pub mod tier {

crates/ecstore/src/bucket/config_parse_mode.rs

@@ -0,0 +1,241 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Handling of stored bucket sub-configurations whose bytes cannot be parsed
+//! (rustfs/backlog#1734): the rollout mode for paths that historically read
+//! them as absent, and the metrics that make them visible.
+//!
+//! The mode only governs those historical degrade paths. Paths that already
+//! refuse an unreadable config (the typed getters, the read-modify-write
+//! guard, Object Lock and default-encryption decisions, delete-time
+//! versioning) refuse in every mode.
+
+use rustfs_config::{DEFAULT_BUCKET_CONFIG_PARSE_MODE, ENV_BUCKET_CONFIG_PARSE_MODE};
+use std::collections::HashMap;
+use std::sync::{LazyLock, Mutex, OnceLock};
+use tracing::error;
+
+/// Counter of stored XML sub-configurations that failed to parse, labeled
+/// `bucket`, `config` and `mode`. Increments on every metadata parse, so it
+/// must read zero fleet-wide before the default mode flips to strict.
+pub const METRIC_BUCKET_METADATA_PARSE_FAILED_TOTAL: &str = "rustfs_bucket_metadata_parse_failed_total";
+
+/// Gauge of buckets whose most recently parsed metadata holds an unreadable
+/// XML sub-configuration, labeled `config`. The counter only moves when a
+/// parse runs; this reflects the current state between parses.
+pub const METRIC_BUCKET_METADATA_UNPARSABLE_CURRENT: &str = "rustfs_bucket_metadata_unparsable_current";
+
+const LOG_COMPONENT: &str = "ecstore";
+const LOG_SUBSYSTEM: &str = "bucket_metadata";
+const EVENT_CONFIG_UNREADABLE: &str = "bucket_metadata_config_unreadable";
+const EVENT_PARSE_MODE_INVALID: &str = "bucket_config_parse_mode_invalid";
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum BucketConfigParseMode {
+    /// Historical degrade paths keep reading an unreadable config as absent;
+    /// the metrics and an error log record every occurrence.
+    Permissive,
+    /// Historical degrade paths refuse instead.
+    Strict,
+}
+
+impl BucketConfigParseMode {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Self::Permissive => "permissive",
+            Self::Strict => "strict",
+        }
+    }
+
+    /// Parse a configured value; unset or blank selects the default. An
+    /// unknown value is an error rather than a silent fallback.
+    pub fn parse(value: Option<&str>) -> Result<Self, String> {
+        let value = value
+            .map(str::trim)
+            .filter(|value| !value.is_empty())
+            .unwrap_or(DEFAULT_BUCKET_CONFIG_PARSE_MODE);
+        match value.to_ascii_lowercase().as_str() {
+            "permissive" => Ok(Self::Permissive),
+            "strict" => Ok(Self::Strict),
+            _ => Err(format!(
+                "invalid {ENV_BUCKET_CONFIG_PARSE_MODE} value {value:?}; expected permissive or strict"
+            )),
+        }
+    }
+}
+
+/// Validate the configured mode; startup calls this so an invalid value fails
+/// the node instead of being guessed at.
+pub fn validate_bucket_config_parse_mode_env() -> Result<BucketConfigParseMode, String> {
+    BucketConfigParseMode::parse(rustfs_utils::get_env_opt_str(ENV_BUCKET_CONFIG_PARSE_MODE).as_deref())
+}
+
+/// The process-wide mode. Startup has already rejected an invalid value; if
+/// this is reached without that validation, an invalid value selects strict,
+/// never permissive.
+pub fn bucket_config_parse_mode() -> BucketConfigParseMode {
+    static MODE: OnceLock<BucketConfigParseMode> = OnceLock::new();
+    *MODE.get_or_init(|| {
+        validate_bucket_config_parse_mode_env().unwrap_or_else(|err| {
+            error!(
+                event = EVENT_PARSE_MODE_INVALID,
+                component = LOG_COMPONENT,
+                subsystem = LOG_SUBSYSTEM,
+                error = %err,
+                "Invalid bucket config parse mode; using strict"
+            );
+            BucketConfigParseMode::Strict
+        })
+    })
+}
+
+/// Which buckets currently hold which unreadable XML configs, so the gauge
+/// reports buckets rather than parse events.
+#[derive(Debug, Default)]
+struct UnreadableConfigTracker {
+    by_bucket: HashMap<String, Vec<&'static str>>,
+}
+
+impl UnreadableConfigTracker {
+    /// Replace `bucket`'s unreadable set and return the current bucket count
+    /// of every config whose count may have changed.
+    fn update(&mut self, bucket: &str, configs: Vec<&'static str>) -> Vec<(&'static str, usize)> {
+        let previous = if configs.is_empty() {
+            self.by_bucket.remove(bucket).unwrap_or_default()
+        } else {
+            self.by_bucket.insert(bucket.to_string(), configs.clone()).unwrap_or_default()
+        };
+        let mut touched = previous;
+        touched.extend(configs);
+        touched.sort_unstable();
+        touched.dedup();
+        touched
+            .into_iter()
+            .map(|config| (config, self.by_bucket.values().filter(|set| set.contains(&config)).count()))
+            .collect()
+    }
+}
+
+static TRACKER: LazyLock<Mutex<UnreadableConfigTracker>> = LazyLock::new(Default::default);
+
+fn publish_gauges(bucket: &str, configs: Vec<&'static str>) {
+    let changed = TRACKER
+        .lock()
+        .unwrap_or_else(std::sync::PoisonError::into_inner)
+        .update(bucket, configs);
+    for (config, count) in changed {
+        metrics::gauge!(METRIC_BUCKET_METADATA_UNPARSABLE_CURRENT, "config" => config).set(count as f64);
+    }
+}
+
+/// Record the outcome of one metadata parse: every unreadable XML config
+/// (config file, stored byte length) counts once, logs at error level, and
+/// replaces the bucket's entry in the current-state gauge.
+pub(crate) fn record_bucket_config_parse_state(bucket: &str, unreadable: &[(&'static str, usize)]) {
+    if !unreadable.is_empty() {
+        let mode = bucket_config_parse_mode();
+        for (config, _) in unreadable {
+            metrics::counter!(
+                METRIC_BUCKET_METADATA_PARSE_FAILED_TOTAL,
+                "bucket" => bucket.to_string(),
+                "config" => *config,
+                "mode" => mode.as_str()
+            )
+            .increment(1);
+        }
+        error!(
+            event = EVENT_CONFIG_UNREADABLE,
+            component = LOG_COMPONENT,
+            subsystem = LOG_SUBSYSTEM,
+            bucket = %bucket,
+            configs = ?unreadable,
+            mode = mode.as_str(),
+            "Stored bucket configuration cannot be parsed"
+        );
+    }
+    publish_gauges(bucket, unreadable.iter().map(|(config, _)| *config).collect());
+}
+
+/// Drop a deleted bucket from the current-state gauge.
+pub(crate) fn forget_bucket_config_parse_state(bucket: &str) {
+    publish_gauges(bucket, Vec::new());
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::bucket::metadata::{BUCKET_TAGGING_CONFIG, BUCKET_VERSIONING_CONFIG};
+
+    #[test]
+    fn parse_mode_defaults_to_permissive_and_rejects_unknown_values() {
+        assert_eq!(BucketConfigParseMode::parse(None), Ok(BucketConfigParseMode::Permissive));
+        assert_eq!(BucketConfigParseMode::parse(Some("  ")), Ok(BucketConfigParseMode::Permissive));
+        assert_eq!(BucketConfigParseMode::parse(Some("permissive")), Ok(BucketConfigParseMode::Permissive));
+        assert_eq!(BucketConfigParseMode::parse(Some(" Strict ")), Ok(BucketConfigParseMode::Strict));
+
+        let err = BucketConfigParseMode::parse(Some("lenient")).expect_err("an unknown mode must not be guessed");
+        assert!(err.contains(ENV_BUCKET_CONFIG_PARSE_MODE), "{err}");
+        assert!(err.contains("permissive") && err.contains("strict"), "must list the valid values: {err}");
+    }
+
+    #[test]
+    fn tracker_counts_buckets_not_parse_events() {
+        let mut tracker = UnreadableConfigTracker::default();
+        assert_eq!(tracker.update("a", vec![BUCKET_VERSIONING_CONFIG]), vec![(BUCKET_VERSIONING_CONFIG, 1)]);
+        // Re-parsing the same state does not double count.
+        assert_eq!(tracker.update("a", vec![BUCKET_VERSIONING_CONFIG]), vec![(BUCKET_VERSIONING_CONFIG, 1)]);
+        assert_eq!(tracker.update("b", vec![BUCKET_VERSIONING_CONFIG]), vec![(BUCKET_VERSIONING_CONFIG, 2)]);
+
+        // A repaired bucket releases its configs; the changed one is reported.
+        let mut changed = tracker.update("a", vec![BUCKET_TAGGING_CONFIG]);
+        changed.sort_unstable();
+        let mut expected = vec![(BUCKET_TAGGING_CONFIG, 1), (BUCKET_VERSIONING_CONFIG, 1)];
+        expected.sort_unstable();
+        assert_eq!(changed, expected);
+
+        assert_eq!(tracker.update("b", Vec::new()), vec![(BUCKET_VERSIONING_CONFIG, 0)]);
+        assert_eq!(tracker.update("never-unreadable", Vec::new()), Vec::new());
+    }
+
+    #[test]
+    fn every_unreadable_config_increments_the_labeled_counter() {
+        let recorder = metrics_util::debugging::DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        metrics::with_local_recorder(&recorder, || {
+            record_bucket_config_parse_state("metric-bucket", &[(BUCKET_VERSIONING_CONFIG, 12), (BUCKET_TAGGING_CONFIG, 3)]);
+            record_bucket_config_parse_state("metric-bucket", &[(BUCKET_VERSIONING_CONFIG, 12)]);
+            record_bucket_config_parse_state("metric-clean-bucket", &[]);
+        });
+
+        let mut versioning = 0;
+        let mut tagging = 0;
+        for (composite, _, _, value) in snapshotter.snapshot().into_vec() {
+            if composite.key().name() != METRIC_BUCKET_METADATA_PARSE_FAILED_TOTAL {
+                continue;
+            }
+            let labels: HashMap<_, _> = composite.key().labels().map(|l| (l.key(), l.value())).collect();
+            assert_eq!(labels.get("bucket"), Some(&"metric-bucket"));
+            assert_eq!(labels.get("mode"), Some(&bucket_config_parse_mode().as_str()));
+            let metrics_util::debugging::DebugValue::Counter(count) = value else {
+                panic!("parse failures must be a counter");
+            };
+            match labels.get("config") {
+                Some(&config) if config == BUCKET_VERSIONING_CONFIG => versioning += count,
+                Some(&config) if config == BUCKET_TAGGING_CONFIG => tagging += count,
+                other => panic!("unexpected config label {other:?}"),
+            }
+        }
+        assert_eq!((versioning, tagging), (2, 1));
+    }
+}

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -82,18 +82,16 @@ use rustfs_config::{
     ENV_TRANSITION_WORKERS, ENV_TRANSITION_WORKERS_ABSOLUTE_MAX,
 };
 use rustfs_data_usage::TierStats;
+use rustfs_filemeta::metadata_keys;
 use rustfs_filemeta::{
-    FileInfo, FileInfoOpts, NULL_VERSION_ID, RestoreStatusOps, TRANSITION_COMPLETE, get_file_info, is_restored_object_on_disk,
+    FileInfo, FileInfoOpts, NULL_VERSION_ID, RestoreStatus, RestoreStatusOps, TRANSITION_COMPLETE, get_file_info,
+    is_restored_object_on_disk,
 };
 use rustfs_scanner_metrics::metrics::{
     IlmAction, Metrics, ScannerLifecycleExpiryStateUpdate, ScannerLifecycleTransitionStateUpdate, global_metrics,
 };
 use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::parse_bool};
-use s3s::dto::{
-    BucketLifecycleConfiguration, ExpirationStatus, ObjectLockConfiguration, RestoreRequest, RestoreRequestType, RestoreStatus,
-    Timestamp,
-};
-use s3s::header::X_AMZ_RESTORE;
+use s3s::dto::{BucketLifecycleConfiguration, ExpirationStatus, ObjectLockConfiguration, RestoreRequest, RestoreRequestType};
 use sha2::{Digest, Sha256};
 use std::any::Any;
 use std::collections::{BTreeMap, HashMap, HashSet};
@@ -779,7 +777,7 @@ fn free_version_physical_topology_generation(api: &ECStore) -> String {
     rustfs_utils::crypto::hex(hasher.finalize().as_slice())
 }
 
-fn free_version_remote_tuple_matches(candidate: &ObjectInfo, expected: &ObjectInfo) -> std::io::Result<bool> {
+pub(crate) fn free_version_remote_tuple_matches(candidate: &ObjectInfo, expected: &ObjectInfo) -> std::io::Result<bool> {
     if candidate.transitioned_object.tier != expected.transitioned_object.tier
         || candidate.transitioned_object.name != expected.transitioned_object.name
     {
@@ -822,7 +820,7 @@ async fn scan_exact_free_version_targets(
     let mut targets = Vec::new();
     for pool in &api.pools {
         for set in &pool.disk_set {
-            let versions = match set.load_file_info_versions_exact(&oi.bucket, &oi.name).await {
+            let versions = match set.load_file_info_versions_for_tier_cleanup(&oi.bucket, &oi.name).await {
                 Ok(Some(versions)) => versions,
                 Ok(None) => continue,
                 Err(err) if is_err_strict_volume_not_found(&err) => continue,
@@ -5174,10 +5172,10 @@ pub async fn put_restore_opts(
     }
     let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), rreq.days.unwrap_or(1));
     meta.insert(
-        X_AMZ_RESTORE.as_str().to_string(),
+        metadata_keys::RESTORE.to_string(),
         RestoreStatus {
             is_restore_in_progress: Some(false),
-            restore_expiry_date: Some(Timestamp::from(restore_expiry)),
+            restore_expiry_date: Some(restore_expiry),
         }
         .to_string(),
     );
@@ -5190,6 +5188,7 @@ pub async fn put_restore_opts(
         // Restore writes stored (possibly encrypted) bytes, so the writer's
         // computed MD5 is not the object's public plaintext ETag.
         preserve_etag: oi.etag.clone(),
+        shard_integrity_write_mode: Some(oi.shard_integrity_write_mode()),
         //expires:           oi.expires,
         ..Default::default()
     })
@@ -5912,6 +5911,7 @@ mod tests {
     use rustfs_config::ENV_MAX_EXPIRY_WORKERS;
     use rustfs_config::ENV_TRANSITION_WORKERS_ABSOLUTE_MAX;
     use rustfs_data_usage::TierStats;
+    use rustfs_filemeta::metadata_keys;
     use rustfs_filemeta::{FileInfo, FileMeta};
     #[cfg(feature = "test-util")]
     use rustfs_s3_client::transition_api::ReaderImpl;
@@ -5921,7 +5921,6 @@ mod tests {
         NoncurrentVersionExpiration, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRetentionMode, ObjectLockRule,
         OutputLocation, RestoreRequest, RestoreRequestType, S3Location, Timestamp, Transition, TransitionStorageClass,
     };
-    use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE};
     use serial_test::serial;
     use sha2::{Digest, Sha256};
     use std::collections::HashMap;
@@ -8100,6 +8099,75 @@ mod tests {
         }
     }
 
+    #[cfg(feature = "test-util")]
+    #[tokio::test]
+    #[serial]
+    async fn tier_overwrite_cleanup_retains_a_minority_live_remote_reference() {
+        let (disk_paths, ecstore) = setup_test_env().await;
+        let bucket = format!("overwrite-minority-{}", Uuid::new_v4());
+        let object = "still-referenced";
+        create_test_bucket(&ecstore, &bucket).await;
+        let (backend, identity) = register_recovery_mock_tier(&ecstore).await;
+        seed_recoverable_free_version(&disk_paths, &bucket, object, None, Some(identity.clone())).await;
+        let page = list_tier_free_versions(Arc::clone(&ecstore), 100, None, None, CancellationToken::new())
+            .await
+            .expect("list persisted cleanup owner");
+        let owner = page.items.into_iter().find(|oi| oi.bucket == bucket).expect("seeded owner");
+        backend
+            .set_put_remote_version(Some(owner.transitioned_object.version_id.clone()))
+            .await;
+        let lease = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM")
+            .await
+            .expect("remote fixture lease");
+        lease
+            .put(
+                &owner.transitioned_object.name,
+                rustfs_s3_client::transition_api::ReaderImpl::Body(bytes::Bytes::from_static(b"old")),
+                3,
+            )
+            .await
+            .expect("seed referenced remote bytes");
+        drop(lease);
+        let path = disk_paths[0].join(&bucket).join(object).join(STORAGE_FORMAT_FILE);
+        let cleanup_metadata = fs::read(&path).await.expect("save completed replica");
+        let mut live = FileInfo::new(object, 2, 2);
+        live.volume = bucket.clone();
+        live.erasure.index = 1;
+        live.data_dir = Some(Uuid::new_v4());
+        live.mod_time = Some(OffsetDateTime::now_utc());
+        live.size = 3;
+        live.add_object_part(1, "149603e6c03516362a8da23f624db945".to_string(), 3, live.mod_time, 3, None, None);
+        live.transition_status = TRANSITION_COMPLETE.to_string();
+        live.transition_tier = "WARM".to_string();
+        live.transitioned_objname = owner.transitioned_object.name.clone();
+        live.transition_version = Some(owner.transitioned_object.version_id.clone());
+        live.transition_version_state = rustfs_filemeta::TransitionVersionState::Exact;
+        rustfs_utils::http::insert_str(&mut live.metadata, rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID, identity);
+        let mut old_metadata = FileMeta::new();
+        old_metadata.add_version(live).expect("prepare minority live source");
+        fs::write(&path, old_metadata.marshal_msg().expect("encode live source"))
+            .await
+            .expect("model one replica retained by an interrupted overwrite");
+
+        let err = super::cleanup_free_version_exact(Arc::clone(&ecstore), &owner, &CancellationToken::new())
+            .await
+            .expect_err("quorum free versions cannot erase a minority live reference");
+        assert_eq!(err.kind(), std::io::ErrorKind::WouldBlock);
+        assert_eq!(backend.remove_count().await, 0);
+        assert!(backend.contains(&owner.transitioned_object.name).await);
+
+        fs::write(&path, cleanup_metadata)
+            .await
+            .expect("complete replica convergence");
+        assert!(
+            super::cleanup_free_version_exact(Arc::clone(&ecstore), &owner, &CancellationToken::new())
+                .await
+                .expect("converged cleanup can delete the exact remote owner")
+        );
+        assert_eq!(backend.remove_count().await, 1);
+        assert!(!backend.contains(&owner.transitioned_object.name).await);
+    }
+
     #[cfg(feature = "test-util")]
     #[tokio::test]
     #[serial]
@@ -12173,7 +12241,7 @@ mod tests {
                 "locked historical null",
                 ObjectInfo {
                     user_defined: Arc::new(HashMap::from([(
-                        X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(),
+                        metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(),
                         "ON".to_string(),
                     )])),
                     ..historical_null.clone()
@@ -12621,9 +12689,8 @@ mod tests {
             .await
             .expect_err("malformed Object Lock metadata must reject lifecycle config resolution");
         assert!(
-            exact_error
-                .to_string()
-                .contains("persisted bucket Object Lock configuration is invalid")
+            crate::bucket::metadata::is_unreadable_config_error(&exact_error),
+            "malformed Object Lock metadata must surface as the typed unreadable-config refusal: {exact_error}"
         );
 
         let runtime_state = install_unconsumed_runtime_expiry_worker(&ecstore, 1).await;
@@ -12952,7 +13019,7 @@ mod tests {
         let lc = latest_expiration_lifecycle();
         let object = current_object_with_metadata(
             ReplicationStatusType::Completed,
-            HashMap::from([(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(), "ON".to_string())]),
+            HashMap::from([(metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(), "ON".to_string())]),
         );
 
         let event = eval_action_from_lifecycle(&lc, None, &object).await;
@@ -12970,10 +13037,10 @@ mod tests {
             ReplicationStatusType::Completed,
             HashMap::from([
                 (
-                    X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
+                    metadata_keys::OBJECT_LOCK_MODE.to_string(),
                     s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
                 ),
-                (X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(), retain_until),
+                (metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), retain_until),
             ]),
         );
 
@@ -12995,10 +13062,10 @@ mod tests {
             ReplicationStatusType::Completed,
             HashMap::from([
                 (
-                    X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
+                    metadata_keys::OBJECT_LOCK_MODE.to_string(),
                     ObjectLockRetentionMode::COMPLIANCE.to_string(),
                 ),
-                (X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(), retain_until),
+                (metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), retain_until),
             ]),
         );
         object.transitioned_object.status = TRANSITION_COMPLETE.to_string();

crates/ecstore/src/bucket/lifecycle/object_lock_boundary.rs

@@ -37,7 +37,7 @@ mod tests {
     #[test]
     fn is_object_locked_by_metadata_preserves_object_lock_parser_behavior() {
         let mut user_defined = HashMap::new();
-        user_defined.insert("x-amz-object-lock-legal-hold".to_string(), "ON".to_string());
+        user_defined.insert(rustfs_filemeta::metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(), "ON".to_string());
 
         assert!(is_object_locked_by_metadata(&user_defined, false));
         assert!(!is_object_locked_by_metadata(&user_defined, true));

crates/ecstore/src/bucket/metadata.rs

@@ -275,6 +275,141 @@ pub const BUCKET_TABLE_RESERVED_PREFIX: &str = ".rustfs-table";
 pub const BUCKET_TABLE_CATALOG_META_PREFIX: &str = "s3tables/catalog";
 pub const BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX: &str = "table-buckets";
 
+/// Refusal to act on a stored sub-configuration whose bytes exist but cannot
+/// be parsed. Carried inside [`Error::other`] so callers can tell it apart
+/// from a storage fault with [`is_unreadable_config_error`].
+#[derive(Debug, Clone)]
+pub struct UnreadableBucketConfig {
+    pub bucket: String,
+    pub config_file: String,
+    /// Length of the stored bytes that failed to parse.
+    pub raw_len: usize,
+}
+
+impl std::fmt::Display for UnreadableBucketConfig {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "persisted bucket configuration {} ({} bytes) for bucket {} cannot be parsed; back up the stored bytes (rustfs inspect bucket-meta), then replace or delete the configuration",
+            self.config_file, self.raw_len, self.bucket
+        )
+    }
+}
+
+impl std::error::Error for UnreadableBucketConfig {}
+
+pub fn is_unreadable_config_error(err: &Error) -> bool {
+    unreadable_config_refusal(err).is_some()
+}
+
+pub fn unreadable_config_refusal(err: &Error) -> Option<&UnreadableBucketConfig> {
+    match err {
+        Error::Io(io) => io.get_ref().and_then(|inner| inner.downcast_ref::<UnreadableBucketConfig>()),
+        _ => None,
+    }
+}
+
+pub(crate) fn unreadable_config_error(bucket: &str, config_file: &str, raw_len: usize) -> Error {
+    Error::other(UnreadableBucketConfig {
+        bucket: bucket.to_string(),
+        config_file: config_file.to_string(),
+        raw_len,
+    })
+}
+
+/// Stored state of one sub-configuration as left by
+/// [`BucketMetadata::parse_all_configs`]: a parse failure keeps the raw bytes
+/// and leaves the typed field `None`, which is kept distinct from "no bytes".
+///
+/// Deliberately has no `Option` conversion or defaulting accessor: folding
+/// `Unreadable` into `Absent` is exactly the failure this type exists to stop.
+#[derive(Debug)]
+pub enum ConfigState<'a, T> {
+    /// No bytes are stored.
+    Absent,
+    /// The stored bytes parsed.
+    Valid(&'a T),
+    /// Bytes are stored but could not be parsed.
+    Unreadable { raw_len: usize },
+}
+
+impl<'a, T> ConfigState<'a, T> {
+    pub fn of(raw: &[u8], parsed: &'a Option<T>) -> Self {
+        match parsed {
+            Some(config) => Self::Valid(config),
+            None if raw.is_empty() => Self::Absent,
+            None => Self::Unreadable { raw_len: raw.len() },
+        }
+    }
+
+    /// `Ok(None)` when absent; a typed [`UnreadableBucketConfig`] refusal
+    /// when the stored bytes cannot be parsed.
+    pub fn require(self, bucket: &str, config_file: &str) -> Result<Option<&'a T>> {
+        match self {
+            Self::Absent => Ok(None),
+            Self::Valid(config) => Ok(Some(config)),
+            Self::Unreadable { raw_len } => Err(unreadable_config_error(bucket, config_file, raw_len)),
+        }
+    }
+}
+
+/// The XML sub-configurations whose parse failure is retained as
+/// [`ConfigState::Unreadable`].
+pub const XML_BUCKET_CONFIG_FILES: [&str; 13] = [
+    BUCKET_NOTIFICATION_CONFIG,
+    BUCKET_LIFECYCLE_CONFIG,
+    OBJECT_LOCK_CONFIG,
+    BUCKET_VERSIONING_CONFIG,
+    BUCKET_SSECONFIG,
+    BUCKET_TAGGING_CONFIG,
+    BUCKET_REPLICATION_CONFIG,
+    BUCKET_CORS_CONFIG,
+    BUCKET_LOGGING_CONFIG,
+    BUCKET_WEBSITE_CONFIG,
+    BUCKET_ACCELERATE_CONFIG,
+    BUCKET_REQUEST_PAYMENT_CONFIG,
+    BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG,
+];
+
+impl BucketMetadata {
+    /// Whether the stored XML sub-configuration `config_file` has bytes that
+    /// cannot be parsed. Non-XML config files report `false`: they carry their
+    /// own unreadable handling (policy, quota, bucket targets).
+    pub fn xml_config_unreadable(&self, config_file: &str) -> bool {
+        self.xml_config_unreadable_len(config_file).is_some()
+    }
+
+    /// Length of the stored bytes of XML sub-configuration `config_file` when
+    /// they cannot be parsed; `None` when it is absent, readable, or not an
+    /// XML config.
+    pub fn xml_config_unreadable_len(&self, config_file: &str) -> Option<usize> {
+        fn unreadable<T>(raw: &[u8], parsed: &Option<T>) -> Option<usize> {
+            match ConfigState::of(raw, parsed) {
+                ConfigState::Unreadable { raw_len } => Some(raw_len),
+                ConfigState::Absent | ConfigState::Valid(_) => None,
+            }
+        }
+        match config_file {
+            BUCKET_NOTIFICATION_CONFIG => unreadable(&self.notification_config_xml, &self.notification_config),
+            BUCKET_LIFECYCLE_CONFIG => unreadable(&self.lifecycle_config_xml, &self.lifecycle_config),
+            OBJECT_LOCK_CONFIG => unreadable(&self.object_lock_config_xml, &self.object_lock_config),
+            BUCKET_VERSIONING_CONFIG => unreadable(&self.versioning_config_xml, &self.versioning_config),
+            BUCKET_SSECONFIG => unreadable(&self.encryption_config_xml, &self.sse_config),
+            BUCKET_TAGGING_CONFIG => unreadable(&self.tagging_config_xml, &self.tagging_config),
+            BUCKET_REPLICATION_CONFIG => unreadable(&self.replication_config_xml, &self.replication_config),
+            BUCKET_CORS_CONFIG => unreadable(&self.cors_config_xml, &self.cors_config),
+            BUCKET_LOGGING_CONFIG => unreadable(&self.logging_config_xml, &self.logging_config),
+            BUCKET_WEBSITE_CONFIG => unreadable(&self.website_config_xml, &self.website_config),
+            BUCKET_ACCELERATE_CONFIG => unreadable(&self.accelerate_config_xml, &self.accelerate_config),
+            BUCKET_REQUEST_PAYMENT_CONFIG => unreadable(&self.request_payment_config_xml, &self.request_payment_config),
+            BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG => {
+                unreadable(&self.public_access_block_config_xml, &self.public_access_block_config)
+            }
+            _ => None,
+        }
+    }
+}
+
 pub fn table_catalog_path_hash(value: &str) -> String {
     let digest = Sha256::digest(value.as_bytes());
     let mut output = String::with_capacity(digest.len() * 2);
@@ -473,6 +608,13 @@ impl BucketMetadata {
         self.lock_enabled || self.object_lock_config.as_ref().is_some_and(|v| v.enabled())
     }
 
+    /// Whether an operation that may skip Object Lock checks must keep them.
+    /// Stored lock bytes that cannot be parsed leave the lock state unknown,
+    /// which must not be read as "no Object Lock".
+    pub fn object_lock_checks_required(&self) -> bool {
+        self.object_locking() || self.xml_config_unreadable(OBJECT_LOCK_CONFIG)
+    }
+
     pub fn table_bucket_enabled(&self) -> bool {
         !self.table_bucket_config_json.is_empty()
     }
@@ -985,17 +1127,25 @@ impl BucketMetadata {
     /// |---|---|
     /// | policy | Fails closed: `get_bucket_policy` re-parses the raw JSON and propagates the error; `get_bucket_policy_raw` returns the stored bytes. |
     /// | object lock | Fails closed in `object_lock_config_state_from_authoritative_metadata`; a retention decision may never be taken on a guess. |
-    /// | versioning | Fails closed in `get_versioning_config`; guessing Unversioned would make delete markers and version ids diverge from what is on disk. |
+    /// | versioning | Fails closed in `get_versioning_config` and the delete-time snapshot; guessing Unversioned would make delete markers and version ids diverge from what is on disk. Object writes lay out versions through `BucketVersioningSys::get_for_write`, which refuses in strict mode and, in the default permissive mode, keeps the historical unversioned write (see `config_parse_mode`). |
     /// | replication | Fails closed in `get_replication_config`. |
     /// | bucket targets | Fails closed in `get_bucket_targets_config`, and `sync_bucket_target_sys` marks the bucket unreadable in `BucketTargetSys` instead of publishing an empty target set (rustfs/backlog#2282). |
     /// | encryption | Fails closed in `get_sse_config`: degrading to "no default encryption" stores plaintext objects the operator required to be encrypted. |
     /// | public access block | Fails closed in `get_public_access_block_config`: degrading grants the anonymous access the operator asked to block. |
     /// | quota | Fails closed in `get_quota_config`; the enforcement path in `quota::checker` already re-parses the raw JSON and refuses on error. |
-    /// | lifecycle | Safe to degrade: no rules means no expiration and no transition, so nothing is deleted or moved on the strength of an unreadable rule set. The bucket keeps serving reads and writes. |
-    /// | notification | Safe to degrade: events are an outbound side channel; no consumer draws a durability or authorization conclusion from their absence. |
-    /// | tagging | Safe to degrade: bucket tags are cost-allocation labels here; object-level tag conditions come from object metadata, not this blob. |
-    /// | CORS | Safe to degrade: an absent CORS configuration rejects cross-origin browser requests, which is already the restrictive direction. |
-    /// | logging, website, accelerate, request payment, bucket ACL | Safe to degrade: each only shapes an optional response or an optional side channel, and none of them authorizes an action or decides whether data is retained. |
+    /// | lifecycle | `get_lifecycle_config` fails closed, so GetBucketLifecycle reports the fault instead of NoSuchLifecycleConfiguration. ILM, scanner and expiry-header consumers still degrade to "no rules": nothing is deleted or moved on the strength of an unreadable rule set, and the bucket keeps serving reads and writes. |
+    /// | notification | `get_notification_config` fails closed, so GetBucketNotificationConfiguration reports the fault and startup leaves that one bucket's rules unchanged instead of clearing them; other buckets are unaffected. |
+    /// | tagging, CORS, logging, website, accelerate, request payment | The getter fails closed so the matching GET API reports the fault instead of "not configured". Per-request consumers (CORS response headers) still degrade to "not configured", which is the restrictive direction. |
+    /// | bucket ACL | Safe to degrade: it only shapes an optional response. |
+    ///
+    /// Independently of the table, `update_config_with` refuses a
+    /// read-modify-write of an unreadable XML config before its mutate step
+    /// runs, so the unreadable bytes are never replaced by a rewrite that saw
+    /// them as absent. Other configs of the same bucket stay writable.
+    ///
+    /// Every unreadable XML config found here is counted in
+    /// `rustfs_bucket_metadata_parse_failed_total`, reflected in
+    /// `rustfs_bucket_metadata_unparsable_current`, and logged at error level.
     pub(super) fn parse_all_configs(&mut self) -> Result<()> {
         if let Err(e) = self.parse_policy_config() {
             tracing::warn!(
@@ -1239,6 +1389,14 @@ impl BucketMetadata {
             );
         }
 
+        if !self.name.is_empty() {
+            let unreadable: Vec<(&'static str, usize)> = XML_BUCKET_CONFIG_FILES
+                .iter()
+                .filter_map(|config| self.xml_config_unreadable_len(config).map(|raw_len| (*config, raw_len)))
+                .collect();
+            super::config_parse_mode::record_bucket_config_parse_state(&self.name, &unreadable);
+        }
+
         Ok(())
     }
 }
@@ -1356,6 +1514,18 @@ where
 mod test {
     use super::*;
 
+    /// rustfs/backlog#1734: `StorageError::clone` rebuilds I/O errors from
+    /// their text. The unreadable-config refusal must stay typed across that
+    /// clone, or a cloned error stops mapping to its retryable S3 response.
+    #[test]
+    fn unreadable_config_refusal_survives_storage_error_clone() {
+        let err = unreadable_config_error("b", BUCKET_TAGGING_CONFIG, 7);
+        let cloned = err.clone();
+        assert!(is_unreadable_config_error(&cloned), "clone lost the typed refusal: {cloned}");
+        assert_eq!(unreadable_config_refusal(&cloned).map(|r| r.raw_len), Some(7));
+        assert!(is_unreadable_config_error(&err));
+    }
+
     /// Decode a whitespace-tolerant hex fixture into bytes.
     fn decode_hex(s: &str) -> Vec<u8> {
         let s: String = s.chars().filter(|c| !c.is_whitespace()).collect();

crates/ecstore/src/bucket/metadata_sys.rs

@@ -13,7 +13,9 @@
 // limitations under the License.
 
 use super::metadata::{
-    BUCKET_TARGETS_FILE, BucketMetadata, load_bucket_incarnation, load_bucket_metadata, save_bucket_incarnation,
+    BUCKET_ACCELERATE_CONFIG, BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG,
+    BUCKET_TAGGING_CONFIG, BUCKET_TARGETS_FILE, BUCKET_WEBSITE_CONFIG, BucketMetadata, ConfigState, load_bucket_incarnation,
+    load_bucket_metadata, save_bucket_incarnation, unreadable_config_error,
 };
 use super::quota::BucketQuota;
 use super::target::BucketTargets;
@@ -150,8 +152,8 @@ enum BucketMetadataAuthority {
 }
 
 pub(crate) fn object_lock_config_state_from_authoritative_metadata(bm: &BucketMetadata) -> Result<ObjectLockConfigState> {
-    if bm.object_lock_config.is_none() && !bm.object_lock_config_xml.is_empty() {
-        return Err(Error::other("persisted bucket Object Lock configuration is invalid"));
+    if let Some(raw_len) = bm.xml_config_unreadable_len(super::metadata::OBJECT_LOCK_CONFIG) {
+        return Err(unreadable_config_error(&bm.name, super::metadata::OBJECT_LOCK_CONFIG, raw_len));
     }
 
     if let Some(config) = bm.object_lock_config.clone() {
@@ -1863,6 +1865,7 @@ impl BucketMetadataSys {
         drop(map);
         let removed_fabricated = self.fabricated_metadata.write().await.remove(bucket);
         self.missing_buckets.insert(bucket.to_string(), ()).await;
+        super::config_parse_mode::forget_bucket_config_parse_state(bucket);
         if removed {
             BucketTargetSys::get().delete(bucket).await;
             clear_bucket_durability(bucket);
@@ -1956,6 +1959,13 @@ impl BucketMetadataSys {
         if !bm.bucket_incarnation_sidecar || bm.bucket_incarnation_id != expected_incarnation_id {
             return Err(Error::BucketNotFound(bucket.to_string()));
         }
+        // `mutate` would see an unreadable config as absent and rebuild it
+        // from nothing; persisting that destroys the only copy of the stored
+        // bytes. Only the rewritten config is checked: `update_config` carries
+        // every other raw config through unchanged.
+        if let Some(raw_len) = bm.xml_config_unreadable_len(config_file) {
+            return Err(unreadable_config_error(bucket, config_file, raw_len));
+        }
 
         let data = mutate(&bm)?;
         let updated = bm.update_config(config_file, data)?;
@@ -2198,12 +2208,11 @@ impl BucketMetadataSys {
             }
         };
 
-        if !bm.versioning_config_xml.is_empty() && bm.versioning_config.is_none() {
-            Err(Error::other("persisted bucket versioning configuration is invalid"))
-        } else if let Some(config) = &bm.versioning_config {
-            Ok((config.clone(), bm.versioning_config_updated_at))
-        } else {
-            Ok((VersioningConfiguration::default(), bm.versioning_config_updated_at))
+        match ConfigState::of(&bm.versioning_config_xml, &bm.versioning_config)
+            .require(bucket, super::metadata::BUCKET_VERSIONING_CONFIG)?
+        {
+            Some(config) => Ok((config.clone(), bm.versioning_config_updated_at)),
+            None => Ok((VersioningConfiguration::default(), bm.versioning_config_updated_at)),
         }
     }
 
@@ -2212,8 +2221,8 @@ impl BucketMetadataSys {
             return Ok(false);
         };
 
-        if metadata.versioning_config.is_none() && !metadata.versioning_config_xml.is_empty() {
-            return Err(Error::other("persisted bucket versioning configuration is invalid"));
+        if let Some(raw_len) = metadata.xml_config_unreadable_len(super::metadata::BUCKET_VERSIONING_CONFIG) {
+            return Err(unreadable_config_error(bucket, super::metadata::BUCKET_VERSIONING_CONFIG, raw_len));
         }
 
         Ok(metadata.versioning_config.is_none() && metadata.versioning_config_xml.is_empty())
@@ -2274,22 +2283,20 @@ impl BucketMetadataSys {
     pub async fn get_tagging_config(&self, bucket: &str) -> Result<(Tagging, OffsetDateTime)> {
         let (bm, _) = self.get_config(bucket).await?;
 
-        if let Some(config) = &bm.tagging_config {
-            Ok((config.clone(), bm.tagging_config_updated_at))
-        } else {
-            Err(Error::ConfigNotFound)
+        match ConfigState::of(&bm.tagging_config_xml, &bm.tagging_config).require(bucket, BUCKET_TAGGING_CONFIG)? {
+            Some(config) => Ok((config.clone(), bm.tagging_config_updated_at)),
+            None => Err(Error::ConfigNotFound),
         }
     }
 
     pub async fn get_public_access_block_config(&self, bucket: &str) -> Result<(PublicAccessBlockConfiguration, OffsetDateTime)> {
         let (bm, _) = self.get_config(bucket).await?;
 
-        if !bm.public_access_block_config_xml.is_empty() && bm.public_access_block_config.is_none() {
-            Err(Error::other("persisted bucket public access block configuration is invalid"))
-        } else if let Some(config) = &bm.public_access_block_config {
-            Ok((config.clone(), bm.public_access_block_config_updated_at))
-        } else {
-            Err(Error::ConfigNotFound)
+        match ConfigState::of(&bm.public_access_block_config_xml, &bm.public_access_block_config)
+            .require(bucket, super::metadata::BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG)?
+        {
+            Some(config) => Ok((config.clone(), bm.public_access_block_config_updated_at)),
+            None => Err(Error::ConfigNotFound),
         }
     }
 
@@ -2571,91 +2578,79 @@ impl BucketMetadataSys {
     pub async fn get_lifecycle_config(&self, bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime)> {
         let (bm, _) = self.get_config(bucket).await?;
 
-        if let Some(config) = &bm.lifecycle_config {
-            if config.rules.is_empty() {
-                Err(Error::ConfigNotFound)
-            } else {
-                Ok((config.clone(), bm.lifecycle_config_updated_at))
-            }
-        } else {
-            Err(Error::ConfigNotFound)
+        match ConfigState::of(&bm.lifecycle_config_xml, &bm.lifecycle_config).require(bucket, BUCKET_LIFECYCLE_CONFIG)? {
+            Some(config) if !config.rules.is_empty() => Ok((config.clone(), bm.lifecycle_config_updated_at)),
+            _ => Err(Error::ConfigNotFound),
         }
     }
 
     pub async fn get_notification_config(&self, bucket: &str) -> Result<Option<NotificationConfiguration>> {
         let bm = match self.get_config(bucket).await {
-            Ok((bm, _)) => bm.notification_config.clone(),
-            Err(err) => {
-                if err == Error::ConfigNotFound {
-                    None
-                } else {
-                    return Err(err);
-                }
-            }
+            Ok((bm, _)) => bm,
+            Err(Error::ConfigNotFound) => return Ok(None),
+            Err(err) => return Err(err),
         };
 
-        Ok(bm)
+        // Unreadable must not read as "no notification configured": that
+        // would silently drop the bucket's event rules.
+        Ok(ConfigState::of(&bm.notification_config_xml, &bm.notification_config)
+            .require(bucket, super::metadata::BUCKET_NOTIFICATION_CONFIG)?
+            .cloned())
     }
 
     pub async fn get_sse_config(&self, bucket: &str) -> Result<(ServerSideEncryptionConfiguration, OffsetDateTime)> {
         let (bm, _) = self.get_config(bucket).await?;
 
-        if !bm.encryption_config_xml.is_empty() && bm.sse_config.is_none() {
-            Err(Error::other("persisted bucket encryption configuration is invalid"))
-        } else if let Some(config) = &bm.sse_config {
-            Ok((config.clone(), bm.encryption_config_updated_at))
-        } else {
-            Err(Error::ConfigNotFound)
+        match ConfigState::of(&bm.encryption_config_xml, &bm.sse_config).require(bucket, super::metadata::BUCKET_SSECONFIG)? {
+            Some(config) => Ok((config.clone(), bm.encryption_config_updated_at)),
+            None => Err(Error::ConfigNotFound),
         }
     }
 
     pub async fn get_cors_config(&self, bucket: &str) -> Result<(CORSConfiguration, OffsetDateTime)> {
         let (bm, _) = self.get_config(bucket).await?;
 
-        if let Some(config) = &bm.cors_config {
-            Ok((config.clone(), bm.cors_config_updated_at))
-        } else {
-            Err(Error::ConfigNotFound)
+        match ConfigState::of(&bm.cors_config_xml, &bm.cors_config).require(bucket, BUCKET_CORS_CONFIG)? {
+            Some(config) => Ok((config.clone(), bm.cors_config_updated_at)),
+            None => Err(Error::ConfigNotFound),
         }
     }
 
     pub async fn get_website_config(&self, bucket: &str) -> Result<(WebsiteConfiguration, OffsetDateTime)> {
         let (bm, _) = self.get_config(bucket).await?;
 
-        if let Some(config) = &bm.website_config {
-            Ok((config.clone(), bm.website_config_updated_at))
-        } else {
-            Err(Error::ConfigNotFound)
+        match ConfigState::of(&bm.website_config_xml, &bm.website_config).require(bucket, BUCKET_WEBSITE_CONFIG)? {
+            Some(config) => Ok((config.clone(), bm.website_config_updated_at)),
+            None => Err(Error::ConfigNotFound),
         }
     }
 
     pub async fn get_logging_config(&self, bucket: &str) -> Result<(BucketLoggingStatus, OffsetDateTime)> {
         let (bm, _) = self.get_config(bucket).await?;
 
-        if let Some(config) = &bm.logging_config {
-            Ok((config.clone(), bm.logging_config_updated_at))
-        } else {
-            Err(Error::ConfigNotFound)
+        match ConfigState::of(&bm.logging_config_xml, &bm.logging_config).require(bucket, BUCKET_LOGGING_CONFIG)? {
+            Some(config) => Ok((config.clone(), bm.logging_config_updated_at)),
+            None => Err(Error::ConfigNotFound),
         }
     }
 
     pub async fn get_accelerate_config(&self, bucket: &str) -> Result<(AccelerateConfiguration, OffsetDateTime)> {
         let (bm, _) = self.get_config(bucket).await?;
 
-        if let Some(config) = &bm.accelerate_config {
-            Ok((config.clone(), bm.accelerate_config_updated_at))
-        } else {
-            Err(Error::ConfigNotFound)
+        match ConfigState::of(&bm.accelerate_config_xml, &bm.accelerate_config).require(bucket, BUCKET_ACCELERATE_CONFIG)? {
+            Some(config) => Ok((config.clone(), bm.accelerate_config_updated_at)),
+            None => Err(Error::ConfigNotFound),
         }
     }
 
     pub async fn get_request_payment_config(&self, bucket: &str) -> Result<(RequestPaymentConfiguration, OffsetDateTime)> {
         let (bm, _) = self.get_config(bucket).await?;
 
-        if let Some(config) = &bm.request_payment_config {
-            Ok((config.clone(), bm.request_payment_config_updated_at))
-        } else {
-            Err(Error::ConfigNotFound)
+        match ConfigState::of(&bm.request_payment_config_xml, &bm.request_payment_config)
+            .require(bucket, BUCKET_REQUEST_PAYMENT_CONFIG)?
+        {
+            Some(config) => Ok((config.clone(), bm.request_payment_config_updated_at)),
+            None => Err(Error::ConfigNotFound),
         }
     }
 
@@ -3826,6 +3821,202 @@ mod tests {
 
         assert!(matches!(err, Error::Io(_)), "malformed persisted policy must surface its parse failure");
     }
+
+    /// Persist `bucket` with the given raw sub-configuration bytes, bypassing
+    /// the parse step the way a newer or foreign writer (or disk damage) would.
+    async fn persist_bucket_with_raw_config(
+        sys: &BucketMetadataSys,
+        dirs: &[tempfile::TempDir],
+        bucket: &str,
+        config_file: &str,
+        raw: &[u8],
+    ) {
+        for dir in dirs {
+            std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created");
+        }
+        let mut bm = BucketMetadata::new(bucket);
+        bm.update_config(config_file, raw.to_vec())
+            .expect("raw config should be stored");
+        sys.persist_new_and_set(bm).await.expect("initial metadata should persist");
+    }
+
+    /// rustfs/backlog#1734: a read-modify-write of a stored config that cannot
+    /// be parsed must be refused before `mutate` runs. Otherwise `mutate` sees
+    /// the unreadable config as absent, rebuilds it from nothing, and the
+    /// write-back destroys the only copy of the original bytes.
+    #[tokio::test]
+    async fn update_config_with_refuses_rewrite_of_unreadable_target_config() {
+        use crate::bucket::metadata::BUCKET_TAGGING_CONFIG;
+
+        let (dirs, ecstore) = isolated_store_over_temp_disks().await;
+        let sys = BucketMetadataSys::new(ecstore);
+        let bucket = "unreadable-tagging-rmw";
+        let corrupt = b"<Tagging><TagSet><Tag><Key>team</Key>".to_vec();
+        persist_bucket_with_raw_config(&sys, &dirs, bucket, BUCKET_TAGGING_CONFIG, &corrupt).await;
+
+        let mutate_calls = std::sync::atomic::AtomicUsize::new(0);
+        let err = sys
+            .update_config_with(bucket, BUCKET_TAGGING_CONFIG, |_| {
+                mutate_calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+                Ok(Vec::new())
+            })
+            .await
+            .expect_err("a rewrite of an unreadable config must be refused");
+
+        assert_eq!(
+            mutate_calls.load(std::sync::atomic::Ordering::SeqCst),
+            0,
+            "mutate must not see an unreadable config as absent"
+        );
+        assert!(
+            crate::bucket::metadata::is_unreadable_config_error(&err),
+            "the refusal must be identifiable as an unreadable-config refusal: {err}"
+        );
+        sys.metadata_map.write().await.clear();
+        let (reloaded, _) = sys.get_config(bucket).await.expect("metadata should reload from disk");
+        assert_eq!(reloaded.tagging_config_xml, corrupt, "the original bytes must stay untouched");
+    }
+
+    /// rustfs/backlog#1734: the refusal is per config. One unreadable config
+    /// must not block a read-modify-write of a different, readable config, and
+    /// that write must carry the unreadable bytes through unchanged.
+    #[tokio::test]
+    async fn update_config_with_rewrites_readable_config_beside_unreadable_one() {
+        use crate::bucket::metadata::{BUCKET_CORS_CONFIG, BUCKET_TAGGING_CONFIG};
+
+        let (dirs, ecstore) = isolated_store_over_temp_disks().await;
+        let sys = BucketMetadataSys::new(ecstore);
+        let bucket = "unreadable-tagging-cors-rmw";
+        let corrupt = b"<Tagging><TagSet><Tag><Key>team</Key>".to_vec();
+        persist_bucket_with_raw_config(&sys, &dirs, bucket, BUCKET_TAGGING_CONFIG, &corrupt).await;
+
+        let xml = br#"<CORSConfiguration><CORSRule><AllowedMethod>GET</AllowedMethod><AllowedOrigin>https://example.test</AllowedOrigin></CORSRule></CORSConfiguration>"#.to_vec();
+        sys.update_config_with(bucket, BUCKET_CORS_CONFIG, move |_| Ok(xml))
+            .await
+            .expect("a readable config must stay writable beside an unreadable one");
+
+        sys.metadata_map.write().await.clear();
+        let (reloaded, _) = sys.get_config(bucket).await.expect("metadata should reload from disk");
+        assert_eq!(
+            reloaded.tagging_config_xml, corrupt,
+            "the unreadable config must be carried through byte-for-byte"
+        );
+        let (stored_cors, _) = sys.get_cors_config(bucket).await.expect("cors should be readable");
+        assert_eq!(stored_cors.cors_rules.len(), 1);
+    }
+
+    /// rustfs/backlog#1734: reading a stored config that cannot be parsed
+    /// must fail, not report the config as absent (which the S3 GET handlers
+    /// turn into NoSuchTagSet / NoSuchLifecycleConfiguration).
+    #[tokio::test]
+    async fn unreadable_tagging_and_lifecycle_reads_fail_instead_of_reading_absent() {
+        use crate::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_TAGGING_CONFIG};
+
+        let (dirs, ecstore) = isolated_store_over_temp_disks().await;
+        let sys = BucketMetadataSys::new(ecstore);
+
+        let tagging_bucket = "unreadable-tagging-read";
+        persist_bucket_with_raw_config(&sys, &dirs, tagging_bucket, BUCKET_TAGGING_CONFIG, b"<Tagging><TagSet>").await;
+        let err = sys
+            .get_tagging_config(tagging_bucket)
+            .await
+            .expect_err("unreadable tagging must not read as a value");
+        assert_ne!(err, Error::ConfigNotFound, "unreadable tagging must not read as absent");
+
+        let lifecycle_bucket = "unreadable-lifecycle-read";
+        persist_bucket_with_raw_config(&sys, &dirs, lifecycle_bucket, BUCKET_LIFECYCLE_CONFIG, b"<LifecycleConfiguration><Rule>")
+            .await;
+        let err = sys
+            .get_lifecycle_config(lifecycle_bucket)
+            .await
+            .expect_err("unreadable lifecycle must not read as a value");
+        assert_ne!(err, Error::ConfigNotFound, "unreadable lifecycle must not read as absent");
+
+        // The genuinely absent case still reads as absent.
+        let absent_bucket = "absent-tagging-read-control";
+        persist_bucket_with_raw_config(&sys, &dirs, absent_bucket, BUCKET_TAGGING_CONFIG, b"").await;
+        assert_eq!(sys.get_tagging_config(absent_bucket).await.expect_err("absent"), Error::ConfigNotFound);
+    }
+
+    /// rustfs/backlog#1734: the configs that gate object writes and deletes
+    /// (versioning, Object Lock, default encryption) and the notification
+    /// config must refuse with the typed unreadable-config error, so the S3
+    /// layer can answer 503 with the bucket and config named instead of a
+    /// generic 500, and notification setup can isolate the one bucket.
+    #[tokio::test]
+    async fn unreadable_gating_configs_refuse_with_the_typed_error() {
+        use crate::bucket::metadata::{BUCKET_VERSIONING_CONFIG, unreadable_config_refusal};
+
+        let (dirs, ecstore) = isolated_store_over_temp_disks().await;
+        let sys = BucketMetadataSys::new(ecstore);
+
+        // `update_config` validates some configs on write, so the corrupt
+        // bytes go straight into the raw fields, as a damaged object would.
+        let persist_corrupt = |bucket: &'static str, corrupt: fn(&mut BucketMetadata)| {
+            for dir in &dirs {
+                std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created");
+            }
+            let mut bm = BucketMetadata::new(bucket);
+            corrupt(&mut bm);
+            sys.persist_new_and_set(bm)
+        };
+
+        persist_corrupt("unreadable-versioning", |bm| {
+            bm.versioning_config_xml = b"<VersioningConfiguration><Status>".to_vec();
+        })
+        .await
+        .expect("corrupt versioning should persist");
+        let err = sys
+            .get_versioning_config("unreadable-versioning")
+            .await
+            .expect_err("unreadable versioning must not read as a value");
+        let refusal = unreadable_config_refusal(&err).unwrap_or_else(|| panic!("expected a typed refusal, got {err}"));
+        assert_eq!(refusal.config_file, BUCKET_VERSIONING_CONFIG);
+        assert_eq!(refusal.raw_len, b"<VersioningConfiguration><Status>".len());
+
+        persist_corrupt("unreadable-lock", |bm| bm.object_lock_config_xml = b"<ObjectLockConfiguration>".to_vec())
+            .await
+            .expect("corrupt Object Lock should persist");
+        let err = sys
+            .get_object_lock_config_state("unreadable-lock")
+            .await
+            .expect_err("unreadable Object Lock must not read as a value");
+        assert!(unreadable_config_refusal(&err).is_some(), "{err}");
+
+        persist_corrupt("unreadable-sse", |bm| {
+            bm.encryption_config_xml = b"<ServerSideEncryptionConfiguration>".to_vec();
+        })
+        .await
+        .expect("corrupt encryption should persist");
+        let err = sys
+            .get_sse_config("unreadable-sse")
+            .await
+            .expect_err("unreadable encryption must not read as a value");
+        assert!(unreadable_config_refusal(&err).is_some(), "{err}");
+
+        persist_corrupt("unreadable-notify", |bm| {
+            bm.notification_config_xml = b"<NotificationConfiguration>".to_vec();
+        })
+        .await
+        .expect("corrupt notification should persist");
+        let err = sys
+            .get_notification_config("unreadable-notify")
+            .await
+            .expect_err("unreadable notification must not read as \"no notification configured\"");
+        assert!(unreadable_config_refusal(&err).is_some(), "{err}");
+
+        // Absent stays absent.
+        persist_corrupt("absent-notification-read", |_| {})
+            .await
+            .expect("plain bucket should persist");
+        assert!(
+            sys.get_notification_config("absent-notification-read")
+                .await
+                .expect("absent")
+                .is_none()
+        );
+    }
+
     /// A tagging rewrite through `update_config_with` (the Swift metadata
     /// POST path) is persisted: it survives a metadata reload from disk, and
     /// an emptied rewrite clears the config in the cached copy too instead of

crates/ecstore/src/bucket/migration.rs

@@ -17,10 +17,10 @@
 use crate::bucket::metadata::BUCKET_METADATA_FILE;
 use crate::bucket::replication::ReplicationMigrationBridge;
 use crate::disk::{BUCKET_META_PREFIX, MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
-use crate::error::Error;
+use crate::error::{Error, Result, is_err_strict_not_found, is_err_strict_volume_not_found};
 use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
 use crate::storage_api_contracts::{
-    bucket::{BucketOperations, BucketOptions},
+    bucket::{BucketInfo, BucketOperations, BucketOptions},
     list::{ListOperations, StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
     object::{DeletedObject, EcstoreObjectIO, EcstoreObjectOperations, ObjectIO, ObjectOperations, ObjectToDelete},
     range::HTTPRangeSpec,
@@ -33,7 +33,7 @@ use rustfs_utils::path::SLASH_SEPARATOR;
 use serde::{Deserialize, Serialize};
 use std::sync::Arc;
 use time::OffsetDateTime;
-use tracing::{debug, info, warn};
+use tracing::{debug, info};
 
 /// IAM config prefix under meta bucket (e.g. config/iam/).
 const IAM_CONFIG_PREFIX: &str = "config/iam";
@@ -53,6 +53,39 @@ type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
 type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
 type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
 
+#[derive(Clone, Debug, thiserror::Error)]
+enum MigrationMetadataError {
+    #[error("empty legacy metadata: {0}")]
+    Empty(String),
+    #[error("incompatible legacy metadata: {0}")]
+    Incompatible(String),
+}
+
+impl From<MigrationMetadataError> for Error {
+    fn from(error: MigrationMetadataError) -> Self {
+        let message = match &error {
+            MigrationMetadataError::Empty(_) => "empty legacy metadata",
+            MigrationMetadataError::Incompatible(_) => "incompatible legacy metadata",
+        };
+        // Keep the record path in the typed source, not in the quorum grouping key.
+        Self::other_with_context(message, error)
+    }
+}
+
+/// Converts a migration failure at the startup boundary, rendering the safe
+/// record path while leaving storage-layer error grouping stable.
+pub fn migration_startup_error(error: Error) -> std::io::Error {
+    if let Error::Io(io_error) = &error
+        && let Some(metadata_error) = io_error
+            .get_ref()
+            .and_then(|context| context.source())
+            .and_then(|source| source.downcast_ref::<MigrationMetadataError>())
+    {
+        return std::io::Error::other(metadata_error.clone());
+    }
+    std::io::Error::other(error)
+}
+
 /// Callback used to decrypt an at-rest config blob during MinIO -> RustFS migration.
 ///
 /// MinIO encrypts IAM identity/service-account files and the server config at rest
@@ -211,7 +244,7 @@ fn normalize_bucket_meta_blob(path: &str, data: &[u8]) -> std::result::Result<Op
 /// Uses list_bucket (from disk volumes) to get bucket names, since list_objects_v2 on the legacy
 /// meta bucket may not work (legacy format differs from object layer expectations).
 /// Skips buckets that already exist in RustFS (idempotent).
-pub async fn try_migrate_bucket_metadata<S>(store: Arc<S>)
+pub async fn try_migrate_bucket_metadata<S>(store: Arc<S>) -> Result<()>
 where
     S: BucketOperations<Error = crate::error::Error>
         + ObjectIO<
@@ -231,25 +264,18 @@ where
             DeletedObject = DeletedObject,
         >,
 {
-    let buckets_list = match store
+    let buckets_list = store
         .list_bucket(&BucketOptions {
             no_metadata: true,
             ..Default::default()
         })
-        .await
-    {
-        Ok(b) => b,
-        Err(e) => {
-            warn!("list buckets failed (skip migration): {e}");
-            return;
-        }
-    };
+        .await?;
 
     let buckets: Vec<String> = buckets_list.into_iter().map(|b| b.name).collect();
 
     if buckets.is_empty() {
         debug!("No migrating bucket metadata found");
-        return;
+        return Ok(());
     }
 
     debug!("Found {} migrating bucket metadata, migrating...", buckets.len());
@@ -263,26 +289,40 @@ where
 
     for bucket in buckets {
         let meta_path = format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{bucket}{SLASH_SEPARATOR}{BUCKET_METADATA_FILE}");
-        migrate_one_if_missing(store.clone(), &opts, &h, &meta_path, &format!("bucket metadata: {bucket}")).await;
+        migrate_one_if_missing(store.clone(), &opts, &h, &meta_path, &format!("bucket metadata: {bucket}")).await?;
 
         let resync_path = format!(
             "{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{bucket}{SLASH_SEPARATOR}{REPLICATION_META_DIR}{SLASH_SEPARATOR}{RESYNC_META_FILE}"
         );
-        migrate_one_if_missing(store.clone(), &opts, &h, &resync_path, &format!("bucket replication resync: {bucket}")).await;
+        migrate_one_if_missing(store.clone(), &opts, &h, &resync_path, &format!("bucket replication resync: {bucket}")).await?;
+    }
+    Ok(())
+}
+
+async fn migration_target_exists<S: EcstoreObjectOperations>(store: &S, path: &str) -> Result<bool> {
+    match store
+        .get_object_info(RUSTFS_META_BUCKET, path, &ObjectOptions::default())
+        .await
+    {
+        Ok(_) => Ok(true),
+        Err(err) if is_err_strict_not_found(&err) || is_err_strict_volume_not_found(&err) => Ok(false),
+        Err(err) => Err(err),
     }
 }
 
-async fn migrate_one_if_missing<S>(store: Arc<S>, opts: &ObjectOptions, headers: &HeaderMap, path: &str, label: &str)
+async fn migrate_one_if_missing<S>(
+    store: Arc<S>,
+    opts: &ObjectOptions,
+    headers: &HeaderMap,
+    path: &str,
+    label: &str,
+) -> Result<()>
 where
     S: EcstoreObjectIO + EcstoreObjectOperations,
 {
-    if store
-        .get_object_info(RUSTFS_META_BUCKET, path, &ObjectOptions::default())
-        .await
-        .is_ok()
-    {
+    if migration_target_exists(store.as_ref(), path).await? {
         debug!("{label} already exists in RustFS, skip");
-        return;
+        return Ok(());
     }
 
     let mut rd = match store
@@ -290,45 +330,34 @@ where
         .await
     {
         Ok(r) => r,
-        Err(e) => {
-            debug!("read migrating {label}: {e}");
-            return;
-        }
+        // Ordinary RustFS deployments have no legacy bucket, and optional
+        // legacy settings (such as replication resync) may not exist.
+        Err(err) if is_err_strict_not_found(&err) || is_err_strict_volume_not_found(&err) => return Ok(()),
+        Err(err) => return Err(err),
     };
 
-    let data = match rd.read_all().await {
-        Ok(d) if !d.is_empty() => d,
-        Ok(_) => return,
-        Err(e) => {
-            debug!("read migrating {label} body: {e}");
-            return;
-        }
-    };
-
-    let data = match normalize_bucket_meta_blob(path, &data) {
-        Ok(Some(normalized)) => normalized,
-        Ok(None) => data,
-        Err(e) => {
-            warn!("skip {label} migration due to incompatible format: {e}");
-            return;
-        }
-    };
+    let data = rd.read_all().await?;
+    if data.is_empty() {
+        return Err(MigrationMetadataError::Empty(path.to_owned()).into());
+    }
+    let data = normalize_bucket_meta_blob(path, &data)
+        .map_err(|_| MigrationMetadataError::Incompatible(path.to_owned()))?
+        .unwrap_or(data);
 
     let mut put_data = PutObjReader::from_vec(data);
-    if let Err(e) = store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, opts).await {
-        warn!("write {label}: {e}");
-    } else {
-        info!("Migrated {label}");
-    }
+    store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, opts).await?;
+    info!("Migrated {label}");
+    Ok(())
 }
 
 /// Migrates IAM config from legacy meta bucket `config/iam/` to RustFS meta bucket.
 /// Lists all objects under the IAM prefix in the source, copies each to the target if not present.
 /// Skips objects that already exist in RustFS (idempotent).
-/// If list_objects_v2 on the legacy bucket fails (e.g. format differs), migration is skipped.
-pub async fn try_migrate_iam_config<S>(store: Arc<S>, decrypt_fn: Option<LegacyBlobDecryptFn>)
+/// An absent legacy bucket is a no-op; migration errors prevent startup readiness.
+pub async fn try_migrate_iam_config<S>(store: Arc<S>, decrypt_fn: Option<LegacyBlobDecryptFn>) -> Result<()>
 where
-    S: ListOperations<
+    S: BucketOperations<Error = crate::error::Error>
+        + ListOperations<
             Error = crate::error::Error,
             ListObjectsV2Info = ListObjectsV2Info,
             ListObjectVersionsInfo = ListObjectVersionsInfo,
@@ -353,6 +382,21 @@ where
             DeletedObject = DeletedObject,
         >,
 {
+    if !legacy_iam_source_exists(
+        store
+            .get_bucket_info(
+                MIGRATING_META_BUCKET,
+                &BucketOptions {
+                    no_metadata: true,
+                    ..Default::default()
+                },
+            )
+            .await,
+    )? {
+        debug!("No legacy IAM volume found");
+        return Ok(());
+    }
+
     let opts = ObjectOptions {
         max_parity: true,
         no_lock: true,
@@ -366,47 +410,36 @@ where
     loop {
         let list_result = match store
             .clone()
-            .list_objects_v2(MIGRATING_META_BUCKET, &prefix, continuation, None, 500, false, None, false)
+            .list_objects_v2(MIGRATING_META_BUCKET, &prefix, continuation.clone(), None, 500, false, None, false)
             .await
         {
             Ok(r) => r,
-            Err(e) => {
-                debug!("list IAM config from legacy bucket failed (skip migration): {e}");
-                return;
-            }
+            Err(err) if is_err_strict_volume_not_found(&err) => return Ok(()),
+            Err(err) => return Err(err),
         };
 
         for obj in list_result.objects {
             let path = &obj.name;
-            if path.is_empty() || path.ends_with('/') {
+            // Unsupported records must not trigger target lookups, reads, or decryption.
+            if path != IAM_FORMAT_FILE_PATH
+                && !is_identity_path(path)
+                && !is_group_path(path)
+                && !is_policy_doc_path(path)
+                && !is_policy_mapping_path(path)
+            {
                 continue;
             }
-            if store
-                .get_object_info(RUSTFS_META_BUCKET, path, &ObjectOptions::default())
-                .await
-                .is_ok()
-            {
+            if migration_target_exists(store.as_ref(), path).await? {
                 debug!("IAM config already exists in RustFS, skip: {path}");
                 continue;
             }
-            let mut rd = match store
+            let mut rd = store
                 .get_object_reader(MIGRATING_META_BUCKET, path, None, h.clone(), &opts)
-                .await
-            {
-                Ok(r) => r,
-                Err(e) => {
-                    debug!("read migrating IAM config {path}: {e}");
-                    continue;
-                }
-            };
-            let data = match rd.read_all().await {
-                Ok(d) if !d.is_empty() => d,
-                Ok(_) => continue,
-                Err(e) => {
-                    debug!("read migrating IAM config {path} body: {e}");
-                    continue;
-                }
-            };
+                .await?;
+            let data = rd.read_all().await?;
+            if data.is_empty() {
+                return Err(MigrationMetadataError::Empty(path.to_owned()).into());
+            }
             // MinIO encrypts IAM identity/service-account files at rest. Decrypt
             // before normalizing; fall back to the raw bytes when no key applies
             // (plaintext blobs, or nothing to decrypt) so existing behavior holds.
@@ -420,22 +453,17 @@ where
                     debug!("skip unsupported IAM config path during migration: {path}");
                     continue;
                 }
-                Err(e) => {
-                    warn!("skip IAM config migration due to incompatible format, path: {path}, err: {e}");
-                    continue;
-                }
+                // Parser errors may contain credential data. Report only the path.
+                Err(_) => return Err(MigrationMetadataError::Incompatible(path.to_owned()).into()),
             };
             let mut put_data = PutObjReader::from_vec(data);
-            if let Err(e) = store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, &opts).await {
-                warn!("write IAM config {path}: {e}");
-            } else {
-                info!("Migrated IAM config: {path}");
-                total_migrated += 1;
-            }
+            store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, &opts).await?;
+            info!("Migrated IAM config: {path}");
+            total_migrated += 1;
         }
 
-        continuation = list_result.next_continuation_token.or(list_result.continuation_token);
-        if !list_result.is_truncated || continuation.is_none() {
+        continuation = next_iam_migration_page(list_result.is_truncated, continuation, list_result.next_continuation_token)?;
+        if continuation.is_none() {
             break;
         }
     }
@@ -443,10 +471,94 @@ where
     if total_migrated > 0 {
         info!("IAM migration complete: {} object(s) migrated", total_migrated);
     }
+    Ok(())
+}
+
+fn legacy_iam_source_exists(result: Result<BucketInfo>) -> Result<bool> {
+    match result {
+        Ok(_) => Ok(true),
+        Err(error) if is_err_strict_volume_not_found(&error) => Ok(false),
+        Err(error) => Err(error),
+    }
+}
+
+fn next_iam_migration_page(truncated: bool, previous: Option<String>, next: Option<String>) -> Result<Option<String>> {
+    if !truncated {
+        return Ok(None);
+    }
+    let next = next.filter(|token| !token.is_empty());
+    if next.is_none() || next == previous {
+        return Err(Error::other("legacy IAM migration listing did not advance"));
+    }
+    Ok(next)
 }
 
 #[cfg(test)]
 mod tests {
+    #[test]
+    fn legacy_iam_source_probe_skips_only_an_absent_volume() {
+        use super::{BucketInfo, Error, legacy_iam_source_exists};
+
+        assert!(!legacy_iam_source_exists(Err(Error::VolumeNotFound)).expect("absent legacy volume is a no-op"));
+        assert!(legacy_iam_source_exists(Ok(BucketInfo::default())).expect("existing legacy volume must be migrated"));
+        assert!(matches!(
+            legacy_iam_source_exists(Err(Error::ErasureReadQuorum)),
+            Err(Error::ErasureReadQuorum)
+        ));
+    }
+
+    #[test]
+    fn migration_errors_group_by_cause_and_retain_typed_record_context() {
+        use super::{Error, MigrationMetadataError};
+
+        for (make_error, message) in [
+            (
+                MigrationMetadataError::Empty as fn(String) -> MigrationMetadataError,
+                "empty legacy metadata",
+            ),
+            (MigrationMetadataError::Incompatible, "incompatible legacy metadata"),
+        ] {
+            let first: Error = make_error("buckets/first/.metadata.bin".into()).into();
+            let second: Error = make_error("buckets/second/.metadata.bin".into()).into();
+            assert_eq!(first, second, "record paths must not fragment error grouping");
+            assert_eq!(first.clone(), second, "cloning must preserve error grouping");
+
+            let io_error = std::io::Error::from(first);
+            let detail = io_error
+                .get_ref()
+                .and_then(|context| context.source())
+                .expect("record context must remain in the error source");
+            assert!(detail.downcast_ref::<MigrationMetadataError>().is_some());
+            assert!(detail.to_string().contains("buckets/first/.metadata.bin"));
+
+            let startup_error = super::migration_startup_error(make_error("buckets/startup/.metadata.bin".into()).into());
+            assert!(
+                startup_error
+                    .get_ref()
+                    .is_some_and(|source| source.is::<MigrationMetadataError>())
+            );
+            assert_eq!(startup_error.to_string(), format!("{message}: buckets/startup/.metadata.bin"));
+        }
+        assert_ne!(
+            Error::from(MigrationMetadataError::Empty("record".into())),
+            Error::from(MigrationMetadataError::Incompatible("record".into())),
+            "different migration failures must remain distinguishable"
+        );
+    }
+
+    #[test]
+    fn truncated_iam_listing_cannot_report_completed_migration() {
+        use super::next_iam_migration_page;
+        assert_eq!(next_iam_migration_page(false, Some("old".into()), None).expect("final page"), None);
+        assert_eq!(
+            next_iam_migration_page(true, Some("old".into()), Some("next".into())).expect("advancing page"),
+            Some("next".into())
+        );
+        for next in [None, Some(String::new()), Some("old".into())] {
+            assert!(next_iam_migration_page(true, Some("old".into()), next).is_err());
+        }
+    }
+
     use super::{normalize_bucket_meta_blob, normalize_iam_config_blob};
     use crate::bucket::replication::{
         BucketReplicationResyncStatus, ReplicationMigrationBridge, ResyncStatusType, TargetReplicationResyncStatus,
@@ -659,6 +771,13 @@ mod tests {
             .collect();
         crate::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), existing).await;
 
+        super::try_migrate_bucket_metadata(ecstore.clone())
+            .await
+            .expect("fresh stores do not require a legacy metadata bucket");
+        super::try_migrate_iam_config(ecstore.clone(), None)
+            .await
+            .expect("fresh stores do not require a legacy IAM bucket");
+
         let meta_path = format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}interop{SLASH_SEPARATOR}{BUCKET_METADATA_FILE}");
         let put_opts = ObjectOptions::default();
 
@@ -680,8 +799,31 @@ mod tests {
             .await
             .expect("seed .minio.sys bucket metadata");
 
-        // --- Run the real startup migration. ---
-        super::try_migrate_bucket_metadata(ecstore.clone()).await;
+        // A partial import must report failure, even if the main bucket
+        // metadata copied successfully before an incompatible resync record.
+        let resync_path = format!("{BUCKET_META_PREFIX}/interop/.replication/resync.bin");
+        ecstore
+            .put_object(
+                MIGRATING_META_BUCKET,
+                &resync_path,
+                &mut PutObjReader::from_vec(b"invalid resync metadata".to_vec()),
+                &put_opts,
+            )
+            .await
+            .expect("seed malformed legacy resync metadata");
+        assert!(
+            super::try_migrate_bucket_metadata(ecstore.clone()).await.is_err(),
+            "incompatible native metadata must not be reported as a completed migration"
+        );
+        ecstore
+            .delete_object(MIGRATING_META_BUCKET, &resync_path, ObjectOptions::default())
+            .await
+            .expect("remove invalid optional legacy resync record");
+
+        // Retry the real startup migration after repairing the source.
+        super::try_migrate_bucket_metadata(ecstore.clone())
+            .await
+            .expect("native bucket metadata migration completes");
 
         // --- The migrated `.rustfs.sys` blob must carry every MinIO config, ---
         // byte-identical to the source (typed XML/JSON parsing of these fields is