test(scanner): report Linux descriptor drift ledger (#7692)

Add a read-only descriptor ledger mode to the Scanner/Heal Linux evidence planner so release operators can separate current-head measured descriptors from old-head measured artifacts and case-level inputs before final bundle assembly.

Co-authored-by: zhi22915 <qiuzgang@gmail.com>

.github/workflows/rustfs-fault-tolerance-test.yml

@@ -0,0 +1,267 @@
+# RustFS Fault-Tolerance (degradation) Test
+#
+# Scenario suite for the 2026-09 degradation report: verifies read/write
+# behavior under drive and node loss against the erasure-coding contract and
+# snapshots health-endpoint responses at every tier.
+#
+#   A  single-node 4 drives (SNMD): hide 1/2/3 drives, restore
+#   B  multi-node 4x1 (one drive per node): stop 1/2/3 nodes, restore
+#   C  multi-node 4x4 (16 drives, EC:4): stop 1 node (read-quorum boundary),
+#      stop 2 nodes, restore
+#   C2 multi-node 4x4 with EC:8: 2 nodes down puts 8 drives online -- reads
+#      satisfy the EC read quorum while the lock majority is broken (the
+#      reported divergence window: reads 503 with lock_quorum_unavailable)
+#
+# Expectations come from product source (default_parity_count, erasure set
+# sizing). By default a "reads refused although the read quorum is met"
+# observation is reported as known-divergence without failing the suite; the
+# strict input turns those into failures once the product behavior changes.
+
+name: RustFS Fault-Tolerance Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      package_url:
+        description: 'Direct .deb URL. Required unless the nightly default is wanted.'
+        required: false
+        type: string
+      strict:
+        description: 'Fail the suite when reads are refused despite a met read quorum'
+        type: boolean
+        default: false
+      cleanup_before:
+        description: 'Reset the nodes before the test (DESTROYS existing data/config)'
+        type: boolean
+        default: true
+      cleanup_after:
+        description: 'Reset the nodes after the test (DESTROYS test data/config)'
+        type: boolean
+        default: true
+  repository_dispatch:
+    # Chain handoff: dispatched when the replication suite finishes, ahead of
+    # the performance suite.
+    types: [rustfs-chain-fault-tolerance]
+
+permissions:
+  contents: read
+
+# The suite stops services and hides drive dirs on the shared fleet; only one
+# functional suite may touch the environment at a time.
+concurrency:
+  group: rustfs-shared-functional-tests
+  cancel-in-progress: false
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
+  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
+  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
+  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
+  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
+
+jobs:
+  fault-tolerance-test:
+    runs-on: smoke-testing
+    timeout-minutes: 480
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    steps:
+      - name: Initialize functional evidence
+        id: evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-ft-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}/evidence"
+          {
+            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'EVIDENCE_DIR=%s/evidence\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+          } >> "${GITHUB_ENV}"
+
+      # auto-testing is private: clone it with the dedicated PF token (not
+      # GITHUB_TOKEN) and retry transient GitHub/network failures.
+      - name: Checkout auto-testing scripts (with retry)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          rm -rf auto-testing
+          for attempt in 1 2 3 4 5; do
+            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+              echo "auto-testing cloned (attempt ${attempt})"
+              exit 0
+            fi
+            rm -rf auto-testing
+            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+            sleep $((attempt * 15))
+          done
+          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+          exit 1
+
+      - name: Show environment
+        run: |
+          uname -a
+          jq --version
+          aws --version
+          df -h /data | tail -1
+
+      - name: Cleanup environment (before)
+        if: ${{ inputs.cleanup_before != 'false' }}
+        run: |
+          ./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}"
+
+      - name: Run fault-tolerance scenarios (A, B, C, C2)
+        id: test
+        run: |
+          ARGS=(--all -y --package-url "${{ inputs.package_url || env.RUSTFS_NIGHTLY_PACKAGE_URL }}" --log-file "${LOG_FILE}")
+          if [ "${{ inputs.strict }}" = "true" ]; then
+            ARGS+=(--strict)
+          fi
+          ./auto-testing/rustfs-fault-tolerance-test.sh "${ARGS[@]}"
+
+      - name: Generate report
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        run: |
+          set -euo pipefail
+          {
+            echo "# RustFS fault-tolerance test report"
+            echo ""
+            echo "- Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+            echo "- Package: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
+            echo "- Strict mode: ${{ inputs.strict || 'false' }}"
+            echo ""
+            echo "## Per-probe results"
+            echo ""
+            echo '```'
+            grep -E '^FT-(CASE|SUMMARY|REPORT)' "${LOG_FILE}" || echo "(no FT-CASE lines found)"
+            echo '```'
+            echo ""
+            echo "## Health snapshots"
+            echo ""
+            for f in "${FUNCTIONAL_ARTIFACTS_DIR}"/evidence/*.code; do
+              [ -e "${f}" ] || continue
+              printf '%s -> %s\n' "$(basename "${f}" .code)" "$(cat "${f}")"
+            done
+          } > "${REPORT_FILE}"
+
+      - name: Manage backlog issues (dedup / label / auto-close)
+        # Signal-based lifecycle: dedups against open issues by label
+        # (fault-tolerance + FT case), labels new issues (functional-test,
+        # category, case IDs, env), and closes fixed issues after a fully
+        # green run. Never acts on cancelled runs. Logic lives in
+        # auto-testing/scripts/issue_manager.py, which parses the
+        # FT-CASE verdict lines from the suite log.
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          SUITE: 'fault-tolerance'
+          SUITE_LABEL: 'Fault-Tolerance'
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
+            exit 0
+          fi
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
+            exit 0
+          fi
+          PACKAGE_URL='${{ inputs.package_url }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
+          fi
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.test.outcome }}" \
+            --report-file "${REPORT_FILE}" \
+            --log "${LOG_FILE}" \
+            --run-url "${RUN_URL}" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
+
+      - name: Upload test logs & evidence
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: rustfs-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/evidence/
+          if-no-files-found: warn
+
+      - name: Cleanup environment (after)
+        if: ${{ always() && inputs.cleanup_after != 'false' }}
+        run: |
+          ./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}" || true
+
+      - name: "Continue functional chain (next: Performance)"
+        # Only chain-triggered runs forward to the next suite; standalone
+        # workflow_dispatch runs stop after their own cleanup. A failed
+        # handoff retries, then files an alert issue in rustfs/backlog.
+        if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -uo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          DISPATCHED=0
+          for attempt in 1 2 3; do
+            if gh api --method POST repos/rustfs/rustfs/dispatches \
+              -f event_type='rustfs-chain-performance' \
+              -F 'client_payload[from_suite]=fault-tolerance'; then
+              echo "dispatched next suite Performance (attempt ${attempt})"
+              DISPATCHED=1
+              break
+            fi
+            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
+            sleep "${attempt}0"
+          done
+          if [ "${DISPATCHED:-0}" -ne 1 ]; then
+            echo "ERROR: functional chain stalled: could not dispatch Performance after 3 attempts" >&2
+            TITLE="[functional][chain] stalled after fault-tolerance (run ${GITHUB_RUN_ID})"
+            BODY_FILE="$(mktemp)"
+            {
+              echo "The functional chain could not hand off from **fault-tolerance** to **Performance** after 3 attempts."
+              echo ""
+              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+              echo "- Expected next event: 'rustfs-chain-performance'"
+              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+              echo "- Recovery: re-dispatch manually with"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-performance'"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+            } > "${BODY_FILE}"
+            gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test \
+              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
+              || echo "could not file the stall alert issue either; check the token" >&2
+            exit 1
+          fi
+
+      - name: Notify on failure
+        if: failure()
+        run: |
+          echo "RustFS fault-tolerance test failed"
+          echo "Package source: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
+          echo "See the uploaded log artifact and FT-CASE lines for details."

.github/workflows/rustfs-functional-chain.yml

@@ -14,7 +14,7 @@
 
 # Functional chain driver: runs the ten functional suites in a fixed order
 # (upgrade -> s3 -> kms -> tier -> storage -> heal -> pool -> security ->
-# replication -> performance). Each suite attempts the next handoff even
+# replication -> fault-tolerance -> performance). Each suite attempts the next handoff even
 # when its tests fail.
 #
 # Each suite workflow can still be dispatched standalone (workflow_dispatch);

.github/workflows/rustfs-heal-test.yml

@@ -280,65 +280,51 @@ jobs:
           fi
           rm -f "${B64_FILE}"
 
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+      - name: Manage backlog issues (dedup / label / auto-close)
+        # Replaces the old per-run failure filing. One entry point that:
+        # - dedups by signal: failing cases are matched against open backlog
+        #   issues by label (category + case ID); covered cases become a
+        #   comment on the existing issue, only uncovered cases file a new one
+        # - labels new issues (functional-test, category, case IDs, env)
+        # - closes fixed issues after a fully green run
+        # - never files or closes on cancelled runs
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'heal'
           SUITE_LABEL: 'Heal'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
             exit 0
           fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          PACKAGE_URL='${{ inputs.package_url }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          echo "filed backlog issue for suite ${SUITE}"
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.test.outcome }}" \
+            --report-file "${REPORT_FILE}" \
+            --log "${LOG_FILE}" \
+            --run-url "${RUN_URL}" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
 
       - name: Upload test logs
         if: ${{ always() && steps.evidence.outcome == 'success' }}

.github/workflows/rustfs-kms-test.yml

@@ -4,9 +4,8 @@ on:
   workflow_dispatch:
     inputs:
       rustfs_version:
-        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
+        description: 'RustFS release tag to test (leave empty to use the latest nightly deb)'
         required: false
-        default: '1.0.0-rc.4-preview.1'
       package_url:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false
@@ -231,65 +230,55 @@ jobs:
           fi
           rm -f "${B64_FILE}"
 
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+      - name: Manage backlog issues (dedup / label / auto-close)
+        # Replaces the old per-run failure filing. One entry point that:
+        # - dedups by signal: failing cases are matched against open backlog
+        #   issues by label (category + case ID); covered cases become a
+        #   comment on the existing issue, only uncovered cases file a new one
+        # - labels new issues (functional-test, category, case IDs, env)
+        # - closes fixed issues after a fully green run
+        # - never files or closes on cancelled runs
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'kms'
           SUITE_LABEL: 'KMS'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
             exit 0
           fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          echo "filed backlog issue for suite ${SUITE}"
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.test.outcome }}" \
+            --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" \
+            --report-file "${REPORT_FILE}" \
+            --log "${LOG_FILE}" \
+            --run-url "${RUN_URL}" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
 
       - name: Upload report and logs
         if: ${{ always() && steps.evidence.outcome == 'success' }}

.github/workflows/rustfs-performance-test.yml

@@ -237,65 +237,48 @@ jobs:
             echo "created ${REPORT_PATH} in rustfs/dashboard"
           fi
 
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.benchmark.outcome == 'failure' || steps.benchmark.outcome == 'cancelled') }}
+      - name: Manage backlog issues (dedup / label / auto-close)
+        # Signal-based lifecycle: dedups against open issues by label,
+        # labels new issues (functional-test, category, case IDs, env),
+        # closes fixed issues after a fully green run. Never acts on
+        # cancelled runs. Logic lives in auto-testing/scripts/.
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'performance'
           SUITE_LABEL: 'Performance'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
             exit 0
           fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          PACKAGE_URL='${{ inputs.package_url }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          echo "filed backlog issue for suite ${SUITE}"
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.benchmark.outcome }}" \
+            --report-file "${REPORT_FILE}" \
+            --log "${LOG_FILE}" \
+            --run-url "${RUN_URL}" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
 
       - name: Upload test logs & results
         if: ${{ always() && steps.evidence.outcome == 'success' }}

.github/workflows/rustfs-pool-expand-test.yml

@@ -556,64 +556,54 @@ jobs:
           fi
           rm -f "${B64_FILE}"
 
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.pool_test.outcome == 'failure' || steps.pool_test.outcome == 'cancelled') }}
+      - name: Manage backlog issues (dedup / label / auto-close)
+        # Replaces the old per-run failure filing. One entry point that:
+        # - dedups by signal: failing cases are matched against open backlog
+        #   issues by label (category + case ID); covered cases become a
+        #   comment on the existing issue, only uncovered cases file a new one
+        # - labels new issues (functional-test, category, case IDs, env)
+        # - closes fixed issues after a fully green run
+        # - never files or closes on cancelled runs
+        if: ${{ always() }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
           SUITE: 'pool'
           SUITE_LABEL: 'Pool expansion'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '${{ env.POOL_ARTIFACT_DIR }}/pool-report.md'
-          LOG_FILE: '${{ env.POOL_ARTIFACT_DIR }}/pool-test.log'
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
             exit 0
           fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          echo "filed backlog issue for suite ${SUITE}"
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.pool_test.outcome }}" \
+            --report-file "${POOL_ARTIFACT_DIR}/pool-report.md" \
+            --log "${POOL_ARTIFACT_DIR}/pool-test.log" \
+            --run-url "${RUN_URL}" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
 
       - name: Upload test logs
         if: always()

.github/workflows/rustfs-replication-test.yml

@@ -18,9 +18,8 @@ on:
   workflow_dispatch:
     inputs:
       rustfs_version:
-        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
+        description: 'RustFS release tag to test (leave empty to use the latest nightly deb)'
         required: false
-        default: '1.0.0-rc.4-preview.1'
       package_url:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false
@@ -239,65 +238,55 @@ jobs:
           fi
           rm -f "${B64_FILE}"
 
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+      - name: Manage backlog issues (dedup / label / auto-close)
+        # Replaces the old per-run failure filing. One entry point that:
+        # - dedups by signal: failing cases are matched against open backlog
+        #   issues by label (category + case ID); covered cases become a
+        #   comment on the existing issue, only uncovered cases file a new one
+        # - labels new issues (functional-test, category, case IDs, env)
+        # - closes fixed issues after a fully green run
+        # - never files or closes on cancelled runs
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'replication'
           SUITE_LABEL: 'Replication (bucket + site)'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
             exit 0
           fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          echo "filed backlog issue for suite ${SUITE}"
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.test.outcome }}" \
+            --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" \
+            --report-file "${REPORT_FILE}" \
+            --log "${LOG_FILE}" \
+            --run-url "${RUN_URL}" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
 
       - name: Upload report and logs
         if: ${{ always() && steps.evidence.outcome == 'success' }}
@@ -329,7 +318,7 @@ jobs:
             '
           done
 
-      - name: "Continue functional chain (next: Performance)"
+      - name: "Continue functional chain (next: Fault tolerance)"
         if: ${{ always() && github.event_name == 'repository_dispatch' }}
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
@@ -342,9 +331,9 @@ jobs:
           DISPATCHED=0
           for attempt in 1 2 3; do
             if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-performance' \
+              -f event_type='rustfs-chain-fault-tolerance' \
               -F 'client_payload[from_suite]=replication'; then
-              echo "dispatched next suite Performance (attempt ${attempt})"
+              echo "dispatched next suite Fault tolerance (attempt ${attempt})"
               DISPATCHED=1
               break
             fi
@@ -352,19 +341,19 @@ jobs:
             sleep "${attempt}0"
           done
           if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch Performance after 3 attempts" >&2
+            echo "ERROR: functional chain stalled: could not dispatch Fault tolerance after 3 attempts" >&2
             TITLE="[functional][chain] stalled after replication (run ${GITHUB_RUN_ID})"
             BODY_FILE="$(mktemp)"
             trap 'rm -f "${BODY_FILE}"' EXIT
             {
-              echo "The functional chain could not hand off from **replication** to **Performance** after 3 attempts."
+              echo "The functional chain could not hand off from **replication** to **Fault tolerance** after 3 attempts."
               echo ""
               echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-performance'"
+              echo "- Expected next event: 'rustfs-chain-fault-tolerance'"
               echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
               echo "- Recovery: re-dispatch manually with"
               FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-performance'"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-fault-tolerance'"
               FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
             } > "${BODY_FILE}"
             gh issue create -R rustfs/backlog --title "${TITLE}" \

.github/workflows/rustfs-s3-compat-test.yml

@@ -4,9 +4,8 @@ on:
   workflow_dispatch:
     inputs:
       rustfs_version:
-        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
+        description: 'RustFS release tag to test (leave empty to use the latest nightly deb)'
         required: false
-        default: '1.0.0-rc.4-preview.1'
       package_url:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false
@@ -208,65 +207,55 @@ jobs:
           fi
           rm -f "${B64_FILE}"
 
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+      - name: Manage backlog issues (dedup / label / auto-close)
+        # Replaces the old per-run failure filing. One entry point that:
+        # - dedups by signal: failing cases are matched against open backlog
+        #   issues by label (category + case ID); covered cases become a
+        #   comment on the existing issue, only uncovered cases file a new one
+        # - labels new issues (functional-test, category, case IDs, env)
+        # - closes fixed issues after a fully green run
+        # - never files or closes on cancelled runs
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 's3'
           SUITE_LABEL: 'S3 compatibility'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
             exit 0
           fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          echo "filed backlog issue for suite ${SUITE}"
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.test.outcome }}" \
+            --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" \
+            --report-file "${REPORT_FILE}" \
+            --log "${LOG_FILE}" \
+            --run-url "${RUN_URL}" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
 
       - name: Upload report and logs
         if: ${{ always() && steps.evidence.outcome == 'success' }}

.github/workflows/rustfs-security-test.yml

@@ -18,9 +18,8 @@ on:
   workflow_dispatch:
     inputs:
       rustfs_version:
-        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
+        description: 'RustFS release tag to test (leave empty to use the latest nightly deb)'
         required: false
-        default: '1.0.0-rc.4-preview.1'
       package_url:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false
@@ -240,65 +239,54 @@ jobs:
           fi
           rm -f "${B64_FILE}"
 
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+      - name: Manage backlog issues (dedup / label / auto-close)
+        # Replaces the old per-run failure filing. One entry point that:
+        # - dedups by signal: failing cases are matched against open backlog
+        #   issues by label (category + case ID); covered cases become a
+        #   comment on the existing issue, only uncovered cases file a new one
+        # - labels new issues (functional-test, category, case IDs, env)
+        # - closes fixed issues after a fully green run
+        # - never files or closes on cancelled runs
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
           SUITE: 'security'
           SUITE_LABEL: 'Security'
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/report.md
-          LOG_FILE: ''
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
             exit 0
           fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          echo "filed backlog issue for suite ${SUITE}"
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.test.outcome }}" \
+            --report-file "${SECURITY_ARTIFACTS_DIR}/report.md" \
+            --log "${SECURITY_ARTIFACTS_DIR}/suite.log" \
+            --run-url "${RUN_URL}" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
 
       - name: Upload report and logs
         if: ${{ always() && steps.evidence.outcome == 'success' }}

.github/workflows/rustfs-storage-test.yml

@@ -4,9 +4,8 @@ on:
   workflow_dispatch:
     inputs:
       rustfs_version:
-        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
+        description: 'RustFS release tag to test (leave empty to use the latest nightly deb)'
         required: false
-        default: '1.0.0-rc.4-preview.1'
       package_url:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false
@@ -223,65 +222,55 @@ jobs:
           fi
           rm -f "${B64_FILE}"
 
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+      - name: Manage backlog issues (dedup / label / auto-close)
+        # Replaces the old per-run failure filing. One entry point that:
+        # - dedups by signal: failing cases are matched against open backlog
+        #   issues by label (category + case ID); covered cases become a
+        #   comment on the existing issue, only uncovered cases file a new one
+        # - labels new issues (functional-test, category, case IDs, env)
+        # - closes fixed issues after a fully green run
+        # - never files or closes on cancelled runs
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'storage'
           SUITE_LABEL: 'Storage engine'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
             exit 0
           fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          echo "filed backlog issue for suite ${SUITE}"
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.test.outcome }}" \
+            --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" \
+            --report-file "${REPORT_FILE}" \
+            --log "${LOG_FILE}" \
+            --run-url "${RUN_URL}" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
 
       - name: Upload report and logs
         if: ${{ always() && steps.evidence.outcome == 'success' }}

.github/workflows/rustfs-tier-test.yml

@@ -4,9 +4,8 @@ on:
   workflow_dispatch:
     inputs:
       rustfs_version:
-        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
+        description: 'RustFS release tag to test (leave empty to use the latest nightly deb)'
         required: false
-        default: '1.0.0-rc.4-preview.1'
       package_url:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false
@@ -495,75 +494,55 @@ jobs:
           fi
           [ "${failed}" -eq 0 ]
 
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled' || steps.evidence_verify.outcome == 'failure' || steps.evidence_verify.outcome == 'cancelled' || steps.gate.outcome == 'failure' || steps.gate.outcome == 'cancelled') }}
+      - name: Manage backlog issues (dedup / label / auto-close)
+        # Replaces the old per-run failure filing. One entry point that:
+        # - dedups by signal: failing cases are matched against open backlog
+        #   issues by label (category + case ID); covered cases become a
+        #   comment on the existing issue, only uncovered cases file a new one
+        # - labels new issues (functional-test, category, case IDs, env)
+        # - closes fixed issues after a fully green run
+        # - never files or closes on cancelled runs
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
           SUITE: 'tier'
           SUITE_LABEL: 'Tier'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          EVIDENCE_DIR: ${{ env.TIER_ARTIFACTS_DIR }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
-          VERIFY_OUTCOME: ${{ steps.evidence_verify.outcome }}
-          GATE_OUTCOME: ${{ steps.gate.outcome }}
-          REPORT_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier-report.md
-          LOG_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier.log
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
             exit 0
           fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo "- Evidence initialization: ${EVIDENCE_OUTCOME}"
-            echo "- Evidence verification: ${VERIFY_OUTCOME}"
-            echo "- Final gate: ${GATE_OUTCOME}"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ "${EVIDENCE_OUTCOME}" != "success" ]; then
-              echo "(the run evidence directory was rejected; its contents were not read)"
-            elif [ ! -d "${EVIDENCE_DIR}" ] || [ -L "${EVIDENCE_DIR}" ]; then
-              echo "(the run evidence directory is missing or unsafe; its contents were not read)"
-            elif [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          echo "filed backlog issue for suite ${SUITE}"
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.test.outcome }}" \
+            --report "${TIER_ARTIFACTS_DIR}/rustfs-tier-cases.md" \
+            --report-file "${TIER_ARTIFACTS_DIR}/rustfs-tier-report.md" \
+            --log "${TIER_ARTIFACTS_DIR}/rustfs-tier.log" \
+            --run-url "${RUN_URL}" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
 
       - name: "Continue functional chain (next: Storage engine)"
         # Only chain-triggered runs forward to the next suite; standalone

.github/workflows/rustfs-upgrade-test.yml

@@ -306,65 +306,62 @@ jobs:
           fi
           rm -f "${B64_FILE}"
 
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+      - name: Manage backlog issues (dedup / label / auto-close)
+        # Replaces the old per-run failure filing. One entry point that:
+        # - dedups by signal: failing cases are matched against open backlog
+        #   issues by label (category + case ID); covered cases become a
+        #   comment on the existing issue, only uncovered cases file a new one
+        # - labels new issues (functional-test, category, case IDs, env)
+        # - closes fixed issues after a fully green run
+        # - never files or closes on cancelled runs
+        if: ${{ always() && steps.evidence.outcome == 'success' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
           SUITE: 'upgrade'
           SUITE_LABEL: 'Upgrade compatibility'
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
             exit 0
           fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+          if [ ! -f auto-testing/scripts/issue_manager.py ]; then
+            echo "issue_manager.py not found in auto-testing checkout; skipping"
             exit 0
           fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+          FROM_URL='${{ inputs.from_url }}'
+          FROM_VERSION='${{ inputs.from_version }}'
+          TO_URL='${{ inputs.to_url }}'
+          TO_VERSION='${{ inputs.to_version }}'
+          PACKAGE_SOURCE=""
+          if [ -n "${TO_URL}" ]; then
+            PACKAGE_SOURCE="to ${TO_URL}"
+          elif [ -n "${TO_VERSION}" ]; then
+            PACKAGE_SOURCE="to version ${TO_VERSION}"
+          else
+            PACKAGE_SOURCE="to ${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          echo "filed backlog issue for suite ${SUITE}"
+          if [ -n "${FROM_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_SOURCE}; from ${FROM_URL}"
+          elif [ -n "${FROM_VERSION}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_SOURCE}; from version ${FROM_VERSION}"
+          fi
+          python3 auto-testing/scripts/issue_manager.py handle \
+            --repo rustfs/backlog \
+            --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
+            --outcome "${{ steps.test.outcome }}" \
+            --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" \
+            --report-file "${REPORT_FILE}" \
+            --log "${LOG_FILE}" \
+            --run-url "${RUN_URL}" \
+            --run-id "${GITHUB_RUN_ID}" \
+            --attempt "${GITHUB_RUN_ATTEMPT}" \
+            --commit "${GITHUB_SHA}" \
+            --trigger "${{ github.event_name }}" \
+            --package-source "${PACKAGE_SOURCE}" \
+            --date "$(date -u +%Y-%m-%d)"
 
       - name: Upload report and logs
         if: ${{ always() && steps.evidence.outcome == 'success' }}

crates/ecstore/src/core/pools.rs

@@ -17609,6 +17609,17 @@ impl ECStore {
         }
         self.persist_decommission_durable_ilm_receipt(source_pool_idx, target_pool_idx, &receipt)
             .await?;
+        self.decommission_durable_ilm_receipt_path_for_test(source_pool_idx, source_path, record)
+            .await
+    }
+
+    #[cfg(all(test, feature = "test-util"))]
+    pub(crate) async fn decommission_durable_ilm_receipt_path_for_test(
+        &self,
+        source_pool_idx: usize,
+        source_path: &str,
+        record: &ValidatedDurableIlmRecord,
+    ) -> Result<String> {
         let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
         Ok(decommission_durable_ilm_receipt_path(&run_token, source_path, record.id_kind, &record.id))
     }

crates/ecstore/src/disk/error.rs

@@ -865,9 +865,8 @@ mod tests {
             io::ErrorKind::PermissionDenied,
             "staging rejected",
         )));
-        let cloned = marked.clone();
         assert!(marked.is_conditional_file_not_committed());
-        assert!(cloned.is_conditional_file_not_committed());
+        assert!(marked.is_conditional_file_not_committed());
         assert!(!DiskError::Timeout.is_conditional_file_not_committed());
         assert!(
             !DiskError::Io(io::Error::new(io::ErrorKind::PermissionDenied, "rename rejected"))

crates/ecstore/src/set_disk/ops/object.rs

@@ -18604,71 +18604,75 @@ mod put_object_tmp_cleanup_tests {
     }
 
     #[tokio::test]
+    #[serial_test::serial(capacity_dirty_scope)]
     async fn cancelled_rename_keeps_namespace_lock_until_publication() {
-        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
-        let bucket = "put-commit-lock-cancelled-rename";
-        let object = "commit-lock-cancelled-rename-object";
-        for disk in &disk_stores {
-            disk.make_volume(bucket).await.expect("bucket volume should be created");
-        }
-
-        let rename_tasks = rename_fanout_barrier::observe_tasks(object);
-        let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
-        let first_store = Arc::clone(&set_disks);
-        let first = tokio::spawn(async move {
-            let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
-            first_store
-                .put_object(bucket, object, &mut reader, &ObjectOptions::default())
-                .await
-        });
-        tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
-            .await
-            .expect("first PUT should pause during the authoritative rename");
-
-        let second_namespace_barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::BeforeNamespace);
-        let second_store = Arc::clone(&set_disks);
-        let second = tokio::spawn(async move {
-            let mut reader = PutObjReader::from_vec(vec![b'2'; TEST_OBJECT_SIZE]);
-            second_store
-                .put_object(bucket, object, &mut reader, &ObjectOptions::default())
-                .await
-        });
-        second_namespace_barrier.release_and_wait_until_namespace_pending().await;
-
-        first.abort();
-        assert!(
-            first
-                .await
-                .expect_err("the first request should be cancelled while rename is parked")
-                .is_cancelled()
-        );
-        tokio::task::yield_now().await;
-        assert!(
-            !second.is_finished(),
-            "the second writer must remain blocked by the cancelled commit owner"
-        );
-
-        rename_barrier.release();
-        drop(rename_barrier);
-        tokio::time::timeout(Duration::from_secs(30), async {
-            while rename_tasks.running() != 0 {
-                tokio::task::yield_now().await;
+        temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("false"))], async {
+            let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+            let bucket = "put-commit-lock-cancelled-rename";
+            let object = "commit-lock-cancelled-rename-object";
+            for disk in &disk_stores {
+                disk.make_volume(bucket).await.expect("bucket volume should be created");
             }
-        })
-        .await
-        .expect("the cancelled owner's rename fanout should drain");
-        second
-            .await
-            .expect("second overwrite task should join")
-            .expect("second overwrite should commit after the cancelled owner reaches publication");
 
-        let mut reader = set_disks
-            .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
+            let rename_tasks = rename_fanout_barrier::observe_tasks(object);
+            let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
+            let first_store = Arc::clone(&set_disks);
+            let first = tokio::spawn(async move {
+                let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
+                first_store
+                    .put_object(bucket, object, &mut reader, &ObjectOptions::default())
+                    .await
+            });
+            tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
+                .await
+                .expect("first PUT should pause during the authoritative rename");
+
+            let second_namespace_barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::BeforeNamespace);
+            let second_store = Arc::clone(&set_disks);
+            let second = tokio::spawn(async move {
+                let mut reader = PutObjReader::from_vec(vec![b'2'; TEST_OBJECT_SIZE]);
+                second_store
+                    .put_object(bucket, object, &mut reader, &ObjectOptions::default())
+                    .await
+            });
+            second_namespace_barrier.release_and_wait_until_namespace_pending().await;
+
+            first.abort();
+            assert!(
+                first
+                    .await
+                    .expect_err("the first request should be cancelled while rename is parked")
+                    .is_cancelled()
+            );
+            tokio::task::yield_now().await;
+            assert!(
+                !second.is_finished(),
+                "the second writer must remain blocked by the cancelled commit owner"
+            );
+
+            rename_barrier.release();
+            drop(rename_barrier);
+            tokio::time::timeout(Duration::from_secs(30), async {
+                while rename_tasks.running() != 0 {
+                    tokio::task::yield_now().await;
+                }
+            })
             .await
-            .expect("the latest overwrite should be readable");
-        let mut body = Vec::new();
-        reader.stream.read_to_end(&mut body).await.expect("latest body should drain");
-        assert_eq!(body, vec![b'2'; TEST_OBJECT_SIZE]);
+            .expect("the cancelled owner's rename fanout should drain");
+            second
+                .await
+                .expect("second overwrite task should join")
+                .expect("second overwrite should commit after the cancelled owner reaches publication");
+
+            let mut reader = set_disks
+                .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
+                .await
+                .expect("the latest overwrite should be readable");
+            let mut body = Vec::new();
+            reader.stream.read_to_end(&mut body).await.expect("latest body should drain");
+            assert_eq!(body, vec![b'2'; TEST_OBJECT_SIZE]);
+        })
+        .await;
     }
 
     #[tokio::test]

crates/ecstore/src/store/init.rs

@@ -9643,7 +9643,13 @@ mod tests {
         com::save_config(store.pools[1].clone(), &manual_task_path, manual_task_bytes.clone())
             .await
             .expect("target task rewrite should invalidate cached metadata before the quorum check");
-        let target_task_set = store.pools[1].get_disks_by_key(&manual_task_path);
+        let manual_task_record =
+            validate_durable_ilm_record(&manual_task_path, &manual_task_bytes).expect("manual task should validate");
+        let manual_task_receipt_path = store
+            .decommission_durable_ilm_receipt_path_for_test(0, &manual_task_path, &manual_task_record)
+            .await
+            .expect("manual task receipt path should resolve");
+        let target_task_set = store.pools[1].get_disks_by_key(&manual_task_receipt_path);
         let original_target_task_disks = {
             let mut disks = target_task_set.disks.write().await;
             let original = disks.clone();

rustfs/src/app/object/get.rs

@@ -2133,7 +2133,7 @@ impl DefaultObjectUsecase {
     }
 
     #[allow(clippy::too_many_arguments)]
-    fn build_reader_blob<R>(
+    async fn build_reader_blob<R>(
         reader: R,
         response_content_length: i64,
         request_id: &str,
@@ -2144,10 +2144,12 @@ impl DefaultObjectUsecase {
         key: &str,
         lifecycle: GetObjectBodyLifecycle,
         resume: Option<GetObjectResumeControl<R>>,
-    ) -> StreamingBlob
+    ) -> S3Result<StreamingBlob>
     where
         R: AsyncRead + Send + Sync + Unpin + 'static,
     {
+        use tokio::io::AsyncReadExt as _;
+
         let streaming_blob_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
         let expected = usize::try_from(response_content_length.max(0)).unwrap_or(usize::MAX);
         let tuned_stream_buffer_size =
@@ -2163,7 +2165,7 @@ impl DefaultObjectUsecase {
             );
         }
         let handoff_start = get_stage_metrics_enabled.then(std::time::Instant::now);
-        let reader = GetObjectStreamingReader::new(
+        let mut reader = GetObjectStreamingReader::new(
             reader,
             bucket,
             key,
@@ -2174,6 +2176,17 @@ impl DefaultObjectUsecase {
             lifecycle,
             resume,
         );
+        let mut prefix = [0_u8; 1];
+        let prefix_len = if expected == 0 {
+            0
+        } else {
+            reader
+                .read_exact(&mut prefix)
+                .await
+                .map_err(|error| map_get_object_reader_error(StorageError::from(error)))?;
+            1
+        };
+        let reader = std::io::Cursor::new(prefix).take(prefix_len).chain(reader);
         let stream = GetObjectReaderStream::new(reader, stream_buffer_size, expected, stream_strategy.as_str(), buffer_source)
             .with_diagnostics(bucket, key, request_id);
         let blob = StreamingBlob::new(stream);
@@ -2187,7 +2200,7 @@ impl DefaultObjectUsecase {
             );
         }
         record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_STREAMING_BLOB, streaming_blob_start);
-        blob
+        Ok(blob)
     }
 
     fn init_get_object_bootstrap(&self, bucket: &str, key: &str, request_id: &str) -> S3Result<GetObjectBootstrap> {
@@ -3161,7 +3174,7 @@ impl DefaultObjectUsecase {
             let (stream_buffer_size, stream_strategy) =
                 Self::select_stream_buffer_strategy(response_content_length, optimal_buffer_size, enable_readahead, has_range);
             record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_STREAM_STRATEGY, stream_strategy_start);
-            return Ok(Self::build_reader_blob(
+            return Self::build_reader_blob(
                 final_stream,
                 response_content_length,
                 request_id,
@@ -3172,7 +3185,8 @@ impl DefaultObjectUsecase {
                 key,
                 lifecycle,
                 resume(info),
-            ));
+            )
+            .await;
         }
 
         if let Some(buffered_body) = buffered_body {
@@ -3239,7 +3253,7 @@ impl DefaultObjectUsecase {
         let (stream_buffer_size, stream_strategy) =
             Self::select_stream_buffer_strategy(response_content_length, optimal_buffer_size, enable_readahead, has_range);
         record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_STREAM_STRATEGY, stream_strategy_start);
-        Ok(Self::build_reader_blob(
+        Self::build_reader_blob(
             final_stream,
             response_content_length,
             request_id,
@@ -3250,7 +3264,8 @@ impl DefaultObjectUsecase {
             key,
             lifecycle,
             resume(info),
-        ))
+        )
+        .await
     }
 
     #[allow(clippy::too_many_arguments)]
@@ -5847,8 +5862,11 @@ mod tests {
     }
 
     impl AsyncRead for ReadProbeReader {
-        fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
+        fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
             self.reads.fetch_add(1, AtomicOrdering::Relaxed);
+            if buf.remaining() > 0 {
+                buf.put_slice(b"x");
+            }
             Poll::Ready(Ok(()))
         }
     }
@@ -6448,12 +6466,11 @@ mod tests {
         )
         .await
         .expect("reservation bypass must construct the normal streaming fallback");
-        let chunk = fallback_body
-            .next()
-            .await
-            .expect("fallback stream must yield a body chunk")
-            .expect("fallback stream must not fail");
-        assert_eq!(chunk, Bytes::from_static(b"body"));
+        let mut received = Vec::new();
+        while let Some(chunk) = fallback_body.next().await {
+            received.extend_from_slice(&chunk.expect("fallback stream must not fail"));
+        }
+        assert_eq!(received, b"body");
         assert!(fallback_reads.load(AtomicOrdering::Relaxed) > 0);
         assert_eq!(readers.load(AtomicOrdering::Relaxed), 0, "cold-fill materialization must remain unopened");
         assert_eq!(coordinator.active_session_count_for_test(), 0);
@@ -7773,6 +7790,151 @@ mod tests {
         )
     }
 
+    #[tokio::test]
+    async fn build_reader_blob_rejects_quorum_failure_before_handoff() {
+        let result = DefaultObjectUsecase::build_reader_blob(
+            FailAtEndReader::new(
+                b"",
+                Some(std::io::Error::other(StorageError::InsufficientReadQuorum(
+                    "test-bucket".to_string(),
+                    "unavailable-object".to_string(),
+                ))),
+            ),
+            5,
+            "req-preheader-quorum",
+            None,
+            64,
+            GetObjectStreamStrategy::Standard,
+            "test-bucket",
+            "unavailable-object",
+            GetObjectBodyLifecycle::disabled(),
+            None,
+        )
+        .await;
+
+        let error = result.expect_err("a read quorum failure before the first byte must reject response construction");
+        assert_eq!(error.code(), &S3ErrorCode::Custom("SlowDownRead".into()));
+        assert_eq!(error.status_code(), Some(StatusCode::SERVICE_UNAVAILABLE));
+        assert_eq!(error.message(), Some("Resource requested is unreadable, please reduce your request rate"));
+    }
+
+    #[tokio::test]
+    async fn build_reader_blob_preserves_primed_byte_for_full_and_range() {
+        for (request_id, content_range) in [("req-preheader-full", None), ("req-preheader-range", Some("bytes 10-14/100"))] {
+            let reads = Arc::new(AtomicUsize::new(0));
+            let mut body = DefaultObjectUsecase::build_reader_blob(
+                DataProbeReader {
+                    reads: Arc::clone(&reads),
+                    data: std::io::Cursor::new(b"hello".to_vec()),
+                },
+                5,
+                request_id,
+                content_range,
+                64,
+                GetObjectStreamStrategy::Standard,
+                "test-bucket",
+                "test-object",
+                GetObjectBodyLifecycle::disabled(),
+                None,
+            )
+            .await
+            .expect("the first byte should be available before response handoff");
+
+            assert_eq!(reads.load(AtomicOrdering::Relaxed), 1, "response construction must prime one byte");
+            let mut received = Vec::new();
+            while let Some(chunk) = body.next().await {
+                received.extend_from_slice(&chunk.expect("the primed body should remain readable"));
+            }
+            assert_eq!(received, b"hello", "the primed byte must be delivered exactly once");
+        }
+    }
+
+    #[tokio::test]
+    async fn build_reader_blob_does_not_poll_empty_object() {
+        let reads = Arc::new(AtomicUsize::new(0));
+        let mut body = DefaultObjectUsecase::build_reader_blob(
+            ReadProbeReader {
+                reads: Arc::clone(&reads),
+            },
+            0,
+            "req-empty-object",
+            None,
+            64,
+            GetObjectStreamStrategy::Standard,
+            "test-bucket",
+            "empty-object",
+            GetObjectBodyLifecycle::disabled(),
+            None,
+        )
+        .await
+        .expect("an empty response should not require a storage read");
+
+        assert_eq!(reads.load(AtomicOrdering::Relaxed), 0);
+        assert!(body.next().await.is_none());
+        assert_eq!(reads.load(AtomicOrdering::Relaxed), 0);
+    }
+
+    #[tokio::test]
+    async fn build_reader_blob_leaves_later_failure_in_body_stream() {
+        let mut body = DefaultObjectUsecase::build_reader_blob(
+            FailAtEndReader::new(b"h", Some(std::io::Error::other("failure after handoff"))),
+            5,
+            "req-postheader-failure",
+            None,
+            64,
+            GetObjectStreamStrategy::Standard,
+            "test-bucket",
+            "later-failure-object",
+            GetObjectBodyLifecycle::disabled(),
+            None,
+        )
+        .await
+        .expect("the available first byte should allow response handoff");
+
+        let first = body
+            .next()
+            .await
+            .expect("the primed byte must be present")
+            .expect("the primed byte must be successful");
+        assert_eq!(first, Bytes::from_static(b"h"));
+        let error = body
+            .next()
+            .await
+            .expect("the later read must produce a body result")
+            .expect_err("a failure after the first byte must stay in the body stream");
+        assert!(error.to_string().contains("failure after handoff"));
+    }
+
+    #[tokio::test]
+    async fn build_reader_blob_resume_offset_includes_primed_byte() {
+        let reopen_count = Arc::new(AtomicUsize::new(0));
+        let control = counting_resume_control(Arc::clone(&reopen_count), |emitted| {
+            assert_eq!(emitted, 1, "resume must start after the byte consumed before response handoff");
+            Ok(FailAtEndReader::new(b"ello", None))
+        });
+        let mut body = DefaultObjectUsecase::build_reader_blob(
+            FailAtEndReader::new(b"h", Some(relocation_read_error())),
+            5,
+            "req-preheader-resume-offset",
+            None,
+            64,
+            GetObjectStreamStrategy::Standard,
+            "test-bucket",
+            "relocated-object",
+            GetObjectBodyLifecycle::disabled(),
+            Some(control),
+        )
+        .await
+        .expect("the first byte should permit response handoff before relocation");
+
+        let mut received = Vec::new();
+        while let Some(chunk) = body.next().await {
+            received.extend_from_slice(&chunk.expect("resume should complete the body"));
+        }
+        assert_eq!(received, b"hello");
+        assert_eq!(reopen_count.load(Ordering::Relaxed), 1);
+    }
+
     #[tokio::test]
     async fn get_object_streaming_reader_resumes_after_relocation_error() {
         use tokio::io::AsyncReadExt;
@@ -9041,7 +9203,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn build_get_object_body_keeps_large_objects_on_streaming_path_without_preread() {
+    async fn build_get_object_body_primes_large_stream_before_handoff() {
         let reads = Arc::new(AtomicUsize::new(0));
         let reader = ReadProbeReader {
             reads: Arc::clone(&reads),
@@ -9074,13 +9236,13 @@ mod tests {
 
         assert_eq!(
             reads.load(AtomicOrdering::Relaxed),
-            0,
-            "large-object response construction should not pre-read object data"
+            1,
+            "large-object response construction should prime exactly one byte"
         );
     }
 
     #[tokio::test]
-    async fn build_get_object_body_keeps_large_encrypted_objects_on_streaming_path_without_preread() {
+    async fn build_get_object_body_primes_large_encrypted_stream_before_handoff() {
         let reads = Arc::new(AtomicUsize::new(0));
         let reader = ReadProbeReader {
             reads: Arc::clone(&reads),
@@ -9113,8 +9275,8 @@ mod tests {
 
         assert_eq!(
             reads.load(AtomicOrdering::Relaxed),
-            0,
-            "large encrypted object response construction should not pre-read object data"
+            1,
+            "large encrypted object response construction should prime exactly one byte"
         );
     }
 
@@ -9284,8 +9446,8 @@ mod tests {
         assert_eq!(fill, rustfs_object_data_cache::ObjectDataCacheFillResult::SkippedSizeMismatch);
         assert_eq!(
             reads.load(AtomicOrdering::Relaxed),
-            0,
-            "size-mismatched rejected fill should construct the fallback stream without pre-reading"
+            1,
+            "size-mismatched rejected fill should prime the fallback stream before handoff"
         );
         assert!(
             matches!(lookup_after_mismatch, rustfs_object_data_cache::ObjectDataCacheLookup::Miss),
@@ -9993,8 +10155,8 @@ mod tests {
 
         assert_eq!(
             reads.load(AtomicOrdering::Relaxed),
-            0,
-            "too-large materialize-fill candidate must not pre-read the fallback reader"
+            1,
+            "too-large materialize-fill candidate must prime the streaming fallback"
         );
     }
 
@@ -10032,8 +10194,8 @@ mod tests {
 
         assert_eq!(
             reads.load(AtomicOrdering::Relaxed),
-            0,
-            "default GetObject response construction should not pre-read small plain object data"
+            1,
+            "default GetObject response construction should prime exactly one byte"
         );
     }
 

rustfs/src/app/object/shared.rs

@@ -465,7 +465,7 @@ mod bucket_default_sse_lookup_tests {
         let err = classify_bucket_default_sse_lookup("bucket", Err(StorageError::ErasureReadQuorum))
             .expect_err("an unreadable metadata subsystem must never degrade to plaintext");
 
-        assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable);
+        assert_eq!(err.code(), &S3ErrorCode::Custom("SlowDownRead".into()));
     }
 
     #[test]

rustfs/src/error.rs

@@ -20,6 +20,8 @@ use s3s::{S3Error, S3ErrorCode};
 
 const MAX_VERSIONS_EXCEEDED_CODE: &str = "MaxVersionsExceeded";
 const MAX_VERSIONS_EXCEEDED_MESSAGE: &str = "You've exceeded the limit on the number of versions you can create on this object";
+const SLOW_DOWN_READ_CODE: &str = "SlowDownRead";
+const SLOW_DOWN_READ_MESSAGE: &str = "Resource requested is unreadable, please reduce your request rate";
 
 /// S3 error code for a request that names a KMS key the KMS does not hold.
 pub const KMS_KEY_NOT_FOUND_ERROR_CODE: &str = "KMS.NotFoundException";
@@ -105,6 +107,7 @@ fn custom_error_status(code: &S3ErrorCode) -> Option<StatusCode> {
         S3ErrorCode::Custom(custom) if &**custom == KMS_KEY_NOT_FOUND_ERROR_CODE || &**custom == MAX_VERSIONS_EXCEEDED_CODE => {
             Some(StatusCode::BAD_REQUEST)
         }
+        S3ErrorCode::Custom(custom) if &**custom == SLOW_DOWN_READ_CODE => Some(StatusCode::SERVICE_UNAVAILABLE),
         _ => None,
     }
 }
@@ -403,6 +406,7 @@ impl ApiError {
             S3ErrorCode::Custom(code) if &**code == MAX_VERSIONS_EXCEEDED_CODE => {
                 MAX_VERSIONS_EXCEEDED_MESSAGE.to_string()
             }
+            S3ErrorCode::Custom(code) if &**code == SLOW_DOWN_READ_CODE => SLOW_DOWN_READ_MESSAGE.to_string(),
             _ => code.as_str().to_string(),
         }
     }
@@ -577,10 +581,10 @@ impl From<StorageError> for ApiError {
             | StorageError::FaultyRemoteDisk
             | StorageError::DiskNotFound
             | StorageError::TooManyOpenFiles => S3ErrorCode::ServiceUnavailable,
-            StorageError::ErasureReadQuorum
-            | StorageError::InsufficientReadQuorum(_, _)
-            | StorageError::ErasureWriteQuorum
-            | StorageError::InsufficientWriteQuorum(_, _) => S3ErrorCode::ServiceUnavailable,
+            StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _) => {
+                S3ErrorCode::Custom(SLOW_DOWN_READ_CODE.into())
+            }
+            StorageError::ErasureWriteQuorum | StorageError::InsufficientWriteQuorum(_, _) => S3ErrorCode::ServiceUnavailable,
             StorageError::NamespaceLockQuorumUnavailable { .. } => S3ErrorCode::ServiceUnavailable,
             StorageError::QuotaExceeded { .. } => S3ErrorCode::InvalidRequest,
             StorageError::MaxVersionsExceeded => S3ErrorCode::Custom(MAX_VERSIONS_EXCEEDED_CODE.into()),
@@ -1288,10 +1292,10 @@ mod tests {
             (StorageError::FaultyRemoteDisk, S3ErrorCode::ServiceUnavailable),
             (StorageError::DiskNotFound, S3ErrorCode::ServiceUnavailable),
             (StorageError::TooManyOpenFiles, S3ErrorCode::ServiceUnavailable),
-            (StorageError::ErasureReadQuorum, S3ErrorCode::ServiceUnavailable),
+            (StorageError::ErasureReadQuorum, S3ErrorCode::Custom(SLOW_DOWN_READ_CODE.into())),
             (
                 StorageError::InsufficientReadQuorum("test".into(), "test".into()),
-                S3ErrorCode::ServiceUnavailable,
+                S3ErrorCode::Custom(SLOW_DOWN_READ_CODE.into()),
             ),
             (StorageError::ErasureWriteQuorum, S3ErrorCode::ServiceUnavailable),
             (
@@ -1429,6 +1433,21 @@ mod tests {
         assert_eq!(s3_error.status_code(), Some(http::StatusCode::SERVICE_UNAVAILABLE));
     }
 
+    #[test]
+    fn read_quorum_failure_matches_minio_slow_down_read_response() {
+        for error in [
+            StorageError::ErasureReadQuorum,
+            StorageError::InsufficientReadQuorum("bucket".into(), "object".into()),
+        ] {
+            let api_error = ApiError::from(error);
+            assert_eq!(api_error.code, S3ErrorCode::Custom(SLOW_DOWN_READ_CODE.into()));
+            assert_eq!(api_error.message, SLOW_DOWN_READ_MESSAGE);
+
+            let s3_error: S3Error = api_error.into();
+            assert_eq!(s3_error.status_code(), Some(StatusCode::SERVICE_UNAVAILABLE));
+        }
+    }
+
     #[test]
     fn quota_exceeded_preserves_existing_s3_error_contract() {
         let api_error: ApiError = StorageError::QuotaExceeded { current: 5, limit: 10 }.into();

rustfs/src/storage/rpc/http_service.rs

@@ -14,7 +14,6 @@
 
 use crate::server::RPC_PREFIX;
 use crate::storage::request_context::spawn_traced;
-use crate::storage::storage_api::DiskError;
 use crate::storage::storage_api::rpc_consumer::http_service::{
     DEFAULT_READ_BUFFER_SIZE, DeleteOptions, DiskStore, NS_SCANNER_PROTOCOL_VERSION, NsScannerCapabilityResponse,
     PUT_FILE_AUTH_TRAILER_LEN, PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_VERSION, PutFileCapabilityResponse, StorageDiskRpcExt as _,
@@ -31,6 +30,7 @@ use crate::storage::storage_api::rpc_consumer::http_service::{
 };
 use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
 use crate::storage::storage_api::tonic_rpc_auth_failure_reason;
+use crate::storage::storage_api::{DiskError, FileReader};
 use bytes::{Bytes, BytesMut};
 use futures_util::{Stream, StreamExt, TryStreamExt, stream};
 use http::{HeaderMap, HeaderValue, Method, Request, Response, StatusCode, Uri};
@@ -67,6 +67,8 @@ const LOG_SUBSYSTEM_NAMESPACE_SCANNER: &str = "namespace_scanner";
 const LOG_SUBSYSTEM_ROUTING: &str = "routing";
 const EVENT_RPC_REQUEST_REJECTED: &str = "rpc_request_rejected";
 const EVENT_RPC_REQUEST_FAILED: &str = "rpc_request_failed";
+const RUSTFS_META_BUCKET: &str = ".rustfs.sys";
+const MIGRATING_META_BUCKET: &str = ".minio.sys";
 const EVENT_RPC_BACKGROUND_TASK_FAILED: &str = "rpc_background_task_failed";
 const RPC_OPERATION_UNKNOWN: &str = "unknown";
 const READ_FILE_STREAM_PATH: &str = "/rustfs/rpc/read_file_stream";
@@ -633,34 +635,32 @@ async fn handle_read_file(req: Request<Incoming>) -> Response<Body> {
         return response_with_status(StatusCode::BAD_REQUEST, "disk not found");
     };
 
-    let file = match disk
-        .read_file_stream(&query.volume, &query.path, query.offset, query.length)
-        .await
-    {
-        Ok(file) => file,
-        Err(e) => {
-            let message = format!("read file err {e}");
-            error!(
-                event = EVENT_RPC_REQUEST_FAILED,
-                component = LOG_COMPONENT_INTERNODE_RPC,
-                subsystem = LOG_SUBSYSTEM_FILE_TRANSFER,
-                operation = INTERNODE_OPERATION_READ_FILE_STREAM,
-                result = "failed",
-                status_code = StatusCode::INTERNAL_SERVER_ERROR.as_u16(),
-                rpc_path = req.uri().path(),
-                method = %req.method(),
-                reason = "read_file_failed",
-                disk = %query.disk,
-                volume = %query.volume,
-                path = %query.path,
-                offset = query.offset,
-                length = query.length,
-                error = %e,
-                "internode rpc request failed"
-            );
-            return response_with_disk_error(&e, message);
-        }
-    };
+    let file =
+        match read_file_stream_with_legacy_meta_fallback(&disk, &query.volume, &query.path, query.offset, query.length).await {
+            Ok(file) => file,
+            Err(e) => {
+                let message = format!("read file err {e}");
+                error!(
+                    event = EVENT_RPC_REQUEST_FAILED,
+                    component = LOG_COMPONENT_INTERNODE_RPC,
+                    subsystem = LOG_SUBSYSTEM_FILE_TRANSFER,
+                    operation = INTERNODE_OPERATION_READ_FILE_STREAM,
+                    result = "failed",
+                    status_code = StatusCode::INTERNAL_SERVER_ERROR.as_u16(),
+                    rpc_path = req.uri().path(),
+                    method = %req.method(),
+                    reason = "read_file_failed",
+                    disk = %query.disk,
+                    volume = %query.volume,
+                    path = %query.path,
+                    offset = query.offset,
+                    length = query.length,
+                    error = %e,
+                    "internode rpc request failed"
+                );
+                return response_with_disk_error(&e, message);
+            }
+        };
 
     runtime_sources::current_internode_metrics().record_incoming_request_for_operation_and_backend(
         INTERNODE_OPERATION_READ_FILE_STREAM,
@@ -800,28 +800,30 @@ async fn handle_walk_dir(req: Request<Incoming>) -> Response<Body> {
     let log_disk_id = args.disk_id.clone();
     let log_skip_total_timeout = args.skip_total_timeout;
     let body = walk_dir_response_body(propagate_completion_errors, move |mut writer| async move {
-        disk.walk_dir(args, &mut writer).await.map_err(|e| {
-            warn!(
-                event = EVENT_RPC_BACKGROUND_TASK_FAILED,
-                component = LOG_COMPONENT_INTERNODE_RPC,
-                subsystem = LOG_SUBSYSTEM_DIRECTORY_WALK,
-                operation = INTERNODE_OPERATION_WALK_DIR,
-                result = "failed",
-                disk = %log_disk,
-                bucket = %log_bucket,
-                base_dir = %log_base_dir,
-                recursive = log_recursive,
-                report_notfound = log_report_notfound,
-                filter_prefix = ?log_filter_prefix,
-                forward_to = ?log_forward_to,
-                limit = log_limit,
-                disk_id = %log_disk_id,
-                skip_total_timeout = log_skip_total_timeout,
-                error = %e,
-                "internode rpc background task failed"
-            );
-            io::Error::other("remote walk_dir failed")
-        })
+        walk_dir_with_legacy_meta_fallback(&disk, args, &mut writer)
+            .await
+            .map_err(|e| {
+                warn!(
+                    event = EVENT_RPC_BACKGROUND_TASK_FAILED,
+                    component = LOG_COMPONENT_INTERNODE_RPC,
+                    subsystem = LOG_SUBSYSTEM_DIRECTORY_WALK,
+                    operation = INTERNODE_OPERATION_WALK_DIR,
+                    result = "failed",
+                    disk = %log_disk,
+                    bucket = %log_bucket,
+                    base_dir = %log_base_dir,
+                    recursive = log_recursive,
+                    report_notfound = log_report_notfound,
+                    filter_prefix = ?log_filter_prefix,
+                    forward_to = ?log_forward_to,
+                    limit = log_limit,
+                    disk_id = %log_disk_id,
+                    skip_total_timeout = log_skip_total_timeout,
+                    error = %e,
+                    "internode rpc background task failed"
+                );
+                io::Error::other("remote walk_dir failed")
+            })
     });
 
     runtime_sources::current_internode_metrics()
@@ -833,6 +835,54 @@ async fn handle_walk_dir(req: Request<Incoming>) -> Response<Body> {
         .expect("failed to build walk dir response")
 }
 
+fn legacy_meta_bucket_alias(volume: &str) -> Option<String> {
+    if volume == MIGRATING_META_BUCKET {
+        return Some(RUSTFS_META_BUCKET.to_string());
+    }
+    volume
+        .strip_prefix(MIGRATING_META_BUCKET)
+        .filter(|rest| rest.starts_with('/'))
+        .map(|rest| format!("{RUSTFS_META_BUCKET}{rest}"))
+}
+
+fn legacy_meta_alias_can_retry(error: &DiskError, volume: &str) -> bool {
+    matches!(error, DiskError::FileNotFound | DiskError::VolumeNotFound) && legacy_meta_bucket_alias(volume).is_some()
+}
+
+async fn read_file_stream_with_legacy_meta_fallback(
+    disk: &DiskStore,
+    volume: &str,
+    path: &str,
+    offset: usize,
+    length: usize,
+) -> Result<FileReader, DiskError> {
+    match disk.read_file_stream(volume, path, offset, length).await {
+        Ok(file) => Ok(file),
+        Err(error) if legacy_meta_alias_can_retry(&error, volume) => {
+            let alias = legacy_meta_bucket_alias(volume).expect("legacy meta alias checked before retry");
+            disk.read_file_stream(&alias, path, offset, length).await
+        }
+        Err(error) => Err(error),
+    }
+}
+
+async fn walk_dir_with_legacy_meta_fallback<W: tokio::io::AsyncWrite + Unpin + Send>(
+    disk: &DiskStore,
+    args: WalkDirOptions,
+    writer: &mut W,
+) -> Result<(), DiskError> {
+    let original_bucket = args.bucket.clone();
+    match disk.walk_dir(args.clone(), writer).await {
+        Ok(()) => Ok(()),
+        Err(error) if legacy_meta_alias_can_retry(&error, &original_bucket) => {
+            let mut retry_args = args;
+            retry_args.bucket = legacy_meta_bucket_alias(&original_bucket).expect("legacy meta alias checked before retry");
+            disk.walk_dir(retry_args, writer).await
+        }
+        Err(error) => Err(error),
+    }
+}
+
 async fn handle_ns_scanner(req: Request<Incoming>) -> Response<Body> {
     let query = match parse_query::<NsScannerQuery>(&req) {
         Ok(query) => query,
@@ -1699,13 +1749,14 @@ mod tests {
         NS_SCANNER_SESSION_SEQUENCE_QUERY, NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY, NsScannerCapabilityResponse,
         NsScannerQuery, PUT_FILE_AUTH_STREAM_PATH, PUT_FILE_CAPABILITY_PATH, PUT_FILE_STREAM_PATH, PutFileQuery,
         READ_FILE_STREAM_PATH, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_PATH, WalkDirQuery, append_walk_dir_completion,
-        internode_http_operation, internode_rpc_subsystem, is_internode_rpc_path, ns_scanner_response_body,
-        ns_scanner_server_epoch_matches, put_body_size_mismatch, put_file_auth_nonce, put_file_capability_response,
-        put_file_server_epoch_accepted, put_file_server_epoch_matches, put_file_stage_error_message, put_file_target_lock,
-        read_file_body_stream, read_file_stream_buffer_size, remote_scanner_claim_rejection, response_with_disk_error,
-        supports_walk_dir_stream_completion, validate_walk_dir_completion_request, verify_internode_rpc_signature,
-        verify_ns_scanner_body_digest, verify_walk_dir_body_digest, walk_dir_response_body, write_authenticated_put_file,
-        write_body_chunks_to_writer, write_put_file_body_chunks_to_writer,
+        internode_http_operation, internode_rpc_subsystem, is_internode_rpc_path, legacy_meta_bucket_alias,
+        ns_scanner_response_body, ns_scanner_server_epoch_matches, put_body_size_mismatch, put_file_auth_nonce,
+        put_file_capability_response, put_file_server_epoch_accepted, put_file_server_epoch_matches,
+        put_file_stage_error_message, put_file_target_lock, read_file_body_stream, read_file_stream_buffer_size,
+        remote_scanner_claim_rejection, response_with_disk_error, supports_walk_dir_stream_completion,
+        validate_walk_dir_completion_request, verify_internode_rpc_signature, verify_ns_scanner_body_digest,
+        verify_walk_dir_body_digest, walk_dir_response_body, write_authenticated_put_file, write_body_chunks_to_writer,
+        write_put_file_body_chunks_to_writer,
     };
     use crate::storage::storage_api::ecstore_rpc::{build_put_file_auth_trailer, gen_signature_headers};
     use crate::storage::storage_api::rpc_consumer::http_service::{DiskAPI as _, DiskOption, DiskStore, Endpoint, new_disk};
@@ -3013,4 +3064,15 @@ mod tests {
         let response = response_with_disk_error(&DiskError::DiskAccessDenied, "permission denied");
         assert!(response.headers().get(rustfs_rio::INTERNODE_DISK_ERROR_HEADER).is_none());
     }
+
+    #[test]
+    fn legacy_meta_bucket_alias_maps_only_legacy_system_metadata() {
+        assert_eq!(legacy_meta_bucket_alias(".minio.sys").as_deref(), Some(".rustfs.sys"));
+        assert_eq!(
+            legacy_meta_bucket_alias(".minio.sys/config/iam").as_deref(),
+            Some(".rustfs.sys/config/iam")
+        );
+        assert_eq!(legacy_meta_bucket_alias(".minio.sys-lookalike"), None);
+        assert_eq!(legacy_meta_bucket_alias("user-bucket"), None);
+    }
 }

scripts/run_scanner_heal_linux_evidence_plan.py

@@ -46,6 +46,10 @@ REQUIRED_GATES = {
 }
 
 
+def is_sha(value: Any) -> bool:
+    return isinstance(value, str) and len(value) == 40 and all(char in "0123456789abcdef" for char in value)
+
+
 def command(*parts: str) -> list[str]:
     return list(parts)
 
@@ -74,6 +78,14 @@ def load_registry() -> dict[str, Any]:
     return registry
 
 
+def read_json_object(path: Path) -> dict[str, Any]:
+    with path.open() as stream:
+        payload = json.load(stream)
+    if not isinstance(payload, dict):
+        raise ValueError(f"expected JSON object: {path}")
+    return payload
+
+
 def validate_registry(registry: dict[str, Any]) -> None:
     gates = {item["gate"] for item in registry.get("release_requirements", [])}
     missing = sorted(REQUIRED_GATES - gates)
@@ -604,6 +616,89 @@ def build_status(plan: dict[str, Any], run_root: Path) -> dict[str, Any]:
     }
 
 
+def descriptor_gate_names(payload: dict[str, Any]) -> list[str]:
+    gates = payload.get("gates")
+    if not isinstance(gates, dict):
+        return []
+    return sorted(gate for gate in gates if isinstance(gate, str))
+
+
+def build_descriptor_ledger(descriptor_paths: list[Path], revision: str) -> dict[str, Any]:
+    entries = []
+    same_head_gates: set[str] = set()
+    old_head_gates: set[str] = set()
+    duplicate_gates: dict[str, list[str]] = {}
+    gate_sources: dict[str, list[str]] = {}
+    for raw_path in descriptor_paths:
+        path = raw_path.resolve()
+        entry: dict[str, Any] = {
+            "path": str(raw_path),
+            "file_name": path.name,
+        }
+        try:
+            if not path.is_file():
+                raise ValueError("descriptor is missing")
+            if path.stat().st_size <= 0:
+                raise ValueError("descriptor is empty")
+            payload = read_json_object(path)
+            evidence = payload.get("evidence")
+            descriptor_revision = payload.get("source_revision")
+            gates = descriptor_gate_names(payload)
+            if evidence != "measured":
+                classification = "case-level only"
+            elif not is_sha(descriptor_revision):
+                classification = "invalid"
+            elif not gates:
+                classification = "case-level only"
+            elif descriptor_revision == revision:
+                classification = "same-head verified"
+                same_head_gates.update(gates)
+            else:
+                classification = "old-head measured, drift-readable"
+                old_head_gates.update(gates)
+            for gate in gates:
+                gate_sources.setdefault(gate, []).append(path.name)
+            entry.update({
+                "status": "present",
+                "classification": classification,
+                "evidence": evidence,
+                "source_revision": descriptor_revision,
+                "gates": gates,
+            })
+        except (ValueError, OSError, json.JSONDecodeError) as error:
+            entry.update({
+                "status": "invalid",
+                "classification": "invalid",
+                "error": str(error),
+                "gates": [],
+            })
+        entries.append(entry)
+    for gate, sources in sorted(gate_sources.items()):
+        if len(sources) > 1:
+            duplicate_gates[gate] = sorted(sources)
+    measured_gates = same_head_gates | old_head_gates
+    return {
+        "schema": 1,
+        "kind": "scanner-heal-descriptor-ledger",
+        "source_revision": revision,
+        "release_approved": False,
+        "entries": entries,
+        "same_head_verified_gates": sorted(same_head_gates),
+        "old_head_measured_gates": sorted(old_head_gates - same_head_gates),
+        "missing_measured_gates": sorted(REQUIRED_GATES - measured_gates),
+        "missing_current_head_gates": sorted(REQUIRED_GATES - same_head_gates),
+        "duplicate_gates": duplicate_gates,
+        "totals": {
+            "descriptors": len(entries),
+            "same_head_verified_gates": len(same_head_gates),
+            "old_head_measured_gates": len(old_head_gates - same_head_gates),
+            "missing_measured_gates": len(REQUIRED_GATES - measured_gates),
+            "missing_current_head_gates": len(REQUIRED_GATES - same_head_gates),
+            "invalid_descriptors": sum(1 for entry in entries if entry["status"] == "invalid"),
+        },
+    }
+
+
 def run_preflight(plan: dict[str, Any]) -> int:
     commands = iter_preflight_commands(plan)
     if not commands:
@@ -672,6 +767,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
     parser.add_argument("--format", choices=("text", "json"), default="text")
     parser.add_argument("--run-preflight", action="store_true")
     parser.add_argument("--status-root", type=Path)
+    parser.add_argument("--descriptor-ledger", type=Path, nargs="+")
     parser.add_argument("--self-test", action="store_true")
     return parser.parse_args(argv)
 
@@ -687,9 +783,15 @@ def main(argv: list[str] | None = None) -> int:
         phases = set(args.phase or ["all"])
         if "all" in phases and len(phases) > 1:
             raise ValueError("--phase all cannot be combined with another phase")
-        if args.status_root is not None and (args.write_plan or args.run_preflight):
-            raise ValueError("--status-root cannot be combined with --write-plan or --run-preflight")
-        plan = build_plan(registry, source_revision(args.source_revision), phases)
+        if args.status_root is not None and (args.write_plan or args.run_preflight or args.descriptor_ledger):
+            raise ValueError("--status-root cannot be combined with --write-plan, --run-preflight, or --descriptor-ledger")
+        if args.descriptor_ledger and (args.write_plan or args.run_preflight):
+            raise ValueError("--descriptor-ledger cannot be combined with --write-plan or --run-preflight")
+        revision = source_revision(args.source_revision)
+        if args.descriptor_ledger:
+            print(json.dumps(build_descriptor_ledger(args.descriptor_ledger, revision), indent=2, sort_keys=True))
+            return 0
+        plan = build_plan(registry, revision, phases)
         if args.status_root is not None:
             status = build_status(plan, args.status_root)
             print(json.dumps(status, indent=2, sort_keys=True))

scripts/test_scanner_heal_linux_evidence_plan.sh

@@ -108,6 +108,60 @@ assert status["release_approved"] is False
 assert status["artifact_totals"]["missing"] == 0
 PY
 
+"${RUSTFS_PYTHON_BIN:-python3}" - "$TMP_DIR" <<'PY'
+import json
+import pathlib
+import sys
+
+root = pathlib.Path(sys.argv[1])
+same_head = root / "same-head.json"
+old_head = root / "old-head.json"
+case_level = root / "case-level.json"
+same_head.write_text(json.dumps({
+    "schema": 1,
+    "evidence": "measured",
+    "source_revision": "a" * 40,
+    "gates": {"G01": {}, "G02": {}},
+}) + "\n")
+old_head.write_text(json.dumps({
+    "schema": 1,
+    "evidence": "measured",
+    "source_revision": "b" * 40,
+    "gates": {"G03": {}, "G09": {}},
+}) + "\n")
+case_level.write_text(json.dumps({
+    "schema": 1,
+    "evidence": "case",
+    "source_revision": "a" * 40,
+    "gates": {"G14": {}},
+}) + "\n")
+PY
+
+"${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" \
+  --source-revision aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
+  --descriptor-ledger "$TMP_DIR/same-head.json" "$TMP_DIR/old-head.json" "$TMP_DIR/case-level.json" \
+  "$TMP_DIR/missing.json" >"$TMP_DIR/ledger.json"
+
+"${RUSTFS_PYTHON_BIN:-python3}" - "$TMP_DIR/ledger.json" <<'PY'
+import json
+import pathlib
+import sys
+
+ledger = json.loads(pathlib.Path(sys.argv[1]).read_text())
+assert ledger["kind"] == "scanner-heal-descriptor-ledger"
+assert ledger["release_approved"] is False
+assert ledger["same_head_verified_gates"] == ["G01", "G02"]
+assert ledger["old_head_measured_gates"] == ["G03", "G09"]
+assert "G14" in ledger["missing_measured_gates"]
+assert "G03" in ledger["missing_current_head_gates"]
+assert ledger["totals"]["invalid_descriptors"] == 1
+classifications = {entry["file_name"]: entry["classification"] for entry in ledger["entries"]}
+assert classifications["same-head.json"] == "same-head verified"
+assert classifications["old-head.json"] == "old-head measured, drift-readable"
+assert classifications["case-level.json"] == "case-level only"
+assert classifications["missing.json"] == "invalid"
+PY
+
 if "${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" \
   --phase performance \
   --run-preflight >/dev/null 2>"$TMP_DIR/no-preflight.err"; then

scripts/test_security_workflow.py

@@ -140,9 +140,9 @@ class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
             self.assertNotIn("        continue-on-error: true", self.steps[name])
         self.assertIn("        if: ${{ always() && steps.evidence.outcome == 'success' }}", self.steps["Generate report"])
         self.assertNotIn("/tmp/rustfs-security", self.source)
-        for name in ("Upload functional report to dashboard", "File failure issue in rustfs/backlog"):
-            report = next(line for line in self.steps[name] if line.strip().startswith("REPORT_FILE:"))
-            self.assertIn("${{ env.SECURITY_ARTIFACTS_DIR }}/report.md", report)
+        for name in ("Upload functional report to dashboard", "Manage backlog issues (dedup / label / auto-close)"):
+            expected = "${{ env.SECURITY_ARTIFACTS_DIR }}/report.md" if name.startswith("Upload") else '--report-file "${SECURITY_ARTIFACTS_DIR}/report.md"'
+            self.assertIn(expected, "\n".join(self.steps[name]))
         for name in ("Upload functional report to dashboard", "Upload report and logs"):
             self.assertIn("        if: ${{ always() && steps.evidence.outcome == 'success' }}", self.steps[name])
         artifact_settings = yaml_block(self.steps["Upload report and logs"], "with", 8)
@@ -268,10 +268,12 @@ class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
             gh.chmod(0o755)
             body = self.directory / "issue-body.md"
             self.env.update(PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", CAPTURE_BODY=str(body))
-            result = self.run_step("File failure issue in rustfs/backlog")
+            result = self.run_step("Manage backlog issues (dedup / label / auto-close)")
             self.assertEqual(result.returncode, 0, result.stderr)
-            self.assertNotIn("OLD RUN REPORT", body.read_text())
-            self.assertIn("https://github.com/rustfs/rustfs/actions/runs/314159", body.read_text())
+            # The manager lives in the private auto-testing checkout; without it
+            # the step must skip without publishing anything.
+            self.assertIn("issue_manager.py not found", result.stdout + result.stderr)
+            self.assertFalse(body.exists())
 
     def test_all_ten_suites_hold_the_shared_lock_for_manual_and_chain_runs(self) -> None:
         for suite in ("upgrade", "s3-compat", "kms", "tier", "storage", "heal", "pool-expand", "security", "replication", "performance"):
@@ -351,7 +353,7 @@ fi
                 job = yaml_block(replication.splitlines(), "replication-test", 2)
                 self.assertFalse(any(line.startswith("    continue-on-error:") for line in job))
                 self.steps = named_steps(job)
-                handoff = "Continue functional chain (next: Performance)"
+                handoff = "Continue functional chain (next: Fault tolerance)"
                 self.assertIn("        if: ${{ always() && github.event_name == 'repository_dispatch' }}", self.steps[handoff])
                 self.assertFalse(any(line.strip().startswith("continue-on-error:") for line in self.steps[handoff]))
                 self.assertIn("        if: always()", self.steps["Cleanup environment (after)"])
@@ -371,11 +373,11 @@ fi
                 self.assertEqual(forwarded.returncode == 0, bool(token) and failed_attempts < 3, forwarded.stderr)
                 calls = dispatches.read_text().splitlines() if dispatches.exists() else []
                 self.assertEqual(calls, [
-                    "api --method POST repos/rustfs/rustfs/dispatches -f event_type=rustfs-chain-performance -F client_payload[from_suite]=replication",
+                    "api --method POST repos/rustfs/rustfs/dispatches -f event_type=rustfs-chain-fault-tolerance -F client_payload[from_suite]=replication",
                 ] * (min(failed_attempts + 1, 3) if token else 0))
                 if failed_attempts == 3:
-                    self.assertIn("could not hand off from **replication** to **Performance**", body.read_text())
-                    self.assertIn("rustfs-chain-performance", body.read_text())
+                    self.assertIn("could not hand off from **replication** to **Fault tolerance**", body.read_text())
+                    self.assertIn("rustfs-chain-fault-tolerance", body.read_text())
                     self.assertEqual(executed.read_text().splitlines().count("issue"), 2 if issue_exit else 1)
                     self.assertFalse(Path(body_path.read_text().strip()).exists())
 
@@ -587,11 +589,10 @@ class FunctionalEvidenceTests(WorkflowSteps, unittest.TestCase):
                 initialized = self.run_step("Initialize functional evidence")
                 self.assertNotEqual(initialized.returncode, 0)
                 self.assertFalse(Path(self.env["GITHUB_ENV"]).exists())
-                issue = self.run_step("File failure issue in rustfs/backlog")
-                self.assertEqual(issue.returncode, 0, issue.stderr)
-                body = Path(self.env["CAPTURE_BODY"]).read_text()
-                self.assertNotIn("OLD RUN EVIDENCE", body)
-                self.assertIn("no report or log file was produced", body)
+                manager = self.run_step("Manage backlog issues (dedup / label / auto-close)")
+                self.assertEqual(manager.returncode, 0, manager.stderr)
+                self.assertIn("issue_manager.py not found", manager.stdout + manager.stderr)
+                self.assertFalse(Path(self.env["CAPTURE_BODY"]).exists())
                 self.assertEqual((existing / "report.md").read_text(), "OLD RUN EVIDENCE")
 
     def test_reports_use_only_current_complete_suite_evidence(self):