ci(functional): retry chain handoffs and alert on stall (#7023)

The repository_dispatch handoff step was continue-on-error with a single
attempt: if the call failed (token lacking contents:write, transient API
error), the chain stalled silently while every job stayed green.

Each handoff now retries 3x and, if all attempts fail, files an alert
issue in rustfs/backlog with the exact recovery command before exiting 1
(still continue-on-error, so suite workflows themselves never fail).

.github/workflows/rustfs-heal-test.yml

@@ -284,21 +284,52 @@ jobs:
 
       - name: "Continue functional chain (next: Pool expansion)"
         # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup.
+        # workflow_dispatch runs stop after their own cleanup. A failed
+        # handoff must never pass silently: it retries, then files an alert
+        # issue in rustfs/backlog so a stalled chain is visible.
         if: ${{ always() && github.event_name == 'repository_dispatch' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
         run: |
-          set -euo pipefail
+          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
+          if [ -z "${{GH_TOKEN:-}}" ]; then
             echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
             exit 1
           fi
-          echo "Dispatching next functional suite: Pool expansion"
+          DISPATCHED=0
-          gh api --method POST repos/rustfs/rustfs/dispatches \
+          for attempt in 1 2 3; do
-            -f event_type='rustfs-chain-pool' \
+            if gh api --method POST repos/rustfs/rustfs/dispatches \
-            -F 'client_payload[from_suite]=heal'
+              -f event_type='rustfs-chain-pool' \
+              -F 'client_payload[from_suite]=heal'; then
+              echo "dispatched next suite Pool expansion (attempt ${{attempt}})"
+              DISPATCHED=1
+              break
+            fi
+            echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2
+            sleep "${{attempt}}0"
+          done
+          if [ "${{DISPATCHED:-0}}" -ne 1 ]; then
+            echo "ERROR: functional chain stalled: could not dispatch Pool expansion after 3 attempts" >&2
+            TITLE="[functional][chain] stalled after heal (run ${{GITHUB_RUN_ID}})"
+            BODY_FILE="$(mktemp)"
+            {
+              echo "The functional chain could not hand off from **heal** to **Pool expansion** after 3 attempts."
+              echo ""
+              echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}"
+              echo "- Expected next event: `rustfs-chain-pool`"
+              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+              echo "- Recovery: re-dispatch manually with"
+              echo "  ```"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-pool'"
+              echo "  ```"
+            } > "${{BODY_FILE}}"
+            gh issue create -R rustfs/backlog --title "${{TITLE}}" \
+              --body-file "${{BODY_FILE}}" --label functional-test \
+              || gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \
+              || echo "could not file the stall alert issue either; check the token" >&2
+            exit 1
+          fi
 
       - name: Notify on failure
         if: failure()

.github/workflows/rustfs-kms-test.yml

@@ -340,21 +340,52 @@ jobs:
 
       - name: "Continue functional chain (next: Tier)"
         # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup.
+        # workflow_dispatch runs stop after their own cleanup. A failed
+        # handoff must never pass silently: it retries, then files an alert
+        # issue in rustfs/backlog so a stalled chain is visible.
         if: ${{ always() && github.event_name == 'repository_dispatch' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
         run: |
-          set -euo pipefail
+          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
+          if [ -z "${{GH_TOKEN:-}}" ]; then
             echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
             exit 1
           fi
-          echo "Dispatching next functional suite: Tier"
+          DISPATCHED=0
-          gh api --method POST repos/rustfs/rustfs/dispatches \
+          for attempt in 1 2 3; do
-            -f event_type='rustfs-chain-tier' \
+            if gh api --method POST repos/rustfs/rustfs/dispatches \
-            -F 'client_payload[from_suite]=kms'
+              -f event_type='rustfs-chain-tier' \
+              -F 'client_payload[from_suite]=kms'; then
+              echo "dispatched next suite Tier (attempt ${{attempt}})"
+              DISPATCHED=1
+              break
+            fi
+            echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2
+            sleep "${{attempt}}0"
+          done
+          if [ "${{DISPATCHED:-0}}" -ne 1 ]; then
+            echo "ERROR: functional chain stalled: could not dispatch Tier after 3 attempts" >&2
+            TITLE="[functional][chain] stalled after kms (run ${{GITHUB_RUN_ID}})"
+            BODY_FILE="$(mktemp)"
+            {
+              echo "The functional chain could not hand off from **kms** to **Tier** after 3 attempts."
+              echo ""
+              echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}"
+              echo "- Expected next event: `rustfs-chain-tier`"
+              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+              echo "- Recovery: re-dispatch manually with"
+              echo "  ```"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-tier'"
+              echo "  ```"
+            } > "${{BODY_FILE}}"
+            gh issue create -R rustfs/backlog --title "${{TITLE}}" \
+              --body-file "${{BODY_FILE}}" --label functional-test \
+              || gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \
+              || echo "could not file the stall alert issue either; check the token" >&2
+            exit 1
+          fi
 
       - name: Notify on failure
         if: failure()

.github/workflows/rustfs-pool-expand-test.yml

@@ -596,21 +596,52 @@ jobs:
 
       - name: "Continue functional chain (next: Security)"
         # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup.
+        # workflow_dispatch runs stop after their own cleanup. A failed
+        # handoff must never pass silently: it retries, then files an alert
+        # issue in rustfs/backlog so a stalled chain is visible.
         if: ${{ always() && github.event_name == 'repository_dispatch' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
         run: |
-          set -euo pipefail
+          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
+          if [ -z "${{GH_TOKEN:-}}" ]; then
             echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
             exit 1
           fi
-          echo "Dispatching next functional suite: Security"
+          DISPATCHED=0
-          gh api --method POST repos/rustfs/rustfs/dispatches \
+          for attempt in 1 2 3; do
-            -f event_type='rustfs-chain-security' \
+            if gh api --method POST repos/rustfs/rustfs/dispatches \
-            -F 'client_payload[from_suite]=pool'
+              -f event_type='rustfs-chain-security' \
+              -F 'client_payload[from_suite]=pool'; then
+              echo "dispatched next suite Security (attempt ${{attempt}})"
+              DISPATCHED=1
+              break
+            fi
+            echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2
+            sleep "${{attempt}}0"
+          done
+          if [ "${{DISPATCHED:-0}}" -ne 1 ]; then
+            echo "ERROR: functional chain stalled: could not dispatch Security after 3 attempts" >&2
+            TITLE="[functional][chain] stalled after pool (run ${{GITHUB_RUN_ID}})"
+            BODY_FILE="$(mktemp)"
+            {
+              echo "The functional chain could not hand off from **pool** to **Security** after 3 attempts."
+              echo ""
+              echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}"
+              echo "- Expected next event: `rustfs-chain-security`"
+              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+              echo "- Recovery: re-dispatch manually with"
+              echo "  ```"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-security'"
+              echo "  ```"
+            } > "${{BODY_FILE}}"
+            gh issue create -R rustfs/backlog --title "${{TITLE}}" \
+              --body-file "${{BODY_FILE}}" --label functional-test \
+              || gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \
+              || echo "could not file the stall alert issue either; check the token" >&2
+            exit 1
+          fi
 
       - name: Notify on failure
         if: failure()

.github/workflows/rustfs-s3-compat-test.yml

@@ -320,21 +320,52 @@ jobs:
 
       - name: "Continue functional chain (next: KMS)"
         # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup.
+        # workflow_dispatch runs stop after their own cleanup. A failed
+        # handoff must never pass silently: it retries, then files an alert
+        # issue in rustfs/backlog so a stalled chain is visible.
         if: ${{ always() && github.event_name == 'repository_dispatch' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
         run: |
-          set -euo pipefail
+          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
+          if [ -z "${{GH_TOKEN:-}}" ]; then
             echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
             exit 1
           fi
-          echo "Dispatching next functional suite: KMS"
+          DISPATCHED=0
-          gh api --method POST repos/rustfs/rustfs/dispatches \
+          for attempt in 1 2 3; do
-            -f event_type='rustfs-chain-kms' \
+            if gh api --method POST repos/rustfs/rustfs/dispatches \
-            -F 'client_payload[from_suite]=s3'
+              -f event_type='rustfs-chain-kms' \
+              -F 'client_payload[from_suite]=s3'; then
+              echo "dispatched next suite KMS (attempt ${{attempt}})"
+              DISPATCHED=1
+              break
+            fi
+            echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2
+            sleep "${{attempt}}0"
+          done
+          if [ "${{DISPATCHED:-0}}" -ne 1 ]; then
+            echo "ERROR: functional chain stalled: could not dispatch KMS after 3 attempts" >&2
+            TITLE="[functional][chain] stalled after s3 (run ${{GITHUB_RUN_ID}})"
+            BODY_FILE="$(mktemp)"
+            {
+              echo "The functional chain could not hand off from **s3** to **KMS** after 3 attempts."
+              echo ""
+              echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}"
+              echo "- Expected next event: `rustfs-chain-kms`"
+              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+              echo "- Recovery: re-dispatch manually with"
+              echo "  ```"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-kms'"
+              echo "  ```"
+            } > "${{BODY_FILE}}"
+            gh issue create -R rustfs/backlog --title "${{TITLE}}" \
+              --body-file "${{BODY_FILE}}" --label functional-test \
+              || gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \
+              || echo "could not file the stall alert issue either; check the token" >&2
+            exit 1
+          fi
 
       - name: Notify on failure
         if: failure()

.github/workflows/rustfs-storage-test.yml

@@ -335,21 +335,52 @@ jobs:
 
       - name: "Continue functional chain (next: Heal)"
         # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup.
+        # workflow_dispatch runs stop after their own cleanup. A failed
+        # handoff must never pass silently: it retries, then files an alert
+        # issue in rustfs/backlog so a stalled chain is visible.
         if: ${{ always() && github.event_name == 'repository_dispatch' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
         run: |
-          set -euo pipefail
+          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
+          if [ -z "${{GH_TOKEN:-}}" ]; then
             echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
             exit 1
           fi
-          echo "Dispatching next functional suite: Heal"
+          DISPATCHED=0
-          gh api --method POST repos/rustfs/rustfs/dispatches \
+          for attempt in 1 2 3; do
-            -f event_type='rustfs-chain-heal' \
+            if gh api --method POST repos/rustfs/rustfs/dispatches \
-            -F 'client_payload[from_suite]=storage'
+              -f event_type='rustfs-chain-heal' \
+              -F 'client_payload[from_suite]=storage'; then
+              echo "dispatched next suite Heal (attempt ${{attempt}})"
+              DISPATCHED=1
+              break
+            fi
+            echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2
+            sleep "${{attempt}}0"
+          done
+          if [ "${{DISPATCHED:-0}}" -ne 1 ]; then
+            echo "ERROR: functional chain stalled: could not dispatch Heal after 3 attempts" >&2
+            TITLE="[functional][chain] stalled after storage (run ${{GITHUB_RUN_ID}})"
+            BODY_FILE="$(mktemp)"
+            {
+              echo "The functional chain could not hand off from **storage** to **Heal** after 3 attempts."
+              echo ""
+              echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}"
+              echo "- Expected next event: `rustfs-chain-heal`"
+              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+              echo "- Recovery: re-dispatch manually with"
+              echo "  ```"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-heal'"
+              echo "  ```"
+            } > "${{BODY_FILE}}"
+            gh issue create -R rustfs/backlog --title "${{TITLE}}" \
+              --body-file "${{BODY_FILE}}" --label functional-test \
+              || gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \
+              || echo "could not file the stall alert issue either; check the token" >&2
+            exit 1
+          fi
 
       - name: Notify on failure
         if: failure()

.github/workflows/rustfs-tier-test.yml

@@ -373,21 +373,52 @@ jobs:
 
       - name: "Continue functional chain (next: Storage engine)"
         # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup.
+        # workflow_dispatch runs stop after their own cleanup. A failed
+        # handoff must never pass silently: it retries, then files an alert
+        # issue in rustfs/backlog so a stalled chain is visible.
         if: ${{ always() && github.event_name == 'repository_dispatch' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
         run: |
-          set -euo pipefail
+          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
+          if [ -z "${{GH_TOKEN:-}}" ]; then
             echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
             exit 1
           fi
-          echo "Dispatching next functional suite: Storage engine"
+          DISPATCHED=0
-          gh api --method POST repos/rustfs/rustfs/dispatches \
+          for attempt in 1 2 3; do
-            -f event_type='rustfs-chain-storage' \
+            if gh api --method POST repos/rustfs/rustfs/dispatches \
-            -F 'client_payload[from_suite]=tier'
+              -f event_type='rustfs-chain-storage' \
+              -F 'client_payload[from_suite]=tier'; then
+              echo "dispatched next suite Storage engine (attempt ${{attempt}})"
+              DISPATCHED=1
+              break
+            fi
+            echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2
+            sleep "${{attempt}}0"
+          done
+          if [ "${{DISPATCHED:-0}}" -ne 1 ]; then
+            echo "ERROR: functional chain stalled: could not dispatch Storage engine after 3 attempts" >&2
+            TITLE="[functional][chain] stalled after tier (run ${{GITHUB_RUN_ID}})"
+            BODY_FILE="$(mktemp)"
+            {
+              echo "The functional chain could not hand off from **tier** to **Storage engine** after 3 attempts."
+              echo ""
+              echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}"
+              echo "- Expected next event: `rustfs-chain-storage`"
+              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+              echo "- Recovery: re-dispatch manually with"
+              echo "  ```"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-storage'"
+              echo "  ```"
+            } > "${{BODY_FILE}}"
+            gh issue create -R rustfs/backlog --title "${{TITLE}}" \
+              --body-file "${{BODY_FILE}}" --label functional-test \
+              || gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \
+              || echo "could not file the stall alert issue either; check the token" >&2
+            exit 1
+          fi
 
       - name: Notify on failure
         if: failure()

.github/workflows/rustfs-upgrade-test.yml

@@ -388,21 +388,52 @@ jobs:
 
       - name: "Continue functional chain (next: S3 compatibility)"
         # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup.
+        # workflow_dispatch runs stop after their own cleanup. A failed
+        # handoff must never pass silently: it retries, then files an alert
+        # issue in rustfs/backlog so a stalled chain is visible.
         if: ${{ always() && github.event_name == 'repository_dispatch' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
         run: |
-          set -euo pipefail
+          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
+          if [ -z "${{GH_TOKEN:-}}" ]; then
             echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
             exit 1
           fi
-          echo "Dispatching next functional suite: S3 compatibility"
+          DISPATCHED=0
-          gh api --method POST repos/rustfs/rustfs/dispatches \
+          for attempt in 1 2 3; do
-            -f event_type='rustfs-chain-s3' \
+            if gh api --method POST repos/rustfs/rustfs/dispatches \
-            -F 'client_payload[from_suite]=upgrade'
+              -f event_type='rustfs-chain-s3' \
+              -F 'client_payload[from_suite]=upgrade'; then
+              echo "dispatched next suite S3 compatibility (attempt ${{attempt}})"
+              DISPATCHED=1
+              break
+            fi
+            echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2
+            sleep "${{attempt}}0"
+          done
+          if [ "${{DISPATCHED:-0}}" -ne 1 ]; then
+            echo "ERROR: functional chain stalled: could not dispatch S3 compatibility after 3 attempts" >&2
+            TITLE="[functional][chain] stalled after upgrade (run ${{GITHUB_RUN_ID}})"
+            BODY_FILE="$(mktemp)"
+            {
+              echo "The functional chain could not hand off from **upgrade** to **S3 compatibility** after 3 attempts."
+              echo ""
+              echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}"
+              echo "- Expected next event: `rustfs-chain-s3`"
+              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+              echo "- Recovery: re-dispatch manually with"
+              echo "  ```"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-s3'"
+              echo "  ```"
+            } > "${{BODY_FILE}}"
+            gh issue create -R rustfs/backlog --title "${{TITLE}}" \
+              --body-file "${{BODY_FILE}}" --label functional-test \
+              || gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \
+              || echo "could not file the stall alert issue either; check the token" >&2
+            exit 1
+          fi
 
       - name: Notify on failure
         if: failure()