name: RustFS Heal Test on: workflow_call: inputs: chain_manifest: description: Verified candidate and chain attempt from the chain driver type: string required: true workflow_dispatch: inputs: package_url: description: 'Direct .deb URL (nightly/R2). Defaults to the latest nightly deb.' required: false type: string stop_node_gb: description: 'Stop the outage node when surviving nodes reach N GiB' required: false default: '15' warp_stop_gb: description: 'Stop warp when surviving nodes reach N GiB' required: false default: '40' cleanup_before: description: 'Reset the nodes before the test (DESTROYS existing data/config)' type: boolean default: true cleanup_after: description: 'Reset the nodes after the test (DESTROYS test data/config)' type: boolean default: true repository_dispatch: # Chain handoff: dispatched when the storage suite finishes. Heal runs # exactly once per chain; the pool expansion workflow no longer embeds # its own heal pass. types: [rustfs-chain-heal] permissions: contents: read # Only one test at a time: both this and the pool-expansion workflow mutate # the same test environment, so they share one concurrency group. concurrency: group: rustfs-shared-functional-tests-v2 cancel-in-progress: false defaults: run: shell: bash env: RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }} RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }} RUSTFS_API_ENDPOINT: ${{ secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT }} RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }} RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }} PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }} jobs: heal-test: runs-on: smoke-testing timeout-minutes: 60 # Standalone manual run, or one link of the nightly functional chain # (storage -> heal -> pool). Pool expansion no longer re-runs heal. if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: - name: Checkout chain tooling uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: persist-credentials: false - name: Initialize functional evidence id: evidence run: | set -euo pipefail umask 077 FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-heal-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch" { printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}" printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}" printf 'RUSTFS_WARP_LOG_FILE=%s/warp.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}" printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}" printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}" } >> "${GITHUB_ENV}" - name: Bind functional candidate id: chain if: ${{ inputs.chain_manifest != '' }} env: CHAIN_MANIFEST: ${{ inputs.chain_manifest }} run: python3 scripts/functional_chain_evidence.py consume - name: Checkout auto-testing scripts uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: repository: rustfs/auto-testing ref: ${{ steps.chain.outputs.testing_sha || 'main' }} token: ${{ secrets.PF_TESTING_GH_TOKEN }} path: auto-testing persist-credentials: false - name: Show environment run: | uname -a jq --version openssl version warp --version || true df -h /data | tail -1 - name: Cleanup environment (before) timeout-minutes: 5 if: ${{ inputs.cleanup_before != 'false' }} run: | set -euo pipefail read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}" SSH_USER="${RUSTFS_SSH_USER:-azureuser}" for node in "${NODES[@]}"; do ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" ' set -euo pipefail SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n" ${SUDO} systemctl stop rustfs 2>/dev/null || true if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then ${SUDO} dpkg -P rustfs fi for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup ' done - name: Install RustFS package & start cluster timeout-minutes: 5 run: | ARGS=(--steps "1,2" -y --endpoint "${{ env.RUSTFS_API_ENDPOINT }}") if [ -n "${{ inputs.package_url }}" ]; then ARGS+=(--package-url "${{ inputs.package_url }}") else ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") fi ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}" - name: Preflight checks timeout-minutes: 5 run: | ARGS=(--preflight --endpoint "${{ env.RUSTFS_API_ENDPOINT }}") if [ -n "${{ inputs.package_url }}" ]; then ARGS+=(--package-url "${{ inputs.package_url }}") else ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") fi ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}" - name: Run heal test (write -> outage -> heal -> verify) timeout-minutes: 45 id: test # Case failures keep the run green: the report and the backlog # issue manager carry the product signal. continue-on-error: true run: | ./auto-testing/rustfs_heal_test.sh \ --steps "3,4,5,6,7" -y \ --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \ --stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \ --warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \ --log-file "${LOG_FILE}" - name: Generate report id: chain_report timeout-minutes: 2 if: ${{ always() && steps.evidence.outcome == 'success' }} run: | set -euo pipefail PACKAGE_URL='${{ inputs.package_url }}' if [ -n "${PACKAGE_URL}" ]; then PACKAGE_SOURCE="${PACKAGE_URL}" else PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}" fi STEPS_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/steps.md" CASE_RESULT=success python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY' || CASE_RESULT=failure import re import sys log_file, out_file = sys.argv[1], sys.argv[2] ansi = re.compile(r'\x1b\[[0-9;]*m') step_re = re.compile(r'^\[HEAL-STEP\]\s+(\d+)\s+(.+?)\s+(PASS|FAIL|SKIP)\s*$') ver_re = re.compile(r'^\[HEAL-VERSION\]\s+(\S+)(?:\s+\(node\s+(\S+)\))?\s*$') result_re = re.compile(r'^\[HEAL-RESULT\]\s+(PASS|FAIL)\s+(.*)$') steps = {} order = [] status_rank = {'SKIP': 0, 'PASS': 1, 'FAIL': 2} version = None version_node = None verdict = None verdict_detail = '' try: with open(log_file, 'r', encoding='utf-8', errors='replace') as fh: for raw in fh: line = ansi.sub('', raw).strip() m = step_re.match(line) if m: n, desc, status = m.group(1), m.group(2), m.group(3) if n not in steps: order.append(n) if n not in steps or status_rank[status] > status_rank[steps[n][1]]: steps[n] = (desc, status) continue m = ver_re.match(line) if m: version, version_node = m.group(1), m.group(2) continue m = result_re.match(line) if m and verdict != 'FAIL': verdict, verdict_detail = m.group(1), m.group(2) except FileNotFoundError: pass with open(out_file, 'w', encoding='utf-8') as out: out.write('## Step Results\n\n') if version: node_note = f' (captured via `rustfs --version` on {version_node})' if version_node else '' out.write(f'- Version under test: **{version}**{node_note}\n') if verdict: out.write(f'- Overall result: **{verdict}** — {verdict_detail}\n') out.write('\n') out.write('| Step | Description | Result |\n') out.write('| --- | --- | --- |\n') for n in sorted(order, key=int): desc, status = steps[n] out.write(f'| {n} | {desc} | {status} |\n') if not order: out.write('| - | - | NOT RUN (no step result lines found) |\n') complete = set(steps) == {str(n) for n in range(1, 8)} sys.exit(0 if complete and verdict != 'FAIL' and all(status == 'PASS' for _, status in steps.values()) else 1) PY RESULT=failure if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then RESULT=success fi # Product gate: failing steps keep the run green — they are reported # below and tracked in rustfs/backlog. Only harness/environment # breakdowns turn the workflow red. STEPS_TOTAL="$(grep -cE '^\[HEAL-STEP\]' "${LOG_FILE}" 2>/dev/null || true)" STEPS_FAIL="$(grep -cE '^\[HEAL-STEP\].* FAIL$' "${LOG_FILE}" 2>/dev/null || true)" STEPS_TOTAL=$(( ${STEPS_TOTAL:-0} + 0 )); STEPS_FAIL=$(( ${STEPS_FAIL:-0} + 0 )) STEPS_PASS=$(( STEPS_TOTAL - STEPS_FAIL )) HARNESS_OK=0 if [ '${{ steps.test.outcome }}' = 'success' ]; then HARNESS_OK=1 elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${STEPS_TOTAL}" -gt 0 ] && [ "${STEPS_PASS}" -ge 1 ]; then HARNESS_OK=1 fi { echo "# RustFS heal test report" echo "" echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" echo "- Attempt: ${GITHUB_RUN_ATTEMPT}" echo "- Workflow Commit: ${GITHUB_SHA}" echo "- Trigger: ${{ github.event_name }}" echo "- Package: ${PACKAGE_SOURCE}" echo "- Test Step Outcome: ${RESULT}" echo "- Suite Step Outcome: ${{ steps.test.outcome }}" echo "" if [ -s "${STEPS_TABLE}" ]; then cat "${STEPS_TABLE}" echo "" fi echo "- Product result: ${STEPS_PASS} passed, ${STEPS_FAIL} failed (failing steps are tracked in rustfs/backlog)" echo "" if [ -s "${STEPS_TABLE}" ]; then echo "## Log tail" echo '```text' tail -n 200 "${LOG_FILE}" 2>/dev/null || true echo '```' else echo "The suite or evidence validation failed. See this run's artifact for partial step results and suite.log." fi } | tee "${REPORT_FILE}" cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" # Red only for harness/environment breakdowns; step failures stay green. [ "${HARNESS_OK}" = "1" ] - name: Upload functional report to dashboard timeout-minutes: 2 if: ${{ always() && steps.evidence.outcome == 'success' }} continue-on-error: true env: GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }} SUITE: heal run: | set -euo pipefail if [ -z "${GH_TOKEN:-}" ]; then echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload" exit 0 fi DATE="$(date -u +%Y-%m-%d)" REPORT_PATH="functional-reports/${SUITE}/${DATE}.md" # Base64-encode the report into a temp file and feed it to jq via # --rawfile: large reports (e.g. pool) exceed the OS argv limit and # make `jq --arg content "${CONTENT}"` fail with "Argument list too long". B64_FILE="$(mktemp)" python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}" SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)" if [ -n "${SHA}" ]; then jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \ '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \ | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null else jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \ '{message:$msg, content:($content|rtrimstr("\n"))}' \ | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null fi rm -f "${B64_FILE}" - name: Manage backlog issues (dedup / label / auto-close) timeout-minutes: 2 # Replaces the old per-run failure filing. One entry point that: # - dedups by signal: failing cases are matched against open backlog # issues by label (category + case ID); covered cases become a # comment on the existing issue, only uncovered cases file a new one # - labels new issues (functional-test, category, case IDs, env) # - closes fixed issues after a fully green run # - never files or closes on cancelled runs if: ${{ always() && steps.evidence.outcome == 'success' }} continue-on-error: true env: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} SUITE: 'heal' SUITE_LABEL: 'Heal' RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | set -euo pipefail if [ -z "${GH_TOKEN:-}" ]; then echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management" exit 0 fi if [ ! -f auto-testing/scripts/issue_manager.py ]; then echo "issue_manager.py not found in auto-testing checkout; skipping" exit 0 fi PACKAGE_URL='${{ inputs.package_url }}' PACKAGE_SOURCE="" if [ -n "${PACKAGE_URL}" ]; then PACKAGE_SOURCE="${PACKAGE_URL}" else PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}" fi python3 auto-testing/scripts/issue_manager.py handle \ --repo rustfs/backlog \ --suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \ --outcome "${{ steps.test.outcome }}" \ --report-file "${REPORT_FILE}" \ --log "${LOG_FILE}" \ --run-url "${RUN_URL}/attempts/${GITHUB_RUN_ATTEMPT}#summary" \ --run-id "${GITHUB_RUN_ID}" \ --attempt "${GITHUB_RUN_ATTEMPT}" \ --commit "${GITHUB_SHA}" \ --trigger "${{ github.event_name }}" \ --package-source "${PACKAGE_SOURCE}" \ --date "$(date -u +%Y-%m-%d)" - name: Upload test logs timeout-minutes: 2 if: ${{ always() && steps.evidence.outcome == 'success' }} uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: name: rustfs-heal-test-${{ github.run_id }}-${{ github.run_attempt }} path: | ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/warp.log ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/steps.md if-no-files-found: error - name: Cleanup environment (after) timeout-minutes: 5 if: ${{ always() && inputs.cleanup_after != 'false' }} run: | set -euo pipefail read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}" SSH_USER="${RUSTFS_SSH_USER:-azureuser}" for node in "${NODES[@]}"; do ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" ' set -euo pipefail SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n" ${SUDO} systemctl stop rustfs 2>/dev/null || true if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then ${SUDO} dpkg -P rustfs fi for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup ' done - name: "Continue functional chain (next: Pool expansion)" # Only chain-triggered runs forward to the next suite; standalone # workflow_dispatch runs stop after their own cleanup. A failed # handoff must never pass silently: it retries, then files an alert # issue in rustfs/backlog so a stalled chain is visible. if: ${{ always() && github.event_name == 'repository_dispatch' }} continue-on-error: true env: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} run: | set -uo pipefail if [ -z "${GH_TOKEN:-}" ]; then echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2 exit 1 fi DISPATCHED=0 for attempt in 1 2 3; do if gh api --method POST repos/rustfs/rustfs/dispatches \ -f event_type='rustfs-chain-pool' \ -F 'client_payload[from_suite]=heal'; then echo "dispatched next suite Pool expansion (attempt ${attempt})" DISPATCHED=1 break fi echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2 sleep "${attempt}0" done if [ "${DISPATCHED:-0}" -ne 1 ]; then echo "ERROR: functional chain stalled: could not dispatch Pool expansion after 3 attempts" >&2 TITLE="[functional][chain] stalled after heal (run ${GITHUB_RUN_ID})" BODY_FILE="$(mktemp)" { echo "The functional chain could not hand off from **heal** to **Pool expansion** after 3 attempts." echo "" echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" echo "- Expected next event: 'rustfs-chain-pool'" echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable." echo "- Recovery: re-dispatch manually with" FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}" echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-pool'" FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}" } > "${BODY_FILE}" gh issue create -R rustfs/backlog --title "${TITLE}" \ --body-file "${BODY_FILE}" --label functional-test \ || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \ || echo "could not file the stall alert issue either; check the token" >&2 exit 1 fi - name: Notify on failure if: failure() run: | echo "RustFS heal test failed" echo "Package source: ${{ inputs.package_url || 'nightly (R2 latest)' }}" echo "See the uploaded log artifact for details." - name: Record chain evidence id: chain_record if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }} env: CHAIN_MANIFEST: ${{ inputs.chain_manifest }} CHAIN_JOB_STATUS: ${{ job.status }} CHAIN_TEST_OUTCOME: ${{ steps.test.outcome }} CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }} run: >- python3 scripts/functional_chain_evidence.py record --suite heal --report "${FUNCTIONAL_ARTIFACTS_DIR}/steps.md" --output "${RUNNER_TEMP}/chain-heal-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/heal.json" - name: Upload chain evidence if: ${{ always() && steps.chain_record.outputs.written == 'true' }} uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: name: functional-chain-heal-${{ github.run_id }}-${{ github.run_attempt }} path: ${{ runner.temp }}/chain-heal-${{ github.run_id }}-${{ github.run_attempt }}/heal.json if-no-files-found: error