mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-25 03:42:14 +00:00
hector
bcf5184a85
A failing product case used to turn the whole workflow red, so the run conclusion carried no signal beyond 'something failed' and the report was suppressed. New semantics across the functional suites: - Suite steps run with continue-on-error: the outcome is still recorded for the report and the backlog issue manager (security/tier already carried the flag). - Generate report always publishes the full per-case table plus a 'Product result: N passed, M failed' summary, and its exit gate is harness health: red only when the suite never reached case level (no case verdicts), failed wholesale (zero passes, >=3 failures), or was cancelled/skipped. performance is unchanged (parked). - tier's structured gate no longer fails on case failures; it keeps red for evidence-init and missing-gate-result breakdowns. - pool/performance keep their existing red sources (install/benchmark). Workflow contract tests updated to the new exit semantics: the security report matrix keys green off the suite outcome, the evidence matrix expects green for failure outcomes with recorded case rows (except performance), the heal staged-rerun block expects the per-step table to always publish, and run steps are now required to carry continue-on-error. Verified locally: actionlint clean; test_security_workflow.py 21/21.
437 lines
20 KiB
YAML
437 lines
20 KiB
YAML
name: RustFS Heal Test
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
package_url:
|
|
description: 'Direct .deb URL (nightly/R2). Defaults to the latest nightly deb.'
|
|
required: false
|
|
type: string
|
|
stop_node_gb:
|
|
description: 'Stop the outage node when surviving nodes reach N GiB'
|
|
required: false
|
|
default: '15'
|
|
warp_stop_gb:
|
|
description: 'Stop warp when surviving nodes reach N GiB'
|
|
required: false
|
|
default: '40'
|
|
cleanup_before:
|
|
description: 'Reset the nodes before the test (DESTROYS existing data/config)'
|
|
type: boolean
|
|
default: true
|
|
cleanup_after:
|
|
description: 'Reset the nodes after the test (DESTROYS test data/config)'
|
|
type: boolean
|
|
default: true
|
|
repository_dispatch:
|
|
# Chain handoff: dispatched when the storage suite finishes. Heal runs
|
|
# exactly once per chain; the pool expansion workflow no longer embeds
|
|
# its own heal pass.
|
|
types: [rustfs-chain-heal]
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
# Only one test at a time: both this and the pool-expansion workflow mutate
|
|
# the same test environment, so they share one concurrency group.
|
|
concurrency:
|
|
group: rustfs-shared-functional-tests
|
|
cancel-in-progress: false
|
|
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
|
|
env:
|
|
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
|
|
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
|
|
RUSTFS_API_ENDPOINT: ${{ secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT }}
|
|
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
|
|
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
|
|
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
|
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
|
|
|
jobs:
|
|
heal-test:
|
|
runs-on: smoke-testing
|
|
timeout-minutes: 480
|
|
# Standalone manual run, or one link of the nightly functional chain
|
|
# (storage -> heal -> pool). Pool expansion no longer re-runs heal.
|
|
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
|
steps:
|
|
- name: Initialize functional evidence
|
|
id: evidence
|
|
run: |
|
|
set -euo pipefail
|
|
umask 077
|
|
FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-heal-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
|
mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
|
|
{
|
|
printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
|
printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
|
printf 'RUSTFS_WARP_LOG_FILE=%s/warp.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
|
printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
|
printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
|
} >> "${GITHUB_ENV}"
|
|
|
|
# auto-testing is private: clone it with the dedicated PF token (not
|
|
# GITHUB_TOKEN) and retry transient GitHub/network failures.
|
|
- name: Checkout auto-testing scripts (with retry)
|
|
env:
|
|
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
|
run: |
|
|
set -euo pipefail
|
|
rm -rf auto-testing
|
|
for attempt in 1 2 3 4 5; do
|
|
if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
|
|
echo "auto-testing cloned (attempt ${attempt})"
|
|
exit 0
|
|
fi
|
|
rm -rf auto-testing
|
|
echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
|
|
sleep $((attempt * 15))
|
|
done
|
|
echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
|
|
exit 1
|
|
|
|
- name: Show environment
|
|
run: |
|
|
uname -a
|
|
jq --version
|
|
openssl version
|
|
warp --version || true
|
|
df -h /data | tail -1
|
|
|
|
- name: Cleanup environment (before)
|
|
if: ${{ inputs.cleanup_before != 'false' }}
|
|
run: |
|
|
set -euo pipefail
|
|
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
|
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
|
for node in "${NODES[@]}"; do
|
|
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
|
set -euo pipefail
|
|
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
|
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
|
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
|
${SUDO} dpkg -P rustfs
|
|
fi
|
|
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
|
${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
|
|
'
|
|
done
|
|
|
|
- name: Install RustFS package & start cluster
|
|
run: |
|
|
ARGS=(--steps "1,2" -y --endpoint "${{ env.RUSTFS_API_ENDPOINT }}")
|
|
if [ -n "${{ inputs.package_url }}" ]; then
|
|
ARGS+=(--package-url "${{ inputs.package_url }}")
|
|
else
|
|
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
|
fi
|
|
./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
|
|
|
|
- name: Preflight checks
|
|
run: |
|
|
ARGS=(--preflight --endpoint "${{ env.RUSTFS_API_ENDPOINT }}")
|
|
if [ -n "${{ inputs.package_url }}" ]; then
|
|
ARGS+=(--package-url "${{ inputs.package_url }}")
|
|
else
|
|
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
|
fi
|
|
./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
|
|
|
|
- name: Run heal test (write -> outage -> heal -> verify)
|
|
id: test
|
|
# Case failures keep the run green: the report and the backlog
|
|
# issue manager carry the product signal.
|
|
continue-on-error: true
|
|
run: |
|
|
./auto-testing/rustfs_heal_test.sh \
|
|
--steps "3,4,5,6,7" -y \
|
|
--endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
|
|
--stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \
|
|
--warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \
|
|
--log-file "${LOG_FILE}"
|
|
|
|
- name: Generate report
|
|
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
|
run: |
|
|
set -euo pipefail
|
|
PACKAGE_URL='${{ inputs.package_url }}'
|
|
if [ -n "${PACKAGE_URL}" ]; then
|
|
PACKAGE_SOURCE="${PACKAGE_URL}"
|
|
else
|
|
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
|
fi
|
|
STEPS_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/steps.md"
|
|
CASE_RESULT=success
|
|
python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY' || CASE_RESULT=failure
|
|
import re
|
|
import sys
|
|
|
|
log_file, out_file = sys.argv[1], sys.argv[2]
|
|
ansi = re.compile(r'\x1b\[[0-9;]*m')
|
|
step_re = re.compile(r'^\[HEAL-STEP\]\s+(\d+)\s+(.+?)\s+(PASS|FAIL|SKIP)\s*$')
|
|
ver_re = re.compile(r'^\[HEAL-VERSION\]\s+(\S+)(?:\s+\(node\s+(\S+)\))?\s*$')
|
|
result_re = re.compile(r'^\[HEAL-RESULT\]\s+(PASS|FAIL)\s+(.*)$')
|
|
|
|
steps = {}
|
|
order = []
|
|
status_rank = {'SKIP': 0, 'PASS': 1, 'FAIL': 2}
|
|
version = None
|
|
version_node = None
|
|
verdict = None
|
|
verdict_detail = ''
|
|
try:
|
|
with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
|
|
for raw in fh:
|
|
line = ansi.sub('', raw).strip()
|
|
m = step_re.match(line)
|
|
if m:
|
|
n, desc, status = m.group(1), m.group(2), m.group(3)
|
|
if n not in steps:
|
|
order.append(n)
|
|
if n not in steps or status_rank[status] > status_rank[steps[n][1]]:
|
|
steps[n] = (desc, status)
|
|
continue
|
|
m = ver_re.match(line)
|
|
if m:
|
|
version, version_node = m.group(1), m.group(2)
|
|
continue
|
|
m = result_re.match(line)
|
|
if m and verdict != 'FAIL':
|
|
verdict, verdict_detail = m.group(1), m.group(2)
|
|
except FileNotFoundError:
|
|
pass
|
|
|
|
with open(out_file, 'w', encoding='utf-8') as out:
|
|
out.write('## Step Results\n\n')
|
|
if version:
|
|
node_note = f' (captured via `rustfs --version` on {version_node})' if version_node else ''
|
|
out.write(f'- Version under test: **{version}**{node_note}\n')
|
|
if verdict:
|
|
out.write(f'- Overall result: **{verdict}** — {verdict_detail}\n')
|
|
out.write('\n')
|
|
out.write('| Step | Description | Result |\n')
|
|
out.write('| --- | --- | --- |\n')
|
|
for n in sorted(order, key=int):
|
|
desc, status = steps[n]
|
|
out.write(f'| {n} | {desc} | {status} |\n')
|
|
if not order:
|
|
out.write('| - | - | NOT RUN (no step result lines found) |\n')
|
|
complete = set(steps) == {str(n) for n in range(1, 8)}
|
|
sys.exit(0 if complete and verdict != 'FAIL' and all(status == 'PASS' for _, status in steps.values()) else 1)
|
|
PY
|
|
RESULT=failure
|
|
if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
|
|
RESULT=success
|
|
fi
|
|
|
|
# Product gate: failing steps keep the run green — they are reported
|
|
# below and tracked in rustfs/backlog. Only harness/environment
|
|
# breakdowns turn the workflow red.
|
|
STEPS_TOTAL="$(grep -cE '^\[HEAL-STEP\]' "${LOG_FILE}" 2>/dev/null || true)"
|
|
STEPS_FAIL="$(grep -cE '^\[HEAL-STEP\].* FAIL$' "${LOG_FILE}" 2>/dev/null || true)"
|
|
STEPS_TOTAL=$(( ${STEPS_TOTAL:-0} + 0 )); STEPS_FAIL=$(( ${STEPS_FAIL:-0} + 0 ))
|
|
STEPS_PASS=$(( STEPS_TOTAL - STEPS_FAIL ))
|
|
HARNESS_OK=0
|
|
if [ '${{ steps.test.outcome }}' = 'success' ]; then
|
|
HARNESS_OK=1
|
|
elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${STEPS_TOTAL}" -gt 0 ] && [ "${STEPS_PASS}" -ge 1 ]; then
|
|
HARNESS_OK=1
|
|
fi
|
|
{
|
|
echo "# RustFS heal test report"
|
|
echo ""
|
|
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
|
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
|
|
echo "- Workflow Commit: ${GITHUB_SHA}"
|
|
echo "- Trigger: ${{ github.event_name }}"
|
|
echo "- Package: ${PACKAGE_SOURCE}"
|
|
echo "- Test Step Outcome: ${RESULT}"
|
|
echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
|
|
echo ""
|
|
if [ -s "${STEPS_TABLE}" ]; then
|
|
cat "${STEPS_TABLE}"
|
|
echo ""
|
|
fi
|
|
echo "- Product result: ${STEPS_PASS} passed, ${STEPS_FAIL} failed (failing steps are tracked in rustfs/backlog)"
|
|
echo ""
|
|
if [ -s "${STEPS_TABLE}" ]; then
|
|
echo "## Log tail"
|
|
echo '```text'
|
|
tail -n 200 "${LOG_FILE}" 2>/dev/null || true
|
|
echo '```'
|
|
else
|
|
echo "The suite or evidence validation failed. See this run's artifact for partial step results and suite.log."
|
|
fi
|
|
} | tee "${REPORT_FILE}"
|
|
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
|
# Red only for harness/environment breakdowns; step failures stay green.
|
|
[ "${HARNESS_OK}" = "1" ]
|
|
|
|
- name: Upload functional report to dashboard
|
|
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
|
continue-on-error: true
|
|
env:
|
|
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
|
SUITE: heal
|
|
run: |
|
|
set -euo pipefail
|
|
if [ -z "${GH_TOKEN:-}" ]; then
|
|
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
|
|
exit 0
|
|
fi
|
|
DATE="$(date -u +%Y-%m-%d)"
|
|
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
|
# Base64-encode the report into a temp file and feed it to jq via
|
|
# --rawfile: large reports (e.g. pool) exceed the OS argv limit and
|
|
# make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
|
|
B64_FILE="$(mktemp)"
|
|
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
|
|
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
|
if [ -n "${SHA}" ]; then
|
|
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
|
|
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
|
|
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
|
else
|
|
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
|
|
'{message:$msg, content:($content|rtrimstr("\n"))}' \
|
|
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
|
fi
|
|
rm -f "${B64_FILE}"
|
|
|
|
- name: Manage backlog issues (dedup / label / auto-close)
|
|
# Replaces the old per-run failure filing. One entry point that:
|
|
# - dedups by signal: failing cases are matched against open backlog
|
|
# issues by label (category + case ID); covered cases become a
|
|
# comment on the existing issue, only uncovered cases file a new one
|
|
# - labels new issues (functional-test, category, case IDs, env)
|
|
# - closes fixed issues after a fully green run
|
|
# - never files or closes on cancelled runs
|
|
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
|
continue-on-error: true
|
|
env:
|
|
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
|
SUITE: 'heal'
|
|
SUITE_LABEL: 'Heal'
|
|
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
|
run: |
|
|
set -euo pipefail
|
|
if [ -z "${GH_TOKEN:-}" ]; then
|
|
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
|
|
exit 0
|
|
fi
|
|
if [ ! -f auto-testing/scripts/issue_manager.py ]; then
|
|
echo "issue_manager.py not found in auto-testing checkout; skipping"
|
|
exit 0
|
|
fi
|
|
PACKAGE_URL='${{ inputs.package_url }}'
|
|
PACKAGE_SOURCE=""
|
|
if [ -n "${PACKAGE_URL}" ]; then
|
|
PACKAGE_SOURCE="${PACKAGE_URL}"
|
|
else
|
|
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
|
fi
|
|
python3 auto-testing/scripts/issue_manager.py handle \
|
|
--repo rustfs/backlog \
|
|
--suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
|
|
--outcome "${{ steps.test.outcome }}" \
|
|
--report-file "${REPORT_FILE}" \
|
|
--log "${LOG_FILE}" \
|
|
--run-url "${RUN_URL}" \
|
|
--run-id "${GITHUB_RUN_ID}" \
|
|
--attempt "${GITHUB_RUN_ATTEMPT}" \
|
|
--commit "${GITHUB_SHA}" \
|
|
--trigger "${{ github.event_name }}" \
|
|
--package-source "${PACKAGE_SOURCE}" \
|
|
--date "$(date -u +%Y-%m-%d)"
|
|
|
|
- name: Upload test logs
|
|
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
|
with:
|
|
name: rustfs-heal-test-${{ github.run_id }}-${{ github.run_attempt }}
|
|
path: |
|
|
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
|
|
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
|
|
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/warp.log
|
|
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/steps.md
|
|
if-no-files-found: error
|
|
|
|
- name: Cleanup environment (after)
|
|
if: ${{ always() && inputs.cleanup_after != 'false' }}
|
|
run: |
|
|
set -euo pipefail
|
|
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
|
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
|
for node in "${NODES[@]}"; do
|
|
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
|
set -euo pipefail
|
|
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
|
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
|
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
|
${SUDO} dpkg -P rustfs
|
|
fi
|
|
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
|
${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
|
|
'
|
|
done
|
|
|
|
- name: "Continue functional chain (next: Pool expansion)"
|
|
# Only chain-triggered runs forward to the next suite; standalone
|
|
# workflow_dispatch runs stop after their own cleanup. A failed
|
|
# handoff must never pass silently: it retries, then files an alert
|
|
# issue in rustfs/backlog so a stalled chain is visible.
|
|
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
|
continue-on-error: true
|
|
env:
|
|
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
|
run: |
|
|
set -uo pipefail
|
|
if [ -z "${GH_TOKEN:-}" ]; then
|
|
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
|
exit 1
|
|
fi
|
|
DISPATCHED=0
|
|
for attempt in 1 2 3; do
|
|
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
|
-f event_type='rustfs-chain-pool' \
|
|
-F 'client_payload[from_suite]=heal'; then
|
|
echo "dispatched next suite Pool expansion (attempt ${attempt})"
|
|
DISPATCHED=1
|
|
break
|
|
fi
|
|
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
|
sleep "${attempt}0"
|
|
done
|
|
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
|
echo "ERROR: functional chain stalled: could not dispatch Pool expansion after 3 attempts" >&2
|
|
TITLE="[functional][chain] stalled after heal (run ${GITHUB_RUN_ID})"
|
|
BODY_FILE="$(mktemp)"
|
|
{
|
|
echo "The functional chain could not hand off from **heal** to **Pool expansion** after 3 attempts."
|
|
echo ""
|
|
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
|
echo "- Expected next event: 'rustfs-chain-pool'"
|
|
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
|
echo "- Recovery: re-dispatch manually with"
|
|
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
|
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-pool'"
|
|
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
|
} > "${BODY_FILE}"
|
|
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
|
--body-file "${BODY_FILE}" --label functional-test \
|
|
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
|
|| echo "could not file the stall alert issue either; check the token" >&2
|
|
exit 1
|
|
fi
|
|
|
|
- name: Notify on failure
|
|
if: failure()
|
|
run: |
|
|
echo "RustFS heal test failed"
|
|
echo "Package source: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
|
|
echo "See the uploaded log artifact for details."
|