mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-22 10:33:25 +00:00
hector
bcf5184a85
A failing product case used to turn the whole workflow red, so the run conclusion carried no signal beyond 'something failed' and the report was suppressed. New semantics across the functional suites: - Suite steps run with continue-on-error: the outcome is still recorded for the report and the backlog issue manager (security/tier already carried the flag). - Generate report always publishes the full per-case table plus a 'Product result: N passed, M failed' summary, and its exit gate is harness health: red only when the suite never reached case level (no case verdicts), failed wholesale (zero passes, >=3 failures), or was cancelled/skipped. performance is unchanged (parked). - tier's structured gate no longer fails on case failures; it keeps red for evidence-init and missing-gate-result breakdowns. - pool/performance keep their existing red sources (install/benchmark). Workflow contract tests updated to the new exit semantics: the security report matrix keys green off the suite outcome, the evidence matrix expects green for failure outcomes with recorded case rows (except performance), the heal staged-rerun block expects the per-step table to always publish, and run steps are now required to carry continue-on-error. Verified locally: actionlint clean; test_security_workflow.py 21/21.
479 lines
21 KiB
YAML
479 lines
21 KiB
YAML
# Copyright 2024 RustFS Team
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
name: RustFS Upgrade Test
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
from_version:
|
|
description: 'OLD RustFS release tag, e.g. 1.0.0-rc.3 (its release must ship a .deb asset). Leave empty for the default.'
|
|
required: false
|
|
default: '1.0.0-rc.3'
|
|
from_url:
|
|
description: 'OLD .deb URL. Overrides from_version.'
|
|
required: false
|
|
type: string
|
|
to_version:
|
|
description: 'NEW RustFS release tag, e.g. 1.0.0-rc.5 (any version with a .deb asset). Leave empty for latest nightly.'
|
|
required: false
|
|
to_url:
|
|
description: 'NEW .deb URL. Overrides to_version / nightly default.'
|
|
required: false
|
|
type: string
|
|
topology:
|
|
description: 'Topology to run (all = SNSD, SNMD, MNMD)'
|
|
type: choice
|
|
options:
|
|
- all
|
|
- single-single
|
|
- single-multi
|
|
- multi-multi
|
|
default: all
|
|
backends:
|
|
description: 'KMS backends to run (local,vault-kv2)'
|
|
required: false
|
|
default: 'local,vault-kv2'
|
|
cleanup_before:
|
|
description: 'Reset the nodes before the test (DESTROYS existing data/config)'
|
|
type: boolean
|
|
default: true
|
|
cleanup_after:
|
|
description: 'Reset the nodes after the test (DESTROYS test data/config)'
|
|
type: boolean
|
|
default: true
|
|
repository_dispatch:
|
|
# Functional-chain entry: dispatched by rustfs-functional-chain.yml.
|
|
types: [rustfs-chain-upgrade]
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
concurrency:
|
|
group: rustfs-shared-functional-tests
|
|
cancel-in-progress: false
|
|
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
|
|
env:
|
|
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
|
|
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
|
|
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
|
|
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
|
|
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
|
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
|
|
|
jobs:
|
|
upgrade-test:
|
|
runs-on: smoke-testing
|
|
timeout-minutes: 420
|
|
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
|
steps:
|
|
- name: Checkout repository (for report parser)
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Initialize functional evidence
|
|
id: evidence
|
|
run: |
|
|
set -euo pipefail
|
|
umask 077
|
|
FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-upgrade-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
|
mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
|
|
{
|
|
printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
|
printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
|
printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
|
printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
|
} >> "${GITHUB_ENV}"
|
|
|
|
# auto-testing is private: clone it with the dedicated PF token (not
|
|
# GITHUB_TOKEN) and retry transient GitHub/network failures.
|
|
- name: Checkout auto-testing scripts (with retry)
|
|
env:
|
|
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
|
run: |
|
|
set -euo pipefail
|
|
rm -rf auto-testing
|
|
for attempt in 1 2 3 4 5; do
|
|
if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
|
|
echo "auto-testing cloned (attempt ${attempt})"
|
|
exit 0
|
|
fi
|
|
rm -rf auto-testing
|
|
echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
|
|
sleep $((attempt * 15))
|
|
done
|
|
echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
|
|
exit 1
|
|
|
|
- name: Show environment
|
|
run: |
|
|
uname -a
|
|
jq --version
|
|
openssl version
|
|
aws --version || true
|
|
docker --version || true
|
|
df -h /data | tail -1
|
|
|
|
- name: Cleanup environment (before)
|
|
if: ${{ inputs.cleanup_before != 'false' || github.event_name != 'workflow_dispatch' }}
|
|
run: |
|
|
set -euo pipefail
|
|
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
|
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
|
for node in "${NODES[@]}"; do
|
|
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
|
set -euo pipefail
|
|
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
|
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
|
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
|
${SUDO} dpkg -P rustfs
|
|
fi
|
|
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
|
${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
|
|
'
|
|
done
|
|
|
|
- name: Ensure docker (Vault container)
|
|
run: |
|
|
if ! command -v docker >/dev/null 2>&1; then
|
|
sudo apt-get update
|
|
sudo apt-get install -y docker.io
|
|
fi
|
|
sudo systemctl enable --now docker
|
|
docker info >/dev/null 2>&1 || sudo docker info >/dev/null 2>&1
|
|
|
|
- name: Run upgrade compatibility suite
|
|
id: test
|
|
# Case failures keep the run green: the report and the backlog
|
|
# issue manager carry the product signal.
|
|
continue-on-error: true
|
|
env:
|
|
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
|
run: |
|
|
set -euo pipefail
|
|
chmod +x auto-testing/rustfs-upgrade-test.sh
|
|
FROM_URL='${{ inputs.from_url }}'
|
|
FROM_VERSION='${{ inputs.from_version }}'
|
|
TO_URL='${{ inputs.to_url }}'
|
|
TO_VERSION='${{ inputs.to_version }}'
|
|
TOPOLOGY='${{ inputs.topology }}'
|
|
BACKENDS='${{ inputs.backends }}'
|
|
ARGS=(-y --log-file "${LOG_FILE}")
|
|
if [ "${TOPOLOGY}" = "all" ] || [ -z "${TOPOLOGY}" ] || [ "${TOPOLOGY}" = "null" ]; then
|
|
ARGS+=(--all-topologies)
|
|
else
|
|
ARGS+=(--topology "${TOPOLOGY}")
|
|
fi
|
|
if [ -n "${BACKENDS}" ] && [ "${BACKENDS}" != "null" ]; then
|
|
ARGS+=(--backends "${BACKENDS}")
|
|
fi
|
|
if [ -n "${FROM_URL}" ]; then
|
|
ARGS+=(--from-url "${FROM_URL}")
|
|
elif [ -n "${FROM_VERSION}" ] && [ "${FROM_VERSION}" != "null" ]; then
|
|
ARGS+=(--from-version "${FROM_VERSION}")
|
|
fi
|
|
if [ -n "${TO_URL}" ]; then
|
|
ARGS+=(--to-url "${TO_URL}")
|
|
elif [ -n "${TO_VERSION}" ] && [ "${TO_VERSION}" != "null" ]; then
|
|
ARGS+=(--to-version "${TO_VERSION}")
|
|
else
|
|
ARGS+=(--to-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
|
|
fi
|
|
# Fail fast with a clear message when a requested release tag has
|
|
# no .deb asset (e.g. 1.0.0-rc.4 ships only zips), instead of
|
|
# letting the suite die mid-run on a 404.
|
|
check_release_asset() {
|
|
local version="$1" tag asset url
|
|
[ -n "${version}" ] && [ "${version}" != "null" ] || return 0
|
|
tag="${version#v}"
|
|
asset="rustfs_${tag//-/.}_amd64.deb"
|
|
url="https://github.com/rustfs/rustfs/releases/download/${tag}/${asset}"
|
|
if ! gh api "repos/rustfs/rustfs/releases/tags/${tag}" --jq '.assets[].name' 2>/dev/null | grep -qxF "${asset}"; then
|
|
echo "ERROR: release ${tag} has no downloadable asset ${asset}:" >&2
|
|
echo " ${url}" >&2
|
|
echo "Pick a tag whose release ships a .deb (check its release assets)." >&2
|
|
exit 1
|
|
fi
|
|
echo "resolved ${tag} -> ${url}"
|
|
}
|
|
if [ -z "${FROM_URL}" ]; then
|
|
check_release_asset "${FROM_VERSION}"
|
|
fi
|
|
if [ -z "${TO_URL}" ]; then
|
|
check_release_asset "${TO_VERSION}"
|
|
fi
|
|
./auto-testing/rustfs-upgrade-test.sh "${ARGS[@]}"
|
|
|
|
- name: Generate report
|
|
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
|
run: |
|
|
set -euo pipefail
|
|
FROM_URL='${{ inputs.from_url }}'
|
|
FROM_VERSION='${{ inputs.from_version }}'
|
|
TO_URL='${{ inputs.to_url }}'
|
|
TO_VERSION='${{ inputs.to_version }}'
|
|
if [ -n "${FROM_URL}" ]; then
|
|
FROM_SOURCE="${FROM_URL}"
|
|
elif [ -n "${FROM_VERSION}" ]; then
|
|
FROM_SOURCE="version ${FROM_VERSION}"
|
|
else
|
|
FROM_SOURCE="release (default)"
|
|
fi
|
|
if [ -n "${TO_URL}" ]; then
|
|
TO_SOURCE="${TO_URL}"
|
|
elif [ -n "${TO_VERSION}" ]; then
|
|
TO_SOURCE="version ${TO_VERSION}"
|
|
else
|
|
TO_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
|
fi
|
|
CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
|
|
MATRIX_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/matrix.md"
|
|
CASE_RESULT=success
|
|
python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" "${MATRIX_TABLE}" || CASE_RESULT=failure
|
|
RESULT=failure
|
|
if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
|
|
RESULT=success
|
|
fi
|
|
|
|
# Product gate: failing cases keep the run green — they are reported
|
|
# below and tracked in rustfs/backlog. Only harness/environment
|
|
# breakdowns (the suite never reached case level, or a wholesale
|
|
# failure with zero passes) turn the workflow red.
|
|
CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
|
|
CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
|
|
CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
|
|
CASES_PASS=$(( CASES_TOTAL - CASES_FAIL ))
|
|
HARNESS_OK=0
|
|
if [ '${{ steps.test.outcome }}' = 'success' ]; then
|
|
HARNESS_OK=1
|
|
elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${CASES_TOTAL}" -gt 0 ] && [ "${CASES_PASS}" -ge 1 ]; then
|
|
HARNESS_OK=1
|
|
fi
|
|
{
|
|
echo "# RustFS upgrade compatibility report"
|
|
echo ""
|
|
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
|
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
|
|
echo "- Workflow Commit: ${GITHUB_SHA}"
|
|
echo "- Trigger: ${{ github.event_name }}"
|
|
echo "- From: ${FROM_SOURCE}"
|
|
echo "- To: ${TO_SOURCE}"
|
|
echo "- Test Step Outcome: ${RESULT}"
|
|
echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
|
|
echo ""
|
|
if [ -s "${MATRIX_TABLE}" ]; then
|
|
cat "${MATRIX_TABLE}"
|
|
echo ""
|
|
fi
|
|
if [ -s "${CASE_TABLE}" ]; then
|
|
cat "${CASE_TABLE}"
|
|
echo ""
|
|
fi
|
|
echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)"
|
|
echo ""
|
|
if [ -s "${CASE_TABLE}" ]; then
|
|
echo "## Log tail"
|
|
echo '```text'
|
|
tail -n 200 "${LOG_FILE}" 2>/dev/null || true
|
|
echo '```'
|
|
else
|
|
echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
|
|
fi
|
|
} | tee "${REPORT_FILE}"
|
|
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
|
# Red only for harness/environment breakdowns; case failures stay green.
|
|
[ "${HARNESS_OK}" = "1" ]
|
|
|
|
- name: Upload functional report to dashboard
|
|
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
|
continue-on-error: true
|
|
env:
|
|
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
|
SUITE: upgrade
|
|
run: |
|
|
set -euo pipefail
|
|
if [ -z "${GH_TOKEN:-}" ]; then
|
|
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
|
|
exit 0
|
|
fi
|
|
DATE="$(date -u +%Y-%m-%d)"
|
|
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
|
# Base64-encode the report into a temp file and feed it to jq via
|
|
# --rawfile: large reports (e.g. pool) exceed the OS argv limit and
|
|
# make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
|
|
B64_FILE="$(mktemp)"
|
|
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
|
|
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
|
if [ -n "${SHA}" ]; then
|
|
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
|
|
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
|
|
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
|
else
|
|
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
|
|
'{message:$msg, content:($content|rtrimstr("\n"))}' \
|
|
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
|
fi
|
|
rm -f "${B64_FILE}"
|
|
|
|
- name: Manage backlog issues (dedup / label / auto-close)
|
|
# Replaces the old per-run failure filing. One entry point that:
|
|
# - dedups by signal: failing cases are matched against open backlog
|
|
# issues by label (category + case ID); covered cases become a
|
|
# comment on the existing issue, only uncovered cases file a new one
|
|
# - labels new issues (functional-test, category, case IDs, env)
|
|
# - closes fixed issues after a fully green run
|
|
# - never files or closes on cancelled runs
|
|
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
|
continue-on-error: true
|
|
env:
|
|
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
|
SUITE: 'upgrade'
|
|
SUITE_LABEL: 'Upgrade compatibility'
|
|
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
|
run: |
|
|
set -euo pipefail
|
|
if [ -z "${GH_TOKEN:-}" ]; then
|
|
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
|
|
exit 0
|
|
fi
|
|
if [ ! -f auto-testing/scripts/issue_manager.py ]; then
|
|
echo "issue_manager.py not found in auto-testing checkout; skipping"
|
|
exit 0
|
|
fi
|
|
FROM_URL='${{ inputs.from_url }}'
|
|
FROM_VERSION='${{ inputs.from_version }}'
|
|
TO_URL='${{ inputs.to_url }}'
|
|
TO_VERSION='${{ inputs.to_version }}'
|
|
PACKAGE_SOURCE=""
|
|
if [ -n "${TO_URL}" ]; then
|
|
PACKAGE_SOURCE="to ${TO_URL}"
|
|
elif [ -n "${TO_VERSION}" ]; then
|
|
PACKAGE_SOURCE="to version ${TO_VERSION}"
|
|
else
|
|
PACKAGE_SOURCE="to ${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
|
fi
|
|
if [ -n "${FROM_URL}" ]; then
|
|
PACKAGE_SOURCE="${PACKAGE_SOURCE}; from ${FROM_URL}"
|
|
elif [ -n "${FROM_VERSION}" ]; then
|
|
PACKAGE_SOURCE="${PACKAGE_SOURCE}; from version ${FROM_VERSION}"
|
|
fi
|
|
python3 auto-testing/scripts/issue_manager.py handle \
|
|
--repo rustfs/backlog \
|
|
--suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
|
|
--outcome "${{ steps.test.outcome }}" \
|
|
--report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" \
|
|
--report-file "${REPORT_FILE}" \
|
|
--log "${LOG_FILE}" \
|
|
--run-url "${RUN_URL}" \
|
|
--run-id "${GITHUB_RUN_ID}" \
|
|
--attempt "${GITHUB_RUN_ATTEMPT}" \
|
|
--commit "${GITHUB_SHA}" \
|
|
--trigger "${{ github.event_name }}" \
|
|
--package-source "${PACKAGE_SOURCE}" \
|
|
--date "$(date -u +%Y-%m-%d)"
|
|
|
|
- name: Upload report and logs
|
|
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
|
with:
|
|
name: rustfs-upgrade-test-${{ github.run_id }}-${{ github.run_attempt }}
|
|
path: |
|
|
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
|
|
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
|
|
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
|
|
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/matrix.md
|
|
if-no-files-found: error
|
|
retention-days: 3
|
|
|
|
- name: Cleanup environment (after)
|
|
if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}
|
|
run: |
|
|
set -euo pipefail
|
|
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
|
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
|
for node in "${NODES[@]}"; do
|
|
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
|
set -euo pipefail
|
|
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
|
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
|
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
|
${SUDO} dpkg -P rustfs
|
|
fi
|
|
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
|
${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
|
|
'
|
|
done
|
|
|
|
- name: "Continue functional chain (next: S3 compatibility)"
|
|
# Only chain-triggered runs forward to the next suite; standalone
|
|
# workflow_dispatch runs stop after their own cleanup. A failed
|
|
# handoff must never pass silently: it retries, then files an alert
|
|
# issue in rustfs/backlog so a stalled chain is visible.
|
|
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
|
continue-on-error: true
|
|
env:
|
|
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
|
run: |
|
|
set -uo pipefail
|
|
if [ -z "${GH_TOKEN:-}" ]; then
|
|
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
|
exit 1
|
|
fi
|
|
DISPATCHED=0
|
|
for attempt in 1 2 3; do
|
|
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
|
-f event_type='rustfs-chain-s3' \
|
|
-F 'client_payload[from_suite]=upgrade'; then
|
|
echo "dispatched next suite S3 compatibility (attempt ${attempt})"
|
|
DISPATCHED=1
|
|
break
|
|
fi
|
|
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
|
sleep "${attempt}0"
|
|
done
|
|
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
|
echo "ERROR: functional chain stalled: could not dispatch S3 compatibility after 3 attempts" >&2
|
|
TITLE="[functional][chain] stalled after upgrade (run ${GITHUB_RUN_ID})"
|
|
BODY_FILE="$(mktemp)"
|
|
{
|
|
echo "The functional chain could not hand off from **upgrade** to **S3 compatibility** after 3 attempts."
|
|
echo ""
|
|
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
|
echo "- Expected next event: 'rustfs-chain-s3'"
|
|
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
|
echo "- Recovery: re-dispatch manually with"
|
|
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
|
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-s3'"
|
|
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
|
} > "${BODY_FILE}"
|
|
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
|
--body-file "${BODY_FILE}" --label functional-test \
|
|
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
|
|| echo "could not file the stall alert issue either; check the token" >&2
|
|
exit 1
|
|
fi
|
|
|
|
- name: Notify on failure
|
|
if: failure()
|
|
run: |
|
|
echo "RustFS upgrade compatibility test failed"
|
|
echo "From: ${{ inputs.from_url || inputs.from_version || 'release (default)' }}"
|
|
echo "To: ${{ inputs.to_url || inputs.to_version || 'nightly (R2 latest)' }}"
|
|
echo "See the uploaded report and logs for details."
|