mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-23 02:53:36 +00:00
hector
bcf5184a85
A failing product case used to turn the whole workflow red, so the run conclusion carried no signal beyond 'something failed' and the report was suppressed. New semantics across the functional suites: - Suite steps run with continue-on-error: the outcome is still recorded for the report and the backlog issue manager (security/tier already carried the flag). - Generate report always publishes the full per-case table plus a 'Product result: N passed, M failed' summary, and its exit gate is harness health: red only when the suite never reached case level (no case verdicts), failed wholesale (zero passes, >=3 failures), or was cancelled/skipped. performance is unchanged (parked). - tier's structured gate no longer fails on case failures; it keeps red for evidence-init and missing-gate-result breakdowns. - pool/performance keep their existing red sources (install/benchmark). Workflow contract tests updated to the new exit semantics: the security report matrix keys green off the suite outcome, the evidence matrix expects green for failure outcomes with recorded case rows (except performance), the heal staged-rerun block expects the per-step table to always publish, and run steps are now required to carry continue-on-error. Verified locally: actionlint clean; test_security_workflow.py 21/21.
595 lines
25 KiB
YAML
595 lines
25 KiB
YAML
name: RustFS Tier Test
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
rustfs_version:
|
|
description: 'RustFS release tag to test (leave empty to use the latest nightly deb)'
|
|
required: false
|
|
package_url:
|
|
description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
|
|
required: false
|
|
type: string
|
|
rc_archive_url:
|
|
description: 'Exact RustFS CLI Linux archive URL.'
|
|
required: false
|
|
default: 'https://github.com/rustfs/cli/releases/download/v0.1.32/rustfs-cli-linux-amd64-v0.1.32.tar.gz'
|
|
type: string
|
|
rc_archive_sha256:
|
|
description: 'Expected SHA-256 of the RustFS CLI archive.'
|
|
required: false
|
|
default: 'ab00d937079dcb6f1c7b41d34bbfaad0eb0bd4f7218672cbcb7c33652d1c46df'
|
|
type: string
|
|
rc_sha256:
|
|
description: 'Expected SHA-256 of the extracted RustFS CLI binary.'
|
|
required: false
|
|
default: '320bdd4223a4d1986c1a098165f2198e92c35c4042b9a4d5e6fa33e9152477df'
|
|
type: string
|
|
force_case_failure:
|
|
description: 'Diagnostic only: rewrite single-single/TIER-101 to FAIL after execution to verify artifact and final-gate behavior.'
|
|
required: false
|
|
default: false
|
|
type: boolean
|
|
repository_dispatch:
|
|
# Chain handoff: dispatched when the KMS suite finishes.
|
|
types: [rustfs-chain-tier]
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
concurrency:
|
|
group: rustfs-shared-functional-tests
|
|
cancel-in-progress: false
|
|
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
|
|
env:
|
|
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
|
|
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
|
|
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
|
|
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
|
|
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
|
RUSTFS_RC_ARCHIVE_URL: ${{ inputs.rc_archive_url || 'https://github.com/rustfs/cli/releases/download/v0.1.32/rustfs-cli-linux-amd64-v0.1.32.tar.gz' }}
|
|
RUSTFS_RC_ARCHIVE_SHA256: ${{ inputs.rc_archive_sha256 || 'ab00d937079dcb6f1c7b41d34bbfaad0eb0bd4f7218672cbcb7c33652d1c46df' }}
|
|
RUSTFS_EXPECTED_RC_SHA256: ${{ inputs.rc_sha256 || '320bdd4223a4d1986c1a098165f2198e92c35c4042b9a4d5e6fa33e9152477df' }}
|
|
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
|
TIER_ARTIFACTS_DIR: /tmp/rustfs-tier-artifacts-${{ github.run_id }}-${{ github.run_attempt }}
|
|
|
|
jobs:
|
|
tier-test:
|
|
runs-on: smoke-testing
|
|
timeout-minutes: 420
|
|
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
|
steps:
|
|
- name: Initialize run evidence directory
|
|
id: evidence
|
|
run: |
|
|
set -euo pipefail
|
|
umask 077
|
|
if ! mkdir -- "${TIER_ARTIFACTS_DIR}"; then
|
|
echo "refusing to reuse tier evidence path: ${TIER_ARTIFACTS_DIR}" >&2
|
|
exit 1
|
|
fi
|
|
test -d "${TIER_ARTIFACTS_DIR}"
|
|
test ! -L "${TIER_ARTIFACTS_DIR}"
|
|
|
|
# auto-testing is private: clone it with the dedicated PF token (not
|
|
# GITHUB_TOKEN) and retry transient GitHub/network failures.
|
|
- name: Checkout auto-testing scripts (with retry)
|
|
env:
|
|
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
|
run: |
|
|
set -euo pipefail
|
|
rm -rf auto-testing
|
|
for attempt in 1 2 3 4 5; do
|
|
if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
|
|
echo "auto-testing cloned (attempt ${attempt})"
|
|
exit 0
|
|
fi
|
|
rm -rf auto-testing
|
|
echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
|
|
sleep $((attempt * 15))
|
|
done
|
|
echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
|
|
exit 1
|
|
|
|
- name: Prepare pinned RustFS CLI
|
|
id: rc
|
|
run: |
|
|
set -euo pipefail
|
|
umask 077
|
|
|
|
case "${RUSTFS_RC_ARCHIVE_URL}" in
|
|
https://*) ;;
|
|
*)
|
|
echo "RustFS CLI archive URL must use HTTPS" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
EXPECTED_ARCHIVE_SHA256="$(printf '%s' "${RUSTFS_RC_ARCHIVE_SHA256}" | tr '[:upper:]' '[:lower:]')"
|
|
EXPECTED_RC_SHA256="$(printf '%s' "${RUSTFS_EXPECTED_RC_SHA256}" | tr '[:upper:]' '[:lower:]')"
|
|
if ! [[ "${EXPECTED_ARCHIVE_SHA256}" =~ ^[0-9a-f]{64}$ ]]; then
|
|
echo "invalid RustFS CLI archive SHA-256" >&2
|
|
exit 1
|
|
fi
|
|
if ! [[ "${EXPECTED_RC_SHA256}" =~ ^[0-9a-f]{64}$ ]]; then
|
|
echo "invalid RustFS CLI binary SHA-256" >&2
|
|
exit 1
|
|
fi
|
|
|
|
RC_ROOT="${RUNNER_TEMP}/rustfs-tier-rc-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
|
RC_ARCHIVE="${RC_ROOT}/rustfs-cli.tar.gz"
|
|
RC_BIN="${RC_ROOT}/rc"
|
|
if ! mkdir -- "${RC_ROOT}"; then
|
|
echo "refusing to reuse RustFS CLI directory: ${RC_ROOT}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
curl --fail --location --retry 3 --retry-all-errors \
|
|
--connect-timeout 15 --max-time 180 \
|
|
--proto '=https' --proto-redir '=https' \
|
|
--output "${RC_ARCHIVE}" "${RUSTFS_RC_ARCHIVE_URL}"
|
|
RC_ARCHIVE_SIZE="$(wc -c < "${RC_ARCHIVE}" | tr -d '[:space:]')"
|
|
if [ "${RC_ARCHIVE_SIZE}" -eq 0 ] || [ "${RC_ARCHIVE_SIZE}" -gt 33554432 ]; then
|
|
echo "RustFS CLI archive size is outside the accepted range: ${RC_ARCHIVE_SIZE}" >&2
|
|
exit 1
|
|
fi
|
|
if ! ACTUAL_ARCHIVE_SHA256="$(openssl dgst -sha256 -r "${RC_ARCHIVE}" | awk '{print $1}')"; then
|
|
echo "failed to calculate RustFS CLI archive SHA-256" >&2
|
|
exit 1
|
|
fi
|
|
if [ "${ACTUAL_ARCHIVE_SHA256}" != "${EXPECTED_ARCHIVE_SHA256}" ]; then
|
|
echo "RustFS CLI archive SHA-256 mismatch: expected ${EXPECTED_ARCHIVE_SHA256}, got ${ACTUAL_ARCHIVE_SHA256}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
ARCHIVE_MEMBERS="$(tar -tzf "${RC_ARCHIVE}")"
|
|
if ! grep -Fxq 'rc' <<< "${ARCHIVE_MEMBERS}"; then
|
|
echo "RustFS CLI archive does not contain the rc entry" >&2
|
|
exit 1
|
|
fi
|
|
if ! tar -xOzf "${RC_ARCHIVE}" rc > "${RC_BIN}"; then
|
|
echo "failed to extract the RustFS CLI binary" >&2
|
|
exit 1
|
|
fi
|
|
chmod 0700 "${RC_BIN}"
|
|
if ! ACTUAL_RC_SHA256="$(openssl dgst -sha256 -r "${RC_BIN}" | awk '{print $1}')"; then
|
|
echo "failed to calculate RustFS CLI binary SHA-256" >&2
|
|
exit 1
|
|
fi
|
|
if [ "${ACTUAL_RC_SHA256}" != "${EXPECTED_RC_SHA256}" ]; then
|
|
echo "RustFS CLI binary SHA-256 mismatch: expected ${EXPECTED_RC_SHA256}, got ${ACTUAL_RC_SHA256}" >&2
|
|
exit 1
|
|
fi
|
|
if ! RC_VERSION_OUTPUT="$(timeout 30 "${RC_BIN}" --version 2>&1)"; then
|
|
echo "failed to execute the pinned RustFS CLI" >&2
|
|
exit 1
|
|
fi
|
|
RC_VERSION="${RC_VERSION_OUTPUT%%$'\n'*}"
|
|
if [ -z "${RC_VERSION}" ]; then
|
|
echo "pinned RustFS CLI returned an empty version" >&2
|
|
exit 1
|
|
fi
|
|
|
|
jq -n \
|
|
--arg schema_version '1' \
|
|
--arg generated_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
|
|
--arg archive_url "${RUSTFS_RC_ARCHIVE_URL}" \
|
|
--arg archive_sha256 "${ACTUAL_ARCHIVE_SHA256}" \
|
|
--arg archive_size "${RC_ARCHIVE_SIZE}" \
|
|
--arg path "${RC_BIN}" \
|
|
--arg version "${RC_VERSION}" \
|
|
--arg sha256 "${ACTUAL_RC_SHA256}" \
|
|
'{
|
|
schema_version: ($schema_version | tonumber),
|
|
generated_at: $generated_at,
|
|
archive: {
|
|
url: $archive_url,
|
|
sha256: $archive_sha256,
|
|
size: ($archive_size | tonumber)
|
|
},
|
|
binary: {
|
|
path: $path,
|
|
version: $version,
|
|
sha256: $sha256
|
|
}
|
|
}' > "${TIER_ARTIFACTS_DIR}/rc-bootstrap.json"
|
|
printf 'path=%s\n' "${RC_BIN}" >> "${GITHUB_OUTPUT}"
|
|
echo "RustFS CLI ready: ${RC_VERSION} (${ACTUAL_RC_SHA256})"
|
|
|
|
- name: Show environment
|
|
run: |
|
|
uname -a
|
|
jq --version
|
|
openssl version
|
|
df -h /data | tail -1
|
|
|
|
- name: Cleanup environment (before)
|
|
run: |
|
|
set -euo pipefail
|
|
sudo docker rm -f rustfs-test-mqtt >/dev/null 2>&1 || true
|
|
sudo rm -f /tmp/rustfs-mosquitto.conf
|
|
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
|
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
|
for node in "${NODES[@]}"; do
|
|
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
|
set -euo pipefail
|
|
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
|
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
|
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
|
${SUDO} dpkg -P rustfs
|
|
fi
|
|
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
|
${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms
|
|
'
|
|
done
|
|
|
|
- name: Ensure MQTT broker + clients
|
|
run: |
|
|
set -euo pipefail
|
|
if ! command -v mosquitto_sub >/dev/null 2>&1; then
|
|
sudo apt-get update
|
|
sudo apt-get install -y mosquitto-clients
|
|
fi
|
|
command -v docker >/dev/null 2>&1 || { echo 'docker not found on runner'; exit 1; }
|
|
sudo docker rm -f rustfs-test-mqtt >/dev/null 2>&1 || true
|
|
cat <<'EOF' | sudo tee /tmp/rustfs-mosquitto.conf >/dev/null
|
|
listener 1883 0.0.0.0
|
|
allow_anonymous true
|
|
EOF
|
|
sudo docker run -d --name rustfs-test-mqtt -p 1883:1883 \
|
|
-v /tmp/rustfs-mosquitto.conf:/mosquitto/config/mosquitto.conf:ro \
|
|
eclipse-mosquitto:2 >/dev/null
|
|
for _ in {1..10}; do
|
|
if ss -tln 2>/dev/null | grep -q ':1883'; then
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
ss -tln 2>/dev/null | grep -q ':1883' || {
|
|
echo 'mosquitto container is not listening on 1883'
|
|
sudo docker logs rustfs-test-mqtt || true
|
|
exit 1
|
|
}
|
|
|
|
- name: Run tier suite
|
|
id: test
|
|
# Case failures keep the run green: the report and the backlog
|
|
# issue manager carry the product signal.
|
|
continue-on-error: true
|
|
env:
|
|
PACKAGE_URL_INPUT: ${{ inputs.package_url }}
|
|
RC_BIN: ${{ steps.rc.outputs.path }}
|
|
RUSTFS_VERSION_INPUT: ${{ inputs.rustfs_version }}
|
|
run: |
|
|
set -euo pipefail
|
|
LOG_FILE="${TIER_ARTIFACTS_DIR}/rustfs-tier.log"
|
|
chmod +x auto-testing/rustfs-tier-test.sh
|
|
PACKAGE_URL="${PACKAGE_URL_INPUT}"
|
|
RUSTFS_VERSION="${RUSTFS_VERSION_INPUT}"
|
|
ARGS=(
|
|
--all-topologies
|
|
-y
|
|
--log-file "${LOG_FILE}"
|
|
--rc-bin "${RC_BIN}"
|
|
--artifacts-dir "${TIER_ARTIFACTS_DIR}"
|
|
)
|
|
if [ -n "${RUSTFS_EXPECTED_RC_SHA256}" ]; then
|
|
ARGS+=(--expected-rc-sha256 "${RUSTFS_EXPECTED_RC_SHA256}")
|
|
fi
|
|
if [ -n "${PACKAGE_URL}" ]; then
|
|
ARGS+=(--package-url "${PACKAGE_URL}")
|
|
elif [ -n "${RUSTFS_VERSION}" ]; then
|
|
ARGS+=(--version "${RUSTFS_VERSION}")
|
|
else
|
|
ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
|
|
fi
|
|
./auto-testing/rustfs-tier-test.sh "${ARGS[@]}"
|
|
|
|
- name: Inject diagnostic case failure
|
|
if: ${{ always() && steps.evidence.outcome == 'success' && inputs.force_case_failure }}
|
|
run: |
|
|
set -euo pipefail
|
|
RESULT_FILE="${TIER_ARTIFACTS_DIR}/cases/single-single--TIER-101.json"
|
|
test -s "${RESULT_FILE}"
|
|
TMP_FILE="$(mktemp "${TIER_ARTIFACTS_DIR}/cases/.forced.XXXXXX")"
|
|
jq '.status = "FAIL" | .case_rc = 97' "${RESULT_FILE}" > "${TMP_FILE}"
|
|
mv "${TMP_FILE}" "${RESULT_FILE}"
|
|
|
|
- name: Generate report
|
|
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
|
env:
|
|
PACKAGE_URL_INPUT: ${{ inputs.package_url }}
|
|
RUSTFS_VERSION_INPUT: ${{ inputs.rustfs_version }}
|
|
TEST_OUTCOME: ${{ steps.test.outcome }}
|
|
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
|
TRIGGER_NAME: ${{ github.event_name }}
|
|
run: |
|
|
set -euo pipefail
|
|
test -d "${TIER_ARTIFACTS_DIR}"
|
|
test ! -L "${TIER_ARTIFACTS_DIR}"
|
|
LOG_FILE="${TIER_ARTIFACTS_DIR}/rustfs-tier.log"
|
|
REPORT_FILE="${TIER_ARTIFACTS_DIR}/rustfs-tier-report.md"
|
|
CASE_TABLE="${TIER_ARTIFACTS_DIR}/rustfs-tier-cases.md"
|
|
GATE_RC_FILE="${TIER_ARTIFACTS_DIR}/rustfs-tier-gate.rc"
|
|
PACKAGE_URL="${PACKAGE_URL_INPUT}"
|
|
RUSTFS_VERSION="${RUSTFS_VERSION_INPUT}"
|
|
if [ -n "${PACKAGE_URL}" ]; then
|
|
PACKAGE_SOURCE="${PACKAGE_URL}"
|
|
elif [ -n "${RUSTFS_VERSION}" ]; then
|
|
PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
|
|
else
|
|
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
|
fi
|
|
if RC_BOOTSTRAP_SUMMARY="$(jq -r '.binary | "\(.version) / \(.sha256)"' "${TIER_ARTIFACTS_DIR}/rc-bootstrap.json" 2>/dev/null)"; then
|
|
:
|
|
else
|
|
RC_BOOTSTRAP_SUMMARY="missing or invalid"
|
|
fi
|
|
set +e
|
|
python3 auto-testing/rustfs_tier_report.py \
|
|
--results-dir "${TIER_ARTIFACTS_DIR}/cases" \
|
|
--provenance "${TIER_ARTIFACTS_DIR}/provenance.json" \
|
|
--output "${CASE_TABLE}"
|
|
CASE_GATE_RC=$?
|
|
set -e
|
|
printf '%s\n' "${CASE_GATE_RC}" > "${GATE_RC_FILE}"
|
|
if [ ! -s "${CASE_TABLE}" ]; then
|
|
{
|
|
echo "## Case Summary"
|
|
echo ""
|
|
echo "Structured report generation failed before producing output (exit ${CASE_GATE_RC})."
|
|
} > "${CASE_TABLE}"
|
|
fi
|
|
{
|
|
echo "# RustFS tier test report"
|
|
echo ""
|
|
echo "- Run: ${RUN_URL}"
|
|
echo "- Trigger: ${TRIGGER_NAME}"
|
|
echo "- Package: ${PACKAGE_SOURCE}"
|
|
echo "- Client bootstrap: ${RC_BOOTSTRAP_SUMMARY}"
|
|
echo "- Test Step Outcome: ${TEST_OUTCOME}"
|
|
echo "- Structured Gate Exit: ${CASE_GATE_RC}"
|
|
echo ""
|
|
cat "${CASE_TABLE}"
|
|
echo ""
|
|
echo "## Log tail"
|
|
echo '```text'
|
|
tail -n 200 "${LOG_FILE}" || true
|
|
echo '```'
|
|
} | tee "${REPORT_FILE}"
|
|
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
|
|
|
- name: Upload functional report to dashboard
|
|
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
|
continue-on-error: true
|
|
env:
|
|
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
|
REPORT_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier-report.md
|
|
SUITE: tier
|
|
run: |
|
|
set -euo pipefail
|
|
if [ -z "${GH_TOKEN:-}" ]; then
|
|
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
|
|
exit 0
|
|
fi
|
|
DATE="$(date -u +%Y-%m-%d)"
|
|
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
|
# Base64-encode the report into a temp file and feed it to jq via
|
|
# --rawfile: large reports (e.g. pool) exceed the OS argv limit and
|
|
# make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
|
|
B64_FILE="$(mktemp)"
|
|
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
|
|
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
|
if [ -n "${SHA}" ]; then
|
|
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
|
|
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
|
|
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
|
else
|
|
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
|
|
'{message:$msg, content:($content|rtrimstr("\n"))}' \
|
|
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
|
fi
|
|
rm -f "${B64_FILE}"
|
|
|
|
- name: Verify required tier evidence
|
|
id: evidence_verify
|
|
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
|
run: |
|
|
set -euo pipefail
|
|
failed=0
|
|
for name in \
|
|
rustfs-tier.log \
|
|
rustfs-tier-report.md \
|
|
rustfs-tier-cases.md \
|
|
rustfs-tier-gate.rc \
|
|
rc-bootstrap.json \
|
|
provenance.json; do
|
|
if [ ! -s "${TIER_ARTIFACTS_DIR}/${name}" ]; then
|
|
echo "required tier evidence is missing or empty: ${name}" >&2
|
|
failed=1
|
|
fi
|
|
done
|
|
for name in cases logs; do
|
|
if [ ! -d "${TIER_ARTIFACTS_DIR}/${name}" ]; then
|
|
echo "required tier evidence directory is missing: ${name}" >&2
|
|
failed=1
|
|
fi
|
|
done
|
|
if ! find "${TIER_ARTIFACTS_DIR}/cases" -maxdepth 1 -type f -name '*.json' -print -quit 2>/dev/null | grep -q .; then
|
|
echo "no atomic tier case result was produced" >&2
|
|
failed=1
|
|
fi
|
|
[ "${failed}" -eq 0 ]
|
|
|
|
- name: Upload report and logs
|
|
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
|
with:
|
|
name: rustfs-tier-test-${{ github.run_id }}-${{ github.run_attempt }}
|
|
path: ${{ env.TIER_ARTIFACTS_DIR }}/
|
|
if-no-files-found: error
|
|
|
|
- name: Cleanup environment (after)
|
|
if: always()
|
|
run: |
|
|
set -euo pipefail
|
|
sudo docker rm -f rustfs-test-mqtt >/dev/null 2>&1 || true
|
|
sudo rm -f /tmp/rustfs-mosquitto.conf
|
|
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
|
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
|
for node in "${NODES[@]}"; do
|
|
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
|
set -euo pipefail
|
|
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
|
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
|
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
|
${SUDO} dpkg -P rustfs
|
|
fi
|
|
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
|
${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms
|
|
'
|
|
done
|
|
|
|
- name: Cleanup pinned RustFS CLI
|
|
if: always()
|
|
run: |
|
|
set -euo pipefail
|
|
RC_ROOT="${RUNNER_TEMP}/rustfs-tier-rc-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
|
rm -f -- "${RC_ROOT}/rustfs-cli.tar.gz" "${RC_ROOT}/rc"
|
|
if [ -d "${RC_ROOT}" ]; then
|
|
rmdir -- "${RC_ROOT}"
|
|
fi
|
|
|
|
- name: Enforce tier suite result
|
|
id: gate
|
|
if: always()
|
|
env:
|
|
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
|
|
TEST_OUTCOME: ${{ steps.test.outcome }}
|
|
GATE_RC_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier-gate.rc
|
|
run: |
|
|
set -euo pipefail
|
|
# Product gate: failing cases keep the run green (the structured gate
|
|
# rc and the report carry the per-case signal, and the backlog issue
|
|
# manager tracks failures). Only harness/environment breakdowns —
|
|
# evidence initialization or a missing structured gate result — turn
|
|
# the workflow red.
|
|
failed=0
|
|
if [ "${EVIDENCE_OUTCOME}" != "success" ]; then
|
|
echo "tier evidence directory initialization is ${EVIDENCE_OUTCOME}, expected success" >&2
|
|
failed=1
|
|
elif [ ! -s "${GATE_RC_FILE}" ]; then
|
|
echo "structured gate result is missing" >&2
|
|
failed=1
|
|
fi
|
|
[ "${failed}" -eq 0 ]
|
|
|
|
- name: Manage backlog issues (dedup / label / auto-close)
|
|
# Replaces the old per-run failure filing. One entry point that:
|
|
# - dedups by signal: failing cases are matched against open backlog
|
|
# issues by label (category + case ID); covered cases become a
|
|
# comment on the existing issue, only uncovered cases file a new one
|
|
# - labels new issues (functional-test, category, case IDs, env)
|
|
# - closes fixed issues after a fully green run
|
|
# - never files or closes on cancelled runs
|
|
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
|
continue-on-error: true
|
|
env:
|
|
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
|
SUITE: 'tier'
|
|
SUITE_LABEL: 'Tier'
|
|
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
|
run: |
|
|
set -euo pipefail
|
|
if [ -z "${GH_TOKEN:-}" ]; then
|
|
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
|
|
exit 0
|
|
fi
|
|
if [ ! -f auto-testing/scripts/issue_manager.py ]; then
|
|
echo "issue_manager.py not found in auto-testing checkout; skipping"
|
|
exit 0
|
|
fi
|
|
PACKAGE_URL='${{ inputs.package_url }}'
|
|
RUSTFS_VERSION='${{ inputs.rustfs_version }}'
|
|
PACKAGE_SOURCE=""
|
|
if [ -n "${PACKAGE_URL}" ]; then
|
|
PACKAGE_SOURCE="${PACKAGE_URL}"
|
|
elif [ -n "${RUSTFS_VERSION}" ]; then
|
|
PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
|
|
else
|
|
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
|
fi
|
|
python3 auto-testing/scripts/issue_manager.py handle \
|
|
--repo rustfs/backlog \
|
|
--suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
|
|
--outcome "${{ steps.test.outcome }}" \
|
|
--report "${TIER_ARTIFACTS_DIR}/rustfs-tier-cases.md" \
|
|
--report-file "${TIER_ARTIFACTS_DIR}/rustfs-tier-report.md" \
|
|
--log "${TIER_ARTIFACTS_DIR}/rustfs-tier.log" \
|
|
--run-url "${RUN_URL}" \
|
|
--run-id "${GITHUB_RUN_ID}" \
|
|
--attempt "${GITHUB_RUN_ATTEMPT}" \
|
|
--commit "${GITHUB_SHA}" \
|
|
--trigger "${{ github.event_name }}" \
|
|
--package-source "${PACKAGE_SOURCE}" \
|
|
--date "$(date -u +%Y-%m-%d)"
|
|
|
|
- name: "Continue functional chain (next: Storage engine)"
|
|
# Only chain-triggered runs forward to the next suite; standalone
|
|
# workflow_dispatch runs stop after their own cleanup. A failed
|
|
# handoff must never pass silently: it retries, then files an alert
|
|
# issue in rustfs/backlog so a stalled chain is visible.
|
|
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
|
continue-on-error: true
|
|
env:
|
|
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
|
run: |
|
|
set -uo pipefail
|
|
if [ -z "${GH_TOKEN:-}" ]; then
|
|
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
|
exit 1
|
|
fi
|
|
DISPATCHED=0
|
|
for attempt in 1 2 3; do
|
|
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
|
-f event_type='rustfs-chain-storage' \
|
|
-F 'client_payload[from_suite]=tier'; then
|
|
echo "dispatched next suite Storage engine (attempt ${attempt})"
|
|
DISPATCHED=1
|
|
break
|
|
fi
|
|
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
|
sleep "${attempt}0"
|
|
done
|
|
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
|
echo "ERROR: functional chain stalled: could not dispatch Storage engine after 3 attempts" >&2
|
|
TITLE="[functional][chain] stalled after tier (run ${GITHUB_RUN_ID})"
|
|
BODY_FILE="$(mktemp)"
|
|
{
|
|
echo "The functional chain could not hand off from **tier** to **Storage engine** after 3 attempts."
|
|
echo ""
|
|
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
|
echo "- Expected next event: 'rustfs-chain-storage'"
|
|
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
|
echo "- Recovery: re-dispatch manually with"
|
|
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
|
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-storage'"
|
|
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
|
} > "${BODY_FILE}"
|
|
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
|
--body-file "${BODY_FILE}" --label functional-test \
|
|
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
|
|| echo "could not file the stall alert issue either; check the token" >&2
|
|
exit 1
|
|
fi
|
|
|
|
- name: Notify on failure
|
|
if: failure()
|
|
run: |
|
|
echo "RustFS tier suite failed"
|
|
echo "See the uploaded report and log artifacts for details."
|