mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-22 10:33:25 +00:00
fix(ci): reject incomplete fault-tolerance runs (#7961)
* fix(ci): reject incomplete fault-tolerance runs * fix(ci): validate fault-tolerance case verdicts
This commit is contained in:
GatewayJ
@@ -4,13 +4,14 @@
|
||||
# behavior under drive and node loss against the erasure-coding contract and
|
||||
# snapshots health-endpoint responses at every tier.
|
||||
#
|
||||
# A single-node 4 drives (SNMD): hide 1/2/3 drives, restore
|
||||
# A single-node 4 drives (SNMD): revoke access to 1/2/3 drives, restore
|
||||
# B multi-node 4x1 (one drive per node): stop 1/2/3 nodes, restore
|
||||
# C multi-node 4x4 (16 drives, EC:4): stop 1 node (read-quorum boundary),
|
||||
# stop 2 nodes, restore
|
||||
# C2 multi-node 4x4 with EC:8: 2 nodes down puts 8 drives online -- reads
|
||||
# satisfy the EC read quorum while the lock majority is broken (the
|
||||
# reported divergence window: reads 503 with lock_quorum_unavailable)
|
||||
# D multi-node 4x1: stop 1/2 nodes, then restore
|
||||
#
|
||||
# Expectations come from product source (default_parity_count, erasure set
|
||||
# sizing). By default a "reads refused although the read quorum is met"
|
||||
@@ -32,6 +33,11 @@ on:
|
||||
description: 'Direct .deb URL. Required unless the nightly default is wanted.'
|
||||
required: false
|
||||
type: string
|
||||
auto_testing_ref:
|
||||
description: 'auto-testing branch, tag, or SHA (defaults to main)'
|
||||
required: false
|
||||
default: main
|
||||
type: string
|
||||
strict:
|
||||
description: 'Fail the suite when reads are refused despite a met read quorum'
|
||||
type: boolean
|
||||
@@ -52,7 +58,7 @@ on:
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# The suite stops services and hides drive dirs on the shared fleet; only one
|
||||
# The suite stops services and revokes drive access on the shared fleet; only one
|
||||
# functional suite may touch the environment at a time.
|
||||
concurrency:
|
||||
group: rustfs-shared-functional-tests-v2
|
||||
@@ -92,6 +98,7 @@ jobs:
|
||||
printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'RESULTS_FILE=%s/results.json\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'EVIDENCE_DIR=%s/evidence\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
@@ -109,16 +116,24 @@ jobs:
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
|
||||
ref: ${{ steps.chain.outputs.testing_sha || inputs.auto_testing_ref || 'main' }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
|
||||
- name: Record auto-testing revision
|
||||
run: |
|
||||
set -euo pipefail
|
||||
AUTO_TESTING_SHA="$(git -C auto-testing rev-parse HEAD)"
|
||||
printf 'AUTO_TESTING_SHA=%s\n' "${AUTO_TESTING_SHA}" >> "${GITHUB_ENV}"
|
||||
echo "auto-testing revision: ${AUTO_TESTING_SHA}"
|
||||
|
||||
- name: Show environment
|
||||
run: |
|
||||
uname -a
|
||||
jq --version
|
||||
aws --version
|
||||
echo "auto-testing: ${AUTO_TESTING_SHA}"
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
@@ -134,14 +149,14 @@ jobs:
|
||||
CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
|
||||
run: python3 scripts/prepare_functional_package.py prepare
|
||||
|
||||
- name: Run fault-tolerance scenarios (A, B, C, C2)
|
||||
- name: Run fault-tolerance scenarios (A, B, C, C2, D)
|
||||
timeout-minutes: 45
|
||||
id: test
|
||||
# Standalone case failures are reported by the backlog manager.
|
||||
# Chain evidence separately requires complete passing cases.
|
||||
continue-on-error: true
|
||||
run: |
|
||||
ARGS=(--all -y --package-url "${{ steps.chain_package.outputs.package_url || inputs.package_url || env.RUSTFS_NIGHTLY_PACKAGE_URL }}" --log-file "${LOG_FILE}")
|
||||
ARGS=(--all -y --package-url "${{ steps.chain_package.outputs.package_url || inputs.package_url || env.RUSTFS_NIGHTLY_PACKAGE_URL }}" --log-file "${LOG_FILE}" --results-file "${RESULTS_FILE}")
|
||||
if [ "${{ inputs.strict }}" = "true" ]; then
|
||||
ARGS+=(--strict)
|
||||
fi
|
||||
@@ -158,12 +173,13 @@ jobs:
|
||||
echo ""
|
||||
echo "- Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Package: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
|
||||
echo "- auto-testing: \`${AUTO_TESTING_SHA}\`"
|
||||
echo "- Strict mode: ${{ inputs.strict || 'false' }}"
|
||||
echo ""
|
||||
echo "## Per-probe results"
|
||||
echo ""
|
||||
echo '```'
|
||||
grep -E '^FT-(CASE|SUMMARY|REPORT)' "${LOG_FILE}" || echo "(no FT-CASE lines found)"
|
||||
grep -E '^FT-(CASE|COMPLETE|SUMMARY|REPORT)' "${LOG_FILE}" || echo "(no FT-CASE lines found)"
|
||||
echo '```'
|
||||
echo ""
|
||||
echo "## Health snapshots"
|
||||
@@ -174,19 +190,30 @@ jobs:
|
||||
done
|
||||
} > "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
# Product gate: failing FT cases keep the run green — the report above and
|
||||
# the backlog issue manager carry the signal. Only harness/environment
|
||||
# breakdowns (no case verdicts at all) turn the workflow red.
|
||||
FT_CASES="$(grep -cE '^FT-CASE:' "${LOG_FILE}" 2>/dev/null || true)"
|
||||
FT_FAILED="$(grep -cE '^FT-CASE: .* verdict=UNEXPECTED' "${LOG_FILE}" 2>/dev/null || true)"
|
||||
FT_PASSED=$(( ${FT_CASES:-0} - ${FT_FAILED:-0} ))
|
||||
HARNESS_OK=0
|
||||
if [ "${{ steps.test.outcome }}" = "success" ]; then
|
||||
HARNESS_OK=1
|
||||
elif [ "${{ steps.test.outcome }}" = "failure" ] && [ "${FT_CASES}" -gt 0 ] && [ "${FT_PASSED}" -ge 1 ]; then
|
||||
HARNESS_OK=1
|
||||
fi
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
printf 'complete=false\n' >> "${GITHUB_OUTPUT}"
|
||||
jq -e --arg testing_sha "${AUTO_TESTING_SHA}" '
|
||||
.schema_version == 1 and
|
||||
.auto_testing_sha == $testing_sha and
|
||||
.selected_scenarios == ["A", "B", "C", "C2", "D"] and
|
||||
.complete == true and
|
||||
.counts.expected == 38 and
|
||||
.counts.seen == 38 and
|
||||
.counts.harness_errors == 0 and
|
||||
(.expected_cases | length) == 38 and
|
||||
(.expected_cases | unique | length) == 38 and
|
||||
(.seen_cases | length) == 38 and
|
||||
(.seen_cases | unique | length) == 38 and
|
||||
(.seen_cases | sort) == (.expected_cases | sort) and
|
||||
(.missing_cases | length) == 0 and
|
||||
(.harness_errors | length) == 0 and
|
||||
(.cases | length) == 38 and
|
||||
([.cases[].case_id] | sort) == (.expected_cases | sort) and
|
||||
([.cases[].case_id] | unique | length) == 38 and
|
||||
all(.cases[]; .verdict == "pass" or .verdict == "known-divergence" or .verdict == "UNEXPECTED") and
|
||||
.counts.unexpected == ([.cases[] | select(.verdict == "UNEXPECTED")] | length) and
|
||||
.counts.known_divergence == ([.cases[] | select(.verdict == "known-divergence")] | length)
|
||||
' "${RESULTS_FILE}" >/dev/null
|
||||
printf 'complete=true\n' >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
timeout-minutes: 2
|
||||
@@ -196,7 +223,7 @@ jobs:
|
||||
# green run. Never acts on cancelled runs. Logic lives in
|
||||
# auto-testing/scripts/issue_manager.py, which parses the
|
||||
# FT-CASE verdict lines from the suite log.
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' && steps.chain_report.outputs.complete == 'true' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
@@ -272,6 +299,7 @@ jobs:
|
||||
name: rustfs-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results.json
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/evidence/
|
||||
if-no-files-found: warn
|
||||
|
||||
Reference in New Issue
Block a user