gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-21 01:53:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ci): retain failed functional chain reports and check runners (#7906)

Browse Source
This commit is contained in:
hector
2026-09-15 18:20:05 +08:00
committed by GitHub
parent 5498b3e5d5
commit f77efe5f2a
19 changed files with 303 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/functional-chain-health.yml
+2 -1
View File
@@ -20,7 +20,8 @@ jobs:
- name: Collect and publish verified chain health
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
run: python3 scripts/functional_chain_health.py --publish --output "${RUNNER_TEMP}/chain-health.json"
NIGHTLY_SOURCE_REF: ${{ vars.NIGHTLY_BRANCH || 'main' }}
run: python3 scripts/functional_chain_health.py --source-ref "${NIGHTLY_SOURCE_REF}" --publish --output "${RUNNER_TEMP}/chain-health.json"
- name: Retain health observation
if: always()
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
.github/workflows/rustfs-functional-chain.yml
+27 -1
View File
@@ -65,6 +65,11 @@ jobs:
path: ${{ runner.temp }}/chain-candidate.json
if-no-files-found: error
- name: Check shared functional runners before scheduling suites
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
run: python3 scripts/check_functional_runners.py smoke-testing pf-testing
upgrade:
needs: [prepare]
if: ${{ always() && needs.prepare.result == 'success' }}
@@ -147,7 +152,7 @@ jobs:
complete-chain:
needs: [prepare, upgrade, s3, kms, tier, storage, heal, pool, security, replication, performance]
if: ${{ always() && needs.prepare.result == 'success' }}
if: ${{ always() && needs.prepare.result != 'skipped' }}
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
@@ -155,12 +160,33 @@ jobs:
with:
persist-credentials: false
- name: Download suite evidence
id: download
continue-on-error: true
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
with:
pattern: functional-chain-*-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/chain-evidence
merge-multiple: true
- name: Summarize every suite including failures and missing evidence
if: always()
env:
CHAIN_MANIFEST: ${{ needs.prepare.outputs.manifest }}
CHAIN_NEEDS: ${{ toJSON(needs) }}
run: >-
python3 scripts/functional_chain_evidence.py summarize
--directory "${RUNNER_TEMP}/chain-evidence"
--output "${RUNNER_TEMP}/chain-report.json"
- name: Retain chain report regardless of test verdict
if: always()
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: functional-chain-report-${{ github.run_id }}-${{ github.run_attempt }}
path: |
${{ runner.temp }}/chain-report.json
${{ runner.temp }}/chain-report.md
if-no-files-found: error
- name: Verify every required suite
if: always()
env:
CHAIN_MANIFEST: ${{ needs.prepare.outputs.manifest }}
CHAIN_NEEDS: ${{ toJSON(needs) }}
.github/workflows/rustfs-heal-test.yml
+3 -3
View File
@@ -303,10 +303,10 @@ jobs:
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
exit 0
fi
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
EXISTING="$(gh issue list -R rustfs/backlog --state all \
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
--json number --jq '.[].number' || true)"
--json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
if [ -n "${EXISTING}" ]; then
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
exit 0
@@ -449,7 +449,7 @@ jobs:
--output "${RUNNER_TEMP}/chain-heal-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/heal.json"
- name: Upload chain evidence
if: ${{ always() && steps.chain_record.outcome == 'success' }}
if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: functional-chain-heal-${{ github.run_id }}-${{ github.run_attempt }}
.github/workflows/rustfs-kms-test.yml
+3 -3
View File
@@ -249,10 +249,10 @@ jobs:
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
exit 0
fi
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
EXISTING="$(gh issue list -R rustfs/backlog --state all \
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
--json number --jq '.[].number' || true)"
--json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
if [ -n "${EXISTING}" ]; then
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
exit 0
@@ -393,7 +393,7 @@ jobs:
--output "${RUNNER_TEMP}/chain-kms-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/kms.json"
- name: Upload chain evidence
if: ${{ always() && steps.chain_record.outcome == 'success' }}
if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: functional-chain-kms-${{ github.run_id }}-${{ github.run_attempt }}
.github/workflows/rustfs-performance-test.yml
+17 -3
View File
@@ -85,7 +85,21 @@ env:
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
jobs:
runner-check:
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Check performance runner before entering its queue
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
run: python3 scripts/check_functional_runners.py pf-testing
performance-test:
needs: runner-check
runs-on: pf-testing
timeout-minutes: 900
env:
@@ -260,10 +274,10 @@ jobs:
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
exit 0
fi
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
EXISTING="$(gh issue list -R rustfs/backlog --state all \
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
--json number --jq '.[].number' || true)"
--json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
if [ -n "${EXISTING}" ]; then
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
exit 0
@@ -348,7 +362,7 @@ jobs:
--output "${RUNNER_TEMP}/chain-performance-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/performance.json"
- name: Upload chain evidence
if: ${{ always() && steps.chain_record.outcome == 'success' }}
if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: functional-chain-performance-${{ github.run_id }}-${{ github.run_attempt }}
.github/workflows/rustfs-pool-expand-test.yml
+17 -6
View File
@@ -69,7 +69,7 @@ env:
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
RUSTFS_API_ENDPOINT: ${{ secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT }}
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES || 'vm000 vm001 vm002' }}
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
# Package used by the nightly run (workflow_dispatch inputs are empty for
@@ -89,7 +89,7 @@ jobs:
RUSTFS_POOL_PROXY_ENDPOINT: http://127.0.0.1:19000
RUSTFS_POOL_WARP_ENDPOINT: http://127.0.0.1:19000
RUSTFS_SHARED_PROXY_ENDPOINT: ${{ secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT }}
RUSTFS_POOL_NODE_ENDPOINTS: ${{ secrets.RUSTFS_POOL_NODE_ENDPOINTS || vars.RUSTFS_POOL_NODE_ENDPOINTS || 'http://rustfs-node1:9000 http://rustfs-node2:9000 http://rustfs-node3:9000' }}
RUSTFS_POOL_NODE_ENDPOINTS: ${{ secrets.RUSTFS_POOL_NODE_ENDPOINTS || vars.RUSTFS_POOL_NODE_ENDPOINTS || 'http://rustfs-node1:9000 http://rustfs-node2:9000 http://rustfs-node3:9000 http://rustfs-node4:9000' }}
steps:
- name: Checkout chain tooling
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -128,6 +128,17 @@ jobs:
warp --version || true
df -h /data | tail -1
- name: Validate pool topology before destructive cleanup
id: topology
run: |
set -euo pipefail
read -r -a NODES <<< "${RUSTFS_NODES}"
read -r -a ENDPOINTS <<< "${RUSTFS_POOL_NODE_ENDPOINTS}"
if [ "${#NODES[@]}" -lt 3 ] || [ "${#ENDPOINTS[@]}" -lt "${#NODES[@]}" ]; then
echo "Pool tests require at least three nodes and a direct endpoint for every node" >&2
exit 1
fi
- name: Cleanup environment (before)
if: ${{ inputs.cleanup_before != 'false' }}
run: |
@@ -581,10 +592,10 @@ jobs:
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
exit 0
fi
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
EXISTING="$(gh issue list -R rustfs/backlog --state all \
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
--json number --jq '.[].number' || true)"
--json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
if [ -n "${EXISTING}" ]; then
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
exit 0
@@ -643,7 +654,7 @@ jobs:
./auto-testing/rustfs_pool_nginx_stage.sh cleanup
- name: Cleanup environment (after)
if: ${{ always() && inputs.cleanup_after != 'false' }}
if: ${{ always() && steps.topology.outcome == 'success' && inputs.cleanup_after != 'false' }}
run: |
set -euo pipefail
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
@@ -731,7 +742,7 @@ jobs:
--output "${RUNNER_TEMP}/chain-pool-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/pool.json"
- name: Upload chain evidence
if: ${{ always() && steps.chain_record.outcome == 'success' }}
if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: functional-chain-pool-${{ github.run_id }}-${{ github.run_attempt }}
.github/workflows/rustfs-replication-test.yml
+3 -3
View File
@@ -257,10 +257,10 @@ jobs:
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
exit 0
fi
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
EXISTING="$(gh issue list -R rustfs/backlog --state all \
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
--json number --jq '.[].number' || true)"
--json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
if [ -n "${EXISTING}" ]; then
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
exit 0
@@ -398,7 +398,7 @@ jobs:
--output "${RUNNER_TEMP}/chain-replication-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/replication.json"
- name: Upload chain evidence
if: ${{ always() && steps.chain_record.outcome == 'success' }}
if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: functional-chain-replication-${{ github.run_id }}-${{ github.run_attempt }}
.github/workflows/rustfs-s3-compat-test.yml
+3 -3
View File
@@ -226,10 +226,10 @@ jobs:
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
exit 0
fi
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
EXISTING="$(gh issue list -R rustfs/backlog --state all \
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
--json number --jq '.[].number' || true)"
--json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
if [ -n "${EXISTING}" ]; then
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
exit 0
@@ -370,7 +370,7 @@ jobs:
--output "${RUNNER_TEMP}/chain-s3-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/s3.json"
- name: Upload chain evidence
if: ${{ always() && steps.chain_record.outcome == 'success' }}
if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: functional-chain-s3-${{ github.run_id }}-${{ github.run_attempt }}
.github/workflows/rustfs-security-test.yml
+3 -3
View File
@@ -259,10 +259,10 @@ jobs:
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
exit 0
fi
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
EXISTING="$(gh issue list -R rustfs/backlog --state all \
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
--json number --jq '.[].number' || true)"
--json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
if [ -n "${EXISTING}" ]; then
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
exit 0
@@ -372,7 +372,7 @@ jobs:
--output "${RUNNER_TEMP}/chain-security-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/security.json"
- name: Upload chain evidence
if: ${{ always() && steps.chain_record.outcome == 'success' }}
if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: functional-chain-security-${{ github.run_id }}-${{ github.run_attempt }}
.github/workflows/rustfs-storage-test.yml
+3 -3
View File
@@ -241,10 +241,10 @@ jobs:
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
exit 0
fi
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
EXISTING="$(gh issue list -R rustfs/backlog --state all \
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
--json number --jq '.[].number' || true)"
--json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
if [ -n "${EXISTING}" ]; then
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
exit 0
@@ -385,7 +385,7 @@ jobs:
--output "${RUNNER_TEMP}/chain-storage-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/storage.json"
- name: Upload chain evidence
if: ${{ always() && steps.chain_record.outcome == 'success' }}
if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: functional-chain-storage-${{ github.run_id }}-${{ github.run_attempt }}
.github/workflows/rustfs-table-test.yml
+2
View File
@@ -225,6 +225,8 @@ jobs:
python3 auto-testing/scripts/issue_manager.py handle \
--repo rustfs/backlog \
--suite table --category table --suite-label "S3 Tables" \
--outcome '${{ steps.test.outcome }}' \
--report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" \
--run-url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
--run-id "${GITHUB_RUN_ID}" \
--attempt "${GITHUB_RUN_ATTEMPT}" \
.github/workflows/rustfs-tier-test.yml
+3 -3
View File
@@ -523,10 +523,10 @@ jobs:
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
exit 0
fi
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
EXISTING="$(gh issue list -R rustfs/backlog --state all \
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
--json number --jq '.[].number' || true)"
--json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
if [ -n "${EXISTING}" ]; then
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
exit 0
@@ -642,7 +642,7 @@ jobs:
--output "${RUNNER_TEMP}/chain-tier-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/tier.json"
- name: Upload chain evidence
if: ${{ always() && steps.chain_record.outcome == 'success' }}
if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: functional-chain-tier-${{ github.run_id }}-${{ github.run_attempt }}
.github/workflows/rustfs-upgrade-test.yml
+3 -3
View File
@@ -324,10 +324,10 @@ jobs:
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
exit 0
fi
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
EXISTING="$(gh issue list -R rustfs/backlog --state all \
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
--json number --jq '.[].number' || true)"
--json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
if [ -n "${EXISTING}" ]; then
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
exit 0
@@ -472,7 +472,7 @@ jobs:
--output "${RUNNER_TEMP}/chain-upgrade-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/upgrade.json"
- name: Upload chain evidence
if: ${{ always() && steps.chain_record.outcome == 'success' }}
if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: functional-chain-upgrade-${{ github.run_id }}-${{ github.run_attempt }}