gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-19 09:05:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ci): retain failed functional chain reports and check runners (#7906)

Browse Source
This commit is contained in:
hector
2026-09-15 18:20:05 +08:00
committed by GitHub
parent 5498b3e5d5
commit f77efe5f2a
19 changed files with 303 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/functional-chain-health.yml
+2 -1
View File
@@ -20,7 +20,8 @@ jobs:
- name: Collect and publish verified chain health
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
run: python3 scripts/functional_chain_health.py --publish --output "${RUNNER_TEMP}/chain-health.json"
NIGHTLY_SOURCE_REF: ${{ vars.NIGHTLY_BRANCH || 'main' }}
run: python3 scripts/functional_chain_health.py --source-ref "${NIGHTLY_SOURCE_REF}" --publish --output "${RUNNER_TEMP}/chain-health.json"
- name: Retain health observation
if: always()
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
.github/workflows/rustfs-functional-chain.yml
+27 -1
View File
@@ -65,6 +65,11 @@ jobs:
path: ${{ runner.temp }}/chain-candidate.json
if-no-files-found: error
- name: Check shared functional runners before scheduling suites
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
run: python3 scripts/check_functional_runners.py smoke-testing pf-testing
upgrade:
needs: [prepare]
if: ${{ always() && needs.prepare.result == 'success' }}
@@ -147,7 +152,7 @@ jobs:
complete-chain:
needs: [prepare, upgrade, s3, kms, tier, storage, heal, pool, security, replication, performance]
if: ${{ always() && needs.prepare.result == 'success' }}
if: ${{ always() && needs.prepare.result != 'skipped' }}
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
@@ -155,12 +160,33 @@ jobs:
with:
persist-credentials: false
- name: Download suite evidence
id: download
continue-on-error: true
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
with:
pattern: functional-chain-*-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/chain-evidence
merge-multiple: true
- name: Summarize every suite including failures and missing evidence
if: always()
env:
CHAIN_MANIFEST: ${{ needs.prepare.outputs.manifest }}
CHAIN_NEEDS: ${{ toJSON(needs) }}
run: >-
python3 scripts/functional_chain_evidence.py summarize
--directory "${RUNNER_TEMP}/chain-evidence"
--output "${RUNNER_TEMP}/chain-report.json"
- name: Retain chain report regardless of test verdict
if: always()
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: functional-chain-report-${{ github.run_id }}-${{ github.run_attempt }}
path: |
${{ runner.temp }}/chain-report.json
${{ runner.temp }}/chain-report.md
if-no-files-found: error
- name: Verify every required suite
if: always()
env:
CHAIN_MANIFEST: ${{ needs.prepare.outputs.manifest }}
CHAIN_NEEDS: ${{ toJSON(needs) }}
.github/workflows/rustfs-heal-test.yml
+3 -3
View File
@@ -303,10 +303,10 @@ jobs:
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
exit 0
fi
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
EXISTING="$(gh issue list -R rustfs/backlog --state all \
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
--json number --jq '.[].number' || true)"
--json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
if [ -n "${EXISTING}" ]; then
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
exit 0
@@ -449,7 +449,7 @@ jobs:
--output "${RUNNER_TEMP}/chain-heal-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/heal.json"
- name: Upload chain evidence
if: ${{ always() && steps.chain_record.outcome == 'success' }}
if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: functional-chain-heal-${{ github.run_id }}-${{ github.run_attempt }}
.github/workflows/rustfs-kms-test.yml
+3 -3
View File
@@ -249,10 +249,10 @@ jobs:
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
exit 0
fi
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
EXISTING="$(gh issue list -R rustfs/backlog --state all \
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
--json number --jq '.[].number' || true)"
--json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
if [ -n "${EXISTING}" ]; then
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
exit 0
@@ -393,7 +393,7 @@ jobs:
--output "${RUNNER_TEMP}/chain-kms-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/kms.json"
- name: Upload chain evidence
if: ${{ always() && steps.chain_record.outcome == 'success' }}
if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: functional-chain-kms-${{ github.run_id }}-${{ github.run_attempt }}
.github/workflows/rustfs-performance-test.yml
+17 -3
View File
@@ -85,7 +85,21 @@ env:
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
jobs:
runner-check:
if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Check performance runner before entering its queue
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
run: python3 scripts/check_functional_runners.py pf-testing
performance-test:
needs: runner-check
runs-on: pf-testing
timeout-minutes: 900
env:
@@ -260,10 +274,10 @@ jobs:
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
exit 0
fi
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
EXISTING="$(gh issue list -R rustfs/backlog --state all \
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
--json number --jq '.[].number' || true)"
--json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
if [ -n "${EXISTING}" ]; then
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
exit 0
@@ -348,7 +362,7 @@ jobs:
--output "${RUNNER_TEMP}/chain-performance-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/performance.json"
- name: Upload chain evidence
if: ${{ always() && steps.chain_record.outcome == 'success' }}
if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: functional-chain-performance-${{ github.run_id }}-${{ github.run_attempt }}
.github/workflows/rustfs-pool-expand-test.yml
+17 -6
View File
@@ -69,7 +69,7 @@ env:
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
RUSTFS_API_ENDPOINT: ${{ secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT }}
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES || 'vm000 vm001 vm002' }}
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
# Package used by the nightly run (workflow_dispatch inputs are empty for
@@ -89,7 +89,7 @@ jobs:
RUSTFS_POOL_PROXY_ENDPOINT: http://127.0.0.1:19000
RUSTFS_POOL_WARP_ENDPOINT: http://127.0.0.1:19000
RUSTFS_SHARED_PROXY_ENDPOINT: ${{ secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT }}
RUSTFS_POOL_NODE_ENDPOINTS: ${{ secrets.RUSTFS_POOL_NODE_ENDPOINTS || vars.RUSTFS_POOL_NODE_ENDPOINTS || 'http://rustfs-node1:9000 http://rustfs-node2:9000 http://rustfs-node3:9000' }}
RUSTFS_POOL_NODE_ENDPOINTS: ${{ secrets.RUSTFS_POOL_NODE_ENDPOINTS || vars.RUSTFS_POOL_NODE_ENDPOINTS || 'http://rustfs-node1:9000 http://rustfs-node2:9000 http://rustfs-node3:9000 http://rustfs-node4:9000' }}
steps:
- name: Checkout chain tooling
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -128,6 +128,17 @@ jobs:
warp --version || true
df -h /data | tail -1
- name: Validate pool topology before destructive cleanup
id: topology
run: |
set -euo pipefail
read -r -a NODES <<< "${RUSTFS_NODES}"
read -r -a ENDPOINTS <<< "${RUSTFS_POOL_NODE_ENDPOINTS}"
if [ "${#NODES[@]}" -lt 3 ] || [ "${#ENDPOINTS[@]}" -lt "${#NODES[@]}" ]; then
echo "Pool tests require at least three nodes and a direct endpoint for every node" >&2
exit 1
fi
- name: Cleanup environment (before)
if: ${{ inputs.cleanup_before != 'false' }}
run: |
@@ -581,10 +592,10 @@ jobs:
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
exit 0
fi
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
EXISTING="$(gh issue list -R rustfs/backlog --state all \
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
--json number --jq '.[].number' || true)"
--json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
if [ -n "${EXISTING}" ]; then
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
exit 0
@@ -643,7 +654,7 @@ jobs:
./auto-testing/rustfs_pool_nginx_stage.sh cleanup
- name: Cleanup environment (after)
if: ${{ always() && inputs.cleanup_after != 'false' }}
if: ${{ always() && steps.topology.outcome == 'success' && inputs.cleanup_after != 'false' }}
run: |
set -euo pipefail
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
@@ -731,7 +742,7 @@ jobs:
--output "${RUNNER_TEMP}/chain-pool-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/pool.json"
- name: Upload chain evidence
if: ${{ always() && steps.chain_record.outcome == 'success' }}
if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: functional-chain-pool-${{ github.run_id }}-${{ github.run_attempt }}
.github/workflows/rustfs-replication-test.yml
+3 -3
View File
@@ -257,10 +257,10 @@ jobs:
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
exit 0
fi
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
EXISTING="$(gh issue list -R rustfs/backlog --state all \
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
--json number --jq '.[].number' || true)"
--json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
if [ -n "${EXISTING}" ]; then
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
exit 0
@@ -398,7 +398,7 @@ jobs:
--output "${RUNNER_TEMP}/chain-replication-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/replication.json"
- name: Upload chain evidence
if: ${{ always() && steps.chain_record.outcome == 'success' }}
if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: functional-chain-replication-${{ github.run_id }}-${{ github.run_attempt }}
.github/workflows/rustfs-s3-compat-test.yml
+3 -3
View File
@@ -226,10 +226,10 @@ jobs:
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
exit 0
fi
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
EXISTING="$(gh issue list -R rustfs/backlog --state all \
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
--json number --jq '.[].number' || true)"
--json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
if [ -n "${EXISTING}" ]; then
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
exit 0
@@ -370,7 +370,7 @@ jobs:
--output "${RUNNER_TEMP}/chain-s3-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/s3.json"
- name: Upload chain evidence
if: ${{ always() && steps.chain_record.outcome == 'success' }}
if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: functional-chain-s3-${{ github.run_id }}-${{ github.run_attempt }}
.github/workflows/rustfs-security-test.yml
+3 -3
View File
@@ -259,10 +259,10 @@ jobs:
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
exit 0
fi
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
EXISTING="$(gh issue list -R rustfs/backlog --state all \
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
--json number --jq '.[].number' || true)"
--json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
if [ -n "${EXISTING}" ]; then
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
exit 0
@@ -372,7 +372,7 @@ jobs:
--output "${RUNNER_TEMP}/chain-security-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/security.json"
- name: Upload chain evidence
if: ${{ always() && steps.chain_record.outcome == 'success' }}
if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: functional-chain-security-${{ github.run_id }}-${{ github.run_attempt }}
.github/workflows/rustfs-storage-test.yml
+3 -3
View File
@@ -241,10 +241,10 @@ jobs:
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
exit 0
fi
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
EXISTING="$(gh issue list -R rustfs/backlog --state all \
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
--json number --jq '.[].number' || true)"
--json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
if [ -n "${EXISTING}" ]; then
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
exit 0
@@ -385,7 +385,7 @@ jobs:
--output "${RUNNER_TEMP}/chain-storage-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/storage.json"
- name: Upload chain evidence
if: ${{ always() && steps.chain_record.outcome == 'success' }}
if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: functional-chain-storage-${{ github.run_id }}-${{ github.run_attempt }}
.github/workflows/rustfs-table-test.yml
+2
View File
@@ -225,6 +225,8 @@ jobs:
python3 auto-testing/scripts/issue_manager.py handle \
--repo rustfs/backlog \
--suite table --category table --suite-label "S3 Tables" \
--outcome '${{ steps.test.outcome }}' \
--report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" \
--run-url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
--run-id "${GITHUB_RUN_ID}" \
--attempt "${GITHUB_RUN_ATTEMPT}" \
.github/workflows/rustfs-tier-test.yml
+3 -3
View File
@@ -523,10 +523,10 @@ jobs:
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
exit 0
fi
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
EXISTING="$(gh issue list -R rustfs/backlog --state all \
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
--json number --jq '.[].number' || true)"
--json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
if [ -n "${EXISTING}" ]; then
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
exit 0
@@ -642,7 +642,7 @@ jobs:
--output "${RUNNER_TEMP}/chain-tier-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/tier.json"
- name: Upload chain evidence
if: ${{ always() && steps.chain_record.outcome == 'success' }}
if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: functional-chain-tier-${{ github.run_id }}-${{ github.run_attempt }}
.github/workflows/rustfs-upgrade-test.yml
+3 -3
View File
@@ -324,10 +324,10 @@ jobs:
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
exit 0
fi
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT})"
EXISTING="$(gh issue list -R rustfs/backlog --state all \
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
--json number --jq '.[].number' || true)"
--json number,title --jq ".[] | select(.title == \"${TITLE}\") | .number")"
if [ -n "${EXISTING}" ]; then
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
exit 0
@@ -472,7 +472,7 @@ jobs:
--output "${RUNNER_TEMP}/chain-upgrade-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/upgrade.json"
- name: Upload chain evidence
if: ${{ always() && steps.chain_record.outcome == 'success' }}
if: ${{ always() && steps.chain_record.outputs.written == 'true' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: functional-chain-upgrade-${{ github.run_id }}-${{ github.run_attempt }}
scripts/check_functional_runners.py
+27
View File
@@ -0,0 +1,27 @@
#!/usr/bin/env python3
"""Fail before dispatching destructive suites to an offline shared runner."""
import sys
from resolve_functional_candidate import REPOSITORY, api, require
def check(labels):
runners = []
for page in range(1, 11):
batch = api(f"repos/{REPOSITORY}/actions/runners?per_page=100&page={page}")["runners"]
runners.extend(batch)
if len(batch) < 100:
break
else:
raise ValueError("runner inventory exceeds the inspection limit")
missing = [label for label in labels if not any(
runner.get("status") == "online" and label in {item["name"] for item in runner.get("labels", [])}
for runner in runners)]
require(not missing, "No online runner for: " + ", ".join(missing)
+ ". Restore the runner service before retrying; no tests were dispatched by this check.")
print("Online functional runner labels: " + ", ".join(labels))
if __name__ == "__main__":
require(len(sys.argv) > 1, "at least one runner label is required")
check(sys.argv[1:])
scripts/functional_chain_evidence.py
+65 -1
View File
@@ -101,12 +101,67 @@ def record(chain, suite, report, output):
result["valid"] = True
except (OSError, ValueError, subprocess.SubprocessError) as exc:
error = exc
result["error"] = str(exc)
output.parent.mkdir(parents=True, exist_ok=False)
output.write_text(json.dumps(result, sort_keys=True) + "\n")
# A failed validation may still produce fresh diagnostics. A failed write
# or directory collision must never authorize uploading a leftover file.
with open(os.environ["GITHUB_OUTPUT"], "a") as step_output:
step_output.write("written=true\n")
if error:
raise error
def summarize(chain, directory, needs):
"""Retain failed/missing lanes without granting complete-success evidence."""
lanes = []
for suite in SUITES:
lane = {"suite": suite, "result": needs.get(suite, {}).get("result", "missing"),
"evidence": None, "error": None}
try:
record = json.loads((directory / (suite + ".json")).read_text())
require(chain is not None and record.get("chain") == chain and record.get("suite") == suite,
"suite evidence identity mismatch")
lane["evidence"] = record
except (OSError, ValueError, AttributeError) as error:
lane["error"] = str(error)
lanes.append(lane)
result = {"schema": 1, "chain": chain, "needs": needs, "lanes": lanes,
"complete": False, "error": None, "completed_at": datetime.now(timezone.utc).isoformat()}
try:
require(chain is not None, "candidate preparation did not complete")
aggregate(chain, directory, {suite: value for suite, value in needs.items() if suite != "prepare"})
result["complete"] = True
except (OSError, ValueError, KeyError, TypeError, AttributeError) as error:
result["error"] = str(error)
return result
def render_summary(result):
lines = ["# RustFS functional chain report", "",
"- All required suites passed with verified evidence: " + str(result["complete"]).lower(),
"- Preparation: " + result["needs"].get("prepare", {}).get("result", "missing")]
if result["chain"]:
chain = result["chain"]
manifest = chain["candidate"]["manifest"]
lines += [f"- Chain run / attempt: {chain['run_id']} / {chain['attempt']}",
f"- Build run / attempt: {manifest['build_run_id']} / {manifest['build_run_attempt']}",
f"- Source: {manifest.get('source_ref', 'main')} @ {manifest['source_sha']}",
f"- Package SHA256: {manifest['package_sha256']}", f"- Test scripts: {chain['testing_sha']}"]
lines += ["", "| Suite | Job result | Evidence | PASS | FAIL | SKIP | UNSUPPORTED | RUNNING |",
"| --- | --- | --- | --- | --- | --- | --- | --- |"]
for lane in result["lanes"]:
record = lane["evidence"] or {}
counts = record.get("counts", {})
state = "valid" if record.get("valid") is True else ("invalid" if record else "missing")
values = [lane["suite"], lane["result"], state] + [str(counts.get(key, "—")) for key in
("PASS", "FAIL", "SKIP", "UNSUPPORTED", "RUNNING")]
lines.append("| " + " | ".join(values) + " |")
lines += ["", "Missing, skipped, cancelled or invalid evidence is NOT a passing test or proof of a fix.",
"See the suite artifacts for case results and diagnostics; this report does not replace the complete-success gate."]
return "\n".join(lines) + "\n"
def aggregate(chain, directory, needs):
require(set(needs) == set(SUITES), "aggregate is missing a required lane")
require(all(value.get("result") == "success" for value in needs.values()), "a required suite did not succeed")
@@ -129,12 +184,21 @@ def validate_records(chain, records):
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("mode", choices=("consume", "record", "aggregate"))
parser.add_argument("mode", choices=("consume", "record", "aggregate", "summarize"))
parser.add_argument("--suite", choices=SUITES)
parser.add_argument("--report", type=Path)
parser.add_argument("--output", type=Path)
parser.add_argument("--directory", type=Path)
args = parser.parse_args()
if args.mode == "summarize":
chain = current_chain() if os.environ.get("CHAIN_MANIFEST") else None
result = summarize(chain, args.directory, json.loads(os.environ["CHAIN_NEEDS"]))
args.output.write_text(json.dumps(result, sort_keys=True) + "\n")
args.output.with_suffix(".md").write_text(render_summary(result))
if os.environ.get("GITHUB_STEP_SUMMARY"):
with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as summary:
summary.write(render_summary(result))
return
chain = current_chain()
if args.mode == "consume":
consume(chain)
scripts/functional_chain_health.py
+7 -4
View File
@@ -63,13 +63,15 @@ def complete_success(run):
return result
def collect(limit=20):
def collect(limit=20, source_ref="main"):
source_ref = source_ref.removeprefix("refs/heads/")
require(bool(source_ref), "expected nightly source ref is empty")
observed = datetime.now(timezone.utc)
workflow = api(f"repos/{REPOSITORY}/actions/workflows/{WORKFLOW}")
runs = api(f"repos/{REPOSITORY}/actions/workflows/{WORKFLOW}/runs?branch=main&per_page={limit}")["workflow_runs"]
runs.sort(key=lambda run: timestamp(run["run_started_at"]), reverse=True)
result = {"schema": 1, "observed_at": observed.isoformat(), "workflow_state": workflow["state"],
"owner": "@overtrue", "scan_limit": limit, "inspection_complete": True,
"owner": "@overtrue", "scan_limit": limit, "inspection_complete": True, "expected_source_ref": source_ref,
"latest_attempt": None, "last_complete_success": {}, "healthy": False}
for index, listed in enumerate(runs):
run = api(f"repos/{REPOSITORY}/actions/runs/{listed['id']}/attempts/{listed['run_attempt']}")
@@ -104,7 +106,7 @@ def collect(limit=20):
complete["fresh"] = observed <= timestamp(complete["expires_at"])
latest = result["latest_attempt"] or {}
result["healthy"] = (result["workflow_state"] == "active" and latest.get("verification") == "complete"
and latest.get("source_ref") == "main" and result["last_complete_success"].get("main", {}).get("fresh") is True)
and latest.get("source_ref") == source_ref and result["last_complete_success"].get(source_ref, {}).get("fresh") is True)
return result
@@ -134,8 +136,9 @@ def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--publish", action="store_true")
parser.add_argument("--source-ref", default="main", help="Expected nightly build source, not the workflow branch")
args = parser.parse_args()
result = collect()
result = collect(source_ref=args.source_ref)
if args.publish:
publish(result)
args.output.write_text(json.dumps(result, indent=2) + "\n")
scripts/test_functional_chain.py
+94
View File
@@ -94,6 +94,35 @@ class EvidenceTests(unittest.TestCase):
self.assertEqual(result["chain"], self.chain)
self.assertEqual(len(result["suites"]), 10)
def test_failure_report_preserves_counts_without_weakening_success_gate(self):
path = self.directory / "s3.json"
record = json.loads(path.read_text())
record.update(valid=False, counts={**record["counts"], "FAIL": 2}, error="failed cases")
path.write_text(json.dumps(record))
needs = {**self.needs, "s3": {"result": "failure"}, "performance": {"result": "cancelled"}}
(self.directory / "performance.json").unlink()
result = evidence.summarize(self.chain, self.directory, needs)
self.assertFalse(result["complete"])
self.assertEqual(result["lanes"][1]["evidence"]["counts"]["FAIL"], 2)
self.assertEqual(result["lanes"][-1]["result"], "cancelled")
self.assertIsNone(result["lanes"][-1]["evidence"])
with self.assertRaises(ValueError):
evidence.aggregate(self.chain, self.directory, needs)
def test_preparation_failure_still_has_all_ten_lanes(self):
result = evidence.summarize(None, self.directory / "absent", {"prepare": {"result": "failure"}})
self.assertFalse(result["complete"])
self.assertEqual(len(result["lanes"]), 10)
self.assertIn("| performance | missing | missing |", evidence.render_summary(result))
self.assertIn("Preparation: failure", evidence.render_summary(result))
def test_malformed_or_cross_attempt_record_is_not_reused(self):
for payload in ("broken json", "[]", json.dumps({"suite": "s3", "chain": {"attempt": 9}})):
(self.directory / "s3.json").write_text(payload)
result = evidence.summarize(self.chain, self.directory, self.needs)
self.assertFalse(result["complete"])
self.assertIsNone(result["lanes"][1]["evidence"])
def test_missing_failed_cancelled_or_skipped_lane_never_passes(self):
for state in ("failure", "cancelled", "skipped", "pending"):
with self.subTest(state=state), self.assertRaises(ValueError):
@@ -172,14 +201,32 @@ class EnvelopeTests(unittest.TestCase):
report.write_text("| Case | Name | Status |\n| --- | --- | --- |\n| KMS-1 | fixture | PASS |\n")
for index, (key, status) in enumerate((("CHAIN_TEST_OUTCOME", "failure"), ("CHAIN_REPORT_OUTCOME", "failure"), ("CHAIN_JOB_STATUS", "cancelled"))):
output = self.root / str(index) / "kms.json"
Path(self.env["GITHUB_OUTPUT"]).unlink(missing_ok=True)
with mock.patch.dict(evidence.os.environ, {**self.env, key: status}), mock.patch.object(evidence.subprocess, "check_output", return_value="d" * 40), self.assertRaises(ValueError):
evidence.record(self.chain, "kms", report, output)
self.assertFalse(json.loads(output.read_text())["valid"])
self.assertIn("error", json.loads(output.read_text()))
self.assertEqual(Path(self.env["GITHUB_OUTPUT"]).read_text(), "written=true\n")
output = self.root / "success" / "kms.json"
with mock.patch.dict(evidence.os.environ, self.env), mock.patch.object(evidence.subprocess, "check_output", return_value="d" * 40):
evidence.record(self.chain, "kms", report, output)
self.assertTrue(json.loads(output.read_text())["valid"])
def test_collision_or_failed_write_never_authorizes_evidence_upload(self):
report = self.root / "cases.md"
report.write_text("| Case | Status |\n| --- | --- |\n| KMS-1 | PASS |\n")
output = self.root / "stale" / "kms.json"
output.parent.mkdir()
output.write_text("OLD RUN EVIDENCE")
with mock.patch.dict(evidence.os.environ, self.env), mock.patch.object(evidence.subprocess, "check_output", return_value="d" * 40):
with self.assertRaises(FileExistsError):
evidence.record(self.chain, "kms", report, output)
self.assertEqual(output.read_text(), "OLD RUN EVIDENCE")
self.assertFalse(Path(self.env["GITHUB_OUTPUT"]).exists())
with mock.patch.object(Path, "write_text", side_effect=OSError("disk full")), self.assertRaises(OSError):
evidence.record(self.chain, "kms", report, self.root / "new" / "kms.json")
self.assertFalse(Path(self.env["GITHUB_OUTPUT"]).exists())
def test_unknown_status_cannot_hide_among_passing_cases(self):
text = "| Case | Name | Status |\n| --- | --- | --- |\n| KMS-1 | fixture | PASS |\n| KMS-2 | fixture | NOT RUN |\n"
with self.assertRaises(ValueError):
@@ -198,6 +245,53 @@ class EnvelopeTests(unittest.TestCase):
complete = "\n".join(yaml_block(lines, "complete-chain", 2))
self.assertIn("needs: [prepare, " + ", ".join(evidence.SUITES) + "]", complete)
self.assertIn("functional_chain_evidence.py aggregate", complete)
self.assertIn("functional_chain_evidence.py summarize", complete)
self.assertIn("functional-chain-report-", complete)
self.assertIn("needs.prepare.result != 'skipped'", complete)
def test_every_lane_retains_failed_evidence_and_deduplicates_its_own_attempt(self):
paths = list((candidate.ROOT / ".github/workflows").glob("rustfs-*-test.yml"))
lanes = [path for path in paths if "name: Upload chain evidence" in path.read_text()]
self.assertEqual(len(lanes), 10)
for path in lanes:
with self.subTest(path=path.name):
text = path.read_text()
self.assertIn("if: ${{ always() && steps.chain_record.outputs.written == 'true' }}", text)
self.assertIn("attempt ${GITHUB_RUN_ATTEMPT})", text)
self.assertIn('select(.title == \\"${TITLE}\\")', text)
def test_table_issue_manager_receives_outcome_and_case_evidence(self):
text = (candidate.ROOT / ".github/workflows/rustfs-table-test.yml").read_text()
self.assertIn("--outcome '${{ steps.test.outcome }}'", text)
self.assertIn('--report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"', text)
def test_pool_topology_is_checked_before_cleanup_with_four_node_defaults(self):
text = (candidate.ROOT / ".github/workflows/rustfs-pool-expand-test.yml").read_text()
self.assertIn("http://rustfs-node4:9000", text)
self.assertIn("vars.RUSTFS_NODES || 'vm000 vm001 vm002'", text)
self.assertLess(text.index("name: Validate pool topology"), text.index("name: Cleanup environment (before)"))
self.assertIn("steps.topology.outcome == 'success' && inputs.cleanup_after", text)
class RunnerTests(unittest.TestCase):
def test_offline_or_missing_runner_fails_before_dispatch(self):
import check_functional_runners as runners
for inventory in ([], [{"status": "offline", "labels": [{"name": "pf-testing"}]}]):
with mock.patch.object(runners, "api", return_value={"runners": inventory}), self.assertRaisesRegex(ValueError, "pf-testing"):
runners.check(["pf-testing"])
def test_busy_online_runner_is_available_and_inventory_is_paginated(self):
import check_functional_runners as runners
pages = [{"runners": [{"status": "online", "labels": []}] * 100},
{"runners": [{"status": "online", "busy": True, "labels": [{"name": "pf-testing"}]}]}]
with mock.patch.object(runners, "api", side_effect=pages) as api:
runners.check(["pf-testing"])
self.assertIn("page=2", api.call_args.args[0])
def test_unavailable_inventory_does_not_assume_online(self):
import check_functional_runners as runners
with mock.patch.object(runners, "api", side_effect=OSError("forbidden")), self.assertRaises(OSError):
runners.check(["pf-testing"])
if __name__ == "__main__":
scripts/test_functional_chain_health.py
+9
View File
@@ -85,6 +85,15 @@ class HealthTests(unittest.TestCase):
with self.assertRaisesRegex(ValueError, "changed during inspection"):
health.collect()
def test_health_follows_configured_build_source_not_workflow_main(self):
run = {**self.run, "status": "completed", "conclusion": "success"}
complete = self.validate()
for expected, healthy in (("release", True), ("refs/heads/release", True), ("main", False)):
responses = [{"state": "active"}, {"workflow_runs": [run]}, run, {"workflow_runs": [run]}]
with mock.patch.object(health, "api", side_effect=responses), mock.patch.object(health, "complete_success", return_value=complete):
result = health.collect(source_ref=expected)
self.assertEqual(result["healthy"], healthy)
def test_publication_uses_the_read_blob_sha(self):
current = {"schema": 1, "observed_at": self.now, "last_complete_success": {}, "healthy": False}
existing = {"sha": "reviewed-blob", "content": base64.b64encode(json.dumps(current).encode()).decode()}
scripts/test_security_workflow.py
+12 -6
View File
@@ -398,11 +398,17 @@ class FunctionalWorkflowTests(unittest.TestCase):
self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", steps["Generate report"])
cleanup = steps["Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"]
condition = next(line.strip() for line in cleanup if line.startswith(" if:"))
self.assertIn(condition, (
"if: always()",
"if: ${{ always() && inputs.cleanup_after != 'false' }}",
"if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}",
))
if suite == "pool-expand":
self.assertEqual(condition, "if: ${{ always() && steps.topology.outcome == 'success' && inputs.cleanup_after != 'false' }}")
self.assertIn(" id: topology", steps["Validate pool topology before destructive cleanup"])
self.assertLess(list(steps).index("Validate pool topology before destructive cleanup"),
list(steps).index("Cleanup environment (before)"))
else:
self.assertIn(condition, (
"if: always()",
"if: ${{ always() && inputs.cleanup_after != 'false' }}",
"if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}",
))
if suite != "performance":
handoff = next(
value for name, value in steps.items() if name.startswith("Continue functional chain")
@@ -567,7 +573,7 @@ class FunctionalEvidenceTests(WorkflowSteps, unittest.TestCase):
self.assertLess(names.index("Checkout repository (for report parser)"), names.index("Checkout auto-testing scripts"))
for name, lines in self.steps.items():
if name == "Upload chain evidence":
self.assertIn(" if: ${{ always() && steps.chain_record.outcome == 'success' }}", lines)
self.assertIn(" if: ${{ always() && steps.chain_record.outputs.written == 'true' }}", lines)
self.assertIn(" if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}", self.steps["Record chain evidence"])
self.assertIn(" if-no-files-found: error", lines)
continue