diff --git a/.github/workflows/rustfs-heal-test.yml b/.github/workflows/rustfs-heal-test.yml index 666c83b4c..28bbb24bc 100644 --- a/.github/workflows/rustfs-heal-test.yml +++ b/.github/workflows/rustfs-heal-test.yml @@ -59,6 +59,20 @@ jobs: # (storage -> heal -> pool). Pool expansion no longer re-runs heal. if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: + - name: Initialize functional evidence + id: evidence + run: | + set -euo pipefail + umask 077 + FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-heal-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" + { + printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'TMPDIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + } >> "${GITHUB_ENV}" + # auto-testing is private: clone it with the dedicated PF token (not # GITHUB_TOKEN) and retry transient GitHub/network failures. - name: Checkout auto-testing scripts (with retry) @@ -114,7 +128,7 @@ jobs: else ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") fi - ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" + ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}" - name: Preflight checks run: | @@ -124,7 +138,7 @@ jobs: else ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") fi - ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" + ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}" - name: Run heal test (write -> outage -> heal -> verify) id: test @@ -134,13 +148,10 @@ jobs: --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \ --stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \ --warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \ - --log-file /tmp/rustfs-heal-test.log + --log-file "${LOG_FILE}" - name: Generate report - if: always() - env: - LOG_FILE: /tmp/rustfs-heal-test.log - REPORT_FILE: /tmp/rustfs-heal-report.md + if: ${{ always() && steps.evidence.outcome == 'success' }} run: | set -euo pipefail PACKAGE_URL='${{ inputs.package_url }}' @@ -149,8 +160,9 @@ jobs: else PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}" fi - STEPS_TABLE="/tmp/rustfs-heal-steps.md" - python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY' + STEPS_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/steps.md" + CASE_RESULT=success + python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY' || CASE_RESULT=failure import re import sys @@ -162,6 +174,7 @@ jobs: steps = {} order = [] + status_rank = {'SKIP': 0, 'PASS': 1, 'FAIL': 2} version = None version_node = None verdict = None @@ -175,14 +188,15 @@ jobs: n, desc, status = m.group(1), m.group(2), m.group(3) if n not in steps: order.append(n) - steps[n] = (desc, status) # later lines win (fail after pass) + if n not in steps or status_rank[status] > status_rank[steps[n][1]]: + steps[n] = (desc, status) continue m = ver_re.match(line) if m: version, version_node = m.group(1), m.group(2) continue m = result_re.match(line) - if m: + if m and verdict != 'FAIL': verdict, verdict_detail = m.group(1), m.group(2) except FileNotFoundError: pass @@ -202,30 +216,43 @@ jobs: out.write(f'| {n} | {desc} | {status} |\n') if not order: out.write('| - | - | NOT RUN (no step result lines found) |\n') + complete = set(steps) == {str(n) for n in range(1, 8)} + sys.exit(0 if complete and verdict != 'FAIL' and all(status == 'PASS' for _, status in steps.values()) else 1) PY + RESULT=failure + if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then + RESULT=success + fi { echo "# RustFS heal test report" echo "" echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo "- Attempt: ${GITHUB_RUN_ATTEMPT}" + echo "- Workflow Commit: ${GITHUB_SHA}" echo "- Trigger: ${{ github.event_name }}" echo "- Package: ${PACKAGE_SOURCE}" - echo "- Test Step Outcome: ${{ steps.test.outcome }}" + echo "- Test Step Outcome: ${RESULT}" + echo "- Suite Step Outcome: ${{ steps.test.outcome }}" echo "" - cat "${STEPS_TABLE}" || true - echo "" - echo "## Log tail" - echo '```text' - tail -n 200 "${LOG_FILE}" || true - echo '```' + if [ "${RESULT}" = "success" ]; then + cat "${STEPS_TABLE}" + echo "" + echo "## Log tail" + echo '```text' + tail -n 200 "${LOG_FILE}" + echo '```' + else + echo "The suite or evidence validation failed. See this run's artifact for partial step results and suite.log." + fi } | tee "${REPORT_FILE}" cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" + [ "${RESULT}" = "success" ] - name: Upload functional report to dashboard - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} continue-on-error: true env: GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }} - REPORT_FILE: /tmp/rustfs-heal-report.md SUITE: heal run: | set -euo pipefail @@ -252,11 +279,10 @@ jobs: continue-on-error: true env: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} + EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }} SUITE: 'heal' SUITE_LABEL: 'Heal' RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - REPORT_FILE: '/tmp/rustfs-heal-report.md' - LOG_FILE: '/tmp/rustfs-heal-test.log' run: | set -euo pipefail if [ -z "${GH_TOKEN:-}" ]; then @@ -284,14 +310,16 @@ jobs: echo "" echo "- Suite: \`${SUITE}\`" echo "- Run: ${RUN_URL}" + echo "- Attempt: ${GITHUB_RUN_ATTEMPT}" + echo "- Workflow Commit: ${GITHUB_SHA}" echo "- Trigger: ${GITHUB_EVENT_NAME}" echo "- Date: $(date -u +%Y-%m-%d)" echo "" echo "## Report (errors and symptoms)" echo "" - if [ -s "${REPORT_FILE}" ]; then + if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then redact < "${REPORT_FILE}" - elif [ -s "${LOG_FILE:-}" ]; then + elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then echo "(report file missing; log tail below)" echo "" tail -n 200 "${LOG_FILE}" | redact @@ -307,14 +335,12 @@ jobs: echo "filed backlog issue for suite ${SUITE}" - name: Upload test logs - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: - name: rustfs-heal-test-${{ github.run_id }} - path: | - /tmp/rustfs-heal-test*.log - /tmp/rustfs-warp.*.log - if-no-files-found: warn + name: rustfs-heal-test-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/ + if-no-files-found: error - name: Cleanup environment (after) if: ${{ always() && inputs.cleanup_after != 'false' }} diff --git a/.github/workflows/rustfs-kms-test.yml b/.github/workflows/rustfs-kms-test.yml index 8647c9268..811cfdc78 100644 --- a/.github/workflows/rustfs-kms-test.yml +++ b/.github/workflows/rustfs-kms-test.yml @@ -52,6 +52,25 @@ jobs: timeout-minutes: 420 if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: + - name: Checkout repository (for report parser) + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + + - name: Initialize functional evidence + id: evidence + run: | + set -euo pipefail + umask 077 + FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-kms-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" + { + printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'TMPDIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + } >> "${GITHUB_ENV}" + # auto-testing is private: clone it with the dedicated PF token (not # GITHUB_TOKEN) and retry transient GitHub/network failures. - name: Checkout auto-testing scripts (with retry) @@ -108,8 +127,6 @@ jobs: - name: Run KMS suite id: test - env: - LOG_FILE: /tmp/rustfs-kms.log run: | set -euo pipefail chmod +x auto-testing/rustfs-kms-test.sh @@ -139,10 +156,7 @@ jobs: ./auto-testing/rustfs-kms-test.sh "${ARGS[@]}" - name: Generate report - if: always() - env: - LOG_FILE: /tmp/rustfs-kms.log - REPORT_FILE: /tmp/rustfs-kms-report.md + if: ${{ always() && steps.evidence.outcome == 'success' }} run: | set -euo pipefail PACKAGE_URL='${{ inputs.package_url }}' @@ -154,79 +168,43 @@ jobs: else PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}" fi - CASE_TABLE="/tmp/rustfs-kms-cases.md" - python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY' - import re - import sys - - log_file, out_file = sys.argv[1], sys.argv[2] - ansi = re.compile(r'\x1b\[[0-9;]*m') - start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$') - done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b') - - rows = [] - index = {} - try: - with open(log_file, 'r', encoding='utf-8', errors='replace') as fh: - for raw in fh: - line = ansi.sub('', raw).strip() - m = start_re.match(line) - if m: - case_id, name = m.group(1), m.group(2) - if case_id not in index: - index[case_id] = len(rows) - rows.append([case_id, name, 'RUNNING']) - continue - m = done_re.match(line) - if m: - status, case_id = m.group(1), m.group(2) - if case_id in index: - rows[index[case_id]][2] = status - else: - rows.append([case_id, case_id, status]) - index[case_id] = len(rows) - 1 - except FileNotFoundError: - rows = [] - - counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0} - for _, _, status in rows: - counts[status] = counts.get(status, 0) + 1 - - with open(out_file, 'w', encoding='utf-8') as out: - out.write('## Case Summary\n\n') - out.write(f"- Total: {len(rows)}\\n") - out.write(f"- PASS: {counts.get('PASS', 0)}\\n") - out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n") - out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n") - out.write('\\n') - out.write('| Case | Name | Status |\\n') - out.write('| --- | --- | --- |\\n') - for case_id, name, status in rows: - out.write(f'| {case_id} | {name} | {status} |\\n') - PY + CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" + CASE_RESULT=success + python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure + RESULT=failure + if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then + RESULT=success + fi { echo "# RustFS KMS test report" echo "" echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo "- Attempt: ${GITHUB_RUN_ATTEMPT}" + echo "- Workflow Commit: ${GITHUB_SHA}" echo "- Trigger: ${{ github.event_name }}" echo "- Package: ${PACKAGE_SOURCE}" - echo "- Test Step Outcome: ${{ steps.test.outcome }}" + echo "- Test Step Outcome: ${RESULT}" + echo "- Suite Step Outcome: ${{ steps.test.outcome }}" echo "" - cat "${CASE_TABLE}" || true - echo "" - echo "## Log tail" - echo '```text' - tail -n 200 "${LOG_FILE}" || true - echo '```' + if [ "${RESULT}" = "success" ]; then + cat "${CASE_TABLE}" + echo "" + echo "## Log tail" + echo '```text' + tail -n 200 "${LOG_FILE}" + echo '```' + else + echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log." + fi } | tee "${REPORT_FILE}" cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" + [ "${RESULT}" = "success" ] - name: Upload functional report to dashboard - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} continue-on-error: true env: GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }} - REPORT_FILE: /tmp/rustfs-kms-report.md SUITE: kms run: | set -euo pipefail @@ -253,11 +231,10 @@ jobs: continue-on-error: true env: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} + EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }} SUITE: 'kms' SUITE_LABEL: 'KMS' RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - REPORT_FILE: '/tmp/rustfs-kms-report.md' - LOG_FILE: '/tmp/rustfs-kms.log' run: | set -euo pipefail if [ -z "${GH_TOKEN:-}" ]; then @@ -285,14 +262,16 @@ jobs: echo "" echo "- Suite: \`${SUITE}\`" echo "- Run: ${RUN_URL}" + echo "- Attempt: ${GITHUB_RUN_ATTEMPT}" + echo "- Workflow Commit: ${GITHUB_SHA}" echo "- Trigger: ${GITHUB_EVENT_NAME}" echo "- Date: $(date -u +%Y-%m-%d)" echo "" echo "## Report (errors and symptoms)" echo "" - if [ -s "${REPORT_FILE}" ]; then + if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then redact < "${REPORT_FILE}" - elif [ -s "${LOG_FILE:-}" ]; then + elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then echo "(report file missing; log tail below)" echo "" tail -n 200 "${LOG_FILE}" | redact @@ -308,14 +287,12 @@ jobs: echo "filed backlog issue for suite ${SUITE}" - name: Upload report and logs - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: - name: rustfs-kms-test-${{ github.run_id }} - path: | - /tmp/rustfs-kms.log - /tmp/rustfs-kms-report.md - if-no-files-found: warn + name: rustfs-kms-test-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/ + if-no-files-found: error - name: Cleanup environment (after) if: always() diff --git a/.github/workflows/rustfs-performance-test.yml b/.github/workflows/rustfs-performance-test.yml index ff3e2978d..22a814a28 100644 --- a/.github/workflows/rustfs-performance-test.yml +++ b/.github/workflows/rustfs-performance-test.yml @@ -76,8 +76,6 @@ env: # Package used by the nightly run (workflow_dispatch inputs are empty for # workflow_run events), i.e. the latest nightly deb published by nightly-gnu.yml. RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }} - # Fixed benchmark result directory so later steps can read summary.md - RUSTFS_RESULT_DIR: /tmp/rustfs-perf-results # Cross-repo token for uploading reports to rustfs/dashboard (set in repo settings) PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} @@ -89,6 +87,22 @@ jobs: # Skipped when nightly failed. if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: + - name: Initialize functional evidence + id: evidence + run: | + set -euo pipefail + umask 077 + FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-performance-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" + { + printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'TMPDIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'RUSTFS_RESULT_DIR=%s/results\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'VERSION_FILE=%s/version.txt\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + } >> "${GITHUB_ENV}" + # auto-testing is private: clone it with the dedicated PF token (not # GITHUB_TOKEN) and retry transient GitHub/network failures. - name: Checkout auto-testing scripts (with retry) @@ -120,7 +134,7 @@ jobs: if: ${{ inputs.cleanup_before != 'false' }} run: | chmod +x auto-testing/rustfs_performance_test.sh - ./auto-testing/rustfs_performance_test.sh --step 1 -y + ./auto-testing/rustfs_performance_test.sh --step 1 -y --log-file "${LOG_FILE:-/dev/null}" - name: Install RustFS package & start cluster (4x4) run: | @@ -130,7 +144,7 @@ jobs: else ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") fi - ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" + ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}" - name: Preflight checks run: | @@ -140,7 +154,7 @@ jobs: else ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") fi - ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" + ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}" - name: Run benchmark (GET/PUT/MIXED) id: benchmark @@ -153,17 +167,15 @@ jobs: --step 5 -y \ --warp-duration "${{ inputs.warp_duration || '5m' }}" \ --warp-concurrency "${{ inputs.warp_concurrency || '64' }}" \ - --log-file /tmp/rustfs-perf-test.log + --log-file "${LOG_FILE}" - name: Analyze results if: ${{ steps.benchmark.conclusion == 'success' }} run: | - ./auto-testing/rustfs_performance_test.sh --step 6 -y + ./auto-testing/rustfs_performance_test.sh --step 6 -y --log-file "${LOG_FILE:-/dev/null}" - name: Collect RustFS version info if: ${{ steps.benchmark.conclusion == 'success' }} - env: - VERSION_FILE: /tmp/rustfs-version.txt run: | set -euo pipefail read -r -a NODES <<< "${RUSTFS_NODES}" @@ -183,7 +195,6 @@ jobs: env: GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }} RESULT_DIR: ${{ env.RUSTFS_RESULT_DIR }} - VERSION_FILE: /tmp/rustfs-version.txt run: | set -euo pipefail if [ -z "${GH_TOKEN:-}" ]; then @@ -191,7 +202,7 @@ jobs: exit 0 fi SUMMARY="${RESULT_DIR}/summary.md" - [ -f "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; } + [ -s "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; } DATE="$(date -u +%Y-%m-%d)" REPORT_PATH="reports/${DATE}.md" { @@ -199,6 +210,8 @@ jobs: echo "" echo "- **Date**: ${DATE}" echo "- **Run**: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo "- **Attempt**: ${GITHUB_RUN_ATTEMPT}" + echo "- **Workflow Commit**: ${GITHUB_SHA}" echo "- **Trigger**: ${{ github.event_name }}" echo "- **Package**: ${{ inputs.package_url || 'nightly (R2 latest)' }}" echo "" @@ -208,8 +221,8 @@ jobs: echo '```text' cat "${VERSION_FILE}" echo '```' - } > /tmp/rustfs-perf-report.md - CONTENT="$(python3 -c 'import base64; print(base64.b64encode(open("/tmp/rustfs-perf-report.md","rb").read()).decode())')" + } > "${REPORT_FILE}" + CONTENT="$(python3 -c 'import base64,sys; print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")" SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)" if [ -n "${SHA}" ]; then jq -n --arg msg "report: ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \ @@ -228,11 +241,10 @@ jobs: continue-on-error: true env: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} + EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }} SUITE: 'performance' SUITE_LABEL: 'Performance' RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - REPORT_FILE: '/tmp/rustfs-perf-report.md' - LOG_FILE: '/tmp/rustfs-perf-test.log' run: | set -euo pipefail if [ -z "${GH_TOKEN:-}" ]; then @@ -260,14 +272,16 @@ jobs: echo "" echo "- Suite: \`${SUITE}\`" echo "- Run: ${RUN_URL}" + echo "- Attempt: ${GITHUB_RUN_ATTEMPT}" + echo "- Workflow Commit: ${GITHUB_SHA}" echo "- Trigger: ${GITHUB_EVENT_NAME}" echo "- Date: $(date -u +%Y-%m-%d)" echo "" echo "## Report (errors and symptoms)" echo "" - if [ -s "${REPORT_FILE}" ]; then + if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then redact < "${REPORT_FILE}" - elif [ -s "${LOG_FILE:-}" ]; then + elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then echo "(report file missing; log tail below)" echo "" tail -n 200 "${LOG_FILE}" | redact @@ -283,20 +297,17 @@ jobs: echo "filed backlog issue for suite ${SUITE}" - name: Upload test logs & results - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: - name: rustfs-perf-test-${{ github.run_id }} - path: | - /tmp/rustfs-perf-test*.log - /tmp/rustfs-perf-results/** - /tmp/rustfs-version.txt - if-no-files-found: warn + name: rustfs-perf-test-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/ + if-no-files-found: error - name: Reset test environment (after) if: ${{ always() && inputs.cleanup_after != 'false' }} run: | - ./auto-testing/rustfs_performance_test.sh --step 7 -y + ./auto-testing/rustfs_performance_test.sh --step 7 -y --log-file "${LOG_FILE:-/dev/null}" - name: Notify on failure if: failure() diff --git a/.github/workflows/rustfs-replication-test.yml b/.github/workflows/rustfs-replication-test.yml index 74c57b6d5..24933d236 100644 --- a/.github/workflows/rustfs-replication-test.yml +++ b/.github/workflows/rustfs-replication-test.yml @@ -65,6 +65,25 @@ jobs: timeout-minutes: 360 if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: + - name: Checkout repository (for report parser) + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + + - name: Initialize functional evidence + id: evidence + run: | + set -euo pipefail + umask 077 + FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-replication-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" + { + printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'TMPDIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + } >> "${GITHUB_ENV}" + # auto-testing is private: clone it with the dedicated PF token (not # GITHUB_TOKEN) and retry transient GitHub/network failures. - name: Checkout auto-testing scripts (with retry) @@ -113,8 +132,6 @@ jobs: - name: Run replication suite id: test - env: - LOG_FILE: /tmp/rustfs-replication.log run: | set -euo pipefail chmod +x auto-testing/rustfs-replication-test.sh @@ -137,10 +154,7 @@ jobs: ./auto-testing/rustfs-replication-test.sh "${ARGS[@]}" - name: Generate report - if: always() - env: - LOG_FILE: /tmp/rustfs-replication.log - REPORT_FILE: /tmp/rustfs-replication-report.md + if: ${{ always() && steps.evidence.outcome == 'success' }} run: | set -euo pipefail PACKAGE_URL='${{ inputs.package_url }}' @@ -162,80 +176,44 @@ jobs: RUSTFS_VERSION_INFO="${DETECTED_VERSION}" fi fi - CASE_TABLE="/tmp/rustfs-replication-cases.md" - python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY' - import re - import sys - - log_file, out_file = sys.argv[1], sys.argv[2] - ansi = re.compile(r'\x1b\[[0-9;]*m') - start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$') - done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b') - - rows = [] - index = {} - try: - with open(log_file, 'r', encoding='utf-8', errors='replace') as fh: - for raw in fh: - line = ansi.sub('', raw).strip() - m = start_re.match(line) - if m: - case_id, name = m.group(1), m.group(2) - if case_id not in index: - index[case_id] = len(rows) - rows.append([case_id, name, 'RUNNING']) - continue - m = done_re.match(line) - if m: - status, case_id = m.group(1), m.group(2) - if case_id in index: - rows[index[case_id]][2] = status - else: - rows.append([case_id, case_id, status]) - index[case_id] = len(rows) - 1 - except FileNotFoundError: - rows = [] - - counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0} - for _, _, status in rows: - counts[status] = counts.get(status, 0) + 1 - - with open(out_file, 'w', encoding='utf-8') as out: - out.write('## Case Summary\n\n') - out.write(f"- Total: {len(rows)}\\n") - out.write(f"- PASS: {counts.get('PASS', 0)}\\n") - out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n") - out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n") - out.write('\\n') - out.write('| Case | Name | Status |\\n') - out.write('| --- | --- | --- |\\n') - for case_id, name, status in rows: - out.write(f'| {case_id} | {name} | {status} |\\n') - PY + CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" + CASE_RESULT=success + python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure + RESULT=failure + if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then + RESULT=success + fi { echo "# RustFS replication test report" echo "" echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo "- Attempt: ${GITHUB_RUN_ATTEMPT}" + echo "- Workflow Commit: ${GITHUB_SHA}" echo "- Trigger: ${{ github.event_name }}" echo "- Package: ${PACKAGE_SOURCE}" echo "- RustFS Version: ${RUSTFS_VERSION_INFO}" - echo "- Test Step Outcome: ${{ steps.test.outcome }}" + echo "- Test Step Outcome: ${RESULT}" + echo "- Suite Step Outcome: ${{ steps.test.outcome }}" echo "" - cat "${CASE_TABLE}" || true - echo "" - echo "## Log tail" - echo '```text' - tail -n 200 "${LOG_FILE}" || true - echo '```' + if [ "${RESULT}" = "success" ]; then + cat "${CASE_TABLE}" + echo "" + echo "## Log tail" + echo '```text' + tail -n 200 "${LOG_FILE}" + echo '```' + else + echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log." + fi } | tee "${REPORT_FILE}" cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" + [ "${RESULT}" = "success" ] - name: Upload functional report to dashboard - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} continue-on-error: true env: GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }} - REPORT_FILE: /tmp/rustfs-replication-report.md SUITE: replication run: | set -euo pipefail @@ -262,11 +240,10 @@ jobs: continue-on-error: true env: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} + EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }} SUITE: 'replication' SUITE_LABEL: 'Replication (bucket + site)' RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - REPORT_FILE: '/tmp/rustfs-replication-report.md' - LOG_FILE: '/tmp/rustfs-replication.log' run: | set -euo pipefail if [ -z "${GH_TOKEN:-}" ]; then @@ -294,14 +271,16 @@ jobs: echo "" echo "- Suite: \`${SUITE}\`" echo "- Run: ${RUN_URL}" + echo "- Attempt: ${GITHUB_RUN_ATTEMPT}" + echo "- Workflow Commit: ${GITHUB_SHA}" echo "- Trigger: ${GITHUB_EVENT_NAME}" echo "- Date: $(date -u +%Y-%m-%d)" echo "" echo "## Report (errors and symptoms)" echo "" - if [ -s "${REPORT_FILE}" ]; then + if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then redact < "${REPORT_FILE}" - elif [ -s "${LOG_FILE:-}" ]; then + elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then echo "(report file missing; log tail below)" echo "" tail -n 200 "${LOG_FILE}" | redact @@ -317,14 +296,12 @@ jobs: echo "filed backlog issue for suite ${SUITE}" - name: Upload report and logs - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: - name: rustfs-replication-${{ github.run_id }} - path: | - /tmp/rustfs-replication.log - /tmp/rustfs-replication-report.md - if-no-files-found: warn + name: rustfs-replication-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/ + if-no-files-found: error - name: Cleanup environment (after) if: always() diff --git a/.github/workflows/rustfs-s3-compat-test.yml b/.github/workflows/rustfs-s3-compat-test.yml index 80856194c..bae7f7035 100644 --- a/.github/workflows/rustfs-s3-compat-test.yml +++ b/.github/workflows/rustfs-s3-compat-test.yml @@ -40,6 +40,25 @@ jobs: timeout-minutes: 360 if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: + - name: Checkout repository (for report parser) + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + + - name: Initialize functional evidence + id: evidence + run: | + set -euo pipefail + umask 077 + FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-s3-compat-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" + { + printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'TMPDIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + } >> "${GITHUB_ENV}" + # auto-testing is private: clone it with the dedicated PF token (not # GITHUB_TOKEN) and retry transient GitHub/network failures. - name: Checkout auto-testing scripts (with retry) @@ -87,8 +106,6 @@ jobs: - name: Run S3 compatibility suite id: test - env: - LOG_FILE: /tmp/rustfs-s3-compat.log run: | set -euo pipefail chmod +x auto-testing/rustfs-s3-compat-test.sh @@ -105,10 +122,7 @@ jobs: ./auto-testing/rustfs-s3-compat-test.sh "${ARGS[@]}" - name: Generate report - if: always() - env: - LOG_FILE: /tmp/rustfs-s3-compat.log - REPORT_FILE: /tmp/rustfs-s3-compat-report.md + if: ${{ always() && steps.evidence.outcome == 'success' }} run: | set -euo pipefail PACKAGE_URL='${{ inputs.package_url }}' @@ -130,83 +144,44 @@ jobs: RUSTFS_VERSION_INFO="${DETECTED_VERSION}" fi fi - CASE_TABLE="/tmp/rustfs-s3-compat-cases.md" - python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY' - import re - import sys - - log_file, out_file = sys.argv[1], sys.argv[2] - ansi = re.compile(r'\x1b\[[0-9;]*m') - start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$') - done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b') - - rows = [] - index = {} - current = None - try: - with open(log_file, 'r', encoding='utf-8', errors='replace') as fh: - for raw in fh: - line = ansi.sub('', raw).strip() - m = start_re.match(line) - if m: - case_id, name = m.group(1), m.group(2) - current = case_id - if case_id not in index: - index[case_id] = len(rows) - rows.append([case_id, name, 'RUNNING']) - continue - m = done_re.match(line) - if m: - status, case_id = m.group(1), m.group(2) - if case_id in index: - rows[index[case_id]][2] = status - else: - rows.append([case_id, case_id, status]) - index[case_id] = len(rows) - 1 - current = None - except FileNotFoundError: - rows = [] - - counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0} - for _, _, status in rows: - counts[status] = counts.get(status, 0) + 1 - - with open(out_file, 'w', encoding='utf-8') as out: - out.write('## Case Summary\n\n') - out.write(f"- Total: {len(rows)}\\n") - out.write(f"- PASS: {counts.get('PASS', 0)}\\n") - out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n") - out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n") - out.write('\\n') - out.write('| Case | Name | Status |\\n') - out.write('| --- | --- | --- |\\n') - for case_id, name, status in rows: - out.write(f'| {case_id} | {name} | {status} |\\n') - PY + CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" + CASE_RESULT=success + python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure + RESULT=failure + if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then + RESULT=success + fi { echo "# RustFS S3 compatibility test report" echo "" echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo "- Attempt: ${GITHUB_RUN_ATTEMPT}" + echo "- Workflow Commit: ${GITHUB_SHA}" echo "- Trigger: ${{ github.event_name }}" echo "- Package: ${PACKAGE_SOURCE}" echo "- RustFS Version: ${RUSTFS_VERSION_INFO}" - echo "- Test Step Outcome: ${{ steps.test.outcome }}" + echo "- Test Step Outcome: ${RESULT}" + echo "- Suite Step Outcome: ${{ steps.test.outcome }}" echo "" - cat "${CASE_TABLE}" || true - echo "" - echo "## Log tail" - echo '```text' - tail -n 200 "${LOG_FILE}" || true - echo '```' + if [ "${RESULT}" = "success" ]; then + cat "${CASE_TABLE}" + echo "" + echo "## Log tail" + echo '```text' + tail -n 200 "${LOG_FILE}" + echo '```' + else + echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log." + fi } | tee "${REPORT_FILE}" cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" + [ "${RESULT}" = "success" ] - name: Upload functional report to dashboard - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} continue-on-error: true env: GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }} - REPORT_FILE: /tmp/rustfs-s3-compat-report.md SUITE: s3 run: | set -euo pipefail @@ -233,11 +208,10 @@ jobs: continue-on-error: true env: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} + EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }} SUITE: 's3' SUITE_LABEL: 'S3 compatibility' RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - REPORT_FILE: '/tmp/rustfs-s3-compat-report.md' - LOG_FILE: '/tmp/rustfs-s3-compat.log' run: | set -euo pipefail if [ -z "${GH_TOKEN:-}" ]; then @@ -265,14 +239,16 @@ jobs: echo "" echo "- Suite: \`${SUITE}\`" echo "- Run: ${RUN_URL}" + echo "- Attempt: ${GITHUB_RUN_ATTEMPT}" + echo "- Workflow Commit: ${GITHUB_SHA}" echo "- Trigger: ${GITHUB_EVENT_NAME}" echo "- Date: $(date -u +%Y-%m-%d)" echo "" echo "## Report (errors and symptoms)" echo "" - if [ -s "${REPORT_FILE}" ]; then + if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then redact < "${REPORT_FILE}" - elif [ -s "${LOG_FILE:-}" ]; then + elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then echo "(report file missing; log tail below)" echo "" tail -n 200 "${LOG_FILE}" | redact @@ -288,14 +264,12 @@ jobs: echo "filed backlog issue for suite ${SUITE}" - name: Upload report and logs - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: - name: rustfs-s3-compat-${{ github.run_id }} - path: | - /tmp/rustfs-s3-compat.log - /tmp/rustfs-s3-compat-report.md - if-no-files-found: warn + name: rustfs-s3-compat-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/ + if-no-files-found: error - name: Cleanup environment (after) if: always() diff --git a/.github/workflows/rustfs-storage-test.yml b/.github/workflows/rustfs-storage-test.yml index 767f734dc..f02f413a1 100644 --- a/.github/workflows/rustfs-storage-test.yml +++ b/.github/workflows/rustfs-storage-test.yml @@ -49,6 +49,25 @@ jobs: timeout-minutes: 360 if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: + - name: Checkout repository (for report parser) + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + + - name: Initialize functional evidence + id: evidence + run: | + set -euo pipefail + umask 077 + FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-storage-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" + { + printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'TMPDIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + } >> "${GITHUB_ENV}" + # auto-testing is private: clone it with the dedicated PF token (not # GITHUB_TOKEN) and retry transient GitHub/network failures. - name: Checkout auto-testing scripts (with retry) @@ -96,8 +115,6 @@ jobs: - name: Run storage engine suite id: test - env: - LOG_FILE: /tmp/rustfs-storage.log run: | set -euo pipefail chmod +x auto-testing/rustfs-storage-test.sh @@ -120,10 +137,7 @@ jobs: ./auto-testing/rustfs-storage-test.sh "${ARGS[@]}" - name: Generate report - if: always() - env: - LOG_FILE: /tmp/rustfs-storage.log - REPORT_FILE: /tmp/rustfs-storage-report.md + if: ${{ always() && steps.evidence.outcome == 'success' }} run: | set -euo pipefail PACKAGE_URL='${{ inputs.package_url }}' @@ -145,83 +159,44 @@ jobs: RUSTFS_VERSION_INFO="${DETECTED_VERSION}" fi fi - CASE_TABLE="/tmp/rustfs-storage-cases.md" - python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY' - import re - import sys - - log_file, out_file = sys.argv[1], sys.argv[2] - ansi = re.compile(r'\x1b\[[0-9;]*m') - start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$') - done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b') - - rows = [] - index = {} - current = None - try: - with open(log_file, 'r', encoding='utf-8', errors='replace') as fh: - for raw in fh: - line = ansi.sub('', raw).strip() - m = start_re.match(line) - if m: - case_id, name = m.group(1), m.group(2) - current = case_id - if case_id not in index: - index[case_id] = len(rows) - rows.append([case_id, name, 'RUNNING']) - continue - m = done_re.match(line) - if m: - status, case_id = m.group(1), m.group(2) - if case_id in index: - rows[index[case_id]][2] = status - else: - rows.append([case_id, case_id, status]) - index[case_id] = len(rows) - 1 - current = None - except FileNotFoundError: - rows = [] - - counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0} - for _, _, status in rows: - counts[status] = counts.get(status, 0) + 1 - - with open(out_file, 'w', encoding='utf-8') as out: - out.write('## Case Summary\n\n') - out.write(f"- Total: {len(rows)}\\n") - out.write(f"- PASS: {counts.get('PASS', 0)}\\n") - out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n") - out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n") - out.write('\\n') - out.write('| Case | Name | Status |\\n') - out.write('| --- | --- | --- |\\n') - for case_id, name, status in rows: - out.write(f'| {case_id} | {name} | {status} |\\n') - PY + CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" + CASE_RESULT=success + python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure + RESULT=failure + if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then + RESULT=success + fi { echo "# RustFS storage engine test report" echo "" echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo "- Attempt: ${GITHUB_RUN_ATTEMPT}" + echo "- Workflow Commit: ${GITHUB_SHA}" echo "- Trigger: ${{ github.event_name }}" echo "- Package: ${PACKAGE_SOURCE}" echo "- RustFS Version: ${RUSTFS_VERSION_INFO}" - echo "- Test Step Outcome: ${{ steps.test.outcome }}" + echo "- Test Step Outcome: ${RESULT}" + echo "- Suite Step Outcome: ${{ steps.test.outcome }}" echo "" - cat "${CASE_TABLE}" || true - echo "" - echo "## Log tail" - echo '```text' - tail -n 200 "${LOG_FILE}" || true - echo '```' + if [ "${RESULT}" = "success" ]; then + cat "${CASE_TABLE}" + echo "" + echo "## Log tail" + echo '```text' + tail -n 200 "${LOG_FILE}" + echo '```' + else + echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log." + fi } | tee "${REPORT_FILE}" cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" + [ "${RESULT}" = "success" ] - name: Upload functional report to dashboard - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} continue-on-error: true env: GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }} - REPORT_FILE: /tmp/rustfs-storage-report.md SUITE: storage run: | set -euo pipefail @@ -248,11 +223,10 @@ jobs: continue-on-error: true env: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} + EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }} SUITE: 'storage' SUITE_LABEL: 'Storage engine' RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - REPORT_FILE: '/tmp/rustfs-storage-report.md' - LOG_FILE: '/tmp/rustfs-storage.log' run: | set -euo pipefail if [ -z "${GH_TOKEN:-}" ]; then @@ -280,14 +254,16 @@ jobs: echo "" echo "- Suite: \`${SUITE}\`" echo "- Run: ${RUN_URL}" + echo "- Attempt: ${GITHUB_RUN_ATTEMPT}" + echo "- Workflow Commit: ${GITHUB_SHA}" echo "- Trigger: ${GITHUB_EVENT_NAME}" echo "- Date: $(date -u +%Y-%m-%d)" echo "" echo "## Report (errors and symptoms)" echo "" - if [ -s "${REPORT_FILE}" ]; then + if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then redact < "${REPORT_FILE}" - elif [ -s "${LOG_FILE:-}" ]; then + elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then echo "(report file missing; log tail below)" echo "" tail -n 200 "${LOG_FILE}" | redact @@ -303,14 +279,12 @@ jobs: echo "filed backlog issue for suite ${SUITE}" - name: Upload report and logs - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: - name: rustfs-storage-${{ github.run_id }} - path: | - /tmp/rustfs-storage.log - /tmp/rustfs-storage-report.md - if-no-files-found: warn + name: rustfs-storage-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/ + if-no-files-found: error - name: Cleanup environment (after) if: always() diff --git a/.github/workflows/rustfs-upgrade-test.yml b/.github/workflows/rustfs-upgrade-test.yml index 0b4c19af1..bb581efa8 100644 --- a/.github/workflows/rustfs-upgrade-test.yml +++ b/.github/workflows/rustfs-upgrade-test.yml @@ -82,6 +82,25 @@ jobs: timeout-minutes: 420 if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: + - name: Checkout repository (for report parser) + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + + - name: Initialize functional evidence + id: evidence + run: | + set -euo pipefail + umask 077 + FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-upgrade-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" + { + printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'TMPDIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + } >> "${GITHUB_ENV}" + # auto-testing is private: clone it with the dedicated PF token (not # GITHUB_TOKEN) and retry transient GitHub/network failures. - name: Checkout auto-testing scripts (with retry) @@ -142,7 +161,6 @@ jobs: - name: Run upgrade compatibility suite id: test env: - LOG_FILE: /tmp/rustfs-upgrade.log GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} run: | set -euo pipefail @@ -200,10 +218,7 @@ jobs: ./auto-testing/rustfs-upgrade-test.sh "${ARGS[@]}" - name: Generate report - if: always() - env: - LOG_FILE: /tmp/rustfs-upgrade.log - REPORT_FILE: /tmp/rustfs-upgrade-report.md + if: ${{ always() && steps.evidence.outcome == 'success' }} run: | set -euo pipefail FROM_URL='${{ inputs.from_url }}' @@ -224,103 +239,47 @@ jobs: else TO_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}" fi - CASE_TABLE="/tmp/rustfs-upgrade-cases.md" - MATRIX_TABLE="/tmp/rustfs-upgrade-matrix.md" - python3 - "${LOG_FILE}" "${CASE_TABLE}" "${MATRIX_TABLE}" <<'PY' - import re - import sys - - log_file, out_file, matrix_file = sys.argv[1], sys.argv[2], sys.argv[3] - ansi = re.compile(r'\x1b\[[0-9;]*m') - start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$') - done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b') - topo_re = re.compile( - r'^\[UPG-TOPO\]\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+PASS=(\d+)\s+FAIL=(\d+)\s*$') - - rows = [] - index = {} - topo_rows = [] - try: - with open(log_file, 'r', encoding='utf-8', errors='replace') as fh: - for raw in fh: - line = ansi.sub('', raw).strip() - m = topo_re.match(line) - if m: - topo_rows.append(m.groups()) - continue - m = start_re.match(line) - if m: - case_id, name = m.group(1), m.group(2) - if case_id not in index: - index[case_id] = len(rows) - rows.append([case_id, name, 'RUNNING']) - continue - m = done_re.match(line) - if m: - status, case_id = m.group(1), m.group(2) - if case_id in index: - rows[index[case_id]][2] = status - else: - rows.append([case_id, case_id, status]) - index[case_id] = len(rows) - 1 - except FileNotFoundError: - rows = [] - - counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0} - for _, _, status in rows: - counts[status] = counts.get(status, 0) + 1 - - with open(out_file, 'w', encoding='utf-8') as out: - out.write('## Case Summary\n\n') - out.write(f"- Total: {len(rows)}\\n") - out.write(f"- PASS: {counts.get('PASS', 0)}\\n") - out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n") - out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n") - out.write('\\n') - out.write('| Case | Name | Status |\\n') - out.write('| --- | --- | --- |\\n') - for case_id, name, status in rows: - out.write(f'| {case_id} | {name} | {status} |\\n') - - # Upgrade matrix: one row per topology/backend with the versions - # captured on the nodes (rustfs --version) and the aggregated - # result. The dashboard renders this table directly. - with open(matrix_file, 'w', encoding='utf-8') as out: - out.write('## Upgrade Matrix\n\n') - out.write('| Topology | KMS Backend | From Version | To Version | Result |\n') - out.write('| --- | --- | --- | --- | --- |\n') - for topo, backend, old_v, new_v, npass, nfail in topo_rows: - result = 'PASS' if nfail == '0' else 'FAIL' - out.write(f'| {topo} | {backend} | {old_v} | {new_v} | {result} (PASS={npass} FAIL={nfail}) |\n') - if not topo_rows: - out.write('| - | - | - | - | NOT RUN (suite failed before upgrade) |\n') - PY + CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" + MATRIX_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/matrix.md" + CASE_RESULT=success + python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" "${MATRIX_TABLE}" || CASE_RESULT=failure + RESULT=failure + if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then + RESULT=success + fi { echo "# RustFS upgrade compatibility report" echo "" echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo "- Attempt: ${GITHUB_RUN_ATTEMPT}" + echo "- Workflow Commit: ${GITHUB_SHA}" echo "- Trigger: ${{ github.event_name }}" echo "- From: ${FROM_SOURCE}" echo "- To: ${TO_SOURCE}" - echo "- Test Step Outcome: ${{ steps.test.outcome }}" + echo "- Test Step Outcome: ${RESULT}" + echo "- Suite Step Outcome: ${{ steps.test.outcome }}" echo "" - cat "${MATRIX_TABLE}" || true - echo "" - cat "${CASE_TABLE}" || true - echo "" - echo "## Log tail" - echo '```text' - tail -n 200 "${LOG_FILE}" || true - echo '```' + if [ "${RESULT}" = "success" ]; then + cat "${MATRIX_TABLE}" + echo "" + cat "${CASE_TABLE}" + echo "" + echo "## Log tail" + echo '```text' + tail -n 200 "${LOG_FILE}" + echo '```' + else + echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log." + fi } | tee "${REPORT_FILE}" cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" + [ "${RESULT}" = "success" ] - name: Upload functional report to dashboard - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} continue-on-error: true env: GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }} - REPORT_FILE: /tmp/rustfs-upgrade-report.md SUITE: upgrade run: | set -euo pipefail @@ -347,11 +306,10 @@ jobs: continue-on-error: true env: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} + EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }} SUITE: 'upgrade' SUITE_LABEL: 'Upgrade compatibility' RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - REPORT_FILE: '/tmp/rustfs-upgrade-report.md' - LOG_FILE: '/tmp/rustfs-upgrade.log' run: | set -euo pipefail if [ -z "${GH_TOKEN:-}" ]; then @@ -379,14 +337,16 @@ jobs: echo "" echo "- Suite: \`${SUITE}\`" echo "- Run: ${RUN_URL}" + echo "- Attempt: ${GITHUB_RUN_ATTEMPT}" + echo "- Workflow Commit: ${GITHUB_SHA}" echo "- Trigger: ${GITHUB_EVENT_NAME}" echo "- Date: $(date -u +%Y-%m-%d)" echo "" echo "## Report (errors and symptoms)" echo "" - if [ -s "${REPORT_FILE}" ]; then + if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then redact < "${REPORT_FILE}" - elif [ -s "${LOG_FILE:-}" ]; then + elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then echo "(report file missing; log tail below)" echo "" tail -n 200 "${LOG_FILE}" | redact @@ -402,14 +362,12 @@ jobs: echo "filed backlog issue for suite ${SUITE}" - name: Upload report and logs - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: - name: rustfs-upgrade-test-${{ github.run_id }} - path: | - /tmp/rustfs-upgrade-report.md - /tmp/rustfs-upgrade.*/* - if-no-files-found: ignore + name: rustfs-upgrade-test-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/ + if-no-files-found: error retention-days: 3 - name: Cleanup environment (after) diff --git a/scripts/functional_case_report.py b/scripts/functional_case_report.py new file mode 100644 index 000000000..7036eaad5 --- /dev/null +++ b/scripts/functional_case_report.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Preserve every functional case execution and its suite context in reports.""" + +from __future__ import annotations + +import argparse +from pathlib import Path +import re + + +def generate_report(log_file: Path, case_file: Path, matrix_file: Path | None = None) -> bool: + ansi = re.compile(r"\x1b\[[0-9;]*m") + start_re = re.compile(r"^---\s+([A-Z][A-Z0-9]*-[0-9]+)\s+(.+?)\s+---$") + done_re = re.compile(r"^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z][A-Z0-9]*-[0-9]+)\b") + context_re = re.compile(r"^(?:\[INFO\]\s+)?==\s+((?:topology|suite):.+?)\s+==$") + topo_re = re.compile(r"^\[UPG-TOPO\]\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+PASS=(\d+)\s+FAIL=(\d+)\s*$") + rows = [] + pending = {} + topo_rows = [] + context = "context not recorded" + complete = True + try: + with log_file.open(encoding="utf-8", errors="replace") as log: + for raw in log: + line = ansi.sub("", raw).strip() + if match := context_re.match(line): + context = match[1] + pending.clear() + elif match := topo_re.match(line): + topo_rows.append(match.groups()) + elif match := start_re.match(line): + case_id, name = match.groups() + pending[case_id] = len(rows) + rows.append([case_id, f"{name} ({context})", "RUNNING"]) + elif match := done_re.match(line): + status, case_id = match.groups() + index = pending.pop(case_id, None) + if index is None: + complete = False + rows.append([case_id, f"{case_id} ({context}; start not recorded)", status]) + else: + rows[index][2] = status + except FileNotFoundError: + pass + + counts = {status: sum(row[2] == status for row in rows) for status in ("PASS", "FAIL", "UNSUPPORTED", "RUNNING")} + with case_file.open("w", encoding="utf-8") as out: + out.write(f"## Case Summary\n\n- Total: {len(rows)}\n") + for status, count in counts.items(): + out.write(f"- {status}: {count}\n") + out.write("\n| Case | Name | Status |\n| --- | --- | --- |\n") + for row in rows: + out.write("| " + " | ".join(value.replace("|", "|") for value in row) + " |\n") + if not rows: + out.write("\nNo case execution was recorded; the log is missing, empty, or stopped before the cases.\n") + + valid = complete and bool(rows) and not counts["FAIL"] and not counts["RUNNING"] + if matrix_file is not None: + with matrix_file.open("w", encoding="utf-8") as out: + out.write("## Upgrade Matrix\n\n| Topology | KMS Backend | From Version | To Version | Result |\n") + out.write("| --- | --- | --- | --- | --- |\n") + for topo, backend, old_v, new_v, npass, nfail in topo_rows: + result = "PASS" if nfail == "0" else "FAIL" + out.write(f"| {topo} | {backend} | {old_v} | {new_v} | {result} (PASS={npass} FAIL={nfail}) |\n") + if not topo_rows: + out.write("| - | - | - | - | NOT RUN (suite failed before upgrade) |\n") + valid = valid and bool(topo_rows) and all(row[-1] == "0" for row in topo_rows) + return valid + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("log_file", type=Path) + parser.add_argument("case_file", type=Path) + parser.add_argument("matrix_file", type=Path, nargs="?") + args = parser.parse_args() + raise SystemExit(0 if generate_report(args.log_file, args.case_file, args.matrix_file) else 1) diff --git a/scripts/test_security_workflow.py b/scripts/test_security_workflow.py index ea2d75487..023c89646 100644 --- a/scripts/test_security_workflow.py +++ b/scripts/test_security_workflow.py @@ -6,11 +6,13 @@ from __future__ import annotations import os import re import subprocess +import sys import tempfile import unittest from pathlib import Path from check_test_wiring import yaml_block +from functional_case_report import generate_report ROOT = Path(__file__).resolve().parents[1] @@ -38,7 +40,36 @@ def shell_body(lines: list[str]) -> str: return "\n".join(shell_lines) -class SecurityWorkflowTests(unittest.TestCase): +class WorkflowSteps: + def render(self, value: str) -> str: + return re.sub(r"\$\{\{\s*(.*?)\s*\}\}", lambda match: self.context[match[1]], value) + + def step_env(self, lines: list[str], indent: int = 8) -> dict[str, str]: + result = {} + for line in yaml_block(lines, "env", indent) or []: + if line.strip() and not line.lstrip().startswith("#"): + key, value = line.strip().split(": ", 1) + result[key] = self.render(value.strip("'\"")) + return result + + def run_step(self, name: str) -> subprocess.CompletedProcess[str]: + lines = self.steps[name] + result = subprocess.run( + ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.render(shell_body(lines))], + cwd=self.directory, env={**self.env, **self.step_env(lines)}, capture_output=True, text=True, + ) + for line in lines: + if line.startswith(" id: "): + self.context[f"steps.{line.split(': ', 1)[1]}.outcome"] = "failure" if result.returncode else "success" + if Path(self.env["GITHUB_ENV"]).exists(): + for line in Path(self.env["GITHUB_ENV"]).read_text().splitlines(): + key, value = line.split("=", 1) + self.env[key] = value + self.context[f"env.{key}"] = value + return result + + +class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase): def setUp(self) -> None: self.source = WORKFLOW.read_text() self.job = yaml_block(self.source.splitlines(), "security-test", 2) @@ -86,33 +117,6 @@ class SecurityWorkflowTests(unittest.TestCase): 'exit "$FAKE_EXIT"\n' ) - def render(self, value: str) -> str: - return re.sub(r"\$\{\{\s*(.*?)\s*\}\}", lambda match: self.context[match[1]], value) - - def step_env(self, lines: list[str], indent: int = 8) -> dict[str, str]: - result = {} - for line in yaml_block(lines, "env", indent) or []: - if line.strip() and not line.lstrip().startswith("#"): - key, value = line.strip().split(": ", 1) - result[key] = self.render(value.strip("'\"")) - return result - - def run_step(self, name: str) -> subprocess.CompletedProcess[str]: - lines = self.steps[name] - result = subprocess.run( - ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.render(shell_body(lines))], - cwd=self.directory, env={**self.env, **self.step_env(lines)}, capture_output=True, text=True, - ) - for line in lines: - if line.startswith(" id: "): - self.context[f"steps.{line.split(': ', 1)[1]}.outcome"] = "failure" if result.returncode else "success" - if Path(self.env["GITHUB_ENV"]).exists(): - for line in Path(self.env["GITHUB_ENV"]).read_text().splitlines(): - key, value = line.split("=", 1) - self.env[key] = value - self.context[f"env.{key}"] = value - return result - def test_workflow_wiring(self) -> None: names = list(self.steps) self.assertLess(names.index("Checkout repository (for the OIDC live gate script)"), names.index("Checkout auto-testing scripts (with retry)")) @@ -225,7 +229,7 @@ class FunctionalWorkflowTests(unittest.TestCase): if suite in self.DIRECT_TESTS: test = steps[self.DIRECT_TESTS[suite]] self.assertNotRegex("\n".join(test), r'''(?m)^ ["']?continue-on-error["']?\s*:''') - self.assertIn(" if: always()", steps["Generate report"]) + self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", steps["Generate report"]) cleanup = steps["Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"] condition = next(line.strip() for line in cleanup if line.startswith(" if:")) self.assertIn(condition, ( @@ -286,5 +290,307 @@ class FunctionalWorkflowTests(unittest.TestCase): self.assertEqual(markers, ["cleanup"] if suite == "replication" else ["cleanup", "dispatch"]) +class FunctionalCaseReportTests(unittest.TestCase): + def report(self, text: str | None, matrix: bool = False) -> tuple[bool, str, str]: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + log = root / "suite.log" + if text is not None: + log.write_text(text) + valid = generate_report(log, root / "cases.md", root / "matrix.md" if matrix else None) + return valid, (root / "cases.md").read_text(), (root / "matrix.md").read_text() if matrix else "" + + def test_repeated_case_executions_preserve_failure_and_context(self): + # log() from rustfs/auto-testing@6120aa0a76de, rustfs-kms-test.sh:131. + log = subprocess.check_output(["bash", "-c", r''' +log() { printf '\033[1;36m[INFO]\033[0m %s\n' "$*"; } +log '== topology: single-single kms-backend: local ==' +printf '\033[32m--- KMS-101 roundtrip ---\033[0m\n[FAIL] KMS-101\n' +log '== topology: single-multi kms-backend: vault-kv2 ==' +printf '%s\n' '--- KMS-101 roundtrip ---' '[PASS] KMS-101' +printf '%s\n' '--- KMS-101 roundtrip ---' '[UNSUPPORTED] KMS-101' +'''], text=True) + valid, cases, _ = self.report(log) + self.assertFalse(valid) + self.assertEqual(cases.count("| KMS-101 |"), 3) + self.assertIn("- Total: 3\n- PASS: 1\n- FAIL: 1\n- UNSUPPORTED: 1\n- RUNNING: 0\n", cases) + self.assertIn("roundtrip (topology: single-single kms-backend: local) | FAIL |", cases) + self.assertIn("roundtrip (topology: single-multi kms-backend: vault-kv2) | PASS |", cases) + self.assertNotIn("\\n", cases) + + def test_missing_empty_unfinished_and_orphan_results_are_not_success(self): + for text in (None, "", "setup failed\n", "--- KMS-101 roundtrip ---\n", "[PASS] KMS-101\n", + "--- KMS-101 first ---\n--- KMS-101 second ---\n[PASS] KMS-101\n", + "--- KMS-101 first ---\n[FAIL] KMS-101\n[PASS] KMS-101\n"): + with self.subTest(log=text): + valid, cases, _ = self.report(text) + self.assertFalse(valid) + self.assertIn("## Case Summary", cases) + valid, cases, _ = self.report("[INFO] == suite: bucket replication (REP-*) ==\n--- REP-101 unsupported ---\n[UNSUPPORTED] REP-101\n") + self.assertTrue(valid) + self.assertIn("suite: bucket replication", cases) + self.assertIn("- UNSUPPORTED: 1\n", cases) + + def test_upgrade_matrix_is_preserved_and_required_for_complete_report(self): + case = "--- UPG-101 upgrade ---\n[PASS] UPG-101\n" + for suffix, expected in (("", False), ("[UPG-TOPO] single-single local v1 v2 PASS=1 FAIL=0\n", True), + ("[UPG-TOPO] single-single local v1 v2 PASS=1 FAIL=1\n", False)): + with self.subTest(matrix=suffix): + valid, _, matrix = self.report(case + suffix, matrix=True) + self.assertEqual(valid, expected) + self.assertIn("| Topology | KMS Backend | From Version | To Version | Result |", matrix) + self.assertIn("| single-single | local | v1 | v2 |" if suffix else "NOT RUN", matrix) + + def test_s3_case_identifiers_include_digits(self): + valid, cases, _ = self.report("--- S3C-101 CreateBucket ---\n[PASS] S3C-101\n") + self.assertTrue(valid) + self.assertIn("| S3C-101 | CreateBucket (context not recorded) | PASS |", cases) + + +class FunctionalEvidenceTests(WorkflowSteps, unittest.TestCase): + SUITES = (*FunctionalWorkflowTests.DIRECT_TESTS, "heal", "performance") + + def prepare(self, suite: str) -> None: + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.directory = Path(self.temp.name) + self.source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text() + self.steps = named_steps(yaml_block(self.source.splitlines(), FunctionalWorkflowTests.JOBS[suite], 2)) + self.context = {expression: "" for expression in re.findall(r"\$\{\{\s*(.*?)\s*\}\}", self.source)} + self.context.update({ + "github.server_url": "https://github.com", "github.repository": "rustfs/rustfs", + "github.run_id": "314159", "github.run_attempt": "2", "github.sha": "0123456789abcdef0123456789abcdef01234567", + "github.event_name": "repository_dispatch", "steps.test.outcome": "success", + "secrets.PF_TESTING_GH_TOKEN": "local-fixture", "env.PF_TESTING_GH_TOKEN": "local-fixture", + }) + self.artifacts = self.directory / f"rustfs-{suite}-314159-2" + self.env = { + **os.environ, "GITHUB_ENV": str(self.directory / "github-env"), "RUNNER_TEMP": self.temp.name, + "GITHUB_STEP_SUMMARY": str(self.directory / "summary.md"), "RUSTFS_NODES": "fixture-node", + "RUSTFS_NIGHTLY_PACKAGE_URL": "https://example.invalid/package.deb", "CAPTURE_BODY": str(self.directory / "issue.md"), + } + for key in ("server_url", "repository", "run_id", "run_attempt", "sha", "event_name"): + self.env[f"GITHUB_{key.upper()}"] = self.context[f"github.{key}"] + (self.directory / "scripts").mkdir() + (self.directory / "scripts/functional_case_report.py").symlink_to(ROOT / "scripts/functional_case_report.py") + fake_bin = self.directory / "bin" + fake_bin.mkdir() + (fake_bin / "python3").symlink_to(sys.executable) + for command, body in ( + ("ssh", 'printf "fixture-version\\n"\n'), + ("gh", 'if [ "$1 $2" = "issue create" ]; then\n' + ' while [ "$#" -gt 0 ]; do\n' + ' if [ "$1" = "--body-file" ]; then cat "$2" > "$CAPTURE_BODY"; fi\n' + ' shift\n' + ' done\n' + 'elif [ "$1 $2" = "api --method" ]; then cat >/dev/null; fi\n'), + ): + script = fake_bin / command + script.write_text("#!/bin/sh\n" + body) + script.chmod(0o755) + self.env["PATH"] = f"{fake_bin}{os.pathsep}{os.environ['PATH']}" + + def test_evidence_wiring_and_failed_initialization_cannot_publish_stale_files(self): + for suite in self.SUITES: + with self.subTest(suite=suite): + self.prepare(suite) + self.assertNotIn("/tmp/rustfs-", self.source) + names = list(self.steps) + self.assertLess(names.index("Initialize functional evidence"), names.index("Checkout auto-testing scripts (with retry)")) + if suite in FunctionalWorkflowTests.DIRECT_TESTS: + self.assertLess(names.index("Checkout repository (for report parser)"), names.index("Checkout auto-testing scripts (with retry)")) + for name, lines in self.steps.items(): + if name in ("Generate report", "Upload functional report to dashboard") or any("uses: actions/upload-artifact@" in line for line in lines): + self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", lines) + if any("uses: actions/upload-artifact@" in line for line in lines): + self.assertIn(" path: ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/", lines) + self.assertIn(" if-no-files-found: error", lines) + self.artifacts.mkdir() + for filename in ("report.md", "suite.log"): + (self.artifacts / filename).write_text("OLD RUN EVIDENCE") + self.env.update(REPORT_FILE=str(self.artifacts / "report.md"), LOG_FILE=str(self.artifacts / "suite.log")) + initialized = self.run_step("Initialize functional evidence") + self.assertNotEqual(initialized.returncode, 0) + self.assertFalse(Path(self.env["GITHUB_ENV"]).exists()) + issue = self.run_step("File failure issue in rustfs/backlog") + self.assertEqual(issue.returncode, 0, issue.stderr) + body = Path(self.env["CAPTURE_BODY"]).read_text() + self.assertNotIn("OLD RUN EVIDENCE", body) + self.assertIn("no report or log file was produced", body) + self.assertEqual((self.artifacts / "report.md").read_text(), "OLD RUN EVIDENCE") + + def test_reports_use_only_current_complete_suite_evidence(self): + for suite in self.SUITES[:-1]: + good = "--- KMS-101 roundtrip ---\n[PASS] KMS-101\n" + partial = "--- KMS-101 roundtrip ---\n[PASS] KMS-101\n--- KMS-102 unfinished ---\n" + if suite == "s3-compat": + good, partial = good.replace("KMS-", "S3C-"), partial.replace("KMS-", "S3C-") + if suite == "upgrade": + good += "[UPG-TOPO] single-single local v1 v2 PASS=1 FAIL=0\n" + if suite == "heal": + good = "".join(f"[HEAL-STEP] {step} fixture PASS\n" for step in range(1, 8)) + partial = "[HEAL-STEP] 1 fixture PASS\n" + for outcome, log in (("success", good), ("failure", good), ("success", partial), ("success", ""), + ("success", None), ("skipped", None), ("cancelled", good)): + with self.subTest(suite=suite, outcome=outcome, log=log): + self.prepare(suite) + stale = self.directory / "old-suite.log" + stale.write_text("OLD RUN EVIDENCE\n" + good) + self.env.update(LOG_FILE=str(stale), REPORT_FILE=str(stale)) + self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0) + self.assertEqual(self.env["LOG_FILE"], str(self.artifacts / "suite.log")) + self.assertEqual(self.env["TMPDIR"], str(self.artifacts)) + if log is not None: + Path(self.env["LOG_FILE"]).write_text(log) + self.context["steps.test.outcome"] = outcome + report = self.run_step("Generate report") + success = outcome == "success" and log == good + self.assertEqual(report.returncode == 0, success, report.stderr) + contents = Path(self.env["REPORT_FILE"]).read_text() + self.assertNotIn("OLD RUN EVIDENCE", contents) + self.assertEqual("| PASS |" in contents, success) + for value in ("actions/runs/314159", "Attempt: 2", "Workflow Commit: " + self.context["github.sha"], + f"Test Step Outcome: {'success' if success else 'failure'}", f"Suite Step Outcome: {outcome}"): + self.assertIn(value, contents) + self.assertEqual(Path(self.env["GITHUB_STEP_SUMMARY"]).read_text(), contents) + evidence = (self.artifacts / ("steps.md" if suite == "heal" else "cases.md")).read_text() + if log in (good, partial): + self.assertIn("| PASS |", evidence) + self.assertNotIn("OLD RUN EVIDENCE", evidence) + + def test_actual_suite_commands_pass_the_current_log_and_scratch_paths(self): + for suite in self.SUITES: + with self.subTest(suite=suite): + self.prepare(suite) + self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0) + scripts = self.directory / "auto-testing" + scripts.mkdir() + filename = f"rustfs_{suite}_test.sh" if suite in ("heal", "performance") else f"rustfs-{suite}-test.sh" + script = scripts / filename + script.write_text( + '#!/bin/bash\nset -euo pipefail\nlog=""\n' + 'while [ "$#" -gt 0 ]; do\n' + ' if [ "$1" = "--log-file" ]; then log="$2"; shift; fi\n' + ' shift\n' + 'done\n' + '[ "$log" = "$LOG_FILE" ] || exit 31\n' + 'printf "CURRENT SUITE LOG\\n" > "$log"\n' + 'scratch=$(mktemp -d "$TMPDIR/fixture.XXXXXX")\n' + 'printf "CURRENT SCRATCH\\n" > "$scratch/trace.log"\n' + 'if [ -n "${RUSTFS_RESULT_DIR:-}" ]; then\n' + ' mkdir -p "$RUSTFS_RESULT_DIR"\n' + ' printf "CURRENT RESULTS\\n" > "$RUSTFS_RESULT_DIR/summary.md"\n' + 'fi\n' + ) + script.chmod(0o755) + name = FunctionalWorkflowTests.DIRECT_TESTS.get(suite) or ( + "Run benchmark (GET/PUT/MIXED)" if suite == "performance" else "Run heal test (write -> outage -> heal -> verify)" + ) + result = self.run_step(name) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual((self.artifacts / "suite.log").read_text(), "CURRENT SUITE LOG\n") + self.assertEqual(len(list(self.artifacts.glob("fixture.*/trace.log"))), 1) + if suite == "performance": + self.assertEqual((self.artifacts / "results/summary.md").read_text(), "CURRENT RESULTS\n") + + def test_heal_accumulates_actual_staged_steps_without_overwriting_failures(self): + self.prepare("heal") + self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0) + script = self.directory / "auto-testing/rustfs_heal_test.sh" + script.parent.mkdir() + # Result printf and full-run condition from auto-testing@6120aa0a76de:143,1163-1168. + script.write_text(r'''#!/bin/bash +set -euo pipefail +SELECTED_STEPS=() +PREFLIGHT=0 +while [ "$#" -gt 0 ]; do + case "$1" in + --steps) IFS=',' read -ra SELECTED_STEPS <<< "$2"; shift ;; + --log-file) LOG_FILE="$2"; shift ;; + --preflight) PREFLIGHT=1 ;; + esac + shift +done +if [ "$PREFLIGHT" -eq 1 ]; then + printf '\n' >> "$INVOKED_STEPS" + exit 0 +fi +printf '%s\n' "${SELECTED_STEPS[*]}" >> "$INVOKED_STEPS" +emit_step_result() { + local n="$1" desc="$2" status="$3" + printf '[HEAL-STEP] %s %s %s\n' "${n}" "${desc}" "${status}" +} +{ + for step in "${SELECTED_STEPS[@]}"; do + emit_step_result "$step" "fixture step $step" PASS + done + want_all=1 + for s in 1 2 3 4 5 6 7; do + [[ " ${SELECTED_STEPS[*]} " == *" ${s} "* ]] || want_all=0 + done + if [ "${want_all}" -eq 1 ]; then + printf '[HEAL-RESULT] PASS all steps passed\n' + fi +} >> "$LOG_FILE" +''') + script.chmod(0o755) + self.env["INVOKED_STEPS"] = str(self.directory / "invoked-steps") + for name in ("Install RustFS package & start cluster", "Preflight checks", "Run heal test (write -> outage -> heal -> verify)"): + result = self.run_step(name) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(Path(self.env["INVOKED_STEPS"]).read_text().splitlines(), ["1 2", "", "3 4 5 6 7"]) + log = Path(self.env["LOG_FILE"]).read_text() + self.assertNotIn("[HEAL-RESULT]", log) + self.assertEqual(log.count("[HEAL-STEP]"), 7) + report = self.run_step("Generate report") + self.assertEqual(report.returncode, 0, report.stderr) + failed_logs = ["\n".join(line for line in log.splitlines() if not line.startswith(f"[HEAL-STEP] {step} ")) + "\n" + for step in range(1, 8)] + failed_logs += [ + log.replace("[HEAL-STEP] 3", "[HEAL-STEP] 3 original failure FAIL\n[HEAL-STEP] 3"), + log + "[HEAL-STEP] 3 later step failure FAIL\n", + log + "[HEAL-RESULT] FAIL earlier failure\n[HEAL-RESULT] PASS later success\n", + log.replace("[HEAL-STEP] 4 fixture step 4 PASS", "[HEAL-STEP] 4 fixture step 4 SKIP"), + ] + for failed_log in failed_logs: + with self.subTest(log=failed_log): + Path(self.env["LOG_FILE"]).write_text(failed_log) + report = self.run_step("Generate report") + self.assertNotEqual(report.returncode, 0, report.stderr) + contents = Path(self.env["REPORT_FILE"]).read_text() + self.assertIn("Test Step Outcome: failure", contents) + self.assertNotIn("| PASS |", contents) + if "original failure" in failed_log: + self.assertIn("| 3 | original failure | FAIL |", (self.artifacts / "steps.md").read_text()) + if "later step failure" in failed_log: + self.assertIn("| 3 | later step failure | FAIL |", (self.artifacts / "steps.md").read_text()) + + def test_performance_results_version_and_report_are_bound_to_the_run(self): + self.prepare("performance") + initialized = self.run_step("Initialize functional evidence") + self.assertEqual(initialized.returncode, 0, initialized.stderr) + self.assertEqual(self.env["RUSTFS_RESULT_DIR"], str(self.artifacts / "results")) + self.assertEqual(self.env["VERSION_FILE"], str(self.artifacts / "version.txt")) + version = self.run_step("Collect RustFS version info") + self.assertEqual(version.returncode, 0, version.stderr) + self.assertIn("fixture-version", Path(self.env["VERSION_FILE"]).read_text()) + old_summary = self.directory / "old-results/summary.md" + old_summary.parent.mkdir() + old_summary.write_text("OLD RUN EVIDENCE") + upload = "Upload report to dashboard (reports/YYYY-MM-DD.md)" + self.assertNotEqual(self.run_step(upload).returncode, 0) + self.assertFalse(Path(self.env["REPORT_FILE"]).exists()) + results = Path(self.env["RUSTFS_RESULT_DIR"]) + results.mkdir() + (results / "summary.md").write_text("CURRENT PERFORMANCE RESULTS\n") + report = self.run_step(upload) + self.assertEqual(report.returncode, 0, report.stderr) + contents = Path(self.env["REPORT_FILE"]).read_text() + for value in ("actions/runs/314159", "**Attempt**: 2", "**Workflow Commit**: " + self.context["github.sha"], + "CURRENT PERFORMANCE RESULTS", "fixture-version"): + self.assertIn(value, contents) + self.assertNotIn("OLD RUN EVIDENCE", contents) + + if __name__ == "__main__": unittest.main()