From c8fe9ff3454c0a1c504d195f9835b39146a73612 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A9=AC=E7=99=BB=E5=B1=B1?= Date: Tue, 1 Sep 2026 20:40:09 +0800 Subject: [PATCH] ci(tier): isolate per-run evidence --- .github/workflows/audit.yml | 5 + .github/workflows/rustfs-tier-test.yml | 199 +++++++++++------- .../security/check_tier_artifact_workflow.sh | 168 +++++++++++++++ 3 files changed, 301 insertions(+), 71 deletions(-) create mode 100755 scripts/security/check_tier_artifact_workflow.sh diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index 58974a46e..78f694aad 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -28,6 +28,7 @@ on: - 'scripts/test_package_versions.sh' - 'scripts/security/check_performance_ab_workflow.sh' - 'scripts/security/check_preview_release_workflow.sh' + - 'scripts/security/check_tier_artifact_workflow.sh' - 'scripts/security/check_workflow_pins.sh' pull_request: types: [ opened, synchronize, reopened, closed ] @@ -43,6 +44,7 @@ on: - 'scripts/test_package_versions.sh' - 'scripts/security/check_performance_ab_workflow.sh' - 'scripts/security/check_preview_release_workflow.sh' + - 'scripts/security/check_tier_artifact_workflow.sh' - 'scripts/security/check_workflow_pins.sh' schedule: # Daily, not weekly. This schedule exists to catch RustSec advisories @@ -150,6 +152,9 @@ jobs: - name: Check performance A/B workflow trust boundary run: ./scripts/security/check_performance_ab_workflow.sh + - name: Check tier evidence workflow isolation + run: ./scripts/security/check_tier_artifact_workflow.sh + - name: Check package version contract run: ./scripts/test_package_versions.sh diff --git a/.github/workflows/rustfs-tier-test.yml b/.github/workflows/rustfs-tier-test.yml index 07cf060ad..c0a84fc47 100644 --- a/.github/workflows/rustfs-tier-test.yml +++ b/.github/workflows/rustfs-tier-test.yml @@ -54,6 +54,18 @@ jobs: timeout-minutes: 420 if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: + - name: Initialize run evidence directory + id: evidence + run: | + set -euo pipefail + umask 077 + if ! mkdir -- "${TIER_ARTIFACTS_DIR}"; then + echo "refusing to reuse tier evidence path: ${TIER_ARTIFACTS_DIR}" >&2 + exit 1 + fi + test -d "${TIER_ARTIFACTS_DIR}" + test ! -L "${TIER_ARTIFACTS_DIR}" + # auto-testing is private: clone it with the dedicated PF token (not # GITHUB_TOKEN) and retry transient GitHub/network failures. - name: Checkout auto-testing scripts (with retry) @@ -133,11 +145,11 @@ jobs: id: test continue-on-error: true env: - LOG_FILE: /tmp/rustfs-tier.log PACKAGE_URL_INPUT: ${{ inputs.package_url }} RUSTFS_VERSION_INPUT: ${{ inputs.rustfs_version }} run: | set -euo pipefail + LOG_FILE="${TIER_ARTIFACTS_DIR}/rustfs-tier.log" chmod +x auto-testing/rustfs-tier-test.sh RC_BIN="$(command -v rc)" PACKAGE_URL="${PACKAGE_URL_INPUT}" @@ -162,7 +174,7 @@ jobs: ./auto-testing/rustfs-tier-test.sh "${ARGS[@]}" - name: Inject diagnostic case failure - if: ${{ always() && inputs.force_case_failure }} + if: ${{ always() && steps.evidence.outcome == 'success' && inputs.force_case_failure }} run: | set -euo pipefail RESULT_FILE="${TIER_ARTIFACTS_DIR}/cases/single-single--TIER-101.json" @@ -172,12 +184,8 @@ jobs: mv "${TMP_FILE}" "${RESULT_FILE}" - name: Generate report - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} env: - LOG_FILE: /tmp/rustfs-tier.log - REPORT_FILE: /tmp/rustfs-tier-report.md - CASE_TABLE: /tmp/rustfs-tier-cases.md - GATE_RC_FILE: /tmp/rustfs-tier-gate.rc PACKAGE_URL_INPUT: ${{ inputs.package_url }} RUSTFS_VERSION_INPUT: ${{ inputs.rustfs_version }} TEST_OUTCOME: ${{ steps.test.outcome }} @@ -185,6 +193,12 @@ jobs: TRIGGER_NAME: ${{ github.event_name }} run: | set -euo pipefail + test -d "${TIER_ARTIFACTS_DIR}" + test ! -L "${TIER_ARTIFACTS_DIR}" + LOG_FILE="${TIER_ARTIFACTS_DIR}/rustfs-tier.log" + REPORT_FILE="${TIER_ARTIFACTS_DIR}/rustfs-tier-report.md" + CASE_TABLE="${TIER_ARTIFACTS_DIR}/rustfs-tier-cases.md" + GATE_RC_FILE="${TIER_ARTIFACTS_DIR}/rustfs-tier-gate.rc" PACKAGE_URL="${PACKAGE_URL_INPUT}" RUSTFS_VERSION="${RUSTFS_VERSION_INPUT}" if [ -n "${PACKAGE_URL}" ]; then @@ -228,11 +242,11 @@ jobs: cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" - name: Upload functional report to dashboard - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} continue-on-error: true env: GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }} - REPORT_FILE: /tmp/rustfs-tier-report.md + REPORT_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier-report.md SUITE: tier run: | set -euo pipefail @@ -254,16 +268,110 @@ jobs: | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null fi + - name: Verify required tier evidence + id: evidence_verify + if: ${{ always() && steps.evidence.outcome == 'success' }} + run: | + set -euo pipefail + failed=0 + for name in \ + rustfs-tier.log \ + rustfs-tier-report.md \ + rustfs-tier-cases.md \ + rustfs-tier-gate.rc \ + provenance.json; do + if [ ! -s "${TIER_ARTIFACTS_DIR}/${name}" ]; then + echo "required tier evidence is missing or empty: ${name}" >&2 + failed=1 + fi + done + for name in cases logs; do + if [ ! -d "${TIER_ARTIFACTS_DIR}/${name}" ]; then + echo "required tier evidence directory is missing: ${name}" >&2 + failed=1 + fi + done + if ! find "${TIER_ARTIFACTS_DIR}/cases" -maxdepth 1 -type f -name '*.json' -print -quit 2>/dev/null | grep -q .; then + echo "no atomic tier case result was produced" >&2 + failed=1 + fi + [ "${failed}" -eq 0 ] + + - name: Upload report and logs + if: ${{ always() && steps.evidence.outcome == 'success' }} + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: rustfs-tier-test-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ env.TIER_ARTIFACTS_DIR }}/ + if-no-files-found: error + + - name: Cleanup environment (after) + if: always() + run: | + set -euo pipefail + sudo docker rm -f rustfs-test-mqtt >/dev/null 2>&1 || true + sudo rm -f /tmp/rustfs-mosquitto.conf + read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}" + SSH_USER="${RUSTFS_SSH_USER:-azureuser}" + for node in "${NODES[@]}"; do + ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" ' + set -euo pipefail + SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n" + ${SUDO} systemctl stop rustfs 2>/dev/null || true + if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then + ${SUDO} dpkg -P rustfs + fi + for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done + ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms + ' + done + + - name: Enforce tier suite result + id: gate + if: always() + env: + EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }} + TEST_OUTCOME: ${{ steps.test.outcome }} + GATE_RC_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier-gate.rc + run: | + set -euo pipefail + failed=0 + if [ "${EVIDENCE_OUTCOME}" != "success" ]; then + echo "tier evidence directory initialization is ${EVIDENCE_OUTCOME}, expected success" >&2 + failed=1 + fi + if [ "${TEST_OUTCOME}" != "success" ]; then + echo "tier suite step outcome is ${TEST_OUTCOME}, expected success" >&2 + failed=1 + fi + if [ "${EVIDENCE_OUTCOME}" != "success" ]; then + echo "structured gate result is unavailable because evidence initialization failed" >&2 + elif [ ! -s "${GATE_RC_FILE}" ]; then + echo "structured gate result is missing" >&2 + failed=1 + else + GATE_RC="$(tr -d '[:space:]' < "${GATE_RC_FILE}")" + if ! [[ "${GATE_RC}" =~ ^[0-9]+$ ]] || [ "${GATE_RC}" -ne 0 ]; then + echo "structured 56-case gate failed with exit ${GATE_RC:-invalid}" >&2 + failed=1 + fi + fi + [ "${failed}" -eq 0 ] + - name: File failure issue in rustfs/backlog - if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }} + if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled' || steps.evidence_verify.outcome == 'failure' || steps.evidence_verify.outcome == 'cancelled' || steps.gate.outcome == 'failure' || steps.gate.outcome == 'cancelled') }} continue-on-error: true env: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} SUITE: 'tier' SUITE_LABEL: 'Tier' RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - REPORT_FILE: '/tmp/rustfs-tier-report.md' - LOG_FILE: '/tmp/rustfs-tier.log' + EVIDENCE_DIR: ${{ env.TIER_ARTIFACTS_DIR }} + EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }} + VERIFY_OUTCOME: ${{ steps.evidence_verify.outcome }} + GATE_OUTCOME: ${{ steps.gate.outcome }} + REPORT_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier-report.md + LOG_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier.log run: | set -euo pipefail if [ -z "${GH_TOKEN:-}" ]; then @@ -293,10 +401,17 @@ jobs: echo "- Run: ${RUN_URL}" echo "- Trigger: ${GITHUB_EVENT_NAME}" echo "- Date: $(date -u +%Y-%m-%d)" + echo "- Evidence initialization: ${EVIDENCE_OUTCOME}" + echo "- Evidence verification: ${VERIFY_OUTCOME}" + echo "- Final gate: ${GATE_OUTCOME}" echo "" echo "## Report (errors and symptoms)" echo "" - if [ -s "${REPORT_FILE}" ]; then + if [ "${EVIDENCE_OUTCOME}" != "success" ]; then + echo "(the run evidence directory was rejected; its contents were not read)" + elif [ ! -d "${EVIDENCE_DIR}" ] || [ -L "${EVIDENCE_DIR}" ]; then + echo "(the run evidence directory is missing or unsafe; its contents were not read)" + elif [ -s "${REPORT_FILE}" ]; then redact < "${REPORT_FILE}" elif [ -s "${LOG_FILE:-}" ]; then echo "(report file missing; log tail below)" @@ -313,64 +428,6 @@ jobs: fi echo "filed backlog issue for suite ${SUITE}" - - name: Upload report and logs - if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - with: - name: rustfs-tier-test-${{ github.run_id }}-${{ github.run_attempt }} - path: | - /tmp/rustfs-tier.log - /tmp/rustfs-tier-report.md - /tmp/rustfs-tier-cases.md - /tmp/rustfs-tier-gate.rc - ${{ env.TIER_ARTIFACTS_DIR }}/ - if-no-files-found: error - - - name: Cleanup environment (after) - if: always() - run: | - set -euo pipefail - sudo docker rm -f rustfs-test-mqtt >/dev/null 2>&1 || true - sudo rm -f /tmp/rustfs-mosquitto.conf - read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}" - SSH_USER="${RUSTFS_SSH_USER:-azureuser}" - for node in "${NODES[@]}"; do - ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" ' - set -euo pipefail - SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n" - ${SUDO} systemctl stop rustfs 2>/dev/null || true - if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then - ${SUDO} dpkg -P rustfs - fi - for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done - ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms - ' - done - - - name: Enforce tier suite result - if: always() - env: - TEST_OUTCOME: ${{ steps.test.outcome }} - GATE_RC_FILE: /tmp/rustfs-tier-gate.rc - run: | - set -euo pipefail - failed=0 - if [ "${TEST_OUTCOME}" != "success" ]; then - echo "tier suite step outcome is ${TEST_OUTCOME}, expected success" >&2 - failed=1 - fi - if [ ! -s "${GATE_RC_FILE}" ]; then - echo "structured gate result is missing" >&2 - failed=1 - else - GATE_RC="$(tr -d '[:space:]' < "${GATE_RC_FILE}")" - if ! [[ "${GATE_RC}" =~ ^[0-9]+$ ]] || [ "${GATE_RC}" -ne 0 ]; then - echo "structured 56-case gate failed with exit ${GATE_RC:-invalid}" >&2 - failed=1 - fi - fi - [ "${failed}" -eq 0 ] - - name: "Continue functional chain (next: Storage engine)" # Only chain-triggered runs forward to the next suite; standalone # workflow_dispatch runs stop after their own cleanup. diff --git a/scripts/security/check_tier_artifact_workflow.sh b/scripts/security/check_tier_artifact_workflow.sh new file mode 100755 index 000000000..57e0d3a5c --- /dev/null +++ b/scripts/security/check_tier_artifact_workflow.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +set -euo pipefail + +WORKFLOW="${1:-.github/workflows/rustfs-tier-test.yml}" + +fail() { + printf 'tier artifact workflow check failed: %s\n' "$*" >&2 + exit 1 +} + +[ -f "${WORKFLOW}" ] || fail "workflow not found: ${WORKFLOW}" + +for fixed_path in \ + /tmp/rustfs-tier.log \ + /tmp/rustfs-tier-report.md \ + /tmp/rustfs-tier-cases.md \ + /tmp/rustfs-tier-gate.rc; do + if grep -Fq -- "${fixed_path}" "${WORKFLOW}"; then + fail "fixed cross-run path remains: ${fixed_path}" + fi +done + +for required in \ + 'TIER_ARTIFACTS_DIR: /tmp/rustfs-tier-artifacts-${{ github.run_id }}-${{ github.run_attempt }}' \ + 'id: evidence' \ + 'umask 077' \ + 'mkdir -- "${TIER_ARTIFACTS_DIR}"' \ + 'rustfs-tier.log' \ + 'rustfs-tier-report.md' \ + 'rustfs-tier-cases.md' \ + 'rustfs-tier-gate.rc' \ + 'provenance.json' \ + 'Verify required tier evidence' \ + 'id: evidence_verify' \ + 'id: gate' \ + 'path: ${{ env.TIER_ARTIFACTS_DIR }}/' \ + 'if-no-files-found: error'; do + grep -Fq -- "${required}" "${WORKFLOW}" || fail "required contract is missing: ${required}" +done + +step_block() { + local name="$1" + awk -v name="${name}" ' + $0 == " - name: " name { capture=1; found=1 } + capture && $0 != " - name: " name && $0 ~ /^ - name: / { exit } + capture { print } + END { if (!found) exit 1 } + ' "${WORKFLOW}" +} + +require_in_block() { + local block="$1" expected="$2" label="$3" + grep -Fqx -- "${expected}" <<< "${block}" \ + || fail "${label} is missing: ${expected}" +} + +verify_block="$(step_block 'Verify required tier evidence')" \ + || fail "required-evidence step was not found" +require_in_block "${verify_block}" \ + " if: \${{ always() && steps.evidence.outcome == 'success' }}" \ + "required-evidence condition" +require_in_block "${verify_block}" \ + ' failed=0' \ + "required-evidence failure accumulator" +require_in_block "${verify_block}" \ + ' [ "${failed}" -eq 0 ]' \ + "required-evidence fail-closed result" +require_in_block "${verify_block}" \ + ' if [ ! -s "${TIER_ARTIFACTS_DIR}/${name}" ]; then' \ + "required-file missing/empty predicate" +require_in_block "${verify_block}" \ + ' if [ ! -d "${TIER_ARTIFACTS_DIR}/${name}" ]; then' \ + "required-directory missing predicate" +for required_name in \ + rustfs-tier.log \ + rustfs-tier-report.md \ + rustfs-tier-cases.md \ + rustfs-tier-gate.rc \ + provenance.json; do + grep -Fq -- "${required_name}" <<< "${verify_block}" \ + || fail "required-evidence step does not verify ${required_name}" +done +require_in_block "${verify_block}" \ + ' if ! find "${TIER_ARTIFACTS_DIR}/cases" -maxdepth 1 -type f -name '\''*.json'\'' -print -quit 2>/dev/null | grep -q .; then' \ + "required atomic-case predicate" +[ "$(grep -Fc ' failed=1' <<< "${verify_block}")" -eq 3 ] \ + || fail "required-evidence step must fail for files, directories, and atomic cases" + +upload_block="$(step_block 'Upload report and logs')" \ + || fail "upload step was not found" +[ -n "${upload_block}" ] || fail "upload step was not found" +require_in_block "${upload_block}" \ + " if: \${{ always() && steps.evidence.outcome == 'success' }}" \ + "artifact upload condition" +[ "$(grep -c '^[[:space:]]*path:' <<< "${upload_block}")" -eq 1 ] \ + || fail "upload step must contain exactly one path" +grep -Fq 'path: ${{ env.TIER_ARTIFACTS_DIR }}/' <<< "${upload_block}" \ + || fail "upload step must archive only the run-scoped evidence directory" +require_in_block "${upload_block}" \ + ' if-no-files-found: error' \ + "artifact upload empty-evidence behavior" + +gate_block="$(step_block 'Enforce tier suite result')" \ + || fail "final gate step was not found" +require_in_block "${gate_block}" ' if: always()' "final gate condition" +require_in_block "${gate_block}" \ + ' EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}' \ + "final gate evidence status" +require_in_block "${gate_block}" \ + ' TEST_OUTCOME: ${{ steps.test.outcome }}' \ + "final gate suite status" +require_in_block "${gate_block}" \ + ' GATE_RC_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier-gate.rc' \ + "final gate structured status path" +require_in_block "${gate_block}" \ + ' if [ "${EVIDENCE_OUTCOME}" != "success" ]; then' \ + "final gate evidence enforcement" +require_in_block "${gate_block}" \ + ' if [ "${TEST_OUTCOME}" != "success" ]; then' \ + "final gate suite enforcement" +require_in_block "${gate_block}" \ + ' elif [ ! -s "${GATE_RC_FILE}" ]; then' \ + "final gate missing-result enforcement" +require_in_block "${gate_block}" \ + ' if ! [[ "${GATE_RC}" =~ ^[0-9]+$ ]] || [ "${GATE_RC}" -ne 0 ]; then' \ + "final gate nonzero-result enforcement" +require_in_block "${gate_block}" \ + ' [ "${failed}" -eq 0 ]' \ + "final gate fail-closed result" + +issue_block="$(step_block 'File failure issue in rustfs/backlog')" \ + || fail "failure-issue step was not found" +require_in_block "${issue_block}" \ + ' EVIDENCE_DIR: ${{ env.TIER_ARTIFACTS_DIR }}' \ + "failure-issue evidence directory" +require_in_block "${issue_block}" \ + ' EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}' \ + "failure-issue initialization status" +require_in_block "${issue_block}" \ + ' VERIFY_OUTCOME: ${{ steps.evidence_verify.outcome }}' \ + "failure-issue verification status" +require_in_block "${issue_block}" \ + ' GATE_OUTCOME: ${{ steps.gate.outcome }}' \ + "failure-issue final-gate status" +require_in_block "${issue_block}" \ + ' if [ "${EVIDENCE_OUTCOME}" != "success" ]; then' \ + "failure-issue rejected-evidence guard" +require_in_block "${issue_block}" \ + ' elif [ ! -d "${EVIDENCE_DIR}" ] || [ -L "${EVIDENCE_DIR}" ]; then' \ + "failure-issue unsafe-evidence guard" +grep -Fq "steps.evidence_verify.outcome == 'failure'" <<< "${issue_block}" \ + || fail "failure-issue condition does not cover evidence verification" +grep -Fq "steps.gate.outcome == 'failure'" <<< "${issue_block}" \ + || fail "failure-issue condition does not cover the final gate" + +step_line() { + local name="$1" + awk -v name="${name}" '$0 == " - name: " name { print NR; exit }' "${WORKFLOW}" +} +verify_line="$(step_line 'Verify required tier evidence')" +gate_line="$(step_line 'Enforce tier suite result')" +issue_line="$(step_line 'File failure issue in rustfs/backlog')" +[ -n "${verify_line}" ] && [ -n "${gate_line}" ] && [ -n "${issue_line}" ] \ + || fail "cannot determine evidence/final-gate/failure-issue order" +[ "${verify_line}" -lt "${gate_line}" ] && [ "${gate_line}" -lt "${issue_line}" ] \ + || fail "failure issue must run after evidence verification and the final gate" + +printf 'tier artifact workflow contract is isolated\n'