From ea01cd339c989d53e243126a3f599a872253eebd Mon Sep 17 00:00:00 2001 From: hector <42570491+majinghe@users.noreply.github.com> Date: Mon, 31 Aug 2026 15:19:46 +0800 Subject: [PATCH] fix(ci): continue functional chain and publish heal/pool reports (#6932) --- .github/workflows/rustfs-heal-test.yml | 206 +++++++++ .github/workflows/rustfs-kms-test.yml | 3 + .github/workflows/rustfs-pool-expand-test.yml | 413 ++++++++++++++++++ .github/workflows/rustfs-s3-compat-test.yml | 68 ++- .github/workflows/rustfs-security-test.yml | 380 ++++++++++++++++ .github/workflows/rustfs-tier-test.yml | 3 + 6 files changed, 1071 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/rustfs-security-test.yml diff --git a/.github/workflows/rustfs-heal-test.yml b/.github/workflows/rustfs-heal-test.yml index 3e366481b..24024d710 100644 --- a/.github/workflows/rustfs-heal-test.yml +++ b/.github/workflows/rustfs-heal-test.yml @@ -43,6 +43,7 @@ env: RUSTFS_API_ENDPOINT: ${{ secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT }} RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }} RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }} + PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }} jobs: @@ -110,6 +111,7 @@ jobs: ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" - name: Run heal test (write -> outage -> heal -> verify) + id: test run: | ./auto-testing/rustfs_heal_test.sh \ --steps "3,4,5,6,7" -y \ @@ -118,6 +120,210 @@ jobs: --warp-stop-gb "${{ inputs.warp_stop_gb }}" \ --log-file /tmp/rustfs-heal-test.log + - name: Generate report + if: always() + env: + LOG_FILE: /tmp/rustfs-heal-test.log + REPORT_FILE: /tmp/rustfs-heal-report.md + run: | + set -euo pipefail + PACKAGE_URL='${{ inputs.package_url }}' + if [ -n "${PACKAGE_URL}" ]; then + PACKAGE_SOURCE="${PACKAGE_URL}" + else + PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}" + fi + { + echo "# RustFS heal test report" + echo "" + echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo "- Trigger: ${{ github.event_name }}" + echo "- Package: ${PACKAGE_SOURCE}" + echo "- Test Step Outcome: ${{ steps.test.outcome }}" + echo "" + echo "## Log tail" + echo '```text' + tail -n 200 "${LOG_FILE}" || true + echo '```' + } | tee "${REPORT_FILE}" + cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" + + - name: Upload functional report to dashboard + if: always() + continue-on-error: true + env: + GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }} + REPORT_FILE: /tmp/rustfs-heal-report.md + SUITE: heal + run: | + set -euo pipefail + if [ -z "${GH_TOKEN:-}" ]; then + echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload" + exit 0 + fi + DATE="$(date -u +%Y-%m-%d)" + REPORT_PATH="functional-reports/${SUITE}/${DATE}.md" + CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")" + SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)" + if [ -n "${SHA}" ]; then + jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \ + '{message:$msg, content:$content, sha:$sha}' \ + | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null + else + jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \ + '{message:$msg, content:$content}' \ + | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null + fi + + cat > /tmp/rustfs-functional-index.html <<'EOF' + + + + + + RustFS Functional Test Reports + + + +
+
+

RustFS Functional Test Reports

+

Select a suite and date to view the build version used in that run.

+
+ +
+
Date: N/A
+
RustFS Version: N/A
+ +
+
+
+ + + + EOF + + INDEX_PATH="functional/index.html" + INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')" + INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)" + if [ -n "${INDEX_SHA}" ]; then + jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \ + '{message:$msg, content:$content, sha:$sha}' \ + | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null + else + jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \ + '{message:$msg, content:$content}' \ + | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null + fi + - name: Upload test logs if: always() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 diff --git a/.github/workflows/rustfs-kms-test.yml b/.github/workflows/rustfs-kms-test.yml index 0cc3dd16a..a00a786fc 100644 --- a/.github/workflows/rustfs-kms-test.yml +++ b/.github/workflows/rustfs-kms-test.yml @@ -253,6 +253,9 @@ jobs: { key: 's3', label: 'S3 Compatibility' }, { key: 'kms', label: 'KMS' }, { key: 'tier', label: 'Tier' }, + { key: 'heal', label: 'Heal' }, + { key: 'pool', label: 'Pool Expansion' }, + { key: 'security', label: 'Security' }, ]; const tabs = document.getElementById('tabs'); const list = document.getElementById('list'); diff --git a/.github/workflows/rustfs-pool-expand-test.yml b/.github/workflows/rustfs-pool-expand-test.yml index 31db6579a..78a555e01 100644 --- a/.github/workflows/rustfs-pool-expand-test.yml +++ b/.github/workflows/rustfs-pool-expand-test.yml @@ -70,6 +70,7 @@ env: RUSTFS_API_ENDPOINT: ${{ secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT }} RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }} RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }} + PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} # Package used by the nightly run (workflow_dispatch inputs are empty for # workflow_run events), i.e. the latest nightly deb published by nightly-gnu.yml. RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }} @@ -137,6 +138,7 @@ jobs: ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" - name: Run heal test (write -> outage -> heal -> verify) + id: test run: | ARGS=(--steps "3,4,5,6,7" -y \ --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \ @@ -150,6 +152,210 @@ jobs: fi ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" + - name: Generate report + if: always() + env: + LOG_FILE: /tmp/rustfs-heal-test.log + REPORT_FILE: /tmp/rustfs-heal-report.md + run: | + set -euo pipefail + PACKAGE_URL='${{ inputs.package_url }}' + if [ -n "${PACKAGE_URL}" ]; then + PACKAGE_SOURCE="${PACKAGE_URL}" + else + PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}" + fi + { + echo "# RustFS heal test report" + echo "" + echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo "- Trigger: ${{ github.event_name }}" + echo "- Package: ${PACKAGE_SOURCE}" + echo "- Test Step Outcome: ${{ steps.test.outcome }}" + echo "" + echo "## Log tail" + echo '```text' + tail -n 200 "${LOG_FILE}" || true + echo '```' + } | tee "${REPORT_FILE}" + cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" + + - name: Upload functional report to dashboard + if: always() + continue-on-error: true + env: + GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }} + REPORT_FILE: /tmp/rustfs-heal-report.md + SUITE: heal + run: | + set -euo pipefail + if [ -z "${GH_TOKEN:-}" ]; then + echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload" + exit 0 + fi + DATE="$(date -u +%Y-%m-%d)" + REPORT_PATH="functional-reports/${SUITE}/${DATE}.md" + CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")" + SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)" + if [ -n "${SHA}" ]; then + jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \ + '{message:$msg, content:$content, sha:$sha}' \ + | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null + else + jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \ + '{message:$msg, content:$content}' \ + | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null + fi + + cat > /tmp/rustfs-functional-index.html <<'EOF' + + + + + + RustFS Functional Test Reports + + + +
+
+

RustFS Functional Test Reports

+

Select a suite and date to view the build version used in that run.

+
+ +
+
Date: N/A
+
RustFS Version: N/A
+ +
+
+
+ + + + EOF + + INDEX_PATH="functional/index.html" + INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')" + INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)" + if [ -n "${INDEX_SHA}" ]; then + jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \ + '{message:$msg, content:$content, sha:$sha}' \ + | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null + else + jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \ + '{message:$msg, content:$content}' \ + | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null + fi + - name: Upload test logs if: always() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 @@ -279,6 +485,213 @@ jobs: fi ./auto-testing/rustfs_pool_expand.sh "${ARGS[@]}" + - name: Generate report + if: always() + env: + LOG_FILE: /tmp/rustfs-pool-test.log + REPORT_FILE: /tmp/rustfs-pool-report.md + run: | + set -euo pipefail + PACKAGE_URL='${{ inputs.package_url }}' + RUSTFS_VERSION='${{ inputs.rustfs_version }}' + if [ -n "${PACKAGE_URL}" ]; then + PACKAGE_SOURCE="${PACKAGE_URL}" + elif [ -n "${RUSTFS_VERSION}" ]; then + PACKAGE_SOURCE="version ${RUSTFS_VERSION}" + else + PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}" + fi + { + echo "# RustFS pool expansion test report" + echo "" + echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo "- Trigger: ${{ github.event_name }}" + echo "- Package: ${PACKAGE_SOURCE}" + echo "- Test Step Outcome: ${{ steps.pool_test.outcome }}" + echo "" + echo "## Log tail" + echo '```text' + tail -n 200 "${LOG_FILE}" || true + echo '```' + } | tee "${REPORT_FILE}" + cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" + + - name: Upload functional report to dashboard + if: always() + continue-on-error: true + env: + GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }} + REPORT_FILE: /tmp/rustfs-pool-report.md + SUITE: pool + run: | + set -euo pipefail + if [ -z "${GH_TOKEN:-}" ]; then + echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload" + exit 0 + fi + DATE="$(date -u +%Y-%m-%d)" + REPORT_PATH="functional-reports/${SUITE}/${DATE}.md" + CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")" + SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)" + if [ -n "${SHA}" ]; then + jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \ + '{message:$msg, content:$content, sha:$sha}' \ + | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null + else + jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \ + '{message:$msg, content:$content}' \ + | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null + fi + + cat > /tmp/rustfs-functional-index.html <<'EOF' + + + + + + RustFS Functional Test Reports + + + +
+
+

RustFS Functional Test Reports

+

Select a suite and date to view the build version used in that run.

+
+ +
+
Date: N/A
+
RustFS Version: N/A
+ +
+
+
+ + + + EOF + + INDEX_PATH="functional/index.html" + INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')" + INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)" + if [ -n "${INDEX_SHA}" ]; then + jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \ + '{message:$msg, content:$content, sha:$sha}' \ + | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null + else + jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \ + '{message:$msg, content:$content}' \ + | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null + fi + - name: Upload test logs if: always() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 diff --git a/.github/workflows/rustfs-s3-compat-test.yml b/.github/workflows/rustfs-s3-compat-test.yml index fffd0621f..b5ac08566 100644 --- a/.github/workflows/rustfs-s3-compat-test.yml +++ b/.github/workflows/rustfs-s3-compat-test.yml @@ -112,6 +112,16 @@ jobs: else PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}" fi + read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}" + SSH_USER="${RUSTFS_SSH_USER:-azureuser}" + RUSTFS_VERSION_INFO="N/A" + if [ "${#NODES[@]}" -gt 0 ]; then + DETECTED_VERSION="$(ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \ + "${SSH_USER}@${NODES[0]}" 'rustfs --version' 2>/dev/null | tr -d '\r' | head -n 1 || true)" + if [ -n "${DETECTED_VERSION}" ]; then + RUSTFS_VERSION_INFO="${DETECTED_VERSION}" + fi + fi CASE_TABLE="/tmp/rustfs-s3-compat-cases.md" python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY' import re @@ -171,6 +181,7 @@ jobs: echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" echo "- Trigger: ${{ github.event_name }}" echo "- Package: ${PACKAGE_SOURCE}" + echo "- RustFS Version: ${RUSTFS_VERSION_INFO}" echo "- Test Step Outcome: ${{ steps.test.outcome }}" echo "" cat "${CASE_TABLE}" || true @@ -227,19 +238,28 @@ jobs: .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; } button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; } button.active { background: var(--accent); color: #fff; border-color: var(--accent); } + .report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); } ul { list-style: none; margin: 0; padding: 0; } li { padding: 10px 0; border-bottom: 1px solid var(--line); } a { color: var(--accent); text-decoration: none; } a:hover { text-decoration: underline; } + .meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; } + .kv { margin: 6px 0; color: var(--text); } + .muted { color: var(--muted); }

RustFS Functional Test Reports

-

S3, KMS, Tier report tabs. Each tab lists reports by date.

+

Select a suite and date to view the build version used in that run.

+
+
Date: N/A
+
RustFS Version: N/A
+ +
+ + + EOF + + INDEX_PATH="functional/index.html" + INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')" + INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)" + if [ -n "${INDEX_SHA}" ]; then + jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \ + '{message:$msg, content:$content, sha:$sha}' \ + | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null + else + jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \ + '{message:$msg, content:$content}' \ + | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null + fi + + - name: Upload report and logs + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: rustfs-security-test-${{ github.run_id }} + path: | + /tmp/rustfs-security-report.md + /tmp/rustfs-security.*/* + if-no-files-found: ignore + retention-days: 3 + + - name: Cleanup environment (after) + if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }} + run: | + set -euo pipefail + read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}" + SSH_USER="${RUSTFS_SSH_USER:-azureuser}" + for node in "${NODES[@]}"; do + ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" ' + set -euo pipefail + SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n" + ${SUDO} systemctl stop rustfs 2>/dev/null || true + if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then + ${SUDO} dpkg -P rustfs + fi + for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done + ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms + ' + done + + - name: Notify on failure + if: failure() + run: | + echo "RustFS security test failed" + echo "Package source: ${{ inputs.package_url || 'nightly (R2 latest)' }}" + echo "See the uploaded report and logs for details." diff --git a/.github/workflows/rustfs-tier-test.yml b/.github/workflows/rustfs-tier-test.yml index 765e0d867..f201ec1ab 100644 --- a/.github/workflows/rustfs-tier-test.yml +++ b/.github/workflows/rustfs-tier-test.yml @@ -274,6 +274,9 @@ jobs: { key: 's3', label: 'S3 Compatibility' }, { key: 'kms', label: 'KMS' }, { key: 'tier', label: 'Tier' }, + { key: 'heal', label: 'Heal' }, + { key: 'pool', label: 'Pool Expansion' }, + { key: 'security', label: 'Security' }, ]; const tabs = document.getElementById('tabs'); const list = document.getElementById('list');