diff --git a/.config/make/tests.mak b/.config/make/tests.mak
index d1297e9ad..726ca04cf 100644
--- a/.config/make/tests.mak
+++ b/.config/make/tests.mak
@@ -41,6 +41,7 @@ script-tests: ## Run shell script tests
$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py
+ $(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
diff --git a/.github/actions/quick-checks/action.yml b/.github/actions/quick-checks/action.yml
index e6cc59775..18c6cb64d 100644
--- a/.github/actions/quick-checks/action.yml
+++ b/.github/actions/quick-checks/action.yml
@@ -100,6 +100,7 @@ runs:
python3 ./scripts/check_test_wiring.py --self-test
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
python3 ./scripts/test_security_workflow.py
+ python3 ./scripts/test_nightly_candidate.py
python3 ./scripts/check_test_wiring.py
- name: Check no planning docs committed
diff --git a/.github/workflows/nightly-gnu.yml b/.github/workflows/nightly-gnu.yml
index 0dedbf465..1864a9715 100644
--- a/.github/workflows/nightly-gnu.yml
+++ b/.github/workflows/nightly-gnu.yml
@@ -166,8 +166,9 @@ jobs:
# e.g. https://dl.rustfs.com/artifacts/rustfs/packages/nightly/... .
# Skipped when the R2 secrets are not configured (artifact-only mode).
- name: Upload DEB to Cloudflare R2
- if: env.R2_ACCESS_KEY_ID != ''
+ id: publish
env:
+ DEB_FILE: ${{ steps.deb.outputs.deb_file }}
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
@@ -182,28 +183,70 @@ jobs:
exit 0
fi
- if ! command -v aws >/dev/null 2>&1; then
- sudo apt-get update && sudo apt-get install -y -qq awscli
- fi
-
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
export AWS_DEFAULT_REGION="auto"
- DEB_FILE="${{ steps.deb.outputs.deb_file }}"
+ SOURCE_SHA="$(git rev-parse HEAD)"
+ if [[ "${SOURCE_SHA}" != "${GITHUB_SHA}" ]]; then
+ echo "Checkout SHA does not match the nightly build run" >&2
+ exit 1
+ fi
+ DEB_SHA256="$(sha256sum "${DEB_FILE}" | cut -d ' ' -f 1)"
+ CANDIDATE_KEY="artifacts/rustfs/packages/nightly/runs/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/${DEB_SHA256}/rustfs.deb"
+ CANDIDATE_URL="https://dl.rustfs.com/${CANDIDATE_KEY}"
+
+ # Old AWS CLI models lack conditional PutObject support. Never fall
+ # back to an overwriting upload for a candidate.
+ AWS_CLI=aws
+ if ! "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null; then
+ sudo apt-get update
+ sudo apt-get install -y -qq python3-venv
+ AWS_CLI_DIR="$(mktemp -d "${RUNNER_TEMP}/nightly-awscli.XXXXXX")"
+ trap 'rm -rf "${AWS_CLI_DIR}"' EXIT
+ python3 -m venv "${AWS_CLI_DIR}"
+ "${AWS_CLI_DIR}/bin/python" -m pip install --disable-pip-version-check 'awscli==1.44.79'
+ AWS_CLI="${AWS_CLI_DIR}/bin/aws"
+ fi
+ "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null
+ "${AWS_CLI}" --version
+ "${AWS_CLI}" s3api put-object --bucket "${R2_BUCKET}" --key "${CANDIDATE_KEY}" \
+ --body "${DEB_FILE}" --if-none-match '*' --endpoint-url "${R2_ENDPOINT}"
+ PUBLISHED_SHA256="$(curl -fsSL --retry 3 --connect-timeout 15 --max-time 300 "${CANDIDATE_URL}" | sha256sum | cut -d ' ' -f 1)"
+ if [[ "${PUBLISHED_SHA256}" != "${DEB_SHA256}" ]]; then
+ echo "Published candidate checksum does not match the built package" >&2
+ exit 1
+ fi
+
R2_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/nightly/"
echo "📤 Uploading ${DEB_FILE} to ${R2_PREFIX}"
- aws s3 cp "${DEB_FILE}" "${R2_PREFIX}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+ "${AWS_CLI}" s3 cp "${DEB_FILE}" "${R2_PREFIX}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
# Stable "latest" alias so tests can fetch the newest nightly
# without knowing today's date.
echo "📤 Uploading latest alias"
- aws s3 cp "${DEB_FILE}" "${R2_PREFIX}rustfs-nightly-latest.deb" \
+ "${AWS_CLI}" s3 cp "${DEB_FILE}" "${R2_PREFIX}rustfs-nightly-latest.deb" \
--endpoint-url "$R2_ENDPOINT" --only-show-errors
echo "✅ R2 upload complete"
+ CANDIDATE_FILE="${RUNNER_TEMP}/nightly-candidate-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
+ jq -n --arg source_sha "${SOURCE_SHA}" \
+ --argjson build_run_id "${GITHUB_RUN_ID}" --argjson build_run_attempt "${GITHUB_RUN_ATTEMPT}" \
+ --arg package_url "${CANDIDATE_URL}" --arg package_sha256 "${DEB_SHA256}" \
+ '{schema: 1, source_sha: $source_sha, build_run_id: $build_run_id, build_run_attempt: $build_run_attempt, package_url: $package_url, package_sha256: $package_sha256}' \
+ > "${CANDIDATE_FILE}"
+ echo "candidate_file=${CANDIDATE_FILE}" >> "${GITHUB_OUTPUT}"
+
+ - name: Upload nightly candidate manifest
+ if: ${{ steps.publish.outputs.candidate_file != '' }}
+ uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+ with:
+ name: nightly-candidate-${{ github.run_id }}-${{ github.run_attempt }}
+ path: ${{ steps.publish.outputs.candidate_file }}
+ if-no-files-found: error
+
# Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
#
# RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
diff --git a/.github/workflows/rustfs-functional-chain.yml b/.github/workflows/rustfs-functional-chain.yml
index 6ce828c1a..10b9c67ad 100644
--- a/.github/workflows/rustfs-functional-chain.yml
+++ b/.github/workflows/rustfs-functional-chain.yml
@@ -14,8 +14,8 @@
# Functional chain driver: runs the ten functional suites in a fixed order
# (upgrade -> s3 -> kms -> tier -> storage -> heal -> pool -> security ->
-# replication, with performance on its own runner in parallel) and guarantees
-# the chain keeps moving even when individual suites fail.
+# replication -> performance). Each suite attempts the next handoff even
+# when its tests fail.
#
# Each suite workflow can still be dispatched standalone (workflow_dispatch);
# only chain-triggered runs forward to the next suite via repository_dispatch,
@@ -59,16 +59,3 @@ jobs:
gh api --method POST repos/rustfs/rustfs/dispatches \
-f event_type='rustfs-chain-upgrade' \
-F 'client_payload[from_suite]=nightly-build'
-
- - name: Dispatch performance suite (parallel, own runner)
- env:
- GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
- run: |
- set -euo pipefail
- if [ -z "${GH_TOKEN:-}" ]; then
- echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch performance" >&2
- exit 1
- fi
- gh api --method POST repos/rustfs/rustfs/dispatches \
- -f event_type='rustfs-chain-performance' \
- -F 'client_payload[from_suite]=nightly-build'
diff --git a/.github/workflows/rustfs-heal-test.yml b/.github/workflows/rustfs-heal-test.yml
index 666c83b4c..4e6b4e190 100644
--- a/.github/workflows/rustfs-heal-test.yml
+++ b/.github/workflows/rustfs-heal-test.yml
@@ -59,6 +59,21 @@ jobs:
# (storage -> heal -> pool). Pool expansion no longer re-runs heal.
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
+ - name: Initialize functional evidence
+ id: evidence
+ run: |
+ set -euo pipefail
+ umask 077
+ FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-heal-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+ mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+ {
+ printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ printf 'RUSTFS_WARP_LOG_FILE=%s/warp.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ } >> "${GITHUB_ENV}"
+
# auto-testing is private: clone it with the dedicated PF token (not
# GITHUB_TOKEN) and retry transient GitHub/network failures.
- name: Checkout auto-testing scripts (with retry)
@@ -114,7 +129,7 @@ jobs:
else
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
fi
- ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
+ ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
- name: Preflight checks
run: |
@@ -124,7 +139,7 @@ jobs:
else
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
fi
- ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
+ ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
- name: Run heal test (write -> outage -> heal -> verify)
id: test
@@ -134,13 +149,10 @@ jobs:
--endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
--stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \
--warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \
- --log-file /tmp/rustfs-heal-test.log
+ --log-file "${LOG_FILE}"
- name: Generate report
- if: always()
- env:
- LOG_FILE: /tmp/rustfs-heal-test.log
- REPORT_FILE: /tmp/rustfs-heal-report.md
+ if: ${{ always() && steps.evidence.outcome == 'success' }}
run: |
set -euo pipefail
PACKAGE_URL='${{ inputs.package_url }}'
@@ -149,8 +161,9 @@ jobs:
else
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
fi
- STEPS_TABLE="/tmp/rustfs-heal-steps.md"
- python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY'
+ STEPS_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/steps.md"
+ CASE_RESULT=success
+ python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY' || CASE_RESULT=failure
import re
import sys
@@ -162,6 +175,7 @@ jobs:
steps = {}
order = []
+ status_rank = {'SKIP': 0, 'PASS': 1, 'FAIL': 2}
version = None
version_node = None
verdict = None
@@ -175,14 +189,15 @@ jobs:
n, desc, status = m.group(1), m.group(2), m.group(3)
if n not in steps:
order.append(n)
- steps[n] = (desc, status) # later lines win (fail after pass)
+ if n not in steps or status_rank[status] > status_rank[steps[n][1]]:
+ steps[n] = (desc, status)
continue
m = ver_re.match(line)
if m:
version, version_node = m.group(1), m.group(2)
continue
m = result_re.match(line)
- if m:
+ if m and verdict != 'FAIL':
verdict, verdict_detail = m.group(1), m.group(2)
except FileNotFoundError:
pass
@@ -202,30 +217,43 @@ jobs:
out.write(f'| {n} | {desc} | {status} |\n')
if not order:
out.write('| - | - | NOT RUN (no step result lines found) |\n')
+ complete = set(steps) == {str(n) for n in range(1, 8)}
+ sys.exit(0 if complete and verdict != 'FAIL' and all(status == 'PASS' for _, status in steps.values()) else 1)
PY
+ RESULT=failure
+ if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+ RESULT=success
+ fi
{
echo "# RustFS heal test report"
echo ""
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+ echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+ echo "- Workflow Commit: ${GITHUB_SHA}"
echo "- Trigger: ${{ github.event_name }}"
echo "- Package: ${PACKAGE_SOURCE}"
- echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+ echo "- Test Step Outcome: ${RESULT}"
+ echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
echo ""
- cat "${STEPS_TABLE}" || true
- echo ""
- echo "## Log tail"
- echo '```text'
- tail -n 200 "${LOG_FILE}" || true
- echo '```'
+ if [ "${RESULT}" = "success" ]; then
+ cat "${STEPS_TABLE}"
+ echo ""
+ echo "## Log tail"
+ echo '```text'
+ tail -n 200 "${LOG_FILE}"
+ echo '```'
+ else
+ echo "The suite or evidence validation failed. See this run's artifact for partial step results and suite.log."
+ fi
} | tee "${REPORT_FILE}"
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+ [ "${RESULT}" = "success" ]
- name: Upload functional report to dashboard
- if: always()
+ if: ${{ always() && steps.evidence.outcome == 'success' }}
continue-on-error: true
env:
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
- REPORT_FILE: /tmp/rustfs-heal-report.md
SUITE: heal
run: |
set -euo pipefail
@@ -235,28 +263,32 @@ jobs:
fi
DATE="$(date -u +%Y-%m-%d)"
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
- CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+ # Base64-encode the report into a temp file and feed it to jq via
+ # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+ # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+ B64_FILE="$(mktemp)"
+ python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
if [ -n "${SHA}" ]; then
- jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
- '{message:$msg, content:$content, sha:$sha}' \
+ jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+ '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
else
- jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
- '{message:$msg, content:$content}' \
+ jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+ '{message:$msg, content:($content|rtrimstr("\n"))}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
fi
+ rm -f "${B64_FILE}"
- name: File failure issue in rustfs/backlog
if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
continue-on-error: true
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
SUITE: 'heal'
SUITE_LABEL: 'Heal'
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
- REPORT_FILE: '/tmp/rustfs-heal-report.md'
- LOG_FILE: '/tmp/rustfs-heal-test.log'
run: |
set -euo pipefail
if [ -z "${GH_TOKEN:-}" ]; then
@@ -284,14 +316,16 @@ jobs:
echo ""
echo "- Suite: \`${SUITE}\`"
echo "- Run: ${RUN_URL}"
+ echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+ echo "- Workflow Commit: ${GITHUB_SHA}"
echo "- Trigger: ${GITHUB_EVENT_NAME}"
echo "- Date: $(date -u +%Y-%m-%d)"
echo ""
echo "## Report (errors and symptoms)"
echo ""
- if [ -s "${REPORT_FILE}" ]; then
+ if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
redact < "${REPORT_FILE}"
- elif [ -s "${LOG_FILE:-}" ]; then
+ elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
echo "(report file missing; log tail below)"
echo ""
tail -n 200 "${LOG_FILE}" | redact
@@ -307,14 +341,16 @@ jobs:
echo "filed backlog issue for suite ${SUITE}"
- name: Upload test logs
- if: always()
+ if: ${{ always() && steps.evidence.outcome == 'success' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
- name: rustfs-heal-test-${{ github.run_id }}
+ name: rustfs-heal-test-${{ github.run_id }}-${{ github.run_attempt }}
path: |
- /tmp/rustfs-heal-test*.log
- /tmp/rustfs-warp.*.log
- if-no-files-found: warn
+ ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+ ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+ ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/warp.log
+ ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/steps.md
+ if-no-files-found: error
- name: Cleanup environment (after)
if: ${{ always() && inputs.cleanup_after != 'false' }}
diff --git a/.github/workflows/rustfs-kms-test.yml b/.github/workflows/rustfs-kms-test.yml
index 8647c9268..c9eb02d00 100644
--- a/.github/workflows/rustfs-kms-test.yml
+++ b/.github/workflows/rustfs-kms-test.yml
@@ -52,6 +52,25 @@ jobs:
timeout-minutes: 420
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
+ - name: Checkout repository (for report parser)
+ uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+ with:
+ persist-credentials: false
+
+ - name: Initialize functional evidence
+ id: evidence
+ run: |
+ set -euo pipefail
+ umask 077
+ FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-kms-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+ mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+ {
+ printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ } >> "${GITHUB_ENV}"
+
# auto-testing is private: clone it with the dedicated PF token (not
# GITHUB_TOKEN) and retry transient GitHub/network failures.
- name: Checkout auto-testing scripts (with retry)
@@ -108,8 +127,6 @@ jobs:
- name: Run KMS suite
id: test
- env:
- LOG_FILE: /tmp/rustfs-kms.log
run: |
set -euo pipefail
chmod +x auto-testing/rustfs-kms-test.sh
@@ -139,10 +156,7 @@ jobs:
./auto-testing/rustfs-kms-test.sh "${ARGS[@]}"
- name: Generate report
- if: always()
- env:
- LOG_FILE: /tmp/rustfs-kms.log
- REPORT_FILE: /tmp/rustfs-kms-report.md
+ if: ${{ always() && steps.evidence.outcome == 'success' }}
run: |
set -euo pipefail
PACKAGE_URL='${{ inputs.package_url }}'
@@ -154,79 +168,43 @@ jobs:
else
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
fi
- CASE_TABLE="/tmp/rustfs-kms-cases.md"
- python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
- import re
- import sys
-
- log_file, out_file = sys.argv[1], sys.argv[2]
- ansi = re.compile(r'\x1b\[[0-9;]*m')
- start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
- done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
-
- rows = []
- index = {}
- try:
- with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
- for raw in fh:
- line = ansi.sub('', raw).strip()
- m = start_re.match(line)
- if m:
- case_id, name = m.group(1), m.group(2)
- if case_id not in index:
- index[case_id] = len(rows)
- rows.append([case_id, name, 'RUNNING'])
- continue
- m = done_re.match(line)
- if m:
- status, case_id = m.group(1), m.group(2)
- if case_id in index:
- rows[index[case_id]][2] = status
- else:
- rows.append([case_id, case_id, status])
- index[case_id] = len(rows) - 1
- except FileNotFoundError:
- rows = []
-
- counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
- for _, _, status in rows:
- counts[status] = counts.get(status, 0) + 1
-
- with open(out_file, 'w', encoding='utf-8') as out:
- out.write('## Case Summary\n\n')
- out.write(f"- Total: {len(rows)}\\n")
- out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
- out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
- out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
- out.write('\\n')
- out.write('| Case | Name | Status |\\n')
- out.write('| --- | --- | --- |\\n')
- for case_id, name, status in rows:
- out.write(f'| {case_id} | {name} | {status} |\\n')
- PY
+ CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+ CASE_RESULT=success
+ python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
+ RESULT=failure
+ if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+ RESULT=success
+ fi
{
echo "# RustFS KMS test report"
echo ""
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+ echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+ echo "- Workflow Commit: ${GITHUB_SHA}"
echo "- Trigger: ${{ github.event_name }}"
echo "- Package: ${PACKAGE_SOURCE}"
- echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+ echo "- Test Step Outcome: ${RESULT}"
+ echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
echo ""
- cat "${CASE_TABLE}" || true
- echo ""
- echo "## Log tail"
- echo '```text'
- tail -n 200 "${LOG_FILE}" || true
- echo '```'
+ if [ "${RESULT}" = "success" ]; then
+ cat "${CASE_TABLE}"
+ echo ""
+ echo "## Log tail"
+ echo '```text'
+ tail -n 200 "${LOG_FILE}"
+ echo '```'
+ else
+ echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+ fi
} | tee "${REPORT_FILE}"
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+ [ "${RESULT}" = "success" ]
- name: Upload functional report to dashboard
- if: always()
+ if: ${{ always() && steps.evidence.outcome == 'success' }}
continue-on-error: true
env:
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
- REPORT_FILE: /tmp/rustfs-kms-report.md
SUITE: kms
run: |
set -euo pipefail
@@ -236,28 +214,32 @@ jobs:
fi
DATE="$(date -u +%Y-%m-%d)"
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
- CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+ # Base64-encode the report into a temp file and feed it to jq via
+ # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+ # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+ B64_FILE="$(mktemp)"
+ python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
if [ -n "${SHA}" ]; then
- jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
- '{message:$msg, content:$content, sha:$sha}' \
+ jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+ '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
else
- jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
- '{message:$msg, content:$content}' \
+ jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+ '{message:$msg, content:($content|rtrimstr("\n"))}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
fi
+ rm -f "${B64_FILE}"
- name: File failure issue in rustfs/backlog
if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
continue-on-error: true
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
SUITE: 'kms'
SUITE_LABEL: 'KMS'
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
- REPORT_FILE: '/tmp/rustfs-kms-report.md'
- LOG_FILE: '/tmp/rustfs-kms.log'
run: |
set -euo pipefail
if [ -z "${GH_TOKEN:-}" ]; then
@@ -285,14 +267,16 @@ jobs:
echo ""
echo "- Suite: \`${SUITE}\`"
echo "- Run: ${RUN_URL}"
+ echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+ echo "- Workflow Commit: ${GITHUB_SHA}"
echo "- Trigger: ${GITHUB_EVENT_NAME}"
echo "- Date: $(date -u +%Y-%m-%d)"
echo ""
echo "## Report (errors and symptoms)"
echo ""
- if [ -s "${REPORT_FILE}" ]; then
+ if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
redact < "${REPORT_FILE}"
- elif [ -s "${LOG_FILE:-}" ]; then
+ elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
echo "(report file missing; log tail below)"
echo ""
tail -n 200 "${LOG_FILE}" | redact
@@ -308,14 +292,15 @@ jobs:
echo "filed backlog issue for suite ${SUITE}"
- name: Upload report and logs
- if: always()
+ if: ${{ always() && steps.evidence.outcome == 'success' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
- name: rustfs-kms-test-${{ github.run_id }}
+ name: rustfs-kms-test-${{ github.run_id }}-${{ github.run_attempt }}
path: |
- /tmp/rustfs-kms.log
- /tmp/rustfs-kms-report.md
- if-no-files-found: warn
+ ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+ ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+ ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
+ if-no-files-found: error
- name: Cleanup environment (after)
if: always()
diff --git a/.github/workflows/rustfs-performance-test.yml b/.github/workflows/rustfs-performance-test.yml
index ff3e2978d..d698b27e0 100644
--- a/.github/workflows/rustfs-performance-test.yml
+++ b/.github/workflows/rustfs-performance-test.yml
@@ -49,17 +49,16 @@ on:
type: boolean
default: true
repository_dispatch:
- # Chain entry: dispatched by rustfs-functional-chain.yml (runs on its own
- # pf-testing runner, in parallel with the shared-VM chain).
+ # Chain handoff: dispatched when the replication suite finishes.
types: [rustfs-chain-performance]
permissions:
contents: read
-# Dedicated pf-testing runner/environment: own concurrency group so perf runs
-# never block (or are blocked by) the pool-expansion / heal tests.
+# The default performance nodes overlap the other suites' remote VMs, even
+# though the runner differs. Hold the shared lock through cleanup as well.
concurrency:
- group: rustfs-performance-test
+ group: rustfs-shared-functional-tests
cancel-in-progress: false
defaults:
@@ -76,8 +75,6 @@ env:
# Package used by the nightly run (workflow_dispatch inputs are empty for
# workflow_run events), i.e. the latest nightly deb published by nightly-gnu.yml.
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
- # Fixed benchmark result directory so later steps can read summary.md
- RUSTFS_RESULT_DIR: /tmp/rustfs-perf-results
# Cross-repo token for uploading reports to rustfs/dashboard (set in repo settings)
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
@@ -89,6 +86,22 @@ jobs:
# Skipped when nightly failed.
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
+ - name: Initialize functional evidence
+ id: evidence
+ run: |
+ set -euo pipefail
+ umask 077
+ FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-performance-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+ mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+ {
+ printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ printf 'RUSTFS_RESULT_DIR=%s/results\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ printf 'VERSION_FILE=%s/version.txt\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ } >> "${GITHUB_ENV}"
+
# auto-testing is private: clone it with the dedicated PF token (not
# GITHUB_TOKEN) and retry transient GitHub/network failures.
- name: Checkout auto-testing scripts (with retry)
@@ -120,7 +133,7 @@ jobs:
if: ${{ inputs.cleanup_before != 'false' }}
run: |
chmod +x auto-testing/rustfs_performance_test.sh
- ./auto-testing/rustfs_performance_test.sh --step 1 -y
+ ./auto-testing/rustfs_performance_test.sh --step 1 -y --log-file "${LOG_FILE:-/dev/null}"
- name: Install RustFS package & start cluster (4x4)
run: |
@@ -130,7 +143,7 @@ jobs:
else
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
fi
- ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}"
+ ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
- name: Preflight checks
run: |
@@ -140,7 +153,7 @@ jobs:
else
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
fi
- ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}"
+ ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
- name: Run benchmark (GET/PUT/MIXED)
id: benchmark
@@ -153,17 +166,15 @@ jobs:
--step 5 -y \
--warp-duration "${{ inputs.warp_duration || '5m' }}" \
--warp-concurrency "${{ inputs.warp_concurrency || '64' }}" \
- --log-file /tmp/rustfs-perf-test.log
+ --log-file "${LOG_FILE}"
- name: Analyze results
if: ${{ steps.benchmark.conclusion == 'success' }}
run: |
- ./auto-testing/rustfs_performance_test.sh --step 6 -y
+ ./auto-testing/rustfs_performance_test.sh --step 6 -y --log-file "${LOG_FILE:-/dev/null}"
- name: Collect RustFS version info
if: ${{ steps.benchmark.conclusion == 'success' }}
- env:
- VERSION_FILE: /tmp/rustfs-version.txt
run: |
set -euo pipefail
read -r -a NODES <<< "${RUSTFS_NODES}"
@@ -183,7 +194,6 @@ jobs:
env:
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
RESULT_DIR: ${{ env.RUSTFS_RESULT_DIR }}
- VERSION_FILE: /tmp/rustfs-version.txt
run: |
set -euo pipefail
if [ -z "${GH_TOKEN:-}" ]; then
@@ -191,7 +201,7 @@ jobs:
exit 0
fi
SUMMARY="${RESULT_DIR}/summary.md"
- [ -f "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; }
+ [ -s "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; }
DATE="$(date -u +%Y-%m-%d)"
REPORT_PATH="reports/${DATE}.md"
{
@@ -199,6 +209,8 @@ jobs:
echo ""
echo "- **Date**: ${DATE}"
echo "- **Run**: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+ echo "- **Attempt**: ${GITHUB_RUN_ATTEMPT}"
+ echo "- **Workflow Commit**: ${GITHUB_SHA}"
echo "- **Trigger**: ${{ github.event_name }}"
echo "- **Package**: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
echo ""
@@ -208,8 +220,8 @@ jobs:
echo '```text'
cat "${VERSION_FILE}"
echo '```'
- } > /tmp/rustfs-perf-report.md
- CONTENT="$(python3 -c 'import base64; print(base64.b64encode(open("/tmp/rustfs-perf-report.md","rb").read()).decode())')"
+ } > "${REPORT_FILE}"
+ CONTENT="$(python3 -c 'import base64,sys; print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
if [ -n "${SHA}" ]; then
jq -n --arg msg "report: ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
@@ -228,11 +240,10 @@ jobs:
continue-on-error: true
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
SUITE: 'performance'
SUITE_LABEL: 'Performance'
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
- REPORT_FILE: '/tmp/rustfs-perf-report.md'
- LOG_FILE: '/tmp/rustfs-perf-test.log'
run: |
set -euo pipefail
if [ -z "${GH_TOKEN:-}" ]; then
@@ -260,14 +271,16 @@ jobs:
echo ""
echo "- Suite: \`${SUITE}\`"
echo "- Run: ${RUN_URL}"
+ echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+ echo "- Workflow Commit: ${GITHUB_SHA}"
echo "- Trigger: ${GITHUB_EVENT_NAME}"
echo "- Date: $(date -u +%Y-%m-%d)"
echo ""
echo "## Report (errors and symptoms)"
echo ""
- if [ -s "${REPORT_FILE}" ]; then
+ if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
redact < "${REPORT_FILE}"
- elif [ -s "${LOG_FILE:-}" ]; then
+ elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
echo "(report file missing; log tail below)"
echo ""
tail -n 200 "${LOG_FILE}" | redact
@@ -283,20 +296,26 @@ jobs:
echo "filed backlog issue for suite ${SUITE}"
- name: Upload test logs & results
- if: always()
+ if: ${{ always() && steps.evidence.outcome == 'success' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
- name: rustfs-perf-test-${{ github.run_id }}
+ name: rustfs-perf-test-${{ github.run_id }}-${{ github.run_attempt }}
path: |
- /tmp/rustfs-perf-test*.log
- /tmp/rustfs-perf-results/**
- /tmp/rustfs-version.txt
- if-no-files-found: warn
+ ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+ ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+ ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/version.txt
+ ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/master.log
+ ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/summary.md
+ ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/summary.tsv
+ ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/get_*.txt
+ ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/put_*.txt
+ ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/mixed_*.txt
+ if-no-files-found: error
- name: Reset test environment (after)
if: ${{ always() && inputs.cleanup_after != 'false' }}
run: |
- ./auto-testing/rustfs_performance_test.sh --step 7 -y
+ ./auto-testing/rustfs_performance_test.sh --step 7 -y --log-file "${LOG_FILE:-/dev/null}"
- name: Notify on failure
if: failure()
diff --git a/.github/workflows/rustfs-pool-expand-test.yml b/.github/workflows/rustfs-pool-expand-test.yml
index 3c9836b2d..c700df7c8 100644
--- a/.github/workflows/rustfs-pool-expand-test.yml
+++ b/.github/workflows/rustfs-pool-expand-test.yml
@@ -539,17 +539,22 @@ jobs:
fi
DATE="$(date -u +%Y-%m-%d)"
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
- CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+ # Base64-encode the report into a temp file and feed it to jq via
+ # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+ # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+ B64_FILE="$(mktemp)"
+ python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
if [ -n "${SHA}" ]; then
- jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
- '{message:$msg, content:$content, sha:$sha}' \
+ jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+ '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
else
- jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
- '{message:$msg, content:$content}' \
+ jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+ '{message:$msg, content:($content|rtrimstr("\n"))}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
fi
+ rm -f "${B64_FILE}"
- name: File failure issue in rustfs/backlog
if: ${{ always() && (failure() || steps.pool_test.outcome == 'failure' || steps.pool_test.outcome == 'cancelled') }}
diff --git a/.github/workflows/rustfs-replication-test.yml b/.github/workflows/rustfs-replication-test.yml
index 74c57b6d5..0faaf7809 100644
--- a/.github/workflows/rustfs-replication-test.yml
+++ b/.github/workflows/rustfs-replication-test.yml
@@ -34,8 +34,7 @@ on:
- site
default: all
repository_dispatch:
- # Chain handoff: dispatched when the security suite finishes. This is the
- # last link of the functional chain.
+ # Chain handoff: dispatched when the security suite finishes.
types: [rustfs-chain-replication]
permissions:
@@ -65,6 +64,25 @@ jobs:
timeout-minutes: 360
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
+ - name: Checkout repository (for report parser)
+ uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+ with:
+ persist-credentials: false
+
+ - name: Initialize functional evidence
+ id: evidence
+ run: |
+ set -euo pipefail
+ umask 077
+ FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-replication-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+ mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+ {
+ printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ } >> "${GITHUB_ENV}"
+
# auto-testing is private: clone it with the dedicated PF token (not
# GITHUB_TOKEN) and retry transient GitHub/network failures.
- name: Checkout auto-testing scripts (with retry)
@@ -113,8 +131,6 @@ jobs:
- name: Run replication suite
id: test
- env:
- LOG_FILE: /tmp/rustfs-replication.log
run: |
set -euo pipefail
chmod +x auto-testing/rustfs-replication-test.sh
@@ -137,10 +153,7 @@ jobs:
./auto-testing/rustfs-replication-test.sh "${ARGS[@]}"
- name: Generate report
- if: always()
- env:
- LOG_FILE: /tmp/rustfs-replication.log
- REPORT_FILE: /tmp/rustfs-replication-report.md
+ if: ${{ always() && steps.evidence.outcome == 'success' }}
run: |
set -euo pipefail
PACKAGE_URL='${{ inputs.package_url }}'
@@ -162,80 +175,44 @@ jobs:
RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
fi
fi
- CASE_TABLE="/tmp/rustfs-replication-cases.md"
- python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
- import re
- import sys
-
- log_file, out_file = sys.argv[1], sys.argv[2]
- ansi = re.compile(r'\x1b\[[0-9;]*m')
- start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
- done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
-
- rows = []
- index = {}
- try:
- with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
- for raw in fh:
- line = ansi.sub('', raw).strip()
- m = start_re.match(line)
- if m:
- case_id, name = m.group(1), m.group(2)
- if case_id not in index:
- index[case_id] = len(rows)
- rows.append([case_id, name, 'RUNNING'])
- continue
- m = done_re.match(line)
- if m:
- status, case_id = m.group(1), m.group(2)
- if case_id in index:
- rows[index[case_id]][2] = status
- else:
- rows.append([case_id, case_id, status])
- index[case_id] = len(rows) - 1
- except FileNotFoundError:
- rows = []
-
- counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
- for _, _, status in rows:
- counts[status] = counts.get(status, 0) + 1
-
- with open(out_file, 'w', encoding='utf-8') as out:
- out.write('## Case Summary\n\n')
- out.write(f"- Total: {len(rows)}\\n")
- out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
- out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
- out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
- out.write('\\n')
- out.write('| Case | Name | Status |\\n')
- out.write('| --- | --- | --- |\\n')
- for case_id, name, status in rows:
- out.write(f'| {case_id} | {name} | {status} |\\n')
- PY
+ CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+ CASE_RESULT=success
+ python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
+ RESULT=failure
+ if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+ RESULT=success
+ fi
{
echo "# RustFS replication test report"
echo ""
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+ echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+ echo "- Workflow Commit: ${GITHUB_SHA}"
echo "- Trigger: ${{ github.event_name }}"
echo "- Package: ${PACKAGE_SOURCE}"
echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
- echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+ echo "- Test Step Outcome: ${RESULT}"
+ echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
echo ""
- cat "${CASE_TABLE}" || true
- echo ""
- echo "## Log tail"
- echo '```text'
- tail -n 200 "${LOG_FILE}" || true
- echo '```'
+ if [ "${RESULT}" = "success" ]; then
+ cat "${CASE_TABLE}"
+ echo ""
+ echo "## Log tail"
+ echo '```text'
+ tail -n 200 "${LOG_FILE}"
+ echo '```'
+ else
+ echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+ fi
} | tee "${REPORT_FILE}"
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+ [ "${RESULT}" = "success" ]
- name: Upload functional report to dashboard
- if: always()
+ if: ${{ always() && steps.evidence.outcome == 'success' }}
continue-on-error: true
env:
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
- REPORT_FILE: /tmp/rustfs-replication-report.md
SUITE: replication
run: |
set -euo pipefail
@@ -245,28 +222,32 @@ jobs:
fi
DATE="$(date -u +%Y-%m-%d)"
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
- CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+ # Base64-encode the report into a temp file and feed it to jq via
+ # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+ # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+ B64_FILE="$(mktemp)"
+ python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
if [ -n "${SHA}" ]; then
- jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
- '{message:$msg, content:$content, sha:$sha}' \
+ jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+ '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
else
- jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
- '{message:$msg, content:$content}' \
+ jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+ '{message:$msg, content:($content|rtrimstr("\n"))}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
fi
+ rm -f "${B64_FILE}"
- name: File failure issue in rustfs/backlog
if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
continue-on-error: true
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
SUITE: 'replication'
SUITE_LABEL: 'Replication (bucket + site)'
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
- REPORT_FILE: '/tmp/rustfs-replication-report.md'
- LOG_FILE: '/tmp/rustfs-replication.log'
run: |
set -euo pipefail
if [ -z "${GH_TOKEN:-}" ]; then
@@ -294,14 +275,16 @@ jobs:
echo ""
echo "- Suite: \`${SUITE}\`"
echo "- Run: ${RUN_URL}"
+ echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+ echo "- Workflow Commit: ${GITHUB_SHA}"
echo "- Trigger: ${GITHUB_EVENT_NAME}"
echo "- Date: $(date -u +%Y-%m-%d)"
echo ""
echo "## Report (errors and symptoms)"
echo ""
- if [ -s "${REPORT_FILE}" ]; then
+ if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
redact < "${REPORT_FILE}"
- elif [ -s "${LOG_FILE:-}" ]; then
+ elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
echo "(report file missing; log tail below)"
echo ""
tail -n 200 "${LOG_FILE}" | redact
@@ -317,14 +300,15 @@ jobs:
echo "filed backlog issue for suite ${SUITE}"
- name: Upload report and logs
- if: always()
+ if: ${{ always() && steps.evidence.outcome == 'success' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
- name: rustfs-replication-${{ github.run_id }}
+ name: rustfs-replication-${{ github.run_id }}-${{ github.run_attempt }}
path: |
- /tmp/rustfs-replication.log
- /tmp/rustfs-replication-report.md
- if-no-files-found: warn
+ ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+ ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+ ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
+ if-no-files-found: error
- name: Cleanup environment (after)
if: always()
@@ -345,13 +329,50 @@ jobs:
'
done
- - name: Chain complete
- # Replication is the last link of the functional chain: nothing to
- # dispatch after it. This step just records that the chain finished.
+ - name: "Continue functional chain (next: Performance)"
if: ${{ always() && github.event_name == 'repository_dispatch' }}
+ env:
+ GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
run: |
- echo "Functional chain complete: replication (final suite) finished."
- echo "from_suite=security trigger=${{ github.event_name }} outcome=${{ steps.test.outcome }}"
+ set -uo pipefail
+ if [ -z "${GH_TOKEN:-}" ]; then
+ echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+ exit 1
+ fi
+ DISPATCHED=0
+ for attempt in 1 2 3; do
+ if gh api --method POST repos/rustfs/rustfs/dispatches \
+ -f event_type='rustfs-chain-performance' \
+ -F 'client_payload[from_suite]=replication'; then
+ echo "dispatched next suite Performance (attempt ${attempt})"
+ DISPATCHED=1
+ break
+ fi
+ echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
+ sleep "${attempt}0"
+ done
+ if [ "${DISPATCHED:-0}" -ne 1 ]; then
+ echo "ERROR: functional chain stalled: could not dispatch Performance after 3 attempts" >&2
+ TITLE="[functional][chain] stalled after replication (run ${GITHUB_RUN_ID})"
+ BODY_FILE="$(mktemp)"
+ trap 'rm -f "${BODY_FILE}"' EXIT
+ {
+ echo "The functional chain could not hand off from **replication** to **Performance** after 3 attempts."
+ echo ""
+ echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+ echo "- Expected next event: 'rustfs-chain-performance'"
+ echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+ echo "- Recovery: re-dispatch manually with"
+ FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
+ echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-performance'"
+ FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
+ } > "${BODY_FILE}"
+ gh issue create -R rustfs/backlog --title "${TITLE}" \
+ --body-file "${BODY_FILE}" --label functional-test \
+ || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
+ || echo "could not file the stall alert issue either; check the token" >&2
+ exit 1
+ fi
- name: Notify on failure
if: failure()
diff --git a/.github/workflows/rustfs-s3-compat-test.yml b/.github/workflows/rustfs-s3-compat-test.yml
index 80856194c..99828537b 100644
--- a/.github/workflows/rustfs-s3-compat-test.yml
+++ b/.github/workflows/rustfs-s3-compat-test.yml
@@ -40,6 +40,25 @@ jobs:
timeout-minutes: 360
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
+ - name: Checkout repository (for report parser)
+ uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+ with:
+ persist-credentials: false
+
+ - name: Initialize functional evidence
+ id: evidence
+ run: |
+ set -euo pipefail
+ umask 077
+ FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-s3-compat-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+ mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+ {
+ printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ } >> "${GITHUB_ENV}"
+
# auto-testing is private: clone it with the dedicated PF token (not
# GITHUB_TOKEN) and retry transient GitHub/network failures.
- name: Checkout auto-testing scripts (with retry)
@@ -87,8 +106,6 @@ jobs:
- name: Run S3 compatibility suite
id: test
- env:
- LOG_FILE: /tmp/rustfs-s3-compat.log
run: |
set -euo pipefail
chmod +x auto-testing/rustfs-s3-compat-test.sh
@@ -105,10 +122,7 @@ jobs:
./auto-testing/rustfs-s3-compat-test.sh "${ARGS[@]}"
- name: Generate report
- if: always()
- env:
- LOG_FILE: /tmp/rustfs-s3-compat.log
- REPORT_FILE: /tmp/rustfs-s3-compat-report.md
+ if: ${{ always() && steps.evidence.outcome == 'success' }}
run: |
set -euo pipefail
PACKAGE_URL='${{ inputs.package_url }}'
@@ -130,83 +144,44 @@ jobs:
RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
fi
fi
- CASE_TABLE="/tmp/rustfs-s3-compat-cases.md"
- python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
- import re
- import sys
-
- log_file, out_file = sys.argv[1], sys.argv[2]
- ansi = re.compile(r'\x1b\[[0-9;]*m')
- start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
- done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
-
- rows = []
- index = {}
- current = None
- try:
- with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
- for raw in fh:
- line = ansi.sub('', raw).strip()
- m = start_re.match(line)
- if m:
- case_id, name = m.group(1), m.group(2)
- current = case_id
- if case_id not in index:
- index[case_id] = len(rows)
- rows.append([case_id, name, 'RUNNING'])
- continue
- m = done_re.match(line)
- if m:
- status, case_id = m.group(1), m.group(2)
- if case_id in index:
- rows[index[case_id]][2] = status
- else:
- rows.append([case_id, case_id, status])
- index[case_id] = len(rows) - 1
- current = None
- except FileNotFoundError:
- rows = []
-
- counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
- for _, _, status in rows:
- counts[status] = counts.get(status, 0) + 1
-
- with open(out_file, 'w', encoding='utf-8') as out:
- out.write('## Case Summary\n\n')
- out.write(f"- Total: {len(rows)}\\n")
- out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
- out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
- out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
- out.write('\\n')
- out.write('| Case | Name | Status |\\n')
- out.write('| --- | --- | --- |\\n')
- for case_id, name, status in rows:
- out.write(f'| {case_id} | {name} | {status} |\\n')
- PY
+ CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+ CASE_RESULT=success
+ python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
+ RESULT=failure
+ if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+ RESULT=success
+ fi
{
echo "# RustFS S3 compatibility test report"
echo ""
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+ echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+ echo "- Workflow Commit: ${GITHUB_SHA}"
echo "- Trigger: ${{ github.event_name }}"
echo "- Package: ${PACKAGE_SOURCE}"
echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
- echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+ echo "- Test Step Outcome: ${RESULT}"
+ echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
echo ""
- cat "${CASE_TABLE}" || true
- echo ""
- echo "## Log tail"
- echo '```text'
- tail -n 200 "${LOG_FILE}" || true
- echo '```'
+ if [ "${RESULT}" = "success" ]; then
+ cat "${CASE_TABLE}"
+ echo ""
+ echo "## Log tail"
+ echo '```text'
+ tail -n 200 "${LOG_FILE}"
+ echo '```'
+ else
+ echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+ fi
} | tee "${REPORT_FILE}"
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+ [ "${RESULT}" = "success" ]
- name: Upload functional report to dashboard
- if: always()
+ if: ${{ always() && steps.evidence.outcome == 'success' }}
continue-on-error: true
env:
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
- REPORT_FILE: /tmp/rustfs-s3-compat-report.md
SUITE: s3
run: |
set -euo pipefail
@@ -216,28 +191,32 @@ jobs:
fi
DATE="$(date -u +%Y-%m-%d)"
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
- CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+ # Base64-encode the report into a temp file and feed it to jq via
+ # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+ # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+ B64_FILE="$(mktemp)"
+ python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
if [ -n "${SHA}" ]; then
- jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
- '{message:$msg, content:$content, sha:$sha}' \
+ jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+ '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
else
- jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
- '{message:$msg, content:$content}' \
+ jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+ '{message:$msg, content:($content|rtrimstr("\n"))}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
fi
+ rm -f "${B64_FILE}"
- name: File failure issue in rustfs/backlog
if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
continue-on-error: true
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
SUITE: 's3'
SUITE_LABEL: 'S3 compatibility'
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
- REPORT_FILE: '/tmp/rustfs-s3-compat-report.md'
- LOG_FILE: '/tmp/rustfs-s3-compat.log'
run: |
set -euo pipefail
if [ -z "${GH_TOKEN:-}" ]; then
@@ -265,14 +244,16 @@ jobs:
echo ""
echo "- Suite: \`${SUITE}\`"
echo "- Run: ${RUN_URL}"
+ echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+ echo "- Workflow Commit: ${GITHUB_SHA}"
echo "- Trigger: ${GITHUB_EVENT_NAME}"
echo "- Date: $(date -u +%Y-%m-%d)"
echo ""
echo "## Report (errors and symptoms)"
echo ""
- if [ -s "${REPORT_FILE}" ]; then
+ if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
redact < "${REPORT_FILE}"
- elif [ -s "${LOG_FILE:-}" ]; then
+ elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
echo "(report file missing; log tail below)"
echo ""
tail -n 200 "${LOG_FILE}" | redact
@@ -288,14 +269,15 @@ jobs:
echo "filed backlog issue for suite ${SUITE}"
- name: Upload report and logs
- if: always()
+ if: ${{ always() && steps.evidence.outcome == 'success' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
- name: rustfs-s3-compat-${{ github.run_id }}
+ name: rustfs-s3-compat-${{ github.run_id }}-${{ github.run_attempt }}
path: |
- /tmp/rustfs-s3-compat.log
- /tmp/rustfs-s3-compat-report.md
- if-no-files-found: warn
+ ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+ ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+ ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
+ if-no-files-found: error
- name: Cleanup environment (after)
if: always()
diff --git a/.github/workflows/rustfs-security-test.yml b/.github/workflows/rustfs-security-test.yml
index 16e10a80c..02fe7d662 100644
--- a/.github/workflows/rustfs-security-test.yml
+++ b/.github/workflows/rustfs-security-test.yml
@@ -77,10 +77,14 @@ jobs:
timeout-minutes: 360
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
+ # Checkout the repository into its own subdirectory. Checking out at
+ # the workspace root would wipe the auto-testing clone above (that is
+ # exactly how run 33934141181 lost rustfs-security-test.sh).
- name: Checkout repository (for the OIDC live gate script)
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
+ path: rustfs-repo
- name: Initialize security evidence
id: evidence
@@ -88,7 +92,7 @@ jobs:
set -euo pipefail
umask 077
SECURITY_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-security-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
- mkdir -- "${SECURITY_ARTIFACTS_DIR}"
+ mkdir -- "${SECURITY_ARTIFACTS_DIR}" "${SECURITY_ARTIFACTS_DIR}-scratch"
printf 'SECURITY_ARTIFACTS_DIR=%s\n' "${SECURITY_ARTIFACTS_DIR}" >> "${GITHUB_ENV}"
# auto-testing is private: clone it with the dedicated PF token (not
@@ -144,8 +148,8 @@ jobs:
continue-on-error: true
env:
REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
- TMPDIR: ${{ env.SECURITY_ARTIFACTS_DIR }}
- RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/scripts/test/oidc_keycloak_live.sh
+ TMPDIR: ${{ env.SECURITY_ARTIFACTS_DIR }}-scratch
+ RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/rustfs-repo/scripts/test/oidc_keycloak_live.sh
run: |
set -euo pipefail
chmod +x auto-testing/rustfs-security-test.sh
@@ -168,7 +172,7 @@ jobs:
else
ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
fi
- GITHUB_STEP_SUMMARY=/dev/null ./auto-testing/rustfs-security-test.sh "${ARGS[@]}"
+ GITHUB_STEP_SUMMARY=/dev/null ./auto-testing/rustfs-security-test.sh "${ARGS[@]}" 2>&1 | tee "${SECURITY_ARTIFACTS_DIR}/suite.log"
- name: Generate report
id: report
@@ -219,17 +223,22 @@ jobs:
fi
DATE="$(date -u +%Y-%m-%d)"
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
- CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+ # Base64-encode the report into a temp file and feed it to jq via
+ # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+ # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+ B64_FILE="$(mktemp)"
+ python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
if [ -n "${SHA}" ]; then
- jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
- '{message:$msg, content:$content, sha:$sha}' \
+ jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+ '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
else
- jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
- '{message:$msg, content:$content}' \
+ jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+ '{message:$msg, content:($content|rtrimstr("\n"))}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
fi
+ rm -f "${B64_FILE}"
- name: File failure issue in rustfs/backlog
if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
@@ -296,7 +305,10 @@ jobs:
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: rustfs-security-test-${{ github.run_id }}-${{ github.run_attempt }}
- path: ${{ env.SECURITY_ARTIFACTS_DIR }}/
+ path: |
+ ${{ env.SECURITY_ARTIFACTS_DIR }}/report.md
+ ${{ env.SECURITY_ARTIFACTS_DIR }}/suite.log
+ ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
if-no-files-found: error
retention-days: 3
diff --git a/.github/workflows/rustfs-storage-test.yml b/.github/workflows/rustfs-storage-test.yml
index 767f734dc..e16fc0058 100644
--- a/.github/workflows/rustfs-storage-test.yml
+++ b/.github/workflows/rustfs-storage-test.yml
@@ -49,6 +49,25 @@ jobs:
timeout-minutes: 360
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
+ - name: Checkout repository (for report parser)
+ uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+ with:
+ persist-credentials: false
+
+ - name: Initialize functional evidence
+ id: evidence
+ run: |
+ set -euo pipefail
+ umask 077
+ FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-storage-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+ mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+ {
+ printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ } >> "${GITHUB_ENV}"
+
# auto-testing is private: clone it with the dedicated PF token (not
# GITHUB_TOKEN) and retry transient GitHub/network failures.
- name: Checkout auto-testing scripts (with retry)
@@ -96,8 +115,6 @@ jobs:
- name: Run storage engine suite
id: test
- env:
- LOG_FILE: /tmp/rustfs-storage.log
run: |
set -euo pipefail
chmod +x auto-testing/rustfs-storage-test.sh
@@ -120,10 +137,7 @@ jobs:
./auto-testing/rustfs-storage-test.sh "${ARGS[@]}"
- name: Generate report
- if: always()
- env:
- LOG_FILE: /tmp/rustfs-storage.log
- REPORT_FILE: /tmp/rustfs-storage-report.md
+ if: ${{ always() && steps.evidence.outcome == 'success' }}
run: |
set -euo pipefail
PACKAGE_URL='${{ inputs.package_url }}'
@@ -145,83 +159,44 @@ jobs:
RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
fi
fi
- CASE_TABLE="/tmp/rustfs-storage-cases.md"
- python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
- import re
- import sys
-
- log_file, out_file = sys.argv[1], sys.argv[2]
- ansi = re.compile(r'\x1b\[[0-9;]*m')
- start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
- done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
-
- rows = []
- index = {}
- current = None
- try:
- with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
- for raw in fh:
- line = ansi.sub('', raw).strip()
- m = start_re.match(line)
- if m:
- case_id, name = m.group(1), m.group(2)
- current = case_id
- if case_id not in index:
- index[case_id] = len(rows)
- rows.append([case_id, name, 'RUNNING'])
- continue
- m = done_re.match(line)
- if m:
- status, case_id = m.group(1), m.group(2)
- if case_id in index:
- rows[index[case_id]][2] = status
- else:
- rows.append([case_id, case_id, status])
- index[case_id] = len(rows) - 1
- current = None
- except FileNotFoundError:
- rows = []
-
- counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
- for _, _, status in rows:
- counts[status] = counts.get(status, 0) + 1
-
- with open(out_file, 'w', encoding='utf-8') as out:
- out.write('## Case Summary\n\n')
- out.write(f"- Total: {len(rows)}\\n")
- out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
- out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
- out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
- out.write('\\n')
- out.write('| Case | Name | Status |\\n')
- out.write('| --- | --- | --- |\\n')
- for case_id, name, status in rows:
- out.write(f'| {case_id} | {name} | {status} |\\n')
- PY
+ CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+ CASE_RESULT=success
+ python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
+ RESULT=failure
+ if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+ RESULT=success
+ fi
{
echo "# RustFS storage engine test report"
echo ""
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+ echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+ echo "- Workflow Commit: ${GITHUB_SHA}"
echo "- Trigger: ${{ github.event_name }}"
echo "- Package: ${PACKAGE_SOURCE}"
echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
- echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+ echo "- Test Step Outcome: ${RESULT}"
+ echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
echo ""
- cat "${CASE_TABLE}" || true
- echo ""
- echo "## Log tail"
- echo '```text'
- tail -n 200 "${LOG_FILE}" || true
- echo '```'
+ if [ "${RESULT}" = "success" ]; then
+ cat "${CASE_TABLE}"
+ echo ""
+ echo "## Log tail"
+ echo '```text'
+ tail -n 200 "${LOG_FILE}"
+ echo '```'
+ else
+ echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+ fi
} | tee "${REPORT_FILE}"
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+ [ "${RESULT}" = "success" ]
- name: Upload functional report to dashboard
- if: always()
+ if: ${{ always() && steps.evidence.outcome == 'success' }}
continue-on-error: true
env:
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
- REPORT_FILE: /tmp/rustfs-storage-report.md
SUITE: storage
run: |
set -euo pipefail
@@ -231,28 +206,32 @@ jobs:
fi
DATE="$(date -u +%Y-%m-%d)"
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
- CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+ # Base64-encode the report into a temp file and feed it to jq via
+ # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+ # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+ B64_FILE="$(mktemp)"
+ python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
if [ -n "${SHA}" ]; then
- jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
- '{message:$msg, content:$content, sha:$sha}' \
+ jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+ '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
else
- jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
- '{message:$msg, content:$content}' \
+ jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+ '{message:$msg, content:($content|rtrimstr("\n"))}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
fi
+ rm -f "${B64_FILE}"
- name: File failure issue in rustfs/backlog
if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
continue-on-error: true
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
SUITE: 'storage'
SUITE_LABEL: 'Storage engine'
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
- REPORT_FILE: '/tmp/rustfs-storage-report.md'
- LOG_FILE: '/tmp/rustfs-storage.log'
run: |
set -euo pipefail
if [ -z "${GH_TOKEN:-}" ]; then
@@ -280,14 +259,16 @@ jobs:
echo ""
echo "- Suite: \`${SUITE}\`"
echo "- Run: ${RUN_URL}"
+ echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+ echo "- Workflow Commit: ${GITHUB_SHA}"
echo "- Trigger: ${GITHUB_EVENT_NAME}"
echo "- Date: $(date -u +%Y-%m-%d)"
echo ""
echo "## Report (errors and symptoms)"
echo ""
- if [ -s "${REPORT_FILE}" ]; then
+ if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
redact < "${REPORT_FILE}"
- elif [ -s "${LOG_FILE:-}" ]; then
+ elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
echo "(report file missing; log tail below)"
echo ""
tail -n 200 "${LOG_FILE}" | redact
@@ -303,14 +284,15 @@ jobs:
echo "filed backlog issue for suite ${SUITE}"
- name: Upload report and logs
- if: always()
+ if: ${{ always() && steps.evidence.outcome == 'success' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
- name: rustfs-storage-${{ github.run_id }}
+ name: rustfs-storage-${{ github.run_id }}-${{ github.run_attempt }}
path: |
- /tmp/rustfs-storage.log
- /tmp/rustfs-storage-report.md
- if-no-files-found: warn
+ ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+ ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+ ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
+ if-no-files-found: error
- name: Cleanup environment (after)
if: always()
diff --git a/.github/workflows/rustfs-tier-test.yml b/.github/workflows/rustfs-tier-test.yml
index 5d9a2c1d7..abe09f101 100644
--- a/.github/workflows/rustfs-tier-test.yml
+++ b/.github/workflows/rustfs-tier-test.yml
@@ -377,17 +377,22 @@ jobs:
fi
DATE="$(date -u +%Y-%m-%d)"
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
- CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+ # Base64-encode the report into a temp file and feed it to jq via
+ # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+ # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+ B64_FILE="$(mktemp)"
+ python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
if [ -n "${SHA}" ]; then
- jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
- '{message:$msg, content:$content, sha:$sha}' \
+ jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+ '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
else
- jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
- '{message:$msg, content:$content}' \
+ jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+ '{message:$msg, content:($content|rtrimstr("\n"))}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
fi
+ rm -f "${B64_FILE}"
- name: Verify required tier evidence
id: evidence_verify
diff --git a/.github/workflows/rustfs-upgrade-test.yml b/.github/workflows/rustfs-upgrade-test.yml
index 0b4c19af1..5a91793a0 100644
--- a/.github/workflows/rustfs-upgrade-test.yml
+++ b/.github/workflows/rustfs-upgrade-test.yml
@@ -82,6 +82,25 @@ jobs:
timeout-minutes: 420
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
+ - name: Checkout repository (for report parser)
+ uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+ with:
+ persist-credentials: false
+
+ - name: Initialize functional evidence
+ id: evidence
+ run: |
+ set -euo pipefail
+ umask 077
+ FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-upgrade-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+ mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
+ {
+ printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+ } >> "${GITHUB_ENV}"
+
# auto-testing is private: clone it with the dedicated PF token (not
# GITHUB_TOKEN) and retry transient GitHub/network failures.
- name: Checkout auto-testing scripts (with retry)
@@ -142,7 +161,6 @@ jobs:
- name: Run upgrade compatibility suite
id: test
env:
- LOG_FILE: /tmp/rustfs-upgrade.log
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
run: |
set -euo pipefail
@@ -200,10 +218,7 @@ jobs:
./auto-testing/rustfs-upgrade-test.sh "${ARGS[@]}"
- name: Generate report
- if: always()
- env:
- LOG_FILE: /tmp/rustfs-upgrade.log
- REPORT_FILE: /tmp/rustfs-upgrade-report.md
+ if: ${{ always() && steps.evidence.outcome == 'success' }}
run: |
set -euo pipefail
FROM_URL='${{ inputs.from_url }}'
@@ -224,103 +239,47 @@ jobs:
else
TO_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
fi
- CASE_TABLE="/tmp/rustfs-upgrade-cases.md"
- MATRIX_TABLE="/tmp/rustfs-upgrade-matrix.md"
- python3 - "${LOG_FILE}" "${CASE_TABLE}" "${MATRIX_TABLE}" <<'PY'
- import re
- import sys
-
- log_file, out_file, matrix_file = sys.argv[1], sys.argv[2], sys.argv[3]
- ansi = re.compile(r'\x1b\[[0-9;]*m')
- start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
- done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
- topo_re = re.compile(
- r'^\[UPG-TOPO\]\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+PASS=(\d+)\s+FAIL=(\d+)\s*$')
-
- rows = []
- index = {}
- topo_rows = []
- try:
- with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
- for raw in fh:
- line = ansi.sub('', raw).strip()
- m = topo_re.match(line)
- if m:
- topo_rows.append(m.groups())
- continue
- m = start_re.match(line)
- if m:
- case_id, name = m.group(1), m.group(2)
- if case_id not in index:
- index[case_id] = len(rows)
- rows.append([case_id, name, 'RUNNING'])
- continue
- m = done_re.match(line)
- if m:
- status, case_id = m.group(1), m.group(2)
- if case_id in index:
- rows[index[case_id]][2] = status
- else:
- rows.append([case_id, case_id, status])
- index[case_id] = len(rows) - 1
- except FileNotFoundError:
- rows = []
-
- counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
- for _, _, status in rows:
- counts[status] = counts.get(status, 0) + 1
-
- with open(out_file, 'w', encoding='utf-8') as out:
- out.write('## Case Summary\n\n')
- out.write(f"- Total: {len(rows)}\\n")
- out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
- out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
- out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
- out.write('\\n')
- out.write('| Case | Name | Status |\\n')
- out.write('| --- | --- | --- |\\n')
- for case_id, name, status in rows:
- out.write(f'| {case_id} | {name} | {status} |\\n')
-
- # Upgrade matrix: one row per topology/backend with the versions
- # captured on the nodes (rustfs --version) and the aggregated
- # result. The dashboard renders this table directly.
- with open(matrix_file, 'w', encoding='utf-8') as out:
- out.write('## Upgrade Matrix\n\n')
- out.write('| Topology | KMS Backend | From Version | To Version | Result |\n')
- out.write('| --- | --- | --- | --- | --- |\n')
- for topo, backend, old_v, new_v, npass, nfail in topo_rows:
- result = 'PASS' if nfail == '0' else 'FAIL'
- out.write(f'| {topo} | {backend} | {old_v} | {new_v} | {result} (PASS={npass} FAIL={nfail}) |\n')
- if not topo_rows:
- out.write('| - | - | - | - | NOT RUN (suite failed before upgrade) |\n')
- PY
+ CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+ MATRIX_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/matrix.md"
+ CASE_RESULT=success
+ python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" "${MATRIX_TABLE}" || CASE_RESULT=failure
+ RESULT=failure
+ if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
+ RESULT=success
+ fi
{
echo "# RustFS upgrade compatibility report"
echo ""
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+ echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+ echo "- Workflow Commit: ${GITHUB_SHA}"
echo "- Trigger: ${{ github.event_name }}"
echo "- From: ${FROM_SOURCE}"
echo "- To: ${TO_SOURCE}"
- echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+ echo "- Test Step Outcome: ${RESULT}"
+ echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
echo ""
- cat "${MATRIX_TABLE}" || true
- echo ""
- cat "${CASE_TABLE}" || true
- echo ""
- echo "## Log tail"
- echo '```text'
- tail -n 200 "${LOG_FILE}" || true
- echo '```'
+ if [ "${RESULT}" = "success" ]; then
+ cat "${MATRIX_TABLE}"
+ echo ""
+ cat "${CASE_TABLE}"
+ echo ""
+ echo "## Log tail"
+ echo '```text'
+ tail -n 200 "${LOG_FILE}"
+ echo '```'
+ else
+ echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
+ fi
} | tee "${REPORT_FILE}"
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+ [ "${RESULT}" = "success" ]
- name: Upload functional report to dashboard
- if: always()
+ if: ${{ always() && steps.evidence.outcome == 'success' }}
continue-on-error: true
env:
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
- REPORT_FILE: /tmp/rustfs-upgrade-report.md
SUITE: upgrade
run: |
set -euo pipefail
@@ -330,28 +289,32 @@ jobs:
fi
DATE="$(date -u +%Y-%m-%d)"
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
- CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+ # Base64-encode the report into a temp file and feed it to jq via
+ # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
+ # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
+ B64_FILE="$(mktemp)"
+ python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
if [ -n "${SHA}" ]; then
- jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
- '{message:$msg, content:$content, sha:$sha}' \
+ jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
+ '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
else
- jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
- '{message:$msg, content:$content}' \
+ jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
+ '{message:$msg, content:($content|rtrimstr("\n"))}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
fi
+ rm -f "${B64_FILE}"
- name: File failure issue in rustfs/backlog
if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
continue-on-error: true
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
SUITE: 'upgrade'
SUITE_LABEL: 'Upgrade compatibility'
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
- REPORT_FILE: '/tmp/rustfs-upgrade-report.md'
- LOG_FILE: '/tmp/rustfs-upgrade.log'
run: |
set -euo pipefail
if [ -z "${GH_TOKEN:-}" ]; then
@@ -379,14 +342,16 @@ jobs:
echo ""
echo "- Suite: \`${SUITE}\`"
echo "- Run: ${RUN_URL}"
+ echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
+ echo "- Workflow Commit: ${GITHUB_SHA}"
echo "- Trigger: ${GITHUB_EVENT_NAME}"
echo "- Date: $(date -u +%Y-%m-%d)"
echo ""
echo "## Report (errors and symptoms)"
echo ""
- if [ -s "${REPORT_FILE}" ]; then
+ if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
redact < "${REPORT_FILE}"
- elif [ -s "${LOG_FILE:-}" ]; then
+ elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
echo "(report file missing; log tail below)"
echo ""
tail -n 200 "${LOG_FILE}" | redact
@@ -402,14 +367,16 @@ jobs:
echo "filed backlog issue for suite ${SUITE}"
- name: Upload report and logs
- if: always()
+ if: ${{ always() && steps.evidence.outcome == 'success' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
- name: rustfs-upgrade-test-${{ github.run_id }}
+ name: rustfs-upgrade-test-${{ github.run_id }}-${{ github.run_attempt }}
path: |
- /tmp/rustfs-upgrade-report.md
- /tmp/rustfs-upgrade.*/*
- if-no-files-found: ignore
+ ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+ ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
+ ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
+ ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/matrix.md
+ if-no-files-found: error
retention-days: 3
- name: Cleanup environment (after)
diff --git a/rustfs/src/app/bucket_list_through.rs b/rustfs/src/app/bucket_list_through.rs
index b8aa0d38e..a41b994f9 100644
--- a/rustfs/src/app/bucket_list_through.rs
+++ b/rustfs/src/app/bucket_list_through.rs
@@ -712,6 +712,8 @@ mod tests {
session_token: None,
}),
tls: TlsConfig::default(),
+ azure: None,
+ gcs: None,
},
filter: FilterConfig {
prefix: filter_prefix.map(str::to_string),
diff --git a/scripts/functional_case_report.py b/scripts/functional_case_report.py
new file mode 100644
index 000000000..7036eaad5
--- /dev/null
+++ b/scripts/functional_case_report.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+"""Preserve every functional case execution and its suite context in reports."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+import re
+
+
+def generate_report(log_file: Path, case_file: Path, matrix_file: Path | None = None) -> bool:
+ ansi = re.compile(r"\x1b\[[0-9;]*m")
+ start_re = re.compile(r"^---\s+([A-Z][A-Z0-9]*-[0-9]+)\s+(.+?)\s+---$")
+ done_re = re.compile(r"^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z][A-Z0-9]*-[0-9]+)\b")
+ context_re = re.compile(r"^(?:\[INFO\]\s+)?==\s+((?:topology|suite):.+?)\s+==$")
+ topo_re = re.compile(r"^\[UPG-TOPO\]\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+PASS=(\d+)\s+FAIL=(\d+)\s*$")
+ rows = []
+ pending = {}
+ topo_rows = []
+ context = "context not recorded"
+ complete = True
+ try:
+ with log_file.open(encoding="utf-8", errors="replace") as log:
+ for raw in log:
+ line = ansi.sub("", raw).strip()
+ if match := context_re.match(line):
+ context = match[1]
+ pending.clear()
+ elif match := topo_re.match(line):
+ topo_rows.append(match.groups())
+ elif match := start_re.match(line):
+ case_id, name = match.groups()
+ pending[case_id] = len(rows)
+ rows.append([case_id, f"{name} ({context})", "RUNNING"])
+ elif match := done_re.match(line):
+ status, case_id = match.groups()
+ index = pending.pop(case_id, None)
+ if index is None:
+ complete = False
+ rows.append([case_id, f"{case_id} ({context}; start not recorded)", status])
+ else:
+ rows[index][2] = status
+ except FileNotFoundError:
+ pass
+
+ counts = {status: sum(row[2] == status for row in rows) for status in ("PASS", "FAIL", "UNSUPPORTED", "RUNNING")}
+ with case_file.open("w", encoding="utf-8") as out:
+ out.write(f"## Case Summary\n\n- Total: {len(rows)}\n")
+ for status, count in counts.items():
+ out.write(f"- {status}: {count}\n")
+ out.write("\n| Case | Name | Status |\n| --- | --- | --- |\n")
+ for row in rows:
+ out.write("| " + " | ".join(value.replace("|", "|") for value in row) + " |\n")
+ if not rows:
+ out.write("\nNo case execution was recorded; the log is missing, empty, or stopped before the cases.\n")
+
+ valid = complete and bool(rows) and not counts["FAIL"] and not counts["RUNNING"]
+ if matrix_file is not None:
+ with matrix_file.open("w", encoding="utf-8") as out:
+ out.write("## Upgrade Matrix\n\n| Topology | KMS Backend | From Version | To Version | Result |\n")
+ out.write("| --- | --- | --- | --- | --- |\n")
+ for topo, backend, old_v, new_v, npass, nfail in topo_rows:
+ result = "PASS" if nfail == "0" else "FAIL"
+ out.write(f"| {topo} | {backend} | {old_v} | {new_v} | {result} (PASS={npass} FAIL={nfail}) |\n")
+ if not topo_rows:
+ out.write("| - | - | - | - | NOT RUN (suite failed before upgrade) |\n")
+ valid = valid and bool(topo_rows) and all(row[-1] == "0" for row in topo_rows)
+ return valid
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("log_file", type=Path)
+ parser.add_argument("case_file", type=Path)
+ parser.add_argument("matrix_file", type=Path, nargs="?")
+ args = parser.parse_args()
+ raise SystemExit(0 if generate_report(args.log_file, args.case_file, args.matrix_file) else 1)
diff --git a/scripts/test/oidc_keycloak_live.sh b/scripts/test/oidc_keycloak_live.sh
index c464f6dcb..73b8eb6da 100755
--- a/scripts/test/oidc_keycloak_live.sh
+++ b/scripts/test/oidc_keycloak_live.sh
@@ -187,7 +187,7 @@ values = {}
for element in root.iter():
values[element.tag.rsplit("}", 1)[-1]] = element.text or ""
for field in ("AccessKeyId", "SecretAccessKey", "SessionToken", "Expiration", "SubjectFromWebIdentityToken"):
- assert values.get(field), values
+ assert values.get(field), f"missing required STS field: {field}"
print("\t".join(values[field] for field in ("AccessKeyId", "SecretAccessKey", "SessionToken")))
PY
)
@@ -218,7 +218,6 @@ TAMPERED_STATUS="$(curl --noproxy '*' -sS \
--data-urlencode DurationSeconds=900 \
--data-urlencode "WebIdentityToken=${TAMPERED_TOKEN}")"
[[ "${TAMPERED_STATUS}" == 403 ]] || {
- cat "${WORK_DIR}/sts-tampered.xml" >&2
echo "expected tampered token to return HTTP 403, got ${TAMPERED_STATUS}" >&2
exit 1
}
@@ -235,7 +234,6 @@ BAD_STATUS="$(curl --noproxy '*' -sS \
--data-urlencode DurationSeconds=900 \
--data-urlencode "WebIdentityToken=${BAD_TOKEN}")"
[[ "${BAD_STATUS}" == 403 ]] || {
- cat "${WORK_DIR}/sts-bad.xml" >&2
echo "expected wrong-audience token to return HTTP 403, got ${BAD_STATUS}" >&2
exit 1
}
diff --git a/scripts/test_nightly_candidate.py b/scripts/test_nightly_candidate.py
new file mode 100644
index 000000000..3e5a50401
--- /dev/null
+++ b/scripts/test_nightly_candidate.py
@@ -0,0 +1,208 @@
+#!/usr/bin/env python3
+"""Exercise the nightly publication step without AWS, network or package builds."""
+
+import hashlib
+import json
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+
+from check_test_wiring import yaml_block
+
+
+ROOT = Path(__file__).resolve().parents[1]
+WORKFLOW = ROOT / ".github/workflows/nightly-gnu.yml"
+
+
+class NightlyCandidateTests(unittest.TestCase):
+ def setUp(self):
+ self.temp = tempfile.TemporaryDirectory()
+ self.addCleanup(self.temp.cleanup)
+ self.root = Path(self.temp.name)
+ self.package = self.root / "rustfs-nightly-2026-09-06.deb"
+ self.package.write_bytes(b"built package bytes\x00\xff")
+ for command in (["git", "init", "-q"], ["git", "add", self.package.name],
+ ["git", "-c", "user.name=Test", "-c", "user.email=test@example.invalid", "commit", "-qm", "fixture"]):
+ subprocess.run(command, cwd=self.root, check=True, capture_output=True)
+ self.sha = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=self.root, text=True).strip()
+ self.digest = hashlib.sha256(self.package.read_bytes()).hexdigest()
+ self.output = self.root / "github-output"
+ self.store = self.root / "store"
+ self.shims = self.root / "fake-tools.sh"
+ self.shims.write_text(r'''aws() {
+ printf '%s\n' "$*" >> "$FAKE_AWS_LOG"
+ if [[ "$1" == --version ]]; then printf 'aws-cli/1.44.79 fixture\n'; return; fi
+ if [[ "$*" == *--generate-cli-skeleton* ]]; then
+ if [[ "$FAKE_MODE" == broken-install || "$FAKE_MODE" =~ ^(old-cli|bootstrap-failure|install-failure)$ && ! -e "$FAKE_INSTALLED" ]]; then
+ printf '{}\n'
+ else
+ printf '{"IfNoneMatch":""}\n'
+ fi
+ return
+ fi
+ if [[ "$1 $2" == 's3api put-object' ]]; then
+ [[ "$FAKE_MODE" != upload-failure ]] || return 42
+ shift 2
+ local key="" body="" condition=""
+ while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --key) key="$2";;
+ --body) body="$2";;
+ --if-none-match) condition="$2";;
+ esac
+ shift 2
+ done
+ [[ -z "$condition" || "$condition" == '*' ]] || return 43
+ if [[ "$condition" == '*' && -e "$FAKE_STORE/$key" ]]; then return 44; fi
+ mkdir -p "$(dirname "$FAKE_STORE/$key")"
+ cp "$body" "$FAKE_STORE/$key"
+ elif [[ "$1 $2" == 's3 cp' ]]; then
+ [[ "$FAKE_MODE" != alias-failure ]] || return 45
+ local destination="${4#s3://test-bucket/}"
+ [[ "$destination" != */ ]] || destination+="$(basename "$3")"
+ mkdir -p "$(dirname "$FAKE_STORE/$destination")"
+ cp "$3" "$FAKE_STORE/$destination"
+ else
+ return 46
+ fi
+}
+curl() {
+ local url="${!#}"
+ printf '%s\n' "$url" >> "$FAKE_CURL_LOG"
+ [[ "$url" == https://dl.rustfs.com/artifacts/rustfs/packages/nightly/runs/* ]] || return 22
+ [[ "$FAKE_MODE" != missing-public-url ]] || return 22
+ if [[ "$FAKE_MODE" == wrong-public-bytes ]]; then printf 'different package'; return; fi
+ cat "$FAKE_STORE/${url#https://dl.rustfs.com/}" || return 22
+ [[ "$FAKE_MODE" != incomplete-download ]] || return 47
+}
+sudo() {
+ [[ "$*" == 'apt-get update' || "$*" == 'apt-get install -y -qq python3-venv' ]] || return 49
+ [[ "$FAKE_MODE" != bootstrap-failure ]] || return 48
+}
+python3() {
+ [[ "$1 $2" == '-m venv' ]] || return 50
+ mkdir -p "$3/bin"
+ cat > "$3/bin/python" <<'SH'
+#!/usr/bin/env bash
+[[ "$*" == '-m pip install --disable-pip-version-check awscli==1.44.79' ]] || exit 51
+[[ "$FAKE_MODE" != install-failure ]] || exit 52
+: > "$FAKE_INSTALLED"
+SH
+ printf '#!/usr/bin/env bash\naws "$@"\n' > "$3/bin/aws"
+ chmod +x "$3/bin/python" "$3/bin/aws"
+}
+''')
+ self.env = dict(os.environ, BASH_ENV=str(self.shims), DEB_FILE=self.package.name,
+ R2_ACCESS_KEY_ID="fake-access", R2_SECRET_ACCESS_KEY="fake-secret", R2_ENDPOINT="https://r2.example.invalid", R2_BUCKET="test-bucket",
+ RUNNER_TEMP=str(self.root), GITHUB_SHA=self.sha, GITHUB_RUN_ID="12345", GITHUB_RUN_ATTEMPT="1", GITHUB_OUTPUT=str(self.output),
+ FAKE_STORE=str(self.store), FAKE_AWS_LOG=str(self.root / "aws.log"), FAKE_CURL_LOG=str(self.root / "curl.log"), FAKE_INSTALLED=str(self.root / "installed"), FAKE_MODE="success")
+ source = WORKFLOW.read_text()
+ job = yaml_block(source.splitlines(), "build", 2)
+ starts = [i for i, line in enumerate(job) if line.startswith(" - name: ")]
+ self.steps = {
+ job[start].split(": ", 1)[1]: job[start:end]
+ for start, end in zip(starts, starts[1:] + [len(job)])
+ }
+ self.publish = self.steps["Upload DEB to Cloudflare R2"]
+ start = self.publish.index(" run: |") + 1
+ self.shell = "\n".join(line[10:] for line in self.publish[start:] if not line.strip() or line.startswith(" "))
+
+ def run_publish(self, **overrides):
+ self.output.unlink(missing_ok=True)
+ return subprocess.run(["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.shell],
+ cwd=self.root, env=dict(self.env, **overrides), capture_output=True, text=True)
+
+ def manifest(self):
+ output = self.output.read_text().strip()
+ self.assertTrue(output.startswith("candidate_file="), output)
+ return json.loads(Path(output.split("=", 1)[1]).read_text())
+
+ def test_success_binds_actual_package_checkout_and_attempt(self):
+ result = self.run_publish()
+ self.assertEqual(result.returncode, 0, result.stderr)
+ manifest = self.manifest()
+ self.assertEqual(manifest, {"schema": 1, "source_sha": self.sha, "build_run_id": 12345, "build_run_attempt": 1,
+ "package_sha256": self.digest, "package_url": f"https://dl.rustfs.com/artifacts/rustfs/packages/nightly/runs/12345/1/{self.digest}/rustfs.deb"})
+ for path in (f"runs/12345/1/{self.digest}/rustfs.deb", self.package.name, "rustfs-nightly-latest.deb"):
+ self.assertEqual((self.store / "artifacts/rustfs/packages/nightly" / path).read_bytes(), self.package.read_bytes())
+ self.assertEqual((self.root / "curl.log").read_text().strip(), manifest["package_url"])
+
+ def test_missing_credentials_remain_artifact_only(self):
+ for key in ("R2_ACCESS_KEY_ID", "R2_SECRET_ACCESS_KEY", "R2_ENDPOINT", "R2_BUCKET"):
+ with self.subTest(missing=key):
+ result = self.run_publish(**{key: ""})
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertFalse(self.output.exists())
+ self.assertFalse((self.root / "aws.log").exists())
+ self.assertEqual(list(self.root.glob("nightly-candidate-*.json")), [])
+
+ def test_publication_failures_never_emit_a_candidate(self):
+ for index, mode in enumerate(("upload-failure", "missing-public-url", "wrong-public-bytes", "incomplete-download", "alias-failure")):
+ with self.subTest(mode=mode):
+ result = self.run_publish(FAKE_MODE=mode, GITHUB_RUN_ID=str(20000 + index))
+ self.assertNotEqual(result.returncode, 0, result.stdout)
+ self.assertFalse(self.output.exists())
+ self.assertEqual(list(self.root.glob("nightly-candidate-*.json")), [])
+
+ def test_old_cli_is_upgraded_in_an_isolated_temporary_environment(self):
+ result = self.run_publish(FAKE_MODE="old-cli")
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertTrue((self.root / "installed").exists())
+ self.assertEqual(self.manifest()["package_sha256"], self.digest)
+ self.assertEqual(list(self.root.glob("nightly-awscli.*")), [])
+
+ def test_failed_cli_bootstrap_cannot_publish(self):
+ for mode in ("bootstrap-failure", "install-failure", "broken-install"):
+ with self.subTest(mode=mode):
+ (self.root / "installed").unlink(missing_ok=True)
+ result = self.run_publish(FAKE_MODE=mode)
+ self.assertNotEqual(result.returncode, 0)
+ self.assertFalse(self.output.exists())
+ self.assertFalse(self.store.exists())
+ self.assertEqual(list(self.root.glob("nightly-awscli.*")), [])
+
+ def test_checkout_sha_mismatch_fails_before_upload(self):
+ result = self.run_publish(GITHUB_SHA="f" * 40)
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("Checkout SHA", result.stderr)
+ self.assertFalse(self.output.exists())
+ self.assertFalse((self.root / "aws.log").exists())
+
+ def test_same_date_builds_and_reruns_keep_distinct_candidates(self):
+ urls = []
+ for run, attempt in (("12345", "1"), ("54321", "1"), ("12345", "2")):
+ result = self.run_publish(GITHUB_RUN_ID=run, GITHUB_RUN_ATTEMPT=attempt)
+ self.assertEqual(result.returncode, 0, result.stderr)
+ urls.append(self.manifest()["package_url"])
+ self.assertEqual(len(set(urls)), 3)
+ self.assertEqual(len(list(self.root.glob("nightly-candidate-*.json"))), 3)
+
+ def test_duplicate_key_is_not_overwritten_or_recertified(self):
+ result = self.run_publish()
+ self.assertEqual(result.returncode, 0, result.stderr)
+ key = self.manifest()["package_url"].removeprefix("https://dl.rustfs.com/")
+ stored = self.store / key
+ stored.write_bytes(b"preexisting conflicting object")
+ (self.root / "nightly-candidate-12345-1.json").unlink()
+ result = self.run_publish()
+ self.assertNotEqual(result.returncode, 0)
+ self.assertEqual(stored.read_bytes(), b"preexisting conflicting object")
+ self.assertFalse(self.output.exists())
+ self.assertEqual(list(self.root.glob("nightly-candidate-*.json")), [])
+
+ def test_manifest_upload_requires_publication_output(self):
+ upload = self.steps["Upload nightly candidate manifest"]
+ self.assertIn(" id: publish", self.publish)
+ self.assertIn(" DEB_FILE: ${{ steps.deb.outputs.deb_file }}", self.publish)
+ self.assertIn(" if: ${{ steps.publish.outputs.candidate_file != '' }}", upload)
+ self.assertIn(" name: nightly-candidate-${{ github.run_id }}-${{ github.run_attempt }}", upload)
+ self.assertIn(" path: ${{ steps.publish.outputs.candidate_file }}", upload)
+ self.assertIn(" if-no-files-found: error", upload)
+ self.assertNotIn(" continue-on-error: true", self.publish)
+ self.assertNotIn(" overwrite: true", upload)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/scripts/test_security_workflow.py b/scripts/test_security_workflow.py
index ea2d75487..b96b3e3f5 100644
--- a/scripts/test_security_workflow.py
+++ b/scripts/test_security_workflow.py
@@ -1,16 +1,20 @@
#!/usr/bin/env python3
-"""Exercise functional workflow failures and security evidence without remote VMs."""
+"""Exercise functional failures, chain dispatch, and security evidence without remote VMs."""
from __future__ import annotations
+import glob
+import json
import os
import re
import subprocess
+import sys
import tempfile
import unittest
from pathlib import Path
from check_test_wiring import yaml_block
+from functional_case_report import generate_report
ROOT = Path(__file__).resolve().parents[1]
@@ -38,7 +42,46 @@ def shell_body(lines: list[str]) -> str:
return "\n".join(shell_lines)
-class SecurityWorkflowTests(unittest.TestCase):
+class WorkflowSteps:
+ def uploaded_files(self) -> set[Path]:
+ upload = next(lines for lines in self.steps.values() if any("uses: actions/upload-artifact@" in line for line in lines))
+ start = upload.index(" path: |") + 1
+ paths = []
+ for line in upload[start:]:
+ if not line.startswith(" "):
+ break
+ paths.extend(Path(path) for path in glob.glob(self.render(line.strip())))
+ return {file for path in paths for file in (path.rglob("*") if path.is_dir() else [path]) if file.is_file()}
+
+ def render(self, value: str) -> str:
+ return re.sub(r"\$\{\{\s*(.*?)\s*\}\}", lambda match: self.context[match[1]], value)
+
+ def step_env(self, lines: list[str], indent: int = 8) -> dict[str, str]:
+ result = {}
+ for line in yaml_block(lines, "env", indent) or []:
+ if line.strip() and not line.lstrip().startswith("#"):
+ key, value = line.strip().split(": ", 1)
+ result[key] = self.render(value.strip("'\""))
+ return result
+
+ def run_step(self, name: str) -> subprocess.CompletedProcess[str]:
+ lines = self.steps[name]
+ result = subprocess.run(
+ ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.render(shell_body(lines))],
+ cwd=self.directory, env={**self.env, **self.step_env(lines)}, capture_output=True, text=True,
+ )
+ for line in lines:
+ if line.startswith(" id: "):
+ self.context[f"steps.{line.split(': ', 1)[1]}.outcome"] = "failure" if result.returncode else "success"
+ if Path(self.env["GITHUB_ENV"]).exists():
+ for line in Path(self.env["GITHUB_ENV"]).read_text().splitlines():
+ key, value = line.split("=", 1)
+ self.env[key] = value
+ self.context[f"env.{key}"] = value
+ return result
+
+
+class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
def setUp(self) -> None:
self.source = WORKFLOW.read_text()
self.job = yaml_block(self.source.splitlines(), "security-test", 2)
@@ -78,6 +121,7 @@ class SecurityWorkflowTests(unittest.TestCase):
'#!/usr/bin/env bash\nset -euo pipefail\n'
'log_dir=$(mktemp -d "$TMPDIR/rustfs-security.XXXXXX")\n'
'echo "CURRENT SUITE LOG" > "$log_dir/suite.log"\n'
+ 'echo "CURRENT SUITE STDOUT"; echo "CURRENT SUITE STDERR" >&2\n'
'case "$FAKE_REPORT" in\n'
f' present) printf "%s\\n" "CURRENT SUITE DIAGNOSTIC" "{CASE_ROW}" > "$REPORT_FILE" ;;\n'
' empty) : > "$REPORT_FILE" ;;\n'
@@ -86,33 +130,6 @@ class SecurityWorkflowTests(unittest.TestCase):
'exit "$FAKE_EXIT"\n'
)
- def render(self, value: str) -> str:
- return re.sub(r"\$\{\{\s*(.*?)\s*\}\}", lambda match: self.context[match[1]], value)
-
- def step_env(self, lines: list[str], indent: int = 8) -> dict[str, str]:
- result = {}
- for line in yaml_block(lines, "env", indent) or []:
- if line.strip() and not line.lstrip().startswith("#"):
- key, value = line.strip().split(": ", 1)
- result[key] = self.render(value.strip("'\""))
- return result
-
- def run_step(self, name: str) -> subprocess.CompletedProcess[str]:
- lines = self.steps[name]
- result = subprocess.run(
- ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.render(shell_body(lines))],
- cwd=self.directory, env={**self.env, **self.step_env(lines)}, capture_output=True, text=True,
- )
- for line in lines:
- if line.startswith(" id: "):
- self.context[f"steps.{line.split(': ', 1)[1]}.outcome"] = "failure" if result.returncode else "success"
- if Path(self.env["GITHUB_ENV"]).exists():
- for line in Path(self.env["GITHUB_ENV"]).read_text().splitlines():
- key, value = line.split("=", 1)
- self.env[key] = value
- self.context[f"env.{key}"] = value
- return result
-
def test_workflow_wiring(self) -> None:
names = list(self.steps)
self.assertLess(names.index("Checkout repository (for the OIDC live gate script)"), names.index("Checkout auto-testing scripts (with retry)"))
@@ -128,7 +145,7 @@ class SecurityWorkflowTests(unittest.TestCase):
for name in ("Upload functional report to dashboard", "Upload report and logs"):
self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", self.steps[name])
artifact_settings = yaml_block(self.steps["Upload report and logs"], "with", 8)
- self.assertIn(" path: ${{ env.SECURITY_ARTIFACTS_DIR }}/", artifact_settings)
+ self.assertIn(" path: |", artifact_settings)
self.assertIn(" if-no-files-found: error", artifact_settings)
def test_suite_report_and_result_matrix(self) -> None:
@@ -147,7 +164,7 @@ class SecurityWorkflowTests(unittest.TestCase):
if outcome != "skipped" or mode == "present":
suite = self.run_step("Run security suite")
self.assertEqual(suite.returncode, exit_code, suite.stderr)
- logs = list(self.artifacts.glob("rustfs-security.*/suite.log"))
+ logs = list(Path(str(self.artifacts) + "-scratch").glob("rustfs-security.*/suite.log"))
self.assertEqual(len(logs), 1)
self.assertEqual(logs[0].read_text(), "CURRENT SUITE LOG\n")
self.context["steps.test.outcome"] = outcome
@@ -169,37 +186,195 @@ class SecurityWorkflowTests(unittest.TestCase):
summary = Path(self.env["GITHUB_STEP_SUMMARY"]).read_text()
self.assertEqual(summary, contents)
self.assertNotIn("UNWRAPPED SUITE SUMMARY", summary)
+ expected = {self.artifacts / "report.md"}
+ if outcome != "skipped" or mode == "present":
+ expected.add(self.artifacts / "suite.log")
+ self.assertEqual((self.artifacts / "suite.log").read_text(), "CURRENT SUITE STDOUT\nCURRENT SUITE STDERR\n")
+ if mode in ("present", "empty"):
+ expected.add(self.artifacts / "suite-report.md")
+ (self.artifacts / "unexpected-token.json").write_text("FAKE-SECRET-CANARY")
+ scratch = Path(str(self.artifacts) + "-scratch")
+ (scratch / "case.out").write_text("FAKE-SECRET-CANARY")
+ self.assertEqual(self.uploaded_files(), expected)
+
+
+ def test_oidc_negative_responses_never_print_issued_credentials(self):
+ source = (ROOT / "scripts/test/oidc_keycloak_live.sh").read_text()
+ for variable, filename in (("TAMPERED_STATUS", "sts-tampered.xml"), ("BAD_STATUS", "sts-bad.xml")):
+ start = source.index('[[ "${' + variable + '}" == 403 ]]')
+ end = source.index("\n", source.index("grep -q 'AccessDenied'", start))
+ guard = source[start:end]
+ credential_xml = "FAKE-ACCESS-CANARYFAKE-SECRET-CANARYFAKE-SESSION-CANARY"
+ for status, body, expected in (("200", credential_xml, 1), ("403", credential_xml, 1),
+ ("403", "AccessDenied", 0)):
+ with self.subTest(variable=variable, status=status, expected=expected):
+ (self.directory / filename).write_text(body)
+ result = subprocess.run(["bash", "-e", "-o", "pipefail", "-c", guard],
+ env={**self.env, variable: status, "WORK_DIR": str(self.directory)},
+ capture_output=True, text=True)
+ self.assertEqual(result.returncode, expected, result.stderr)
+ for canary in ("FAKE-ACCESS-CANARY", "FAKE-SECRET-CANARY", "FAKE-SESSION-CANARY"):
+ self.assertNotIn(canary, result.stdout + result.stderr)
+ if status == "200":
+ self.assertIn("HTTP 403, got 200", result.stderr)
+
+ def test_oidc_incomplete_credentials_report_only_the_missing_field(self):
+ source = (ROOT / "scripts/test/oidc_keycloak_live.sh").read_text()
+ start = source.index("IFS=$'\\t' read -r STS_ACCESS_KEY")
+ end = source.index("\n)\n", start) + 3
+ extract = source[start:end]
+ values = {"AccessKeyId": "FAKE-ACCESS-CANARY", "SecretAccessKey": "FAKE-SECRET-CANARY",
+ "SessionToken": "FAKE-SESSION-CANARY", "Expiration": "2099-01-01T00:00:00Z",
+ "SubjectFromWebIdentityToken": "alice"}
+ for missing in (None, "Expiration", "SubjectFromWebIdentityToken"):
+ with self.subTest(missing=missing):
+ xml = "" + "".join(f"<{key}>{value}" for key, value in values.items() if key != missing) + ""
+ (self.directory / "sts-good.xml").write_text(xml)
+ result = subprocess.run(["bash", "-e", "-o", "pipefail", "-c", extract],
+ env={**self.env, "WORK_DIR": str(self.directory)}, capture_output=True, text=True)
+ self.assertEqual(result.returncode, 1 if missing else 0, result.stderr)
+ for canary in ("FAKE-ACCESS-CANARY", "FAKE-SECRET-CANARY", "FAKE-SESSION-CANARY"):
+ self.assertNotIn(canary, result.stdout + result.stderr)
+ if missing:
+ self.assertIn(f"missing required STS field: {missing}", result.stderr)
def test_existing_evidence_directory_is_rejected(self) -> None:
- self.artifacts.mkdir()
- stale = self.artifacts / "suite-report.md"
- stale.write_text("OLD RUN REPORT")
- self.assertNotEqual(self.run_step("Initialize security evidence").returncode, 0)
- self.assertEqual(stale.read_text(), "OLD RUN REPORT")
- self.assertFalse(Path(self.env["GITHUB_ENV"]).exists())
- (self.artifacts / "report.md").write_text("OLD RUN REPORT")
- self.context.update({
- "env.SECURITY_ARTIFACTS_DIR": str(self.artifacts), "secrets.PF_TESTING_GH_TOKEN": "fake-local-token",
- })
- fake_bin = self.directory / "bin"
- fake_bin.mkdir()
- gh = fake_bin / "gh"
- gh.write_text(
- '#!/usr/bin/env bash\nset -euo pipefail\n'
- 'if [ "$1 $2" = "issue create" ]; then\n'
- ' while [ "$#" -gt 0 ]; do\n'
- ' if [ "$1" = "--body-file" ]; then cat "$2" > "$CAPTURE_BODY"; fi\n'
- ' shift\n'
- ' done\n'
- 'fi\n'
- )
- gh.chmod(0o755)
- body = self.directory / "issue-body.md"
- self.env.update(PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", CAPTURE_BODY=str(body))
- result = self.run_step("File failure issue in rustfs/backlog")
- self.assertEqual(result.returncode, 0, result.stderr)
- self.assertNotIn("OLD RUN REPORT", body.read_text())
- self.assertIn("https://github.com/rustfs/rustfs/actions/runs/314159", body.read_text())
+ for suffix in ("", "-scratch"):
+ self.setUp()
+ existing = Path(str(self.artifacts) + suffix)
+ existing.mkdir()
+ stale = existing / "suite-report.md"
+ stale.write_text("OLD RUN REPORT")
+ self.assertNotEqual(self.run_step("Initialize security evidence").returncode, 0)
+ self.assertEqual(stale.read_text(), "OLD RUN REPORT")
+ self.assertFalse(Path(self.env["GITHUB_ENV"]).exists())
+ (self.artifacts / "report.md").write_text("OLD RUN REPORT")
+ self.context.update({
+ "env.SECURITY_ARTIFACTS_DIR": str(self.artifacts), "secrets.PF_TESTING_GH_TOKEN": "fake-local-token",
+ })
+ fake_bin = self.directory / "bin"
+ fake_bin.mkdir()
+ gh = fake_bin / "gh"
+ gh.write_text(
+ '#!/usr/bin/env bash\nset -euo pipefail\n'
+ 'if [ "$1 $2" = "issue create" ]; then\n'
+ ' while [ "$#" -gt 0 ]; do\n'
+ ' if [ "$1" = "--body-file" ]; then cat "$2" > "$CAPTURE_BODY"; fi\n'
+ ' shift\n'
+ ' done\n'
+ 'fi\n'
+ )
+ gh.chmod(0o755)
+ body = self.directory / "issue-body.md"
+ self.env.update(PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", CAPTURE_BODY=str(body))
+ result = self.run_step("File failure issue in rustfs/backlog")
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertNotIn("OLD RUN REPORT", body.read_text())
+ self.assertIn("https://github.com/rustfs/rustfs/actions/runs/314159", body.read_text())
+
+ def test_all_ten_suites_hold_the_shared_lock_for_manual_and_chain_runs(self) -> None:
+ for suite in ("upgrade", "s3-compat", "kms", "tier", "storage", "heal", "pool-expand", "security", "replication", "performance"):
+ with self.subTest(suite=suite):
+ source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text().splitlines()
+ # Workflow-level concurrency covers every job, including cleanup,
+ # regardless of trigger or the runner hosting the job.
+ self.assertEqual([
+ line.strip() for line in yaml_block(source, "concurrency", 0)
+ if line.strip() and not line.lstrip().startswith("#")
+ ], [
+ "group: rustfs-shared-functional-tests", "cancel-in-progress: false",
+ ])
+ self.assertIsNotNone(yaml_block(source, "workflow_dispatch", 2))
+ self.assertIsNotNone(yaml_block(source, "repository_dispatch", 2))
+ cleanup_name = "Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"
+ cleanup = named_steps(yaml_block(source, "jobs", 0))[cleanup_name]
+ self.assertTrue(any(line.startswith(" if:") and "always()" in line for line in cleanup))
+
+ def test_root_dispatches_only_upgrade_and_replication_hands_off_after_failure(self) -> None:
+ for failed_attempts, issue_exit, token in ((0, 0, "fixture"), (2, 0, "fixture"), (3, 0, "fixture"), (3, 7, "fixture"), (0, 0, "")):
+ with self.subTest(failed_attempts=failed_attempts, issue_exit=issue_exit, token=bool(token)):
+ self.setUp()
+ fake_bin = self.directory / "bin"
+ fake_bin.mkdir()
+ commands = {
+ "gh": '''#!/usr/bin/env bash
+set -euo pipefail
+if [ "$1" = api ]; then
+ printf '%s\\n' "$*" >> "$DISPATCHES"
+ attempt=$(wc -l < "$DISPATCHES")
+ [ "$attempt" -gt "$FAILED_ATTEMPTS" ]
+elif [ "$1 $2" = 'issue create' ]; then
+ printf 'issue\\n' >> "$EXECUTED"
+ while [ "$#" -gt 0 ]; do
+ if [ "$1" = --body-file ]; then
+ cat "$2" > "$CAPTURE_BODY"
+ printf '%s\\n' "$2" > "$CAPTURE_BODY_PATH"
+ fi
+ shift
+ done
+ exit "$ISSUE_EXIT"
+else
+ exit 99
+fi
+''',
+ "sleep": '#!/bin/sh\nprintf "sleep %s\\n" "$1" >> "$EXECUTED"\n',
+ "ssh": '#!/bin/sh\nprintf "cleanup\\n" >> "$EXECUTED"\n',
+ }
+ for name, contents in commands.items():
+ command = fake_bin / name
+ command.write_text(contents)
+ command.chmod(0o755)
+ dispatches = self.directory / "dispatches"
+ executed = self.directory / "executed"
+ body = self.directory / "issue-body.md"
+ body_path = self.directory / "issue-body-path"
+ self.env.update(
+ PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", DISPATCHES=str(dispatches),
+ EXECUTED=str(executed), CAPTURE_BODY=str(body), CAPTURE_BODY_PATH=str(body_path),
+ FAILED_ATTEMPTS="0", ISSUE_EXIT=str(issue_exit),
+ RUSTFS_NODES="fixture-node", RUSTFS_SSH_USER="fixture-user",
+ RUSTFS_NIGHTLY_PACKAGE_URL="https://example.invalid/package.deb",
+ )
+ self.context.update({"secrets.PF_TESTING_GH_TOKEN": "fixture", "inputs.suite": "all"})
+ driver = (ROOT / ".github/workflows/rustfs-functional-chain.yml").read_text()
+ self.steps = named_steps(yaml_block(driver.splitlines(), "start-chain", 2))
+ self.assertEqual(list(self.steps), ["Dispatch first suite (upgrade)"])
+ started = self.run_step("Dispatch first suite (upgrade)")
+ self.assertEqual(started.returncode, 0, started.stderr)
+ self.assertEqual(dispatches.read_text().splitlines(), [
+ "api --method POST repos/rustfs/rustfs/dispatches -f event_type=rustfs-chain-upgrade -F client_payload[from_suite]=nightly-build",
+ ])
+ dispatches.unlink()
+
+ replication = (ROOT / ".github/workflows/rustfs-replication-test.yml").read_text()
+ job = yaml_block(replication.splitlines(), "replication-test", 2)
+ self.assertFalse(any(line.startswith(" continue-on-error:") for line in job))
+ self.steps = named_steps(job)
+ handoff = "Continue functional chain (next: Performance)"
+ self.assertIn(" if: ${{ always() && github.event_name == 'repository_dispatch' }}", self.steps[handoff])
+ self.assertFalse(any(line.strip().startswith("continue-on-error:") for line in self.steps[handoff]))
+ self.assertIn(" if: always()", self.steps["Cleanup environment (after)"])
+ self.assertLess(list(self.steps).index("Cleanup environment (after)"), list(self.steps).index(handoff))
+ suite = self.directory / "auto-testing/rustfs-replication-test.sh"
+ suite.write_text('#!/bin/sh\nprintf "suite failed\\n" >> "$EXECUTED"\nexit 17\n')
+ failed = self.run_step("Run replication suite")
+ self.assertEqual(failed.returncode, 17, failed.stderr)
+ cleaned = self.run_step("Cleanup environment (after)")
+ self.assertEqual(cleaned.returncode, 0, cleaned.stderr)
+ self.assertEqual(executed.read_text().splitlines(), ["suite failed", "cleanup"])
+ self.env["FAILED_ATTEMPTS"] = str(failed_attempts)
+ self.context["secrets.PF_TESTING_GH_TOKEN"] = token
+ forwarded = self.run_step(handoff)
+ self.assertEqual(forwarded.returncode == 0, bool(token) and failed_attempts < 3, forwarded.stderr)
+ calls = dispatches.read_text().splitlines() if dispatches.exists() else []
+ self.assertEqual(calls, [
+ "api --method POST repos/rustfs/rustfs/dispatches -f event_type=rustfs-chain-performance -F client_payload[from_suite]=replication",
+ ] * (min(failed_attempts + 1, 3) if token else 0))
+ if failed_attempts == 3:
+ self.assertIn("could not hand off from **replication** to **Performance**", body.read_text())
+ self.assertIn("rustfs-chain-performance", body.read_text())
+ self.assertEqual(executed.read_text().splitlines().count("issue"), 2 if issue_exit else 1)
+ self.assertFalse(Path(body_path.read_text().strip()).exists())
class FunctionalWorkflowTests(unittest.TestCase):
@@ -225,7 +400,7 @@ class FunctionalWorkflowTests(unittest.TestCase):
if suite in self.DIRECT_TESTS:
test = steps[self.DIRECT_TESTS[suite]]
self.assertNotRegex("\n".join(test), r'''(?m)^ ["']?continue-on-error["']?\s*:''')
- self.assertIn(" if: always()", steps["Generate report"])
+ self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", steps["Generate report"])
cleanup = steps["Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"]
condition = next(line.strip() for line in cleanup if line.startswith(" if:"))
self.assertIn(condition, (
@@ -234,7 +409,7 @@ class FunctionalWorkflowTests(unittest.TestCase):
"if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}",
))
if suite != "performance":
- handoff = steps["Chain complete"] if suite == "replication" else next(
+ handoff = next(
value for name, value in steps.items() if name.startswith("Continue functional chain")
)
self.assertIn(" if: ${{ always() && github.event_name == 'repository_dispatch' }}", handoff)
@@ -277,13 +452,387 @@ class FunctionalWorkflowTests(unittest.TestCase):
self.assertIn("partial suite diagnostics", failed.stdout)
cleanup = execute("Cleanup environment (after)")
self.assertEqual(cleanup.returncode, 0, cleanup.stderr)
- handoff_name = "Chain complete" if suite == "replication" else next(
+ handoff_name = next(
name for name in steps if name.startswith("Continue functional chain")
)
handoff = execute(handoff_name)
self.assertEqual(handoff.returncode, 0, handoff.stderr)
markers = (root / "executed").read_text().splitlines()
- self.assertEqual(markers, ["cleanup"] if suite == "replication" else ["cleanup", "dispatch"])
+ self.assertEqual(markers, ["cleanup", "dispatch"])
+
+
+class FunctionalCaseReportTests(unittest.TestCase):
+ def report(self, text: str | None, matrix: bool = False) -> tuple[bool, str, str]:
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ log = root / "suite.log"
+ if text is not None:
+ log.write_text(text)
+ valid = generate_report(log, root / "cases.md", root / "matrix.md" if matrix else None)
+ return valid, (root / "cases.md").read_text(), (root / "matrix.md").read_text() if matrix else ""
+
+ def test_repeated_case_executions_preserve_failure_and_context(self):
+ # log() from rustfs/auto-testing@6120aa0a76de, rustfs-kms-test.sh:131.
+ log = subprocess.check_output(["bash", "-c", r'''
+log() { printf '\033[1;36m[INFO]\033[0m %s\n' "$*"; }
+log '== topology: single-single kms-backend: local =='
+printf '\033[32m--- KMS-101 roundtrip ---\033[0m\n[FAIL] KMS-101\n'
+log '== topology: single-multi kms-backend: vault-kv2 =='
+printf '%s\n' '--- KMS-101 roundtrip ---' '[PASS] KMS-101'
+printf '%s\n' '--- KMS-101 roundtrip ---' '[UNSUPPORTED] KMS-101'
+'''], text=True)
+ valid, cases, _ = self.report(log)
+ self.assertFalse(valid)
+ self.assertEqual(cases.count("| KMS-101 |"), 3)
+ self.assertIn("- Total: 3\n- PASS: 1\n- FAIL: 1\n- UNSUPPORTED: 1\n- RUNNING: 0\n", cases)
+ self.assertIn("roundtrip (topology: single-single kms-backend: local) | FAIL |", cases)
+ self.assertIn("roundtrip (topology: single-multi kms-backend: vault-kv2) | PASS |", cases)
+ self.assertNotIn("\\n", cases)
+
+ def test_missing_empty_unfinished_and_orphan_results_are_not_success(self):
+ for text in (None, "", "setup failed\n", "--- KMS-101 roundtrip ---\n", "[PASS] KMS-101\n",
+ "--- KMS-101 first ---\n--- KMS-101 second ---\n[PASS] KMS-101\n",
+ "--- KMS-101 first ---\n[FAIL] KMS-101\n[PASS] KMS-101\n"):
+ with self.subTest(log=text):
+ valid, cases, _ = self.report(text)
+ self.assertFalse(valid)
+ self.assertIn("## Case Summary", cases)
+ valid, cases, _ = self.report("[INFO] == suite: bucket replication (REP-*) ==\n--- REP-101 unsupported ---\n[UNSUPPORTED] REP-101\n")
+ self.assertTrue(valid)
+ self.assertIn("suite: bucket replication", cases)
+ self.assertIn("- UNSUPPORTED: 1\n", cases)
+
+ def test_upgrade_matrix_is_preserved_and_required_for_complete_report(self):
+ case = "--- UPG-101 upgrade ---\n[PASS] UPG-101\n"
+ for suffix, expected in (("", False), ("[UPG-TOPO] single-single local v1 v2 PASS=1 FAIL=0\n", True),
+ ("[UPG-TOPO] single-single local v1 v2 PASS=1 FAIL=1\n", False)):
+ with self.subTest(matrix=suffix):
+ valid, _, matrix = self.report(case + suffix, matrix=True)
+ self.assertEqual(valid, expected)
+ self.assertIn("| Topology | KMS Backend | From Version | To Version | Result |", matrix)
+ self.assertIn("| single-single | local | v1 | v2 |" if suffix else "NOT RUN", matrix)
+
+ def test_s3_case_identifiers_include_digits(self):
+ valid, cases, _ = self.report("--- S3C-101 CreateBucket ---\n[PASS] S3C-101\n")
+ self.assertTrue(valid)
+ self.assertIn("| S3C-101 | CreateBucket (context not recorded) | PASS |", cases)
+
+
+class FunctionalEvidenceTests(WorkflowSteps, unittest.TestCase):
+ SUITES = (*FunctionalWorkflowTests.DIRECT_TESTS, "heal", "performance")
+
+ def prepare(self, suite: str) -> None:
+ self.temp = tempfile.TemporaryDirectory()
+ self.addCleanup(self.temp.cleanup)
+ self.directory = Path(self.temp.name)
+ self.source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text()
+ self.steps = named_steps(yaml_block(self.source.splitlines(), FunctionalWorkflowTests.JOBS[suite], 2))
+ self.context = {expression: "" for expression in re.findall(r"\$\{\{\s*(.*?)\s*\}\}", self.source)}
+ self.context.update({
+ "github.server_url": "https://github.com", "github.repository": "rustfs/rustfs",
+ "github.run_id": "314159", "github.run_attempt": "2", "github.sha": "0123456789abcdef0123456789abcdef01234567",
+ "github.event_name": "repository_dispatch", "steps.test.outcome": "success",
+ "secrets.PF_TESTING_GH_TOKEN": "local-fixture", "env.PF_TESTING_GH_TOKEN": "local-fixture",
+ })
+ self.artifacts = self.directory / f"rustfs-{suite}-314159-2"
+ self.env = {
+ **os.environ, "GITHUB_ENV": str(self.directory / "github-env"), "RUNNER_TEMP": self.temp.name,
+ "GITHUB_STEP_SUMMARY": str(self.directory / "summary.md"), "RUSTFS_NODES": "fixture-node",
+ "RUSTFS_NIGHTLY_PACKAGE_URL": "https://example.invalid/package.deb", "CAPTURE_BODY": str(self.directory / "issue.md"),
+ }
+ for key in ("server_url", "repository", "run_id", "run_attempt", "sha", "event_name"):
+ self.env[f"GITHUB_{key.upper()}"] = self.context[f"github.{key}"]
+ (self.directory / "scripts").mkdir()
+ (self.directory / "scripts/functional_case_report.py").symlink_to(ROOT / "scripts/functional_case_report.py")
+ fake_bin = self.directory / "bin"
+ fake_bin.mkdir()
+ (fake_bin / "python3").symlink_to(sys.executable)
+ for command, body in (
+ ("ssh", 'printf "fixture-version\\n"\n'),
+ ("gh", 'if [ "$1 $2" = "issue create" ]; then\n'
+ ' while [ "$#" -gt 0 ]; do\n'
+ ' if [ "$1" = "--body-file" ]; then cat "$2" > "$CAPTURE_BODY"; fi\n'
+ ' shift\n'
+ ' done\n'
+ 'elif [ "$1 $2" = "api --method" ]; then cat >/dev/null; fi\n'),
+ ):
+ script = fake_bin / command
+ script.write_text("#!/bin/sh\n" + body)
+ script.chmod(0o755)
+ self.env["PATH"] = f"{fake_bin}{os.pathsep}{os.environ['PATH']}"
+
+ def test_evidence_wiring_and_failed_initialization_cannot_publish_stale_files(self):
+ for suite, suffix in ((suite, suffix) for suite in self.SUITES for suffix in ("", "-scratch")):
+ with self.subTest(suite=suite, collision=suffix or "artifact"):
+ self.prepare(suite)
+ self.assertNotIn("/tmp/rustfs-", self.source)
+ names = list(self.steps)
+ self.assertLess(names.index("Initialize functional evidence"), names.index("Checkout auto-testing scripts (with retry)"))
+ if suite in FunctionalWorkflowTests.DIRECT_TESTS:
+ self.assertLess(names.index("Checkout repository (for report parser)"), names.index("Checkout auto-testing scripts (with retry)"))
+ for name, lines in self.steps.items():
+ if name in ("Generate report", "Upload functional report to dashboard") or any("uses: actions/upload-artifact@" in line for line in lines):
+ self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", lines)
+ if any("uses: actions/upload-artifact@" in line for line in lines):
+ self.assertIn(" path: |", lines)
+ self.assertIn(" if-no-files-found: error", lines)
+ existing = Path(str(self.artifacts) + suffix)
+ existing.mkdir()
+ for filename in ("report.md", "suite.log"):
+ (existing / filename).write_text("OLD RUN EVIDENCE")
+ self.env.update(REPORT_FILE=str(existing / "report.md"), LOG_FILE=str(existing / "suite.log"))
+ initialized = self.run_step("Initialize functional evidence")
+ self.assertNotEqual(initialized.returncode, 0)
+ self.assertFalse(Path(self.env["GITHUB_ENV"]).exists())
+ issue = self.run_step("File failure issue in rustfs/backlog")
+ self.assertEqual(issue.returncode, 0, issue.stderr)
+ body = Path(self.env["CAPTURE_BODY"]).read_text()
+ self.assertNotIn("OLD RUN EVIDENCE", body)
+ self.assertIn("no report or log file was produced", body)
+ self.assertEqual((existing / "report.md").read_text(), "OLD RUN EVIDENCE")
+
+ def test_reports_use_only_current_complete_suite_evidence(self):
+ for suite in self.SUITES[:-1]:
+ good = "--- KMS-101 roundtrip ---\n[PASS] KMS-101\n"
+ partial = "--- KMS-101 roundtrip ---\n[PASS] KMS-101\n--- KMS-102 unfinished ---\n"
+ if suite == "s3-compat":
+ good, partial = good.replace("KMS-", "S3C-"), partial.replace("KMS-", "S3C-")
+ if suite == "upgrade":
+ good += "[UPG-TOPO] single-single local v1 v2 PASS=1 FAIL=0\n"
+ if suite == "heal":
+ good = "".join(f"[HEAL-STEP] {step} fixture PASS\n" for step in range(1, 8))
+ partial = "[HEAL-STEP] 1 fixture PASS\n"
+ for outcome, log in (("success", good), ("failure", good), ("success", partial), ("success", ""),
+ ("success", None), ("skipped", None), ("cancelled", good)):
+ with self.subTest(suite=suite, outcome=outcome, log=log):
+ self.prepare(suite)
+ stale = self.directory / "old-suite.log"
+ stale.write_text("OLD RUN EVIDENCE\n" + good)
+ self.env.update(LOG_FILE=str(stale), REPORT_FILE=str(stale))
+ self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
+ self.assertEqual(self.env["LOG_FILE"], str(self.artifacts / "suite.log"))
+ self.assertEqual(self.env["TMPDIR"], str(self.artifacts) + "-scratch")
+ if log is not None:
+ Path(self.env["LOG_FILE"]).write_text(log)
+ self.context["steps.test.outcome"] = outcome
+ report = self.run_step("Generate report")
+ success = outcome == "success" and log == good
+ self.assertEqual(report.returncode == 0, success, report.stderr)
+ contents = Path(self.env["REPORT_FILE"]).read_text()
+ self.assertNotIn("OLD RUN EVIDENCE", contents)
+ self.assertEqual("| PASS |" in contents, success)
+ for value in ("actions/runs/314159", "Attempt: 2", "Workflow Commit: " + self.context["github.sha"],
+ f"Test Step Outcome: {'success' if success else 'failure'}", f"Suite Step Outcome: {outcome}"):
+ self.assertIn(value, contents)
+ self.assertEqual(Path(self.env["GITHUB_STEP_SUMMARY"]).read_text(), contents)
+ evidence = (self.artifacts / ("steps.md" if suite == "heal" else "cases.md")).read_text()
+ if log in (good, partial):
+ self.assertIn("| PASS |", evidence)
+ self.assertNotIn("OLD RUN EVIDENCE", evidence)
+
+ def test_actual_suite_commands_pass_the_current_log_and_scratch_paths(self):
+ for suite in self.SUITES:
+ with self.subTest(suite=suite):
+ self.prepare(suite)
+ self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
+ if suite == "heal":
+ self.assertEqual(self.env["RUSTFS_WARP_LOG_FILE"], str(self.artifacts / "warp.log"))
+ scripts = self.directory / "auto-testing"
+ scripts.mkdir()
+ filename = f"rustfs_{suite}_test.sh" if suite in ("heal", "performance") else f"rustfs-{suite}-test.sh"
+ script = scripts / filename
+ script.write_text(
+ '#!/bin/bash\nset -euo pipefail\nlog=""\n'
+ 'while [ "$#" -gt 0 ]; do\n'
+ ' if [ "$1" = "--log-file" ]; then log="$2"; shift; fi\n'
+ ' shift\n'
+ 'done\n'
+ '[ "$log" = "$LOG_FILE" ] || exit 31\n'
+ 'printf "CURRENT SUITE LOG\\n" > "$log"\n'
+ 'scratch=$(mktemp -d "$TMPDIR/fixture.XXXXXX")\n'
+ 'printf "CURRENT SCRATCH\\n" > "$scratch/trace.log"\n'
+ 'if [ -n "${RUSTFS_RESULT_DIR:-}" ]; then\n'
+ ' mkdir -p "$RUSTFS_RESULT_DIR"\n'
+ ' printf "CURRENT RESULTS\\n" > "$RUSTFS_RESULT_DIR/summary.md"\n'
+ 'fi\n'
+ )
+ script.chmod(0o755)
+ name = FunctionalWorkflowTests.DIRECT_TESTS.get(suite) or (
+ "Run benchmark (GET/PUT/MIXED)" if suite == "performance" else "Run heal test (write -> outage -> heal -> verify)"
+ )
+ result = self.run_step(name)
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual((self.artifacts / "suite.log").read_text(), "CURRENT SUITE LOG\n")
+ self.assertEqual(len(list(Path(self.env["TMPDIR"]).glob("fixture.*/trace.log"))), 1)
+ self.assertEqual(list(self.artifacts.glob("fixture.*")), [])
+ if suite == "performance":
+ self.assertEqual((self.artifacts / "results/summary.md").read_text(), "CURRENT RESULTS\n")
+
+ def test_upload_allowlist_preserves_diagnostics_without_scratch(self):
+ extra = {
+ "kms": ["cases.md"], "storage": ["cases.md"], "s3-compat": ["cases.md"],
+ "upgrade": ["cases.md", "matrix.md"], "replication": ["cases.md"], "heal": ["steps.md", "warp.log"],
+ "performance": ["version.txt", "results/master.log", "results/summary.md", "results/summary.tsv",
+ "results/get_1KiB.txt", "results/put_1MiB.txt", "results/mixed_4MiB.txt"],
+ }
+ for suite in self.SUITES:
+ with self.subTest(suite=suite):
+ self.prepare(suite)
+ self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
+ expected = {self.artifacts / name for name in ["report.md", "suite.log", *extra[suite]]}
+ for path in expected:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text("PARTIAL FAILURE DIAGNOSTIC")
+ for directory in (self.artifacts, Path(self.env["TMPDIR"]), self.artifacts / "results"):
+ directory.mkdir(exist_ok=True)
+ (directory / "init.json").write_text('{"root_token":"FAKE-SECRET-CANARY"}')
+ self.assertEqual(self.uploaded_files(), expected)
+ self.assertTrue(all("FAKE-SECRET-CANARY" not in path.read_text() for path in self.uploaded_files()))
+
+ def test_kms_failure_after_vault_init_keeps_only_failure_evidence(self):
+ self.prepare("kms")
+ self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
+ scripts = self.directory / "auto-testing"
+ scripts.mkdir()
+ # Vault file writes and EXIT cleanup from auto-testing@06cd3c097350:23-24,57,479-487.
+ # The Docker boundary returns synthetic credentials; setup fails before vault_stop.
+ (scripts / "rustfs-kms-test.sh").write_text(r'''#!/bin/bash
+set -Eeuo pipefail
+TEST_TMP="$(mktemp -d "${TMPDIR:-/tmp}/rustfs-test.XXXXXX")"
+trap 'rm -rf "${TEST_TMP}"' EXIT
+VAULT_DATA_DIR="${RUSTFS_VAULT_DATA_DIR:-${TMPDIR:-/tmp}/rustfs-vault-data}"
+VAULT_CONTAINER="rustfs-vault"
+heal_run() { "$@"; }
+while [ "$#" -gt 0 ]; do
+ if [ "$1" = "--log-file" ]; then LOG_FILE="$2"; shift; fi
+ shift
+done
+mkdir -p "${VAULT_DATA_DIR}"
+tmp_init="${TEST_TMP}/vault-init.json"
+tmp_err="${TEST_TMP}/vault-init.stderr"
+heal_run docker exec -e "VAULT_ADDR=http://127.0.0.1:8200" "${VAULT_CONTAINER}" \
+ vault operator init -key-shares=1 -key-threshold=1 -format=json \
+ > "${tmp_init}" 2>"${tmp_err}"
+cat "${tmp_init}" | heal_run tee "${VAULT_DATA_DIR}/init.json" >/dev/null
+printf '%s\n' 'vault initialized; root token acquired' 'fixture setup failed after init' > "${LOG_FILE}"
+exit 42
+''')
+ docker = self.directory / "bin/docker"
+ docker.write_text("""#!/bin/sh
+[ "$1" = exec ] || exit 99
+printf '%s\\n' '{"root_token":"FAKE-ROOT-CANARY","unseal_keys_b64":["FAKE-UNSEAL-CANARY"]}'
+""")
+ docker.chmod(0o755)
+ result = self.run_step("Run KMS suite")
+ self.assertEqual(result.returncode, 42, result.stderr)
+ vault_init = Path(self.env["TMPDIR"]) / "rustfs-vault-data/init.json"
+ self.assertEqual(json.loads(vault_init.read_text()), {"root_token": "FAKE-ROOT-CANARY", "unseal_keys_b64": ["FAKE-UNSEAL-CANARY"]})
+ self.assertNotIn(self.artifacts, vault_init.parents)
+ self.assertEqual(list(Path(self.env["TMPDIR"]).glob("rustfs-test.*")), [])
+ self.assertNotEqual(self.run_step("Generate report").returncode, 0)
+ self.assertEqual(self.uploaded_files(), {self.artifacts / name for name in ("suite.log", "cases.md", "report.md")})
+ self.assertIn("fixture setup failed after init", (self.artifacts / "suite.log").read_text())
+ for path in self.uploaded_files():
+ self.assertNotIn("FAKE-ROOT-CANARY", path.read_text())
+ self.assertNotIn("FAKE-UNSEAL-CANARY", path.read_text())
+
+ def test_heal_accumulates_actual_staged_steps_without_overwriting_failures(self):
+ self.prepare("heal")
+ self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
+ script = self.directory / "auto-testing/rustfs_heal_test.sh"
+ script.parent.mkdir()
+ # Result printf and full-run condition from auto-testing@6120aa0a76de:143,1163-1168.
+ script.write_text(r'''#!/bin/bash
+set -euo pipefail
+SELECTED_STEPS=()
+PREFLIGHT=0
+while [ "$#" -gt 0 ]; do
+ case "$1" in
+ --steps) IFS=',' read -ra SELECTED_STEPS <<< "$2"; shift ;;
+ --log-file) LOG_FILE="$2"; shift ;;
+ --preflight) PREFLIGHT=1 ;;
+ esac
+ shift
+done
+if [ "$PREFLIGHT" -eq 1 ]; then
+ printf '\n' >> "$INVOKED_STEPS"
+ exit 0
+fi
+printf '%s\n' "${SELECTED_STEPS[*]}" >> "$INVOKED_STEPS"
+emit_step_result() {
+ local n="$1" desc="$2" status="$3"
+ printf '[HEAL-STEP] %s %s %s\n' "${n}" "${desc}" "${status}"
+}
+{
+ for step in "${SELECTED_STEPS[@]}"; do
+ emit_step_result "$step" "fixture step $step" PASS
+ done
+ want_all=1
+ for s in 1 2 3 4 5 6 7; do
+ [[ " ${SELECTED_STEPS[*]} " == *" ${s} "* ]] || want_all=0
+ done
+ if [ "${want_all}" -eq 1 ]; then
+ printf '[HEAL-RESULT] PASS all steps passed\n'
+ fi
+} >> "$LOG_FILE"
+''')
+ script.chmod(0o755)
+ self.env["INVOKED_STEPS"] = str(self.directory / "invoked-steps")
+ for name in ("Install RustFS package & start cluster", "Preflight checks", "Run heal test (write -> outage -> heal -> verify)"):
+ result = self.run_step(name)
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(Path(self.env["INVOKED_STEPS"]).read_text().splitlines(), ["1 2", "", "3 4 5 6 7"])
+ log = Path(self.env["LOG_FILE"]).read_text()
+ self.assertNotIn("[HEAL-RESULT]", log)
+ self.assertEqual(log.count("[HEAL-STEP]"), 7)
+ report = self.run_step("Generate report")
+ self.assertEqual(report.returncode, 0, report.stderr)
+ failed_logs = ["\n".join(line for line in log.splitlines() if not line.startswith(f"[HEAL-STEP] {step} ")) + "\n"
+ for step in range(1, 8)]
+ failed_logs += [
+ log.replace("[HEAL-STEP] 3", "[HEAL-STEP] 3 original failure FAIL\n[HEAL-STEP] 3"),
+ log + "[HEAL-STEP] 3 later step failure FAIL\n",
+ log + "[HEAL-RESULT] FAIL earlier failure\n[HEAL-RESULT] PASS later success\n",
+ log.replace("[HEAL-STEP] 4 fixture step 4 PASS", "[HEAL-STEP] 4 fixture step 4 SKIP"),
+ ]
+ for failed_log in failed_logs:
+ with self.subTest(log=failed_log):
+ Path(self.env["LOG_FILE"]).write_text(failed_log)
+ report = self.run_step("Generate report")
+ self.assertNotEqual(report.returncode, 0, report.stderr)
+ contents = Path(self.env["REPORT_FILE"]).read_text()
+ self.assertIn("Test Step Outcome: failure", contents)
+ self.assertNotIn("| PASS |", contents)
+ if "original failure" in failed_log:
+ self.assertIn("| 3 | original failure | FAIL |", (self.artifacts / "steps.md").read_text())
+ if "later step failure" in failed_log:
+ self.assertIn("| 3 | later step failure | FAIL |", (self.artifacts / "steps.md").read_text())
+
+ def test_performance_results_version_and_report_are_bound_to_the_run(self):
+ self.prepare("performance")
+ initialized = self.run_step("Initialize functional evidence")
+ self.assertEqual(initialized.returncode, 0, initialized.stderr)
+ self.assertEqual(self.env["RUSTFS_RESULT_DIR"], str(self.artifacts / "results"))
+ self.assertEqual(self.env["VERSION_FILE"], str(self.artifacts / "version.txt"))
+ version = self.run_step("Collect RustFS version info")
+ self.assertEqual(version.returncode, 0, version.stderr)
+ self.assertIn("fixture-version", Path(self.env["VERSION_FILE"]).read_text())
+ old_summary = self.directory / "old-results/summary.md"
+ old_summary.parent.mkdir()
+ old_summary.write_text("OLD RUN EVIDENCE")
+ upload = "Upload report to dashboard (reports/YYYY-MM-DD.md)"
+ self.assertNotEqual(self.run_step(upload).returncode, 0)
+ self.assertFalse(Path(self.env["REPORT_FILE"]).exists())
+ results = Path(self.env["RUSTFS_RESULT_DIR"])
+ results.mkdir()
+ (results / "summary.md").write_text("CURRENT PERFORMANCE RESULTS\n")
+ report = self.run_step(upload)
+ self.assertEqual(report.returncode, 0, report.stderr)
+ contents = Path(self.env["REPORT_FILE"]).read_text()
+ for value in ("actions/runs/314159", "**Attempt**: 2", "**Workflow Commit**: " + self.context["github.sha"],
+ "CURRENT PERFORMANCE RESULTS", "fixture-version"):
+ self.assertIn(value, contents)
+ self.assertNotIn("OLD RUN EVIDENCE", contents)
if __name__ == "__main__":