gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-06 03:59:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: integrate frozen main for MRF staging delivery

Browse Source 
Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
This commit is contained in:
houseme
2026-09-06 00:59:01 +08:00
parent 59a1cbc587 a6b5da64f2
commit 3ebc65cc45
19 changed files with 1483 additions and 603 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.config/make/tests.mak
+1
View File
@@ -41,6 +41,7 @@ script-tests: ## Run shell script tests
$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
.github/actions/quick-checks/action.yml
+1
View File
@@ -100,6 +100,7 @@ runs:
python3 ./scripts/check_test_wiring.py --self-test
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
python3 ./scripts/test_security_workflow.py
python3 ./scripts/test_nightly_candidate.py
python3 ./scripts/check_test_wiring.py
- name: Check no planning docs committed
.github/workflows/nightly-gnu.yml
+51 -8
View File
@@ -166,8 +166,9 @@ jobs:
# e.g. https://dl.rustfs.com/artifacts/rustfs/packages/nightly/... .
# Skipped when the R2 secrets are not configured (artifact-only mode).
- name: Upload DEB to Cloudflare R2
if: env.R2_ACCESS_KEY_ID != ''
id: publish
env:
DEB_FILE: ${{ steps.deb.outputs.deb_file }}
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
@@ -182,28 +183,70 @@ jobs:
exit 0
fi
if ! command -v aws >/dev/null 2>&1; then
sudo apt-get update && sudo apt-get install -y -qq awscli
fi
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
export AWS_DEFAULT_REGION="auto"
DEB_FILE="${{ steps.deb.outputs.deb_file }}"
SOURCE_SHA="$(git rev-parse HEAD)"
if [[ "${SOURCE_SHA}" != "${GITHUB_SHA}" ]]; then
echo "Checkout SHA does not match the nightly build run" >&2
exit 1
fi
DEB_SHA256="$(sha256sum "${DEB_FILE}" | cut -d ' ' -f 1)"
CANDIDATE_KEY="artifacts/rustfs/packages/nightly/runs/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/${DEB_SHA256}/rustfs.deb"
CANDIDATE_URL="https://dl.rustfs.com/${CANDIDATE_KEY}"
# Old AWS CLI models lack conditional PutObject support. Never fall
# back to an overwriting upload for a candidate.
AWS_CLI=aws
if ! "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null; then
sudo apt-get update
sudo apt-get install -y -qq python3-venv
AWS_CLI_DIR="$(mktemp -d "${RUNNER_TEMP}/nightly-awscli.XXXXXX")"
trap 'rm -rf "${AWS_CLI_DIR}"' EXIT
python3 -m venv "${AWS_CLI_DIR}"
"${AWS_CLI_DIR}/bin/python" -m pip install --disable-pip-version-check 'awscli==1.44.79'
AWS_CLI="${AWS_CLI_DIR}/bin/aws"
fi
"${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null
"${AWS_CLI}" --version
"${AWS_CLI}" s3api put-object --bucket "${R2_BUCKET}" --key "${CANDIDATE_KEY}" \
--body "${DEB_FILE}" --if-none-match '*' --endpoint-url "${R2_ENDPOINT}"
PUBLISHED_SHA256="$(curl -fsSL --retry 3 --connect-timeout 15 --max-time 300 "${CANDIDATE_URL}" | sha256sum | cut -d ' ' -f 1)"
if [[ "${PUBLISHED_SHA256}" != "${DEB_SHA256}" ]]; then
echo "Published candidate checksum does not match the built package" >&2
exit 1
fi
R2_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/nightly/"
echo "📤 Uploading ${DEB_FILE} to ${R2_PREFIX}"
aws s3 cp "${DEB_FILE}" "${R2_PREFIX}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
"${AWS_CLI}" s3 cp "${DEB_FILE}" "${R2_PREFIX}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
# Stable "latest" alias so tests can fetch the newest nightly
# without knowing today's date.
echo "📤 Uploading latest alias"
aws s3 cp "${DEB_FILE}" "${R2_PREFIX}rustfs-nightly-latest.deb" \
"${AWS_CLI}" s3 cp "${DEB_FILE}" "${R2_PREFIX}rustfs-nightly-latest.deb" \
--endpoint-url "$R2_ENDPOINT" --only-show-errors
echo "✅ R2 upload complete"
CANDIDATE_FILE="${RUNNER_TEMP}/nightly-candidate-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
jq -n --arg source_sha "${SOURCE_SHA}" \
--argjson build_run_id "${GITHUB_RUN_ID}" --argjson build_run_attempt "${GITHUB_RUN_ATTEMPT}" \
--arg package_url "${CANDIDATE_URL}" --arg package_sha256 "${DEB_SHA256}" \
'{schema: 1, source_sha: $source_sha, build_run_id: $build_run_id, build_run_attempt: $build_run_attempt, package_url: $package_url, package_sha256: $package_sha256}' \
> "${CANDIDATE_FILE}"
echo "candidate_file=${CANDIDATE_FILE}" >> "${GITHUB_OUTPUT}"
- name: Upload nightly candidate manifest
if: ${{ steps.publish.outputs.candidate_file != '' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: nightly-candidate-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ steps.publish.outputs.candidate_file }}
if-no-files-found: error
# Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
#
# RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
.github/workflows/rustfs-functional-chain.yml
+2 -15
View File
@@ -14,8 +14,8 @@
# Functional chain driver: runs the ten functional suites in a fixed order
# (upgrade -> s3 -> kms -> tier -> storage -> heal -> pool -> security ->
# replication, with performance on its own runner in parallel) and guarantees
# the chain keeps moving even when individual suites fail.
# replication -> performance). Each suite attempts the next handoff even
# when its tests fail.
#
# Each suite workflow can still be dispatched standalone (workflow_dispatch);
# only chain-triggered runs forward to the next suite via repository_dispatch,
@@ -59,16 +59,3 @@ jobs:
gh api --method POST repos/rustfs/rustfs/dispatches \
-f event_type='rustfs-chain-upgrade' \
-F 'client_payload[from_suite]=nightly-build'
- name: Dispatch performance suite (parallel, own runner)
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
run: |
set -euo pipefail
if [ -z "${GH_TOKEN:-}" ]; then
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch performance" >&2
exit 1
fi
gh api --method POST repos/rustfs/rustfs/dispatches \
-f event_type='rustfs-chain-performance' \
-F 'client_payload[from_suite]=nightly-build'
.github/workflows/rustfs-heal-test.yml
+70 -34
View File
@@ -59,6 +59,21 @@ jobs:
# (storage -> heal -> pool). Pool expansion no longer re-runs heal.
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
- name: Initialize functional evidence
id: evidence
run: |
set -euo pipefail
umask 077
FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-heal-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
{
printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
printf 'RUSTFS_WARP_LOG_FILE=%s/warp.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
} >> "${GITHUB_ENV}"
# auto-testing is private: clone it with the dedicated PF token (not
# GITHUB_TOKEN) and retry transient GitHub/network failures.
- name: Checkout auto-testing scripts (with retry)
@@ -114,7 +129,7 @@ jobs:
else
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
fi
./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
- name: Preflight checks
run: |
@@ -124,7 +139,7 @@ jobs:
else
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
fi
./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
- name: Run heal test (write -> outage -> heal -> verify)
id: test
@@ -134,13 +149,10 @@ jobs:
--endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
--stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \
--warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \
--log-file /tmp/rustfs-heal-test.log
--log-file "${LOG_FILE}"
- name: Generate report
if: always()
env:
LOG_FILE: /tmp/rustfs-heal-test.log
REPORT_FILE: /tmp/rustfs-heal-report.md
if: ${{ always() && steps.evidence.outcome == 'success' }}
run: |
set -euo pipefail
PACKAGE_URL='${{ inputs.package_url }}'
@@ -149,8 +161,9 @@ jobs:
else
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
fi
STEPS_TABLE="/tmp/rustfs-heal-steps.md"
python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY'
STEPS_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/steps.md"
CASE_RESULT=success
python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY' || CASE_RESULT=failure
import re
import sys
@@ -162,6 +175,7 @@ jobs:
steps = {}
order = []
status_rank = {'SKIP': 0, 'PASS': 1, 'FAIL': 2}
version = None
version_node = None
verdict = None
@@ -175,14 +189,15 @@ jobs:
n, desc, status = m.group(1), m.group(2), m.group(3)
if n not in steps:
order.append(n)
steps[n] = (desc, status) # later lines win (fail after pass)
if n not in steps or status_rank[status] > status_rank[steps[n][1]]:
steps[n] = (desc, status)
continue
m = ver_re.match(line)
if m:
version, version_node = m.group(1), m.group(2)
continue
m = result_re.match(line)
if m:
if m and verdict != 'FAIL':
verdict, verdict_detail = m.group(1), m.group(2)
except FileNotFoundError:
pass
@@ -202,30 +217,43 @@ jobs:
out.write(f'| {n} | {desc} | {status} |\n')
if not order:
out.write('| - | - | NOT RUN (no step result lines found) |\n')
complete = set(steps) == {str(n) for n in range(1, 8)}
sys.exit(0 if complete and verdict != 'FAIL' and all(status == 'PASS' for _, status in steps.values()) else 1)
PY
RESULT=failure
if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
RESULT=success
fi
{
echo "# RustFS heal test report"
echo ""
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
echo "- Workflow Commit: ${GITHUB_SHA}"
echo "- Trigger: ${{ github.event_name }}"
echo "- Package: ${PACKAGE_SOURCE}"
echo "- Test Step Outcome: ${{ steps.test.outcome }}"
echo "- Test Step Outcome: ${RESULT}"
echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
echo ""
cat "${STEPS_TABLE}" || true
echo ""
echo "## Log tail"
echo '```text'
tail -n 200 "${LOG_FILE}" || true
echo '```'
if [ "${RESULT}" = "success" ]; then
cat "${STEPS_TABLE}"
echo ""
echo "## Log tail"
echo '```text'
tail -n 200 "${LOG_FILE}"
echo '```'
else
echo "The suite or evidence validation failed. See this run's artifact for partial step results and suite.log."
fi
} | tee "${REPORT_FILE}"
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
[ "${RESULT}" = "success" ]
- name: Upload functional report to dashboard
if: always()
if: ${{ always() && steps.evidence.outcome == 'success' }}
continue-on-error: true
env:
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
REPORT_FILE: /tmp/rustfs-heal-report.md
SUITE: heal
run: |
set -euo pipefail
@@ -235,28 +263,32 @@ jobs:
fi
DATE="$(date -u +%Y-%m-%d)"
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
# Base64-encode the report into a temp file and feed it to jq via
# --rawfile: large reports (e.g. pool) exceed the OS argv limit and
# make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
B64_FILE="$(mktemp)"
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
if [ -n "${SHA}" ]; then
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
'{message:$msg, content:$content, sha:$sha}' \
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
else
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
'{message:$msg, content:$content}' \
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
'{message:$msg, content:($content|rtrimstr("\n"))}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
fi
rm -f "${B64_FILE}"
- name: File failure issue in rustfs/backlog
if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
continue-on-error: true
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
SUITE: 'heal'
SUITE_LABEL: 'Heal'
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
REPORT_FILE: '/tmp/rustfs-heal-report.md'
LOG_FILE: '/tmp/rustfs-heal-test.log'
run: |
set -euo pipefail
if [ -z "${GH_TOKEN:-}" ]; then
@@ -284,14 +316,16 @@ jobs:
echo ""
echo "- Suite: \`${SUITE}\`"
echo "- Run: ${RUN_URL}"
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
echo "- Workflow Commit: ${GITHUB_SHA}"
echo "- Trigger: ${GITHUB_EVENT_NAME}"
echo "- Date: $(date -u +%Y-%m-%d)"
echo ""
echo "## Report (errors and symptoms)"
echo ""
if [ -s "${REPORT_FILE}" ]; then
if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
redact < "${REPORT_FILE}"
elif [ -s "${LOG_FILE:-}" ]; then
elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
echo "(report file missing; log tail below)"
echo ""
tail -n 200 "${LOG_FILE}" | redact
@@ -307,14 +341,16 @@ jobs:
echo "filed backlog issue for suite ${SUITE}"
- name: Upload test logs
if: always()
if: ${{ always() && steps.evidence.outcome == 'success' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: rustfs-heal-test-${{ github.run_id }}
name: rustfs-heal-test-${{ github.run_id }}-${{ github.run_attempt }}
path: |
/tmp/rustfs-heal-test*.log
/tmp/rustfs-warp.*.log
if-no-files-found: warn
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/warp.log
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/steps.md
if-no-files-found: error
- name: Cleanup environment (after)
if: ${{ always() && inputs.cleanup_after != 'false' }}
.github/workflows/rustfs-kms-test.yml
+64 -79
View File
@@ -52,6 +52,25 @@ jobs:
timeout-minutes: 420
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
- name: Checkout repository (for report parser)
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Initialize functional evidence
id: evidence
run: |
set -euo pipefail
umask 077
FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-kms-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
{
printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
} >> "${GITHUB_ENV}"
# auto-testing is private: clone it with the dedicated PF token (not
# GITHUB_TOKEN) and retry transient GitHub/network failures.
- name: Checkout auto-testing scripts (with retry)
@@ -108,8 +127,6 @@ jobs:
- name: Run KMS suite
id: test
env:
LOG_FILE: /tmp/rustfs-kms.log
run: |
set -euo pipefail
chmod +x auto-testing/rustfs-kms-test.sh
@@ -139,10 +156,7 @@ jobs:
./auto-testing/rustfs-kms-test.sh "${ARGS[@]}"
- name: Generate report
if: always()
env:
LOG_FILE: /tmp/rustfs-kms.log
REPORT_FILE: /tmp/rustfs-kms-report.md
if: ${{ always() && steps.evidence.outcome == 'success' }}
run: |
set -euo pipefail
PACKAGE_URL='${{ inputs.package_url }}'
@@ -154,79 +168,43 @@ jobs:
else
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
fi
CASE_TABLE="/tmp/rustfs-kms-cases.md"
python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
import re
import sys
log_file, out_file = sys.argv[1], sys.argv[2]
ansi = re.compile(r'\x1b\[[0-9;]*m')
start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
rows = []
index = {}
try:
with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
for raw in fh:
line = ansi.sub('', raw).strip()
m = start_re.match(line)
if m:
case_id, name = m.group(1), m.group(2)
if case_id not in index:
index[case_id] = len(rows)
rows.append([case_id, name, 'RUNNING'])
continue
m = done_re.match(line)
if m:
status, case_id = m.group(1), m.group(2)
if case_id in index:
rows[index[case_id]][2] = status
else:
rows.append([case_id, case_id, status])
index[case_id] = len(rows) - 1
except FileNotFoundError:
rows = []
counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
for _, _, status in rows:
counts[status] = counts.get(status, 0) + 1
with open(out_file, 'w', encoding='utf-8') as out:
out.write('## Case Summary\n\n')
out.write(f"- Total: {len(rows)}\\n")
out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
out.write('\\n')
out.write('| Case | Name | Status |\\n')
out.write('| --- | --- | --- |\\n')
for case_id, name, status in rows:
out.write(f'| {case_id} | {name} | {status} |\\n')
PY
CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
CASE_RESULT=success
python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
RESULT=failure
if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
RESULT=success
fi
{
echo "# RustFS KMS test report"
echo ""
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
echo "- Workflow Commit: ${GITHUB_SHA}"
echo "- Trigger: ${{ github.event_name }}"
echo "- Package: ${PACKAGE_SOURCE}"
echo "- Test Step Outcome: ${{ steps.test.outcome }}"
echo "- Test Step Outcome: ${RESULT}"
echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
echo ""
cat "${CASE_TABLE}" || true
echo ""
echo "## Log tail"
echo '```text'
tail -n 200 "${LOG_FILE}" || true
echo '```'
if [ "${RESULT}" = "success" ]; then
cat "${CASE_TABLE}"
echo ""
echo "## Log tail"
echo '```text'
tail -n 200 "${LOG_FILE}"
echo '```'
else
echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
fi
} | tee "${REPORT_FILE}"
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
[ "${RESULT}" = "success" ]
- name: Upload functional report to dashboard
if: always()
if: ${{ always() && steps.evidence.outcome == 'success' }}
continue-on-error: true
env:
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
REPORT_FILE: /tmp/rustfs-kms-report.md
SUITE: kms
run: |
set -euo pipefail
@@ -236,28 +214,32 @@ jobs:
fi
DATE="$(date -u +%Y-%m-%d)"
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
# Base64-encode the report into a temp file and feed it to jq via
# --rawfile: large reports (e.g. pool) exceed the OS argv limit and
# make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
B64_FILE="$(mktemp)"
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
if [ -n "${SHA}" ]; then
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
'{message:$msg, content:$content, sha:$sha}' \
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
else
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
'{message:$msg, content:$content}' \
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
'{message:$msg, content:($content|rtrimstr("\n"))}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
fi
rm -f "${B64_FILE}"
- name: File failure issue in rustfs/backlog
if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
continue-on-error: true
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
SUITE: 'kms'
SUITE_LABEL: 'KMS'
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
REPORT_FILE: '/tmp/rustfs-kms-report.md'
LOG_FILE: '/tmp/rustfs-kms.log'
run: |
set -euo pipefail
if [ -z "${GH_TOKEN:-}" ]; then
@@ -285,14 +267,16 @@ jobs:
echo ""
echo "- Suite: \`${SUITE}\`"
echo "- Run: ${RUN_URL}"
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
echo "- Workflow Commit: ${GITHUB_SHA}"
echo "- Trigger: ${GITHUB_EVENT_NAME}"
echo "- Date: $(date -u +%Y-%m-%d)"
echo ""
echo "## Report (errors and symptoms)"
echo ""
if [ -s "${REPORT_FILE}" ]; then
if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
redact < "${REPORT_FILE}"
elif [ -s "${LOG_FILE:-}" ]; then
elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
echo "(report file missing; log tail below)"
echo ""
tail -n 200 "${LOG_FILE}" | redact
@@ -308,14 +292,15 @@ jobs:
echo "filed backlog issue for suite ${SUITE}"
- name: Upload report and logs
if: always()
if: ${{ always() && steps.evidence.outcome == 'success' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: rustfs-kms-test-${{ github.run_id }}
name: rustfs-kms-test-${{ github.run_id }}-${{ github.run_attempt }}
path: |
/tmp/rustfs-kms.log
/tmp/rustfs-kms-report.md
if-no-files-found: warn
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
if-no-files-found: error
- name: Cleanup environment (after)
if: always()
.github/workflows/rustfs-performance-test.yml
+48 -29
View File
@@ -49,17 +49,16 @@ on:
type: boolean
default: true
repository_dispatch:
# Chain entry: dispatched by rustfs-functional-chain.yml (runs on its own
# pf-testing runner, in parallel with the shared-VM chain).
# Chain handoff: dispatched when the replication suite finishes.
types: [rustfs-chain-performance]
permissions:
contents: read
# Dedicated pf-testing runner/environment: own concurrency group so perf runs
# never block (or are blocked by) the pool-expansion / heal tests.
# The default performance nodes overlap the other suites' remote VMs, even
# though the runner differs. Hold the shared lock through cleanup as well.
concurrency:
group: rustfs-performance-test
group: rustfs-shared-functional-tests
cancel-in-progress: false
defaults:
@@ -76,8 +75,6 @@ env:
# Package used by the nightly run (workflow_dispatch inputs are empty for
# workflow_run events), i.e. the latest nightly deb published by nightly-gnu.yml.
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
# Fixed benchmark result directory so later steps can read summary.md
RUSTFS_RESULT_DIR: /tmp/rustfs-perf-results
# Cross-repo token for uploading reports to rustfs/dashboard (set in repo settings)
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
@@ -89,6 +86,22 @@ jobs:
# Skipped when nightly failed.
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
- name: Initialize functional evidence
id: evidence
run: |
set -euo pipefail
umask 077
FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-performance-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
{
printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
printf 'RUSTFS_RESULT_DIR=%s/results\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
printf 'VERSION_FILE=%s/version.txt\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
} >> "${GITHUB_ENV}"
# auto-testing is private: clone it with the dedicated PF token (not
# GITHUB_TOKEN) and retry transient GitHub/network failures.
- name: Checkout auto-testing scripts (with retry)
@@ -120,7 +133,7 @@ jobs:
if: ${{ inputs.cleanup_before != 'false' }}
run: |
chmod +x auto-testing/rustfs_performance_test.sh
./auto-testing/rustfs_performance_test.sh --step 1 -y
./auto-testing/rustfs_performance_test.sh --step 1 -y --log-file "${LOG_FILE:-/dev/null}"
- name: Install RustFS package & start cluster (4x4)
run: |
@@ -130,7 +143,7 @@ jobs:
else
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
fi
./auto-testing/rustfs_performance_test.sh "${ARGS[@]}"
./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
- name: Preflight checks
run: |
@@ -140,7 +153,7 @@ jobs:
else
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
fi
./auto-testing/rustfs_performance_test.sh "${ARGS[@]}"
./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
- name: Run benchmark (GET/PUT/MIXED)
id: benchmark
@@ -153,17 +166,15 @@ jobs:
--step 5 -y \
--warp-duration "${{ inputs.warp_duration || '5m' }}" \
--warp-concurrency "${{ inputs.warp_concurrency || '64' }}" \
--log-file /tmp/rustfs-perf-test.log
--log-file "${LOG_FILE}"
- name: Analyze results
if: ${{ steps.benchmark.conclusion == 'success' }}
run: |
./auto-testing/rustfs_performance_test.sh --step 6 -y
./auto-testing/rustfs_performance_test.sh --step 6 -y --log-file "${LOG_FILE:-/dev/null}"
- name: Collect RustFS version info
if: ${{ steps.benchmark.conclusion == 'success' }}
env:
VERSION_FILE: /tmp/rustfs-version.txt
run: |
set -euo pipefail
read -r -a NODES <<< "${RUSTFS_NODES}"
@@ -183,7 +194,6 @@ jobs:
env:
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
RESULT_DIR: ${{ env.RUSTFS_RESULT_DIR }}
VERSION_FILE: /tmp/rustfs-version.txt
run: |
set -euo pipefail
if [ -z "${GH_TOKEN:-}" ]; then
@@ -191,7 +201,7 @@ jobs:
exit 0
fi
SUMMARY="${RESULT_DIR}/summary.md"
[ -f "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; }
[ -s "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; }
DATE="$(date -u +%Y-%m-%d)"
REPORT_PATH="reports/${DATE}.md"
{
@@ -199,6 +209,8 @@ jobs:
echo ""
echo "- **Date**: ${DATE}"
echo "- **Run**: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
echo "- **Attempt**: ${GITHUB_RUN_ATTEMPT}"
echo "- **Workflow Commit**: ${GITHUB_SHA}"
echo "- **Trigger**: ${{ github.event_name }}"
echo "- **Package**: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
echo ""
@@ -208,8 +220,8 @@ jobs:
echo '```text'
cat "${VERSION_FILE}"
echo '```'
} > /tmp/rustfs-perf-report.md
CONTENT="$(python3 -c 'import base64; print(base64.b64encode(open("/tmp/rustfs-perf-report.md","rb").read()).decode())')"
} > "${REPORT_FILE}"
CONTENT="$(python3 -c 'import base64,sys; print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
if [ -n "${SHA}" ]; then
jq -n --arg msg "report: ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
@@ -228,11 +240,10 @@ jobs:
continue-on-error: true
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
SUITE: 'performance'
SUITE_LABEL: 'Performance'
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
REPORT_FILE: '/tmp/rustfs-perf-report.md'
LOG_FILE: '/tmp/rustfs-perf-test.log'
run: |
set -euo pipefail
if [ -z "${GH_TOKEN:-}" ]; then
@@ -260,14 +271,16 @@ jobs:
echo ""
echo "- Suite: \`${SUITE}\`"
echo "- Run: ${RUN_URL}"
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
echo "- Workflow Commit: ${GITHUB_SHA}"
echo "- Trigger: ${GITHUB_EVENT_NAME}"
echo "- Date: $(date -u +%Y-%m-%d)"
echo ""
echo "## Report (errors and symptoms)"
echo ""
if [ -s "${REPORT_FILE}" ]; then
if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
redact < "${REPORT_FILE}"
elif [ -s "${LOG_FILE:-}" ]; then
elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
echo "(report file missing; log tail below)"
echo ""
tail -n 200 "${LOG_FILE}" | redact
@@ -283,20 +296,26 @@ jobs:
echo "filed backlog issue for suite ${SUITE}"
- name: Upload test logs & results
if: always()
if: ${{ always() && steps.evidence.outcome == 'success' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: rustfs-perf-test-${{ github.run_id }}
name: rustfs-perf-test-${{ github.run_id }}-${{ github.run_attempt }}
path: |
/tmp/rustfs-perf-test*.log
/tmp/rustfs-perf-results/**
/tmp/rustfs-version.txt
if-no-files-found: warn
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/version.txt
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/master.log
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/summary.md
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/summary.tsv
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/get_*.txt
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/put_*.txt
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/mixed_*.txt
if-no-files-found: error
- name: Reset test environment (after)
if: ${{ always() && inputs.cleanup_after != 'false' }}
run: |
./auto-testing/rustfs_performance_test.sh --step 7 -y
./auto-testing/rustfs_performance_test.sh --step 7 -y --log-file "${LOG_FILE:-/dev/null}"
- name: Notify on failure
if: failure()
.github/workflows/rustfs-pool-expand-test.yml
+10 -5
View File
@@ -539,17 +539,22 @@ jobs:
fi
DATE="$(date -u +%Y-%m-%d)"
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
# Base64-encode the report into a temp file and feed it to jq via
# --rawfile: large reports (e.g. pool) exceed the OS argv limit and
# make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
B64_FILE="$(mktemp)"
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
if [ -n "${SHA}" ]; then
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
'{message:$msg, content:$content, sha:$sha}' \
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
else
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
'{message:$msg, content:$content}' \
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
'{message:$msg, content:($content|rtrimstr("\n"))}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
fi
rm -f "${B64_FILE}"
- name: File failure issue in rustfs/backlog
if: ${{ always() && (failure() || steps.pool_test.outcome == 'failure' || steps.pool_test.outcome == 'cancelled') }}
.github/workflows/rustfs-replication-test.yml
+107 -86
View File
@@ -34,8 +34,7 @@ on:
- site
default: all
repository_dispatch:
# Chain handoff: dispatched when the security suite finishes. This is the
# last link of the functional chain.
# Chain handoff: dispatched when the security suite finishes.
types: [rustfs-chain-replication]
permissions:
@@ -65,6 +64,25 @@ jobs:
timeout-minutes: 360
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
- name: Checkout repository (for report parser)
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Initialize functional evidence
id: evidence
run: |
set -euo pipefail
umask 077
FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-replication-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
{
printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
} >> "${GITHUB_ENV}"
# auto-testing is private: clone it with the dedicated PF token (not
# GITHUB_TOKEN) and retry transient GitHub/network failures.
- name: Checkout auto-testing scripts (with retry)
@@ -113,8 +131,6 @@ jobs:
- name: Run replication suite
id: test
env:
LOG_FILE: /tmp/rustfs-replication.log
run: |
set -euo pipefail
chmod +x auto-testing/rustfs-replication-test.sh
@@ -137,10 +153,7 @@ jobs:
./auto-testing/rustfs-replication-test.sh "${ARGS[@]}"
- name: Generate report
if: always()
env:
LOG_FILE: /tmp/rustfs-replication.log
REPORT_FILE: /tmp/rustfs-replication-report.md
if: ${{ always() && steps.evidence.outcome == 'success' }}
run: |
set -euo pipefail
PACKAGE_URL='${{ inputs.package_url }}'
@@ -162,80 +175,44 @@ jobs:
RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
fi
fi
CASE_TABLE="/tmp/rustfs-replication-cases.md"
python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
import re
import sys
log_file, out_file = sys.argv[1], sys.argv[2]
ansi = re.compile(r'\x1b\[[0-9;]*m')
start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
rows = []
index = {}
try:
with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
for raw in fh:
line = ansi.sub('', raw).strip()
m = start_re.match(line)
if m:
case_id, name = m.group(1), m.group(2)
if case_id not in index:
index[case_id] = len(rows)
rows.append([case_id, name, 'RUNNING'])
continue
m = done_re.match(line)
if m:
status, case_id = m.group(1), m.group(2)
if case_id in index:
rows[index[case_id]][2] = status
else:
rows.append([case_id, case_id, status])
index[case_id] = len(rows) - 1
except FileNotFoundError:
rows = []
counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
for _, _, status in rows:
counts[status] = counts.get(status, 0) + 1
with open(out_file, 'w', encoding='utf-8') as out:
out.write('## Case Summary\n\n')
out.write(f"- Total: {len(rows)}\\n")
out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
out.write('\\n')
out.write('| Case | Name | Status |\\n')
out.write('| --- | --- | --- |\\n')
for case_id, name, status in rows:
out.write(f'| {case_id} | {name} | {status} |\\n')
PY
CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
CASE_RESULT=success
python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
RESULT=failure
if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
RESULT=success
fi
{
echo "# RustFS replication test report"
echo ""
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
echo "- Workflow Commit: ${GITHUB_SHA}"
echo "- Trigger: ${{ github.event_name }}"
echo "- Package: ${PACKAGE_SOURCE}"
echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
echo "- Test Step Outcome: ${{ steps.test.outcome }}"
echo "- Test Step Outcome: ${RESULT}"
echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
echo ""
cat "${CASE_TABLE}" || true
echo ""
echo "## Log tail"
echo '```text'
tail -n 200 "${LOG_FILE}" || true
echo '```'
if [ "${RESULT}" = "success" ]; then
cat "${CASE_TABLE}"
echo ""
echo "## Log tail"
echo '```text'
tail -n 200 "${LOG_FILE}"
echo '```'
else
echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
fi
} | tee "${REPORT_FILE}"
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
[ "${RESULT}" = "success" ]
- name: Upload functional report to dashboard
if: always()
if: ${{ always() && steps.evidence.outcome == 'success' }}
continue-on-error: true
env:
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
REPORT_FILE: /tmp/rustfs-replication-report.md
SUITE: replication
run: |
set -euo pipefail
@@ -245,28 +222,32 @@ jobs:
fi
DATE="$(date -u +%Y-%m-%d)"
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
# Base64-encode the report into a temp file and feed it to jq via
# --rawfile: large reports (e.g. pool) exceed the OS argv limit and
# make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
B64_FILE="$(mktemp)"
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
if [ -n "${SHA}" ]; then
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
'{message:$msg, content:$content, sha:$sha}' \
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
else
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
'{message:$msg, content:$content}' \
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
'{message:$msg, content:($content|rtrimstr("\n"))}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
fi
rm -f "${B64_FILE}"
- name: File failure issue in rustfs/backlog
if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
continue-on-error: true
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
SUITE: 'replication'
SUITE_LABEL: 'Replication (bucket + site)'
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
REPORT_FILE: '/tmp/rustfs-replication-report.md'
LOG_FILE: '/tmp/rustfs-replication.log'
run: |
set -euo pipefail
if [ -z "${GH_TOKEN:-}" ]; then
@@ -294,14 +275,16 @@ jobs:
echo ""
echo "- Suite: \`${SUITE}\`"
echo "- Run: ${RUN_URL}"
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
echo "- Workflow Commit: ${GITHUB_SHA}"
echo "- Trigger: ${GITHUB_EVENT_NAME}"
echo "- Date: $(date -u +%Y-%m-%d)"
echo ""
echo "## Report (errors and symptoms)"
echo ""
if [ -s "${REPORT_FILE}" ]; then
if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
redact < "${REPORT_FILE}"
elif [ -s "${LOG_FILE:-}" ]; then
elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
echo "(report file missing; log tail below)"
echo ""
tail -n 200 "${LOG_FILE}" | redact
@@ -317,14 +300,15 @@ jobs:
echo "filed backlog issue for suite ${SUITE}"
- name: Upload report and logs
if: always()
if: ${{ always() && steps.evidence.outcome == 'success' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: rustfs-replication-${{ github.run_id }}
name: rustfs-replication-${{ github.run_id }}-${{ github.run_attempt }}
path: |
/tmp/rustfs-replication.log
/tmp/rustfs-replication-report.md
if-no-files-found: warn
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
if-no-files-found: error
- name: Cleanup environment (after)
if: always()
@@ -345,13 +329,50 @@ jobs:
'
done
- name: Chain complete
# Replication is the last link of the functional chain: nothing to
# dispatch after it. This step just records that the chain finished.
- name: "Continue functional chain (next: Performance)"
if: ${{ always() && github.event_name == 'repository_dispatch' }}
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
run: |
echo "Functional chain complete: replication (final suite) finished."
echo "from_suite=security trigger=${{ github.event_name }} outcome=${{ steps.test.outcome }}"
set -uo pipefail
if [ -z "${GH_TOKEN:-}" ]; then
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
exit 1
fi
DISPATCHED=0
for attempt in 1 2 3; do
if gh api --method POST repos/rustfs/rustfs/dispatches \
-f event_type='rustfs-chain-performance' \
-F 'client_payload[from_suite]=replication'; then
echo "dispatched next suite Performance (attempt ${attempt})"
DISPATCHED=1
break
fi
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
sleep "${attempt}0"
done
if [ "${DISPATCHED:-0}" -ne 1 ]; then
echo "ERROR: functional chain stalled: could not dispatch Performance after 3 attempts" >&2
TITLE="[functional][chain] stalled after replication (run ${GITHUB_RUN_ID})"
BODY_FILE="$(mktemp)"
trap 'rm -f "${BODY_FILE}"' EXIT
{
echo "The functional chain could not hand off from **replication** to **Performance** after 3 attempts."
echo ""
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
echo "- Expected next event: 'rustfs-chain-performance'"
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
echo "- Recovery: re-dispatch manually with"
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-performance'"
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
} > "${BODY_FILE}"
gh issue create -R rustfs/backlog --title "${TITLE}" \
--body-file "${BODY_FILE}" --label functional-test \
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|| echo "could not file the stall alert issue either; check the token" >&2
exit 1
fi
- name: Notify on failure
if: failure()
.github/workflows/rustfs-s3-compat-test.yml
+64 -82
View File
@@ -40,6 +40,25 @@ jobs:
timeout-minutes: 360
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
- name: Checkout repository (for report parser)
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Initialize functional evidence
id: evidence
run: |
set -euo pipefail
umask 077
FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-s3-compat-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
{
printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
} >> "${GITHUB_ENV}"
# auto-testing is private: clone it with the dedicated PF token (not
# GITHUB_TOKEN) and retry transient GitHub/network failures.
- name: Checkout auto-testing scripts (with retry)
@@ -87,8 +106,6 @@ jobs:
- name: Run S3 compatibility suite
id: test
env:
LOG_FILE: /tmp/rustfs-s3-compat.log
run: |
set -euo pipefail
chmod +x auto-testing/rustfs-s3-compat-test.sh
@@ -105,10 +122,7 @@ jobs:
./auto-testing/rustfs-s3-compat-test.sh "${ARGS[@]}"
- name: Generate report
if: always()
env:
LOG_FILE: /tmp/rustfs-s3-compat.log
REPORT_FILE: /tmp/rustfs-s3-compat-report.md
if: ${{ always() && steps.evidence.outcome == 'success' }}
run: |
set -euo pipefail
PACKAGE_URL='${{ inputs.package_url }}'
@@ -130,83 +144,44 @@ jobs:
RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
fi
fi
CASE_TABLE="/tmp/rustfs-s3-compat-cases.md"
python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
import re
import sys
log_file, out_file = sys.argv[1], sys.argv[2]
ansi = re.compile(r'\x1b\[[0-9;]*m')
start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
rows = []
index = {}
current = None
try:
with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
for raw in fh:
line = ansi.sub('', raw).strip()
m = start_re.match(line)
if m:
case_id, name = m.group(1), m.group(2)
current = case_id
if case_id not in index:
index[case_id] = len(rows)
rows.append([case_id, name, 'RUNNING'])
continue
m = done_re.match(line)
if m:
status, case_id = m.group(1), m.group(2)
if case_id in index:
rows[index[case_id]][2] = status
else:
rows.append([case_id, case_id, status])
index[case_id] = len(rows) - 1
current = None
except FileNotFoundError:
rows = []
counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
for _, _, status in rows:
counts[status] = counts.get(status, 0) + 1
with open(out_file, 'w', encoding='utf-8') as out:
out.write('## Case Summary\n\n')
out.write(f"- Total: {len(rows)}\\n")
out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
out.write('\\n')
out.write('| Case | Name | Status |\\n')
out.write('| --- | --- | --- |\\n')
for case_id, name, status in rows:
out.write(f'| {case_id} | {name} | {status} |\\n')
PY
CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
CASE_RESULT=success
python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
RESULT=failure
if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
RESULT=success
fi
{
echo "# RustFS S3 compatibility test report"
echo ""
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
echo "- Workflow Commit: ${GITHUB_SHA}"
echo "- Trigger: ${{ github.event_name }}"
echo "- Package: ${PACKAGE_SOURCE}"
echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
echo "- Test Step Outcome: ${{ steps.test.outcome }}"
echo "- Test Step Outcome: ${RESULT}"
echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
echo ""
cat "${CASE_TABLE}" || true
echo ""
echo "## Log tail"
echo '```text'
tail -n 200 "${LOG_FILE}" || true
echo '```'
if [ "${RESULT}" = "success" ]; then
cat "${CASE_TABLE}"
echo ""
echo "## Log tail"
echo '```text'
tail -n 200 "${LOG_FILE}"
echo '```'
else
echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
fi
} | tee "${REPORT_FILE}"
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
[ "${RESULT}" = "success" ]
- name: Upload functional report to dashboard
if: always()
if: ${{ always() && steps.evidence.outcome == 'success' }}
continue-on-error: true
env:
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
REPORT_FILE: /tmp/rustfs-s3-compat-report.md
SUITE: s3
run: |
set -euo pipefail
@@ -216,28 +191,32 @@ jobs:
fi
DATE="$(date -u +%Y-%m-%d)"
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
# Base64-encode the report into a temp file and feed it to jq via
# --rawfile: large reports (e.g. pool) exceed the OS argv limit and
# make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
B64_FILE="$(mktemp)"
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
if [ -n "${SHA}" ]; then
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
'{message:$msg, content:$content, sha:$sha}' \
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
else
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
'{message:$msg, content:$content}' \
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
'{message:$msg, content:($content|rtrimstr("\n"))}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
fi
rm -f "${B64_FILE}"
- name: File failure issue in rustfs/backlog
if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
continue-on-error: true
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
SUITE: 's3'
SUITE_LABEL: 'S3 compatibility'
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
REPORT_FILE: '/tmp/rustfs-s3-compat-report.md'
LOG_FILE: '/tmp/rustfs-s3-compat.log'
run: |
set -euo pipefail
if [ -z "${GH_TOKEN:-}" ]; then
@@ -265,14 +244,16 @@ jobs:
echo ""
echo "- Suite: \`${SUITE}\`"
echo "- Run: ${RUN_URL}"
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
echo "- Workflow Commit: ${GITHUB_SHA}"
echo "- Trigger: ${GITHUB_EVENT_NAME}"
echo "- Date: $(date -u +%Y-%m-%d)"
echo ""
echo "## Report (errors and symptoms)"
echo ""
if [ -s "${REPORT_FILE}" ]; then
if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
redact < "${REPORT_FILE}"
elif [ -s "${LOG_FILE:-}" ]; then
elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
echo "(report file missing; log tail below)"
echo ""
tail -n 200 "${LOG_FILE}" | redact
@@ -288,14 +269,15 @@ jobs:
echo "filed backlog issue for suite ${SUITE}"
- name: Upload report and logs
if: always()
if: ${{ always() && steps.evidence.outcome == 'success' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: rustfs-s3-compat-${{ github.run_id }}
name: rustfs-s3-compat-${{ github.run_id }}-${{ github.run_attempt }}
path: |
/tmp/rustfs-s3-compat.log
/tmp/rustfs-s3-compat-report.md
if-no-files-found: warn
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
if-no-files-found: error
- name: Cleanup environment (after)
if: always()
.github/workflows/rustfs-security-test.yml
+22 -10
View File
@@ -77,10 +77,14 @@ jobs:
timeout-minutes: 360
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
# Checkout the repository into its own subdirectory. Checking out at
# the workspace root would wipe the auto-testing clone above (that is
# exactly how run 33934141181 lost rustfs-security-test.sh).
- name: Checkout repository (for the OIDC live gate script)
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
path: rustfs-repo
- name: Initialize security evidence
id: evidence
@@ -88,7 +92,7 @@ jobs:
set -euo pipefail
umask 077
SECURITY_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-security-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
mkdir -- "${SECURITY_ARTIFACTS_DIR}"
mkdir -- "${SECURITY_ARTIFACTS_DIR}" "${SECURITY_ARTIFACTS_DIR}-scratch"
printf 'SECURITY_ARTIFACTS_DIR=%s\n' "${SECURITY_ARTIFACTS_DIR}" >> "${GITHUB_ENV}"
# auto-testing is private: clone it with the dedicated PF token (not
@@ -144,8 +148,8 @@ jobs:
continue-on-error: true
env:
REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
TMPDIR: ${{ env.SECURITY_ARTIFACTS_DIR }}
RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/scripts/test/oidc_keycloak_live.sh
TMPDIR: ${{ env.SECURITY_ARTIFACTS_DIR }}-scratch
RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/rustfs-repo/scripts/test/oidc_keycloak_live.sh
run: |
set -euo pipefail
chmod +x auto-testing/rustfs-security-test.sh
@@ -168,7 +172,7 @@ jobs:
else
ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
fi
GITHUB_STEP_SUMMARY=/dev/null ./auto-testing/rustfs-security-test.sh "${ARGS[@]}"
GITHUB_STEP_SUMMARY=/dev/null ./auto-testing/rustfs-security-test.sh "${ARGS[@]}" 2>&1 | tee "${SECURITY_ARTIFACTS_DIR}/suite.log"
- name: Generate report
id: report
@@ -219,17 +223,22 @@ jobs:
fi
DATE="$(date -u +%Y-%m-%d)"
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
# Base64-encode the report into a temp file and feed it to jq via
# --rawfile: large reports (e.g. pool) exceed the OS argv limit and
# make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
B64_FILE="$(mktemp)"
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
if [ -n "${SHA}" ]; then
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
'{message:$msg, content:$content, sha:$sha}' \
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
else
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
'{message:$msg, content:$content}' \
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
'{message:$msg, content:($content|rtrimstr("\n"))}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
fi
rm -f "${B64_FILE}"
- name: File failure issue in rustfs/backlog
if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
@@ -296,7 +305,10 @@ jobs:
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: rustfs-security-test-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ env.SECURITY_ARTIFACTS_DIR }}/
path: |
${{ env.SECURITY_ARTIFACTS_DIR }}/report.md
${{ env.SECURITY_ARTIFACTS_DIR }}/suite.log
${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
if-no-files-found: error
retention-days: 3
.github/workflows/rustfs-storage-test.yml
+64 -82
View File
@@ -49,6 +49,25 @@ jobs:
timeout-minutes: 360
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
- name: Checkout repository (for report parser)
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Initialize functional evidence
id: evidence
run: |
set -euo pipefail
umask 077
FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-storage-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
{
printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
} >> "${GITHUB_ENV}"
# auto-testing is private: clone it with the dedicated PF token (not
# GITHUB_TOKEN) and retry transient GitHub/network failures.
- name: Checkout auto-testing scripts (with retry)
@@ -96,8 +115,6 @@ jobs:
- name: Run storage engine suite
id: test
env:
LOG_FILE: /tmp/rustfs-storage.log
run: |
set -euo pipefail
chmod +x auto-testing/rustfs-storage-test.sh
@@ -120,10 +137,7 @@ jobs:
./auto-testing/rustfs-storage-test.sh "${ARGS[@]}"
- name: Generate report
if: always()
env:
LOG_FILE: /tmp/rustfs-storage.log
REPORT_FILE: /tmp/rustfs-storage-report.md
if: ${{ always() && steps.evidence.outcome == 'success' }}
run: |
set -euo pipefail
PACKAGE_URL='${{ inputs.package_url }}'
@@ -145,83 +159,44 @@ jobs:
RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
fi
fi
CASE_TABLE="/tmp/rustfs-storage-cases.md"
python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
import re
import sys
log_file, out_file = sys.argv[1], sys.argv[2]
ansi = re.compile(r'\x1b\[[0-9;]*m')
start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
rows = []
index = {}
current = None
try:
with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
for raw in fh:
line = ansi.sub('', raw).strip()
m = start_re.match(line)
if m:
case_id, name = m.group(1), m.group(2)
current = case_id
if case_id not in index:
index[case_id] = len(rows)
rows.append([case_id, name, 'RUNNING'])
continue
m = done_re.match(line)
if m:
status, case_id = m.group(1), m.group(2)
if case_id in index:
rows[index[case_id]][2] = status
else:
rows.append([case_id, case_id, status])
index[case_id] = len(rows) - 1
current = None
except FileNotFoundError:
rows = []
counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
for _, _, status in rows:
counts[status] = counts.get(status, 0) + 1
with open(out_file, 'w', encoding='utf-8') as out:
out.write('## Case Summary\n\n')
out.write(f"- Total: {len(rows)}\\n")
out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
out.write('\\n')
out.write('| Case | Name | Status |\\n')
out.write('| --- | --- | --- |\\n')
for case_id, name, status in rows:
out.write(f'| {case_id} | {name} | {status} |\\n')
PY
CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
CASE_RESULT=success
python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
RESULT=failure
if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
RESULT=success
fi
{
echo "# RustFS storage engine test report"
echo ""
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
echo "- Workflow Commit: ${GITHUB_SHA}"
echo "- Trigger: ${{ github.event_name }}"
echo "- Package: ${PACKAGE_SOURCE}"
echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
echo "- Test Step Outcome: ${{ steps.test.outcome }}"
echo "- Test Step Outcome: ${RESULT}"
echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
echo ""
cat "${CASE_TABLE}" || true
echo ""
echo "## Log tail"
echo '```text'
tail -n 200 "${LOG_FILE}" || true
echo '```'
if [ "${RESULT}" = "success" ]; then
cat "${CASE_TABLE}"
echo ""
echo "## Log tail"
echo '```text'
tail -n 200 "${LOG_FILE}"
echo '```'
else
echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
fi
} | tee "${REPORT_FILE}"
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
[ "${RESULT}" = "success" ]
- name: Upload functional report to dashboard
if: always()
if: ${{ always() && steps.evidence.outcome == 'success' }}
continue-on-error: true
env:
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
REPORT_FILE: /tmp/rustfs-storage-report.md
SUITE: storage
run: |
set -euo pipefail
@@ -231,28 +206,32 @@ jobs:
fi
DATE="$(date -u +%Y-%m-%d)"
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
# Base64-encode the report into a temp file and feed it to jq via
# --rawfile: large reports (e.g. pool) exceed the OS argv limit and
# make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
B64_FILE="$(mktemp)"
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
if [ -n "${SHA}" ]; then
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
'{message:$msg, content:$content, sha:$sha}' \
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
else
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
'{message:$msg, content:$content}' \
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
'{message:$msg, content:($content|rtrimstr("\n"))}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
fi
rm -f "${B64_FILE}"
- name: File failure issue in rustfs/backlog
if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
continue-on-error: true
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
SUITE: 'storage'
SUITE_LABEL: 'Storage engine'
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
REPORT_FILE: '/tmp/rustfs-storage-report.md'
LOG_FILE: '/tmp/rustfs-storage.log'
run: |
set -euo pipefail
if [ -z "${GH_TOKEN:-}" ]; then
@@ -280,14 +259,16 @@ jobs:
echo ""
echo "- Suite: \`${SUITE}\`"
echo "- Run: ${RUN_URL}"
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
echo "- Workflow Commit: ${GITHUB_SHA}"
echo "- Trigger: ${GITHUB_EVENT_NAME}"
echo "- Date: $(date -u +%Y-%m-%d)"
echo ""
echo "## Report (errors and symptoms)"
echo ""
if [ -s "${REPORT_FILE}" ]; then
if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
redact < "${REPORT_FILE}"
elif [ -s "${LOG_FILE:-}" ]; then
elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
echo "(report file missing; log tail below)"
echo ""
tail -n 200 "${LOG_FILE}" | redact
@@ -303,14 +284,15 @@ jobs:
echo "filed backlog issue for suite ${SUITE}"
- name: Upload report and logs
if: always()
if: ${{ always() && steps.evidence.outcome == 'success' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: rustfs-storage-${{ github.run_id }}
name: rustfs-storage-${{ github.run_id }}-${{ github.run_attempt }}
path: |
/tmp/rustfs-storage.log
/tmp/rustfs-storage-report.md
if-no-files-found: warn
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
if-no-files-found: error
- name: Cleanup environment (after)
if: always()
.github/workflows/rustfs-tier-test.yml
+10 -5
View File
@@ -377,17 +377,22 @@ jobs:
fi
DATE="$(date -u +%Y-%m-%d)"
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
# Base64-encode the report into a temp file and feed it to jq via
# --rawfile: large reports (e.g. pool) exceed the OS argv limit and
# make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
B64_FILE="$(mktemp)"
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
if [ -n "${SHA}" ]; then
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
'{message:$msg, content:$content, sha:$sha}' \
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
else
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
'{message:$msg, content:$content}' \
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
'{message:$msg, content:($content|rtrimstr("\n"))}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
fi
rm -f "${B64_FILE}"
- name: Verify required tier evidence
id: evidence_verify
.github/workflows/rustfs-upgrade-test.yml
+68 -101
View File
@@ -82,6 +82,25 @@ jobs:
timeout-minutes: 420
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
- name: Checkout repository (for report parser)
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Initialize functional evidence
id: evidence
run: |
set -euo pipefail
umask 077
FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-upgrade-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
{
printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
} >> "${GITHUB_ENV}"
# auto-testing is private: clone it with the dedicated PF token (not
# GITHUB_TOKEN) and retry transient GitHub/network failures.
- name: Checkout auto-testing scripts (with retry)
@@ -142,7 +161,6 @@ jobs:
- name: Run upgrade compatibility suite
id: test
env:
LOG_FILE: /tmp/rustfs-upgrade.log
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
run: |
set -euo pipefail
@@ -200,10 +218,7 @@ jobs:
./auto-testing/rustfs-upgrade-test.sh "${ARGS[@]}"
- name: Generate report
if: always()
env:
LOG_FILE: /tmp/rustfs-upgrade.log
REPORT_FILE: /tmp/rustfs-upgrade-report.md
if: ${{ always() && steps.evidence.outcome == 'success' }}
run: |
set -euo pipefail
FROM_URL='${{ inputs.from_url }}'
@@ -224,103 +239,47 @@ jobs:
else
TO_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
fi
CASE_TABLE="/tmp/rustfs-upgrade-cases.md"
MATRIX_TABLE="/tmp/rustfs-upgrade-matrix.md"
python3 - "${LOG_FILE}" "${CASE_TABLE}" "${MATRIX_TABLE}" <<'PY'
import re
import sys
log_file, out_file, matrix_file = sys.argv[1], sys.argv[2], sys.argv[3]
ansi = re.compile(r'\x1b\[[0-9;]*m')
start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
topo_re = re.compile(
r'^\[UPG-TOPO\]\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+PASS=(\d+)\s+FAIL=(\d+)\s*$')
rows = []
index = {}
topo_rows = []
try:
with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
for raw in fh:
line = ansi.sub('', raw).strip()
m = topo_re.match(line)
if m:
topo_rows.append(m.groups())
continue
m = start_re.match(line)
if m:
case_id, name = m.group(1), m.group(2)
if case_id not in index:
index[case_id] = len(rows)
rows.append([case_id, name, 'RUNNING'])
continue
m = done_re.match(line)
if m:
status, case_id = m.group(1), m.group(2)
if case_id in index:
rows[index[case_id]][2] = status
else:
rows.append([case_id, case_id, status])
index[case_id] = len(rows) - 1
except FileNotFoundError:
rows = []
counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
for _, _, status in rows:
counts[status] = counts.get(status, 0) + 1
with open(out_file, 'w', encoding='utf-8') as out:
out.write('## Case Summary\n\n')
out.write(f"- Total: {len(rows)}\\n")
out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
out.write('\\n')
out.write('| Case | Name | Status |\\n')
out.write('| --- | --- | --- |\\n')
for case_id, name, status in rows:
out.write(f'| {case_id} | {name} | {status} |\\n')
# Upgrade matrix: one row per topology/backend with the versions
# captured on the nodes (rustfs --version) and the aggregated
# result. The dashboard renders this table directly.
with open(matrix_file, 'w', encoding='utf-8') as out:
out.write('## Upgrade Matrix\n\n')
out.write('| Topology | KMS Backend | From Version | To Version | Result |\n')
out.write('| --- | --- | --- | --- | --- |\n')
for topo, backend, old_v, new_v, npass, nfail in topo_rows:
result = 'PASS' if nfail == '0' else 'FAIL'
out.write(f'| {topo} | {backend} | {old_v} | {new_v} | {result} (PASS={npass} FAIL={nfail}) |\n')
if not topo_rows:
out.write('| - | - | - | - | NOT RUN (suite failed before upgrade) |\n')
PY
CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
MATRIX_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/matrix.md"
CASE_RESULT=success
python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" "${MATRIX_TABLE}" || CASE_RESULT=failure
RESULT=failure
if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
RESULT=success
fi
{
echo "# RustFS upgrade compatibility report"
echo ""
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
echo "- Workflow Commit: ${GITHUB_SHA}"
echo "- Trigger: ${{ github.event_name }}"
echo "- From: ${FROM_SOURCE}"
echo "- To: ${TO_SOURCE}"
echo "- Test Step Outcome: ${{ steps.test.outcome }}"
echo "- Test Step Outcome: ${RESULT}"
echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
echo ""
cat "${MATRIX_TABLE}" || true
echo ""
cat "${CASE_TABLE}" || true
echo ""
echo "## Log tail"
echo '```text'
tail -n 200 "${LOG_FILE}" || true
echo '```'
if [ "${RESULT}" = "success" ]; then
cat "${MATRIX_TABLE}"
echo ""
cat "${CASE_TABLE}"
echo ""
echo "## Log tail"
echo '```text'
tail -n 200 "${LOG_FILE}"
echo '```'
else
echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
fi
} | tee "${REPORT_FILE}"
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
[ "${RESULT}" = "success" ]
- name: Upload functional report to dashboard
if: always()
if: ${{ always() && steps.evidence.outcome == 'success' }}
continue-on-error: true
env:
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
REPORT_FILE: /tmp/rustfs-upgrade-report.md
SUITE: upgrade
run: |
set -euo pipefail
@@ -330,28 +289,32 @@ jobs:
fi
DATE="$(date -u +%Y-%m-%d)"
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
# Base64-encode the report into a temp file and feed it to jq via
# --rawfile: large reports (e.g. pool) exceed the OS argv limit and
# make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
B64_FILE="$(mktemp)"
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
if [ -n "${SHA}" ]; then
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
'{message:$msg, content:$content, sha:$sha}' \
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
else
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
'{message:$msg, content:$content}' \
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
'{message:$msg, content:($content|rtrimstr("\n"))}' \
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
fi
rm -f "${B64_FILE}"
- name: File failure issue in rustfs/backlog
if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
continue-on-error: true
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
SUITE: 'upgrade'
SUITE_LABEL: 'Upgrade compatibility'
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
REPORT_FILE: '/tmp/rustfs-upgrade-report.md'
LOG_FILE: '/tmp/rustfs-upgrade.log'
run: |
set -euo pipefail
if [ -z "${GH_TOKEN:-}" ]; then
@@ -379,14 +342,16 @@ jobs:
echo ""
echo "- Suite: \`${SUITE}\`"
echo "- Run: ${RUN_URL}"
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
echo "- Workflow Commit: ${GITHUB_SHA}"
echo "- Trigger: ${GITHUB_EVENT_NAME}"
echo "- Date: $(date -u +%Y-%m-%d)"
echo ""
echo "## Report (errors and symptoms)"
echo ""
if [ -s "${REPORT_FILE}" ]; then
if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
redact < "${REPORT_FILE}"
elif [ -s "${LOG_FILE:-}" ]; then
elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
echo "(report file missing; log tail below)"
echo ""
tail -n 200 "${LOG_FILE}" | redact
@@ -402,14 +367,16 @@ jobs:
echo "filed backlog issue for suite ${SUITE}"
- name: Upload report and logs
if: always()
if: ${{ always() && steps.evidence.outcome == 'success' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: rustfs-upgrade-test-${{ github.run_id }}
name: rustfs-upgrade-test-${{ github.run_id }}-${{ github.run_attempt }}
path: |
/tmp/rustfs-upgrade-report.md
/tmp/rustfs-upgrade.*/*
if-no-files-found: ignore
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/matrix.md
if-no-files-found: error
retention-days: 3
- name: Cleanup environment (after)
rustfs/src/app/bucket_list_through.rs
+2
View File
@@ -712,6 +712,8 @@ mod tests {
session_token: None,
}),
tls: TlsConfig::default(),
azure: None,
gcs: None,
},
filter: FilterConfig {
prefix: filter_prefix.map(str::to_string),
scripts/functional_case_report.py
+77
View File
@@ -0,0 +1,77 @@
#!/usr/bin/env python3
"""Preserve every functional case execution and its suite context in reports."""
from __future__ import annotations
import argparse
from pathlib import Path
import re
def generate_report(log_file: Path, case_file: Path, matrix_file: Path | None = None) -> bool:
ansi = re.compile(r"\x1b\[[0-9;]*m")
start_re = re.compile(r"^---\s+([A-Z][A-Z0-9]*-[0-9]+)\s+(.+?)\s+---$")
done_re = re.compile(r"^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z][A-Z0-9]*-[0-9]+)\b")
context_re = re.compile(r"^(?:\[INFO\]\s+)?==\s+((?:topology|suite):.+?)\s+==$")
topo_re = re.compile(r"^\[UPG-TOPO\]\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+PASS=(\d+)\s+FAIL=(\d+)\s*$")
rows = []
pending = {}
topo_rows = []
context = "context not recorded"
complete = True
try:
with log_file.open(encoding="utf-8", errors="replace") as log:
for raw in log:
line = ansi.sub("", raw).strip()
if match := context_re.match(line):
context = match[1]
pending.clear()
elif match := topo_re.match(line):
topo_rows.append(match.groups())
elif match := start_re.match(line):
case_id, name = match.groups()
pending[case_id] = len(rows)
rows.append([case_id, f"{name} ({context})", "RUNNING"])
elif match := done_re.match(line):
status, case_id = match.groups()
index = pending.pop(case_id, None)
if index is None:
complete = False
rows.append([case_id, f"{case_id} ({context}; start not recorded)", status])
else:
rows[index][2] = status
except FileNotFoundError:
pass
counts = {status: sum(row[2] == status for row in rows) for status in ("PASS", "FAIL", "UNSUPPORTED", "RUNNING")}
with case_file.open("w", encoding="utf-8") as out:
out.write(f"## Case Summary\n\n- Total: {len(rows)}\n")
for status, count in counts.items():
out.write(f"- {status}: {count}\n")
out.write("\n| Case | Name | Status |\n| --- | --- | --- |\n")
for row in rows:
out.write("| " + " | ".join(value.replace("|", "&#124;") for value in row) + " |\n")
if not rows:
out.write("\nNo case execution was recorded; the log is missing, empty, or stopped before the cases.\n")
valid = complete and bool(rows) and not counts["FAIL"] and not counts["RUNNING"]
if matrix_file is not None:
with matrix_file.open("w", encoding="utf-8") as out:
out.write("## Upgrade Matrix\n\n| Topology | KMS Backend | From Version | To Version | Result |\n")
out.write("| --- | --- | --- | --- | --- |\n")
for topo, backend, old_v, new_v, npass, nfail in topo_rows:
result = "PASS" if nfail == "0" else "FAIL"
out.write(f"| {topo} | {backend} | {old_v} | {new_v} | {result} (PASS={npass} FAIL={nfail}) |\n")
if not topo_rows:
out.write("| - | - | - | - | NOT RUN (suite failed before upgrade) |\n")
valid = valid and bool(topo_rows) and all(row[-1] == "0" for row in topo_rows)
return valid
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("log_file", type=Path)
parser.add_argument("case_file", type=Path)
parser.add_argument("matrix_file", type=Path, nargs="?")
args = parser.parse_args()
raise SystemExit(0 if generate_report(args.log_file, args.case_file, args.matrix_file) else 1)
scripts/test/oidc_keycloak_live.sh
+1 -3
View File
@@ -187,7 +187,7 @@ values = {}
for element in root.iter():
values[element.tag.rsplit("}", 1)[-1]] = element.text or ""
for field in ("AccessKeyId", "SecretAccessKey", "SessionToken", "Expiration", "SubjectFromWebIdentityToken"):
assert values.get(field), values
assert values.get(field), f"missing required STS field: {field}"
print("\t".join(values[field] for field in ("AccessKeyId", "SecretAccessKey", "SessionToken")))
PY
)
@@ -218,7 +218,6 @@ TAMPERED_STATUS="$(curl --noproxy '*' -sS \
--data-urlencode DurationSeconds=900 \
--data-urlencode "WebIdentityToken=${TAMPERED_TOKEN}")"
[[ "${TAMPERED_STATUS}" == 403 ]] || {
cat "${WORK_DIR}/sts-tampered.xml" >&2
echo "expected tampered token to return HTTP 403, got ${TAMPERED_STATUS}" >&2
exit 1
}
@@ -235,7 +234,6 @@ BAD_STATUS="$(curl --noproxy '*' -sS \
--data-urlencode DurationSeconds=900 \
--data-urlencode "WebIdentityToken=${BAD_TOKEN}")"
[[ "${BAD_STATUS}" == 403 ]] || {
cat "${WORK_DIR}/sts-bad.xml" >&2
echo "expected wrong-audience token to return HTTP 403, got ${BAD_STATUS}" >&2
exit 1
}
scripts/test_nightly_candidate.py
+208
View File
@@ -0,0 +1,208 @@
#!/usr/bin/env python3
"""Exercise the nightly publication step without AWS, network or package builds."""
import hashlib
import json
import os
from pathlib import Path
import subprocess
import tempfile
import unittest
from check_test_wiring import yaml_block
ROOT = Path(__file__).resolve().parents[1]
WORKFLOW = ROOT / ".github/workflows/nightly-gnu.yml"
class NightlyCandidateTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name)
self.package = self.root / "rustfs-nightly-2026-09-06.deb"
self.package.write_bytes(b"built package bytes\x00\xff")
for command in (["git", "init", "-q"], ["git", "add", self.package.name],
["git", "-c", "user.name=Test", "-c", "user.email=test@example.invalid", "commit", "-qm", "fixture"]):
subprocess.run(command, cwd=self.root, check=True, capture_output=True)
self.sha = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=self.root, text=True).strip()
self.digest = hashlib.sha256(self.package.read_bytes()).hexdigest()
self.output = self.root / "github-output"
self.store = self.root / "store"
self.shims = self.root / "fake-tools.sh"
self.shims.write_text(r'''aws() {
printf '%s\n' "$*" >> "$FAKE_AWS_LOG"
if [[ "$1" == --version ]]; then printf 'aws-cli/1.44.79 fixture\n'; return; fi
if [[ "$*" == *--generate-cli-skeleton* ]]; then
if [[ "$FAKE_MODE" == broken-install || "$FAKE_MODE" =~ ^(old-cli|bootstrap-failure|install-failure)$ && ! -e "$FAKE_INSTALLED" ]]; then
printf '{}\n'
else
printf '{"IfNoneMatch":""}\n'
fi
return
fi
if [[ "$1 $2" == 's3api put-object' ]]; then
[[ "$FAKE_MODE" != upload-failure ]] || return 42
shift 2
local key="" body="" condition=""
while [[ $# -gt 0 ]]; do
case "$1" in
--key) key="$2";;
--body) body="$2";;
--if-none-match) condition="$2";;
esac
shift 2
done
[[ -z "$condition" || "$condition" == '*' ]] || return 43
if [[ "$condition" == '*' && -e "$FAKE_STORE/$key" ]]; then return 44; fi
mkdir -p "$(dirname "$FAKE_STORE/$key")"
cp "$body" "$FAKE_STORE/$key"
elif [[ "$1 $2" == 's3 cp' ]]; then
[[ "$FAKE_MODE" != alias-failure ]] || return 45
local destination="${4#s3://test-bucket/}"
[[ "$destination" != */ ]] || destination+="$(basename "$3")"
mkdir -p "$(dirname "$FAKE_STORE/$destination")"
cp "$3" "$FAKE_STORE/$destination"
else
return 46
fi
}
curl() {
local url="${!#}"
printf '%s\n' "$url" >> "$FAKE_CURL_LOG"
[[ "$url" == https://dl.rustfs.com/artifacts/rustfs/packages/nightly/runs/* ]] || return 22
[[ "$FAKE_MODE" != missing-public-url ]] || return 22
if [[ "$FAKE_MODE" == wrong-public-bytes ]]; then printf 'different package'; return; fi
cat "$FAKE_STORE/${url#https://dl.rustfs.com/}" || return 22
[[ "$FAKE_MODE" != incomplete-download ]] || return 47
}
sudo() {
[[ "$*" == 'apt-get update' || "$*" == 'apt-get install -y -qq python3-venv' ]] || return 49
[[ "$FAKE_MODE" != bootstrap-failure ]] || return 48
}
python3() {
[[ "$1 $2" == '-m venv' ]] || return 50
mkdir -p "$3/bin"
cat > "$3/bin/python" <<'SH'
#!/usr/bin/env bash
[[ "$*" == '-m pip install --disable-pip-version-check awscli==1.44.79' ]] || exit 51
[[ "$FAKE_MODE" != install-failure ]] || exit 52
: > "$FAKE_INSTALLED"
SH
printf '#!/usr/bin/env bash\naws "$@"\n' > "$3/bin/aws"
chmod +x "$3/bin/python" "$3/bin/aws"
}
''')
self.env = dict(os.environ, BASH_ENV=str(self.shims), DEB_FILE=self.package.name,
R2_ACCESS_KEY_ID="fake-access", R2_SECRET_ACCESS_KEY="fake-secret", R2_ENDPOINT="https://r2.example.invalid", R2_BUCKET="test-bucket",
RUNNER_TEMP=str(self.root), GITHUB_SHA=self.sha, GITHUB_RUN_ID="12345", GITHUB_RUN_ATTEMPT="1", GITHUB_OUTPUT=str(self.output),
FAKE_STORE=str(self.store), FAKE_AWS_LOG=str(self.root / "aws.log"), FAKE_CURL_LOG=str(self.root / "curl.log"), FAKE_INSTALLED=str(self.root / "installed"), FAKE_MODE="success")
source = WORKFLOW.read_text()
job = yaml_block(source.splitlines(), "build", 2)
starts = [i for i, line in enumerate(job) if line.startswith(" - name: ")]
self.steps = {
job[start].split(": ", 1)[1]: job[start:end]
for start, end in zip(starts, starts[1:] + [len(job)])
}
self.publish = self.steps["Upload DEB to Cloudflare R2"]
start = self.publish.index(" run: |") + 1
self.shell = "\n".join(line[10:] for line in self.publish[start:] if not line.strip() or line.startswith(" "))
def run_publish(self, **overrides):
self.output.unlink(missing_ok=True)
return subprocess.run(["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.shell],
cwd=self.root, env=dict(self.env, **overrides), capture_output=True, text=True)
def manifest(self):
output = self.output.read_text().strip()
self.assertTrue(output.startswith("candidate_file="), output)
return json.loads(Path(output.split("=", 1)[1]).read_text())
def test_success_binds_actual_package_checkout_and_attempt(self):
result = self.run_publish()
self.assertEqual(result.returncode, 0, result.stderr)
manifest = self.manifest()
self.assertEqual(manifest, {"schema": 1, "source_sha": self.sha, "build_run_id": 12345, "build_run_attempt": 1,
"package_sha256": self.digest, "package_url": f"https://dl.rustfs.com/artifacts/rustfs/packages/nightly/runs/12345/1/{self.digest}/rustfs.deb"})
for path in (f"runs/12345/1/{self.digest}/rustfs.deb", self.package.name, "rustfs-nightly-latest.deb"):
self.assertEqual((self.store / "artifacts/rustfs/packages/nightly" / path).read_bytes(), self.package.read_bytes())
self.assertEqual((self.root / "curl.log").read_text().strip(), manifest["package_url"])
def test_missing_credentials_remain_artifact_only(self):
for key in ("R2_ACCESS_KEY_ID", "R2_SECRET_ACCESS_KEY", "R2_ENDPOINT", "R2_BUCKET"):
with self.subTest(missing=key):
result = self.run_publish(**{key: ""})
self.assertEqual(result.returncode, 0, result.stderr)
self.assertFalse(self.output.exists())
self.assertFalse((self.root / "aws.log").exists())
self.assertEqual(list(self.root.glob("nightly-candidate-*.json")), [])
def test_publication_failures_never_emit_a_candidate(self):
for index, mode in enumerate(("upload-failure", "missing-public-url", "wrong-public-bytes", "incomplete-download", "alias-failure")):
with self.subTest(mode=mode):
result = self.run_publish(FAKE_MODE=mode, GITHUB_RUN_ID=str(20000 + index))
self.assertNotEqual(result.returncode, 0, result.stdout)
self.assertFalse(self.output.exists())
self.assertEqual(list(self.root.glob("nightly-candidate-*.json")), [])
def test_old_cli_is_upgraded_in_an_isolated_temporary_environment(self):
result = self.run_publish(FAKE_MODE="old-cli")
self.assertEqual(result.returncode, 0, result.stderr)
self.assertTrue((self.root / "installed").exists())
self.assertEqual(self.manifest()["package_sha256"], self.digest)
self.assertEqual(list(self.root.glob("nightly-awscli.*")), [])
def test_failed_cli_bootstrap_cannot_publish(self):
for mode in ("bootstrap-failure", "install-failure", "broken-install"):
with self.subTest(mode=mode):
(self.root / "installed").unlink(missing_ok=True)
result = self.run_publish(FAKE_MODE=mode)
self.assertNotEqual(result.returncode, 0)
self.assertFalse(self.output.exists())
self.assertFalse(self.store.exists())
self.assertEqual(list(self.root.glob("nightly-awscli.*")), [])
def test_checkout_sha_mismatch_fails_before_upload(self):
result = self.run_publish(GITHUB_SHA="f" * 40)
self.assertNotEqual(result.returncode, 0)
self.assertIn("Checkout SHA", result.stderr)
self.assertFalse(self.output.exists())
self.assertFalse((self.root / "aws.log").exists())
def test_same_date_builds_and_reruns_keep_distinct_candidates(self):
urls = []
for run, attempt in (("12345", "1"), ("54321", "1"), ("12345", "2")):
result = self.run_publish(GITHUB_RUN_ID=run, GITHUB_RUN_ATTEMPT=attempt)
self.assertEqual(result.returncode, 0, result.stderr)
urls.append(self.manifest()["package_url"])
self.assertEqual(len(set(urls)), 3)
self.assertEqual(len(list(self.root.glob("nightly-candidate-*.json"))), 3)
def test_duplicate_key_is_not_overwritten_or_recertified(self):
result = self.run_publish()
self.assertEqual(result.returncode, 0, result.stderr)
key = self.manifest()["package_url"].removeprefix("https://dl.rustfs.com/")
stored = self.store / key
stored.write_bytes(b"preexisting conflicting object")
(self.root / "nightly-candidate-12345-1.json").unlink()
result = self.run_publish()
self.assertNotEqual(result.returncode, 0)
self.assertEqual(stored.read_bytes(), b"preexisting conflicting object")
self.assertFalse(self.output.exists())
self.assertEqual(list(self.root.glob("nightly-candidate-*.json")), [])
def test_manifest_upload_requires_publication_output(self):
upload = self.steps["Upload nightly candidate manifest"]
self.assertIn(" id: publish", self.publish)
self.assertIn(" DEB_FILE: ${{ steps.deb.outputs.deb_file }}", self.publish)
self.assertIn(" if: ${{ steps.publish.outputs.candidate_file != '' }}", upload)
self.assertIn(" name: nightly-candidate-${{ github.run_id }}-${{ github.run_attempt }}", upload)
self.assertIn(" path: ${{ steps.publish.outputs.candidate_file }}", upload)
self.assertIn(" if-no-files-found: error", upload)
self.assertNotIn(" continue-on-error: true", self.publish)
self.assertNotIn(" overwrite: true", upload)
if __name__ == "__main__":
unittest.main()
scripts/test_security_workflow.py
+613 -64
View File
@@ -1,16 +1,20 @@
#!/usr/bin/env python3
"""Exercise functional workflow failures and security evidence without remote VMs."""
"""Exercise functional failures, chain dispatch, and security evidence without remote VMs."""
from __future__ import annotations
import glob
import json
import os
import re
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
from check_test_wiring import yaml_block
from functional_case_report import generate_report
ROOT = Path(__file__).resolve().parents[1]
@@ -38,7 +42,46 @@ def shell_body(lines: list[str]) -> str:
return "\n".join(shell_lines)
class SecurityWorkflowTests(unittest.TestCase):
class WorkflowSteps:
def uploaded_files(self) -> set[Path]:
upload = next(lines for lines in self.steps.values() if any("uses: actions/upload-artifact@" in line for line in lines))
start = upload.index(" path: |") + 1
paths = []
for line in upload[start:]:
if not line.startswith(" "):
break
paths.extend(Path(path) for path in glob.glob(self.render(line.strip())))
return {file for path in paths for file in (path.rglob("*") if path.is_dir() else [path]) if file.is_file()}
def render(self, value: str) -> str:
return re.sub(r"\$\{\{\s*(.*?)\s*\}\}", lambda match: self.context[match[1]], value)
def step_env(self, lines: list[str], indent: int = 8) -> dict[str, str]:
result = {}
for line in yaml_block(lines, "env", indent) or []:
if line.strip() and not line.lstrip().startswith("#"):
key, value = line.strip().split(": ", 1)
result[key] = self.render(value.strip("'\""))
return result
def run_step(self, name: str) -> subprocess.CompletedProcess[str]:
lines = self.steps[name]
result = subprocess.run(
["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.render(shell_body(lines))],
cwd=self.directory, env={**self.env, **self.step_env(lines)}, capture_output=True, text=True,
)
for line in lines:
if line.startswith(" id: "):
self.context[f"steps.{line.split(': ', 1)[1]}.outcome"] = "failure" if result.returncode else "success"
if Path(self.env["GITHUB_ENV"]).exists():
for line in Path(self.env["GITHUB_ENV"]).read_text().splitlines():
key, value = line.split("=", 1)
self.env[key] = value
self.context[f"env.{key}"] = value
return result
class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
def setUp(self) -> None:
self.source = WORKFLOW.read_text()
self.job = yaml_block(self.source.splitlines(), "security-test", 2)
@@ -78,6 +121,7 @@ class SecurityWorkflowTests(unittest.TestCase):
'#!/usr/bin/env bash\nset -euo pipefail\n'
'log_dir=$(mktemp -d "$TMPDIR/rustfs-security.XXXXXX")\n'
'echo "CURRENT SUITE LOG" > "$log_dir/suite.log"\n'
'echo "CURRENT SUITE STDOUT"; echo "CURRENT SUITE STDERR" >&2\n'
'case "$FAKE_REPORT" in\n'
f' present) printf "%s\\n" "CURRENT SUITE DIAGNOSTIC" "{CASE_ROW}" > "$REPORT_FILE" ;;\n'
' empty) : > "$REPORT_FILE" ;;\n'
@@ -86,33 +130,6 @@ class SecurityWorkflowTests(unittest.TestCase):
'exit "$FAKE_EXIT"\n'
)
def render(self, value: str) -> str:
return re.sub(r"\$\{\{\s*(.*?)\s*\}\}", lambda match: self.context[match[1]], value)
def step_env(self, lines: list[str], indent: int = 8) -> dict[str, str]:
result = {}
for line in yaml_block(lines, "env", indent) or []:
if line.strip() and not line.lstrip().startswith("#"):
key, value = line.strip().split(": ", 1)
result[key] = self.render(value.strip("'\""))
return result
def run_step(self, name: str) -> subprocess.CompletedProcess[str]:
lines = self.steps[name]
result = subprocess.run(
["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.render(shell_body(lines))],
cwd=self.directory, env={**self.env, **self.step_env(lines)}, capture_output=True, text=True,
)
for line in lines:
if line.startswith(" id: "):
self.context[f"steps.{line.split(': ', 1)[1]}.outcome"] = "failure" if result.returncode else "success"
if Path(self.env["GITHUB_ENV"]).exists():
for line in Path(self.env["GITHUB_ENV"]).read_text().splitlines():
key, value = line.split("=", 1)
self.env[key] = value
self.context[f"env.{key}"] = value
return result
def test_workflow_wiring(self) -> None:
names = list(self.steps)
self.assertLess(names.index("Checkout repository (for the OIDC live gate script)"), names.index("Checkout auto-testing scripts (with retry)"))
@@ -128,7 +145,7 @@ class SecurityWorkflowTests(unittest.TestCase):
for name in ("Upload functional report to dashboard", "Upload report and logs"):
self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", self.steps[name])
artifact_settings = yaml_block(self.steps["Upload report and logs"], "with", 8)
self.assertIn(" path: ${{ env.SECURITY_ARTIFACTS_DIR }}/", artifact_settings)
self.assertIn(" path: |", artifact_settings)
self.assertIn(" if-no-files-found: error", artifact_settings)
def test_suite_report_and_result_matrix(self) -> None:
@@ -147,7 +164,7 @@ class SecurityWorkflowTests(unittest.TestCase):
if outcome != "skipped" or mode == "present":
suite = self.run_step("Run security suite")
self.assertEqual(suite.returncode, exit_code, suite.stderr)
logs = list(self.artifacts.glob("rustfs-security.*/suite.log"))
logs = list(Path(str(self.artifacts) + "-scratch").glob("rustfs-security.*/suite.log"))
self.assertEqual(len(logs), 1)
self.assertEqual(logs[0].read_text(), "CURRENT SUITE LOG\n")
self.context["steps.test.outcome"] = outcome
@@ -169,37 +186,195 @@ class SecurityWorkflowTests(unittest.TestCase):
summary = Path(self.env["GITHUB_STEP_SUMMARY"]).read_text()
self.assertEqual(summary, contents)
self.assertNotIn("UNWRAPPED SUITE SUMMARY", summary)
expected = {self.artifacts / "report.md"}
if outcome != "skipped" or mode == "present":
expected.add(self.artifacts / "suite.log")
self.assertEqual((self.artifacts / "suite.log").read_text(), "CURRENT SUITE STDOUT\nCURRENT SUITE STDERR\n")
if mode in ("present", "empty"):
expected.add(self.artifacts / "suite-report.md")
(self.artifacts / "unexpected-token.json").write_text("FAKE-SECRET-CANARY")
scratch = Path(str(self.artifacts) + "-scratch")
(scratch / "case.out").write_text("FAKE-SECRET-CANARY")
self.assertEqual(self.uploaded_files(), expected)
def test_oidc_negative_responses_never_print_issued_credentials(self):
source = (ROOT / "scripts/test/oidc_keycloak_live.sh").read_text()
for variable, filename in (("TAMPERED_STATUS", "sts-tampered.xml"), ("BAD_STATUS", "sts-bad.xml")):
start = source.index('[[ "${' + variable + '}" == 403 ]]')
end = source.index("\n", source.index("grep -q '<Code>AccessDenied</Code>'", start))
guard = source[start:end]
credential_xml = "<Credentials><AccessKeyId>FAKE-ACCESS-CANARY</AccessKeyId><SecretAccessKey>FAKE-SECRET-CANARY</SecretAccessKey><SessionToken>FAKE-SESSION-CANARY</SessionToken></Credentials>"
for status, body, expected in (("200", credential_xml, 1), ("403", credential_xml, 1),
("403", "<Code>AccessDenied</Code>", 0)):
with self.subTest(variable=variable, status=status, expected=expected):
(self.directory / filename).write_text(body)
result = subprocess.run(["bash", "-e", "-o", "pipefail", "-c", guard],
env={**self.env, variable: status, "WORK_DIR": str(self.directory)},
capture_output=True, text=True)
self.assertEqual(result.returncode, expected, result.stderr)
for canary in ("FAKE-ACCESS-CANARY", "FAKE-SECRET-CANARY", "FAKE-SESSION-CANARY"):
self.assertNotIn(canary, result.stdout + result.stderr)
if status == "200":
self.assertIn("HTTP 403, got 200", result.stderr)
def test_oidc_incomplete_credentials_report_only_the_missing_field(self):
source = (ROOT / "scripts/test/oidc_keycloak_live.sh").read_text()
start = source.index("IFS=$'\\t' read -r STS_ACCESS_KEY")
end = source.index("\n)\n", start) + 3
extract = source[start:end]
values = {"AccessKeyId": "FAKE-ACCESS-CANARY", "SecretAccessKey": "FAKE-SECRET-CANARY",
"SessionToken": "FAKE-SESSION-CANARY", "Expiration": "2099-01-01T00:00:00Z",
"SubjectFromWebIdentityToken": "alice"}
for missing in (None, "Expiration", "SubjectFromWebIdentityToken"):
with self.subTest(missing=missing):
xml = "<Credentials>" + "".join(f"<{key}>{value}</{key}>" for key, value in values.items() if key != missing) + "</Credentials>"
(self.directory / "sts-good.xml").write_text(xml)
result = subprocess.run(["bash", "-e", "-o", "pipefail", "-c", extract],
env={**self.env, "WORK_DIR": str(self.directory)}, capture_output=True, text=True)
self.assertEqual(result.returncode, 1 if missing else 0, result.stderr)
for canary in ("FAKE-ACCESS-CANARY", "FAKE-SECRET-CANARY", "FAKE-SESSION-CANARY"):
self.assertNotIn(canary, result.stdout + result.stderr)
if missing:
self.assertIn(f"missing required STS field: {missing}", result.stderr)
def test_existing_evidence_directory_is_rejected(self) -> None:
self.artifacts.mkdir()
stale = self.artifacts / "suite-report.md"
stale.write_text("OLD RUN REPORT")
self.assertNotEqual(self.run_step("Initialize security evidence").returncode, 0)
self.assertEqual(stale.read_text(), "OLD RUN REPORT")
self.assertFalse(Path(self.env["GITHUB_ENV"]).exists())
(self.artifacts / "report.md").write_text("OLD RUN REPORT")
self.context.update({
"env.SECURITY_ARTIFACTS_DIR": str(self.artifacts), "secrets.PF_TESTING_GH_TOKEN": "fake-local-token",
})
fake_bin = self.directory / "bin"
fake_bin.mkdir()
gh = fake_bin / "gh"
gh.write_text(
'#!/usr/bin/env bash\nset -euo pipefail\n'
'if [ "$1 $2" = "issue create" ]; then\n'
' while [ "$#" -gt 0 ]; do\n'
' if [ "$1" = "--body-file" ]; then cat "$2" > "$CAPTURE_BODY"; fi\n'
' shift\n'
' done\n'
'fi\n'
)
gh.chmod(0o755)
body = self.directory / "issue-body.md"
self.env.update(PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", CAPTURE_BODY=str(body))
result = self.run_step("File failure issue in rustfs/backlog")
self.assertEqual(result.returncode, 0, result.stderr)
self.assertNotIn("OLD RUN REPORT", body.read_text())
self.assertIn("https://github.com/rustfs/rustfs/actions/runs/314159", body.read_text())
for suffix in ("", "-scratch"):
self.setUp()
existing = Path(str(self.artifacts) + suffix)
existing.mkdir()
stale = existing / "suite-report.md"
stale.write_text("OLD RUN REPORT")
self.assertNotEqual(self.run_step("Initialize security evidence").returncode, 0)
self.assertEqual(stale.read_text(), "OLD RUN REPORT")
self.assertFalse(Path(self.env["GITHUB_ENV"]).exists())
(self.artifacts / "report.md").write_text("OLD RUN REPORT")
self.context.update({
"env.SECURITY_ARTIFACTS_DIR": str(self.artifacts), "secrets.PF_TESTING_GH_TOKEN": "fake-local-token",
})
fake_bin = self.directory / "bin"
fake_bin.mkdir()
gh = fake_bin / "gh"
gh.write_text(
'#!/usr/bin/env bash\nset -euo pipefail\n'
'if [ "$1 $2" = "issue create" ]; then\n'
' while [ "$#" -gt 0 ]; do\n'
' if [ "$1" = "--body-file" ]; then cat "$2" > "$CAPTURE_BODY"; fi\n'
' shift\n'
' done\n'
'fi\n'
)
gh.chmod(0o755)
body = self.directory / "issue-body.md"
self.env.update(PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", CAPTURE_BODY=str(body))
result = self.run_step("File failure issue in rustfs/backlog")
self.assertEqual(result.returncode, 0, result.stderr)
self.assertNotIn("OLD RUN REPORT", body.read_text())
self.assertIn("https://github.com/rustfs/rustfs/actions/runs/314159", body.read_text())
def test_all_ten_suites_hold_the_shared_lock_for_manual_and_chain_runs(self) -> None:
for suite in ("upgrade", "s3-compat", "kms", "tier", "storage", "heal", "pool-expand", "security", "replication", "performance"):
with self.subTest(suite=suite):
source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text().splitlines()
# Workflow-level concurrency covers every job, including cleanup,
# regardless of trigger or the runner hosting the job.
self.assertEqual([
line.strip() for line in yaml_block(source, "concurrency", 0)
if line.strip() and not line.lstrip().startswith("#")
], [
"group: rustfs-shared-functional-tests", "cancel-in-progress: false",
])
self.assertIsNotNone(yaml_block(source, "workflow_dispatch", 2))
self.assertIsNotNone(yaml_block(source, "repository_dispatch", 2))
cleanup_name = "Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"
cleanup = named_steps(yaml_block(source, "jobs", 0))[cleanup_name]
self.assertTrue(any(line.startswith(" if:") and "always()" in line for line in cleanup))
def test_root_dispatches_only_upgrade_and_replication_hands_off_after_failure(self) -> None:
for failed_attempts, issue_exit, token in ((0, 0, "fixture"), (2, 0, "fixture"), (3, 0, "fixture"), (3, 7, "fixture"), (0, 0, "")):
with self.subTest(failed_attempts=failed_attempts, issue_exit=issue_exit, token=bool(token)):
self.setUp()
fake_bin = self.directory / "bin"
fake_bin.mkdir()
commands = {
"gh": '''#!/usr/bin/env bash
set -euo pipefail
if [ "$1" = api ]; then
printf '%s\\n' "$*" >> "$DISPATCHES"
attempt=$(wc -l < "$DISPATCHES")
[ "$attempt" -gt "$FAILED_ATTEMPTS" ]
elif [ "$1 $2" = 'issue create' ]; then
printf 'issue\\n' >> "$EXECUTED"
while [ "$#" -gt 0 ]; do
if [ "$1" = --body-file ]; then
cat "$2" > "$CAPTURE_BODY"
printf '%s\\n' "$2" > "$CAPTURE_BODY_PATH"
fi
shift
done
exit "$ISSUE_EXIT"
else
exit 99
fi
''',
"sleep": '#!/bin/sh\nprintf "sleep %s\\n" "$1" >> "$EXECUTED"\n',
"ssh": '#!/bin/sh\nprintf "cleanup\\n" >> "$EXECUTED"\n',
}
for name, contents in commands.items():
command = fake_bin / name
command.write_text(contents)
command.chmod(0o755)
dispatches = self.directory / "dispatches"
executed = self.directory / "executed"
body = self.directory / "issue-body.md"
body_path = self.directory / "issue-body-path"
self.env.update(
PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", DISPATCHES=str(dispatches),
EXECUTED=str(executed), CAPTURE_BODY=str(body), CAPTURE_BODY_PATH=str(body_path),
FAILED_ATTEMPTS="0", ISSUE_EXIT=str(issue_exit),
RUSTFS_NODES="fixture-node", RUSTFS_SSH_USER="fixture-user",
RUSTFS_NIGHTLY_PACKAGE_URL="https://example.invalid/package.deb",
)
self.context.update({"secrets.PF_TESTING_GH_TOKEN": "fixture", "inputs.suite": "all"})
driver = (ROOT / ".github/workflows/rustfs-functional-chain.yml").read_text()
self.steps = named_steps(yaml_block(driver.splitlines(), "start-chain", 2))
self.assertEqual(list(self.steps), ["Dispatch first suite (upgrade)"])
started = self.run_step("Dispatch first suite (upgrade)")
self.assertEqual(started.returncode, 0, started.stderr)
self.assertEqual(dispatches.read_text().splitlines(), [
"api --method POST repos/rustfs/rustfs/dispatches -f event_type=rustfs-chain-upgrade -F client_payload[from_suite]=nightly-build",
])
dispatches.unlink()
replication = (ROOT / ".github/workflows/rustfs-replication-test.yml").read_text()
job = yaml_block(replication.splitlines(), "replication-test", 2)
self.assertFalse(any(line.startswith(" continue-on-error:") for line in job))
self.steps = named_steps(job)
handoff = "Continue functional chain (next: Performance)"
self.assertIn(" if: ${{ always() && github.event_name == 'repository_dispatch' }}", self.steps[handoff])
self.assertFalse(any(line.strip().startswith("continue-on-error:") for line in self.steps[handoff]))
self.assertIn(" if: always()", self.steps["Cleanup environment (after)"])
self.assertLess(list(self.steps).index("Cleanup environment (after)"), list(self.steps).index(handoff))
suite = self.directory / "auto-testing/rustfs-replication-test.sh"
suite.write_text('#!/bin/sh\nprintf "suite failed\\n" >> "$EXECUTED"\nexit 17\n')
failed = self.run_step("Run replication suite")
self.assertEqual(failed.returncode, 17, failed.stderr)
cleaned = self.run_step("Cleanup environment (after)")
self.assertEqual(cleaned.returncode, 0, cleaned.stderr)
self.assertEqual(executed.read_text().splitlines(), ["suite failed", "cleanup"])
self.env["FAILED_ATTEMPTS"] = str(failed_attempts)
self.context["secrets.PF_TESTING_GH_TOKEN"] = token
forwarded = self.run_step(handoff)
self.assertEqual(forwarded.returncode == 0, bool(token) and failed_attempts < 3, forwarded.stderr)
calls = dispatches.read_text().splitlines() if dispatches.exists() else []
self.assertEqual(calls, [
"api --method POST repos/rustfs/rustfs/dispatches -f event_type=rustfs-chain-performance -F client_payload[from_suite]=replication",
] * (min(failed_attempts + 1, 3) if token else 0))
if failed_attempts == 3:
self.assertIn("could not hand off from **replication** to **Performance**", body.read_text())
self.assertIn("rustfs-chain-performance", body.read_text())
self.assertEqual(executed.read_text().splitlines().count("issue"), 2 if issue_exit else 1)
self.assertFalse(Path(body_path.read_text().strip()).exists())
class FunctionalWorkflowTests(unittest.TestCase):
@@ -225,7 +400,7 @@ class FunctionalWorkflowTests(unittest.TestCase):
if suite in self.DIRECT_TESTS:
test = steps[self.DIRECT_TESTS[suite]]
self.assertNotRegex("\n".join(test), r'''(?m)^ ["']?continue-on-error["']?\s*:''')
self.assertIn(" if: always()", steps["Generate report"])
self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", steps["Generate report"])
cleanup = steps["Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"]
condition = next(line.strip() for line in cleanup if line.startswith(" if:"))
self.assertIn(condition, (
@@ -234,7 +409,7 @@ class FunctionalWorkflowTests(unittest.TestCase):
"if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}",
))
if suite != "performance":
handoff = steps["Chain complete"] if suite == "replication" else next(
handoff = next(
value for name, value in steps.items() if name.startswith("Continue functional chain")
)
self.assertIn(" if: ${{ always() && github.event_name == 'repository_dispatch' }}", handoff)
@@ -277,13 +452,387 @@ class FunctionalWorkflowTests(unittest.TestCase):
self.assertIn("partial suite diagnostics", failed.stdout)
cleanup = execute("Cleanup environment (after)")
self.assertEqual(cleanup.returncode, 0, cleanup.stderr)
handoff_name = "Chain complete" if suite == "replication" else next(
handoff_name = next(
name for name in steps if name.startswith("Continue functional chain")
)
handoff = execute(handoff_name)
self.assertEqual(handoff.returncode, 0, handoff.stderr)
markers = (root / "executed").read_text().splitlines()
self.assertEqual(markers, ["cleanup"] if suite == "replication" else ["cleanup", "dispatch"])
self.assertEqual(markers, ["cleanup", "dispatch"])
class FunctionalCaseReportTests(unittest.TestCase):
def report(self, text: str | None, matrix: bool = False) -> tuple[bool, str, str]:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
log = root / "suite.log"
if text is not None:
log.write_text(text)
valid = generate_report(log, root / "cases.md", root / "matrix.md" if matrix else None)
return valid, (root / "cases.md").read_text(), (root / "matrix.md").read_text() if matrix else ""
def test_repeated_case_executions_preserve_failure_and_context(self):
# log() from rustfs/auto-testing@6120aa0a76de, rustfs-kms-test.sh:131.
log = subprocess.check_output(["bash", "-c", r'''
log() { printf '\033[1;36m[INFO]\033[0m %s\n' "$*"; }
log '== topology: single-single kms-backend: local =='
printf '\033[32m--- KMS-101 roundtrip ---\033[0m\n[FAIL] KMS-101\n'
log '== topology: single-multi kms-backend: vault-kv2 =='
printf '%s\n' '--- KMS-101 roundtrip ---' '[PASS] KMS-101'
printf '%s\n' '--- KMS-101 roundtrip ---' '[UNSUPPORTED] KMS-101'
'''], text=True)
valid, cases, _ = self.report(log)
self.assertFalse(valid)
self.assertEqual(cases.count("| KMS-101 |"), 3)
self.assertIn("- Total: 3\n- PASS: 1\n- FAIL: 1\n- UNSUPPORTED: 1\n- RUNNING: 0\n", cases)
self.assertIn("roundtrip (topology: single-single kms-backend: local) | FAIL |", cases)
self.assertIn("roundtrip (topology: single-multi kms-backend: vault-kv2) | PASS |", cases)
self.assertNotIn("\\n", cases)
def test_missing_empty_unfinished_and_orphan_results_are_not_success(self):
for text in (None, "", "setup failed\n", "--- KMS-101 roundtrip ---\n", "[PASS] KMS-101\n",
"--- KMS-101 first ---\n--- KMS-101 second ---\n[PASS] KMS-101\n",
"--- KMS-101 first ---\n[FAIL] KMS-101\n[PASS] KMS-101\n"):
with self.subTest(log=text):
valid, cases, _ = self.report(text)
self.assertFalse(valid)
self.assertIn("## Case Summary", cases)
valid, cases, _ = self.report("[INFO] == suite: bucket replication (REP-*) ==\n--- REP-101 unsupported ---\n[UNSUPPORTED] REP-101\n")
self.assertTrue(valid)
self.assertIn("suite: bucket replication", cases)
self.assertIn("- UNSUPPORTED: 1\n", cases)
def test_upgrade_matrix_is_preserved_and_required_for_complete_report(self):
case = "--- UPG-101 upgrade ---\n[PASS] UPG-101\n"
for suffix, expected in (("", False), ("[UPG-TOPO] single-single local v1 v2 PASS=1 FAIL=0\n", True),
("[UPG-TOPO] single-single local v1 v2 PASS=1 FAIL=1\n", False)):
with self.subTest(matrix=suffix):
valid, _, matrix = self.report(case + suffix, matrix=True)
self.assertEqual(valid, expected)
self.assertIn("| Topology | KMS Backend | From Version | To Version | Result |", matrix)
self.assertIn("| single-single | local | v1 | v2 |" if suffix else "NOT RUN", matrix)
def test_s3_case_identifiers_include_digits(self):
valid, cases, _ = self.report("--- S3C-101 CreateBucket ---\n[PASS] S3C-101\n")
self.assertTrue(valid)
self.assertIn("| S3C-101 | CreateBucket (context not recorded) | PASS |", cases)
class FunctionalEvidenceTests(WorkflowSteps, unittest.TestCase):
SUITES = (*FunctionalWorkflowTests.DIRECT_TESTS, "heal", "performance")
def prepare(self, suite: str) -> None:
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.directory = Path(self.temp.name)
self.source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text()
self.steps = named_steps(yaml_block(self.source.splitlines(), FunctionalWorkflowTests.JOBS[suite], 2))
self.context = {expression: "" for expression in re.findall(r"\$\{\{\s*(.*?)\s*\}\}", self.source)}
self.context.update({
"github.server_url": "https://github.com", "github.repository": "rustfs/rustfs",
"github.run_id": "314159", "github.run_attempt": "2", "github.sha": "0123456789abcdef0123456789abcdef01234567",
"github.event_name": "repository_dispatch", "steps.test.outcome": "success",
"secrets.PF_TESTING_GH_TOKEN": "local-fixture", "env.PF_TESTING_GH_TOKEN": "local-fixture",
})
self.artifacts = self.directory / f"rustfs-{suite}-314159-2"
self.env = {
**os.environ, "GITHUB_ENV": str(self.directory / "github-env"), "RUNNER_TEMP": self.temp.name,
"GITHUB_STEP_SUMMARY": str(self.directory / "summary.md"), "RUSTFS_NODES": "fixture-node",
"RUSTFS_NIGHTLY_PACKAGE_URL": "https://example.invalid/package.deb", "CAPTURE_BODY": str(self.directory / "issue.md"),
}
for key in ("server_url", "repository", "run_id", "run_attempt", "sha", "event_name"):
self.env[f"GITHUB_{key.upper()}"] = self.context[f"github.{key}"]
(self.directory / "scripts").mkdir()
(self.directory / "scripts/functional_case_report.py").symlink_to(ROOT / "scripts/functional_case_report.py")
fake_bin = self.directory / "bin"
fake_bin.mkdir()
(fake_bin / "python3").symlink_to(sys.executable)
for command, body in (
("ssh", 'printf "fixture-version\\n"\n'),
("gh", 'if [ "$1 $2" = "issue create" ]; then\n'
' while [ "$#" -gt 0 ]; do\n'
' if [ "$1" = "--body-file" ]; then cat "$2" > "$CAPTURE_BODY"; fi\n'
' shift\n'
' done\n'
'elif [ "$1 $2" = "api --method" ]; then cat >/dev/null; fi\n'),
):
script = fake_bin / command
script.write_text("#!/bin/sh\n" + body)
script.chmod(0o755)
self.env["PATH"] = f"{fake_bin}{os.pathsep}{os.environ['PATH']}"
def test_evidence_wiring_and_failed_initialization_cannot_publish_stale_files(self):
for suite, suffix in ((suite, suffix) for suite in self.SUITES for suffix in ("", "-scratch")):
with self.subTest(suite=suite, collision=suffix or "artifact"):
self.prepare(suite)
self.assertNotIn("/tmp/rustfs-", self.source)
names = list(self.steps)
self.assertLess(names.index("Initialize functional evidence"), names.index("Checkout auto-testing scripts (with retry)"))
if suite in FunctionalWorkflowTests.DIRECT_TESTS:
self.assertLess(names.index("Checkout repository (for report parser)"), names.index("Checkout auto-testing scripts (with retry)"))
for name, lines in self.steps.items():
if name in ("Generate report", "Upload functional report to dashboard") or any("uses: actions/upload-artifact@" in line for line in lines):
self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", lines)
if any("uses: actions/upload-artifact@" in line for line in lines):
self.assertIn(" path: |", lines)
self.assertIn(" if-no-files-found: error", lines)
existing = Path(str(self.artifacts) + suffix)
existing.mkdir()
for filename in ("report.md", "suite.log"):
(existing / filename).write_text("OLD RUN EVIDENCE")
self.env.update(REPORT_FILE=str(existing / "report.md"), LOG_FILE=str(existing / "suite.log"))
initialized = self.run_step("Initialize functional evidence")
self.assertNotEqual(initialized.returncode, 0)
self.assertFalse(Path(self.env["GITHUB_ENV"]).exists())
issue = self.run_step("File failure issue in rustfs/backlog")
self.assertEqual(issue.returncode, 0, issue.stderr)
body = Path(self.env["CAPTURE_BODY"]).read_text()
self.assertNotIn("OLD RUN EVIDENCE", body)
self.assertIn("no report or log file was produced", body)
self.assertEqual((existing / "report.md").read_text(), "OLD RUN EVIDENCE")
def test_reports_use_only_current_complete_suite_evidence(self):
for suite in self.SUITES[:-1]:
good = "--- KMS-101 roundtrip ---\n[PASS] KMS-101\n"
partial = "--- KMS-101 roundtrip ---\n[PASS] KMS-101\n--- KMS-102 unfinished ---\n"
if suite == "s3-compat":
good, partial = good.replace("KMS-", "S3C-"), partial.replace("KMS-", "S3C-")
if suite == "upgrade":
good += "[UPG-TOPO] single-single local v1 v2 PASS=1 FAIL=0\n"
if suite == "heal":
good = "".join(f"[HEAL-STEP] {step} fixture PASS\n" for step in range(1, 8))
partial = "[HEAL-STEP] 1 fixture PASS\n"
for outcome, log in (("success", good), ("failure", good), ("success", partial), ("success", ""),
("success", None), ("skipped", None), ("cancelled", good)):
with self.subTest(suite=suite, outcome=outcome, log=log):
self.prepare(suite)
stale = self.directory / "old-suite.log"
stale.write_text("OLD RUN EVIDENCE\n" + good)
self.env.update(LOG_FILE=str(stale), REPORT_FILE=str(stale))
self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
self.assertEqual(self.env["LOG_FILE"], str(self.artifacts / "suite.log"))
self.assertEqual(self.env["TMPDIR"], str(self.artifacts) + "-scratch")
if log is not None:
Path(self.env["LOG_FILE"]).write_text(log)
self.context["steps.test.outcome"] = outcome
report = self.run_step("Generate report")
success = outcome == "success" and log == good
self.assertEqual(report.returncode == 0, success, report.stderr)
contents = Path(self.env["REPORT_FILE"]).read_text()
self.assertNotIn("OLD RUN EVIDENCE", contents)
self.assertEqual("| PASS |" in contents, success)
for value in ("actions/runs/314159", "Attempt: 2", "Workflow Commit: " + self.context["github.sha"],
f"Test Step Outcome: {'success' if success else 'failure'}", f"Suite Step Outcome: {outcome}"):
self.assertIn(value, contents)
self.assertEqual(Path(self.env["GITHUB_STEP_SUMMARY"]).read_text(), contents)
evidence = (self.artifacts / ("steps.md" if suite == "heal" else "cases.md")).read_text()
if log in (good, partial):
self.assertIn("| PASS |", evidence)
self.assertNotIn("OLD RUN EVIDENCE", evidence)
def test_actual_suite_commands_pass_the_current_log_and_scratch_paths(self):
for suite in self.SUITES:
with self.subTest(suite=suite):
self.prepare(suite)
self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
if suite == "heal":
self.assertEqual(self.env["RUSTFS_WARP_LOG_FILE"], str(self.artifacts / "warp.log"))
scripts = self.directory / "auto-testing"
scripts.mkdir()
filename = f"rustfs_{suite}_test.sh" if suite in ("heal", "performance") else f"rustfs-{suite}-test.sh"
script = scripts / filename
script.write_text(
'#!/bin/bash\nset -euo pipefail\nlog=""\n'
'while [ "$#" -gt 0 ]; do\n'
' if [ "$1" = "--log-file" ]; then log="$2"; shift; fi\n'
' shift\n'
'done\n'
'[ "$log" = "$LOG_FILE" ] || exit 31\n'
'printf "CURRENT SUITE LOG\\n" > "$log"\n'
'scratch=$(mktemp -d "$TMPDIR/fixture.XXXXXX")\n'
'printf "CURRENT SCRATCH\\n" > "$scratch/trace.log"\n'
'if [ -n "${RUSTFS_RESULT_DIR:-}" ]; then\n'
' mkdir -p "$RUSTFS_RESULT_DIR"\n'
' printf "CURRENT RESULTS\\n" > "$RUSTFS_RESULT_DIR/summary.md"\n'
'fi\n'
)
script.chmod(0o755)
name = FunctionalWorkflowTests.DIRECT_TESTS.get(suite) or (
"Run benchmark (GET/PUT/MIXED)" if suite == "performance" else "Run heal test (write -> outage -> heal -> verify)"
)
result = self.run_step(name)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual((self.artifacts / "suite.log").read_text(), "CURRENT SUITE LOG\n")
self.assertEqual(len(list(Path(self.env["TMPDIR"]).glob("fixture.*/trace.log"))), 1)
self.assertEqual(list(self.artifacts.glob("fixture.*")), [])
if suite == "performance":
self.assertEqual((self.artifacts / "results/summary.md").read_text(), "CURRENT RESULTS\n")
def test_upload_allowlist_preserves_diagnostics_without_scratch(self):
extra = {
"kms": ["cases.md"], "storage": ["cases.md"], "s3-compat": ["cases.md"],
"upgrade": ["cases.md", "matrix.md"], "replication": ["cases.md"], "heal": ["steps.md", "warp.log"],
"performance": ["version.txt", "results/master.log", "results/summary.md", "results/summary.tsv",
"results/get_1KiB.txt", "results/put_1MiB.txt", "results/mixed_4MiB.txt"],
}
for suite in self.SUITES:
with self.subTest(suite=suite):
self.prepare(suite)
self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
expected = {self.artifacts / name for name in ["report.md", "suite.log", *extra[suite]]}
for path in expected:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("PARTIAL FAILURE DIAGNOSTIC")
for directory in (self.artifacts, Path(self.env["TMPDIR"]), self.artifacts / "results"):
directory.mkdir(exist_ok=True)
(directory / "init.json").write_text('{"root_token":"FAKE-SECRET-CANARY"}')
self.assertEqual(self.uploaded_files(), expected)
self.assertTrue(all("FAKE-SECRET-CANARY" not in path.read_text() for path in self.uploaded_files()))
def test_kms_failure_after_vault_init_keeps_only_failure_evidence(self):
self.prepare("kms")
self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
scripts = self.directory / "auto-testing"
scripts.mkdir()
# Vault file writes and EXIT cleanup from auto-testing@06cd3c097350:23-24,57,479-487.
# The Docker boundary returns synthetic credentials; setup fails before vault_stop.
(scripts / "rustfs-kms-test.sh").write_text(r'''#!/bin/bash
set -Eeuo pipefail
TEST_TMP="$(mktemp -d "${TMPDIR:-/tmp}/rustfs-test.XXXXXX")"
trap 'rm -rf "${TEST_TMP}"' EXIT
VAULT_DATA_DIR="${RUSTFS_VAULT_DATA_DIR:-${TMPDIR:-/tmp}/rustfs-vault-data}"
VAULT_CONTAINER="rustfs-vault"
heal_run() { "$@"; }
while [ "$#" -gt 0 ]; do
if [ "$1" = "--log-file" ]; then LOG_FILE="$2"; shift; fi
shift
done
mkdir -p "${VAULT_DATA_DIR}"
tmp_init="${TEST_TMP}/vault-init.json"
tmp_err="${TEST_TMP}/vault-init.stderr"
heal_run docker exec -e "VAULT_ADDR=http://127.0.0.1:8200" "${VAULT_CONTAINER}" \
vault operator init -key-shares=1 -key-threshold=1 -format=json \
> "${tmp_init}" 2>"${tmp_err}"
cat "${tmp_init}" | heal_run tee "${VAULT_DATA_DIR}/init.json" >/dev/null
printf '%s\n' 'vault initialized; root token acquired' 'fixture setup failed after init' > "${LOG_FILE}"
exit 42
''')
docker = self.directory / "bin/docker"
docker.write_text("""#!/bin/sh
[ "$1" = exec ] || exit 99
printf '%s\\n' '{"root_token":"FAKE-ROOT-CANARY","unseal_keys_b64":["FAKE-UNSEAL-CANARY"]}'
""")
docker.chmod(0o755)
result = self.run_step("Run KMS suite")
self.assertEqual(result.returncode, 42, result.stderr)
vault_init = Path(self.env["TMPDIR"]) / "rustfs-vault-data/init.json"
self.assertEqual(json.loads(vault_init.read_text()), {"root_token": "FAKE-ROOT-CANARY", "unseal_keys_b64": ["FAKE-UNSEAL-CANARY"]})
self.assertNotIn(self.artifacts, vault_init.parents)
self.assertEqual(list(Path(self.env["TMPDIR"]).glob("rustfs-test.*")), [])
self.assertNotEqual(self.run_step("Generate report").returncode, 0)
self.assertEqual(self.uploaded_files(), {self.artifacts / name for name in ("suite.log", "cases.md", "report.md")})
self.assertIn("fixture setup failed after init", (self.artifacts / "suite.log").read_text())
for path in self.uploaded_files():
self.assertNotIn("FAKE-ROOT-CANARY", path.read_text())
self.assertNotIn("FAKE-UNSEAL-CANARY", path.read_text())
def test_heal_accumulates_actual_staged_steps_without_overwriting_failures(self):
self.prepare("heal")
self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
script = self.directory / "auto-testing/rustfs_heal_test.sh"
script.parent.mkdir()
# Result printf and full-run condition from auto-testing@6120aa0a76de:143,1163-1168.
script.write_text(r'''#!/bin/bash
set -euo pipefail
SELECTED_STEPS=()
PREFLIGHT=0
while [ "$#" -gt 0 ]; do
case "$1" in
--steps) IFS=',' read -ra SELECTED_STEPS <<< "$2"; shift ;;
--log-file) LOG_FILE="$2"; shift ;;
--preflight) PREFLIGHT=1 ;;
esac
shift
done
if [ "$PREFLIGHT" -eq 1 ]; then
printf '\n' >> "$INVOKED_STEPS"
exit 0
fi
printf '%s\n' "${SELECTED_STEPS[*]}" >> "$INVOKED_STEPS"
emit_step_result() {
local n="$1" desc="$2" status="$3"
printf '[HEAL-STEP] %s %s %s\n' "${n}" "${desc}" "${status}"
}
{
for step in "${SELECTED_STEPS[@]}"; do
emit_step_result "$step" "fixture step $step" PASS
done
want_all=1
for s in 1 2 3 4 5 6 7; do
[[ " ${SELECTED_STEPS[*]} " == *" ${s} "* ]] || want_all=0
done
if [ "${want_all}" -eq 1 ]; then
printf '[HEAL-RESULT] PASS all steps passed\n'
fi
} >> "$LOG_FILE"
''')
script.chmod(0o755)
self.env["INVOKED_STEPS"] = str(self.directory / "invoked-steps")
for name in ("Install RustFS package & start cluster", "Preflight checks", "Run heal test (write -> outage -> heal -> verify)"):
result = self.run_step(name)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(Path(self.env["INVOKED_STEPS"]).read_text().splitlines(), ["1 2", "", "3 4 5 6 7"])
log = Path(self.env["LOG_FILE"]).read_text()
self.assertNotIn("[HEAL-RESULT]", log)
self.assertEqual(log.count("[HEAL-STEP]"), 7)
report = self.run_step("Generate report")
self.assertEqual(report.returncode, 0, report.stderr)
failed_logs = ["\n".join(line for line in log.splitlines() if not line.startswith(f"[HEAL-STEP] {step} ")) + "\n"
for step in range(1, 8)]
failed_logs += [
log.replace("[HEAL-STEP] 3", "[HEAL-STEP] 3 original failure FAIL\n[HEAL-STEP] 3"),
log + "[HEAL-STEP] 3 later step failure FAIL\n",
log + "[HEAL-RESULT] FAIL earlier failure\n[HEAL-RESULT] PASS later success\n",
log.replace("[HEAL-STEP] 4 fixture step 4 PASS", "[HEAL-STEP] 4 fixture step 4 SKIP"),
]
for failed_log in failed_logs:
with self.subTest(log=failed_log):
Path(self.env["LOG_FILE"]).write_text(failed_log)
report = self.run_step("Generate report")
self.assertNotEqual(report.returncode, 0, report.stderr)
contents = Path(self.env["REPORT_FILE"]).read_text()
self.assertIn("Test Step Outcome: failure", contents)
self.assertNotIn("| PASS |", contents)
if "original failure" in failed_log:
self.assertIn("| 3 | original failure | FAIL |", (self.artifacts / "steps.md").read_text())
if "later step failure" in failed_log:
self.assertIn("| 3 | later step failure | FAIL |", (self.artifacts / "steps.md").read_text())
def test_performance_results_version_and_report_are_bound_to_the_run(self):
self.prepare("performance")
initialized = self.run_step("Initialize functional evidence")
self.assertEqual(initialized.returncode, 0, initialized.stderr)
self.assertEqual(self.env["RUSTFS_RESULT_DIR"], str(self.artifacts / "results"))
self.assertEqual(self.env["VERSION_FILE"], str(self.artifacts / "version.txt"))
version = self.run_step("Collect RustFS version info")
self.assertEqual(version.returncode, 0, version.stderr)
self.assertIn("fixture-version", Path(self.env["VERSION_FILE"]).read_text())
old_summary = self.directory / "old-results/summary.md"
old_summary.parent.mkdir()
old_summary.write_text("OLD RUN EVIDENCE")
upload = "Upload report to dashboard (reports/YYYY-MM-DD.md)"
self.assertNotEqual(self.run_step(upload).returncode, 0)
self.assertFalse(Path(self.env["REPORT_FILE"]).exists())
results = Path(self.env["RUSTFS_RESULT_DIR"])
results.mkdir()
(results / "summary.md").write_text("CURRENT PERFORMANCE RESULTS\n")
report = self.run_step(upload)
self.assertEqual(report.returncode, 0, report.stderr)
contents = Path(self.env["REPORT_FILE"]).read_text()
for value in ("actions/runs/314159", "**Attempt**: 2", "**Workflow Commit**: " + self.context["github.sha"],
"CURRENT PERFORMANCE RESULTS", "fixture-version"):
self.assertIn(value, contents)
self.assertNotIn("OLD RUN EVIDENCE", contents)
if __name__ == "__main__":