diff --git a/.config/make/tests.mak b/.config/make/tests.mak index d1297e9ad..726ca04cf 100644 --- a/.config/make/tests.mak +++ b/.config/make/tests.mak @@ -41,6 +41,7 @@ script-tests: ## Run shell script tests $(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test $(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test $(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py + $(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py $(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py bash -n ./scripts/validate_object_data_cache_cold_stampede.sh $(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test diff --git a/.github/actions/quick-checks/action.yml b/.github/actions/quick-checks/action.yml index e6cc59775..c9a957f10 100644 --- a/.github/actions/quick-checks/action.yml +++ b/.github/actions/quick-checks/action.yml @@ -94,12 +94,17 @@ runs: shell: bash run: ./scripts/check_embedded_secrets.sh + - name: Run script contract tests + shell: bash + run: make script-tests + - name: Check test wiring shell: bash run: | python3 ./scripts/check_test_wiring.py --self-test python3 ./scripts/check_scheduled_validation_freshness.py --self-test python3 ./scripts/test_security_workflow.py + python3 ./scripts/test_nightly_candidate.py python3 ./scripts/check_test_wiring.py - name: Check no planning docs committed diff --git a/.github/workflows/e2e-upgrade.yml b/.github/workflows/e2e-upgrade.yml index ed420cd06..8cc1d53a5 100644 --- a/.github/workflows/e2e-upgrade.yml +++ b/.github/workflows/e2e-upgrade.yml @@ -19,7 +19,9 @@ on: paths: - ".github/workflows/e2e-upgrade.yml" - "crates/e2e_test/src/common.rs" + - "crates/e2e_test/src/fake_s3_target/**" - "crates/e2e_test/src/lib.rs" + - "crates/e2e_test/src/replication_extension_test.rs" - "crates/e2e_test/src/upgrade_compatibility_test.rs" - "crates/ecstore/**" - "crates/filemeta/**" @@ -44,9 +46,9 @@ concurrency: env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 - UPGRADE_SOURCE_VERSION: 1.0.0-rc.2 - UPGRADE_SOURCE_ASSET: rustfs-linux-x86_64-gnu-v1.0.0-rc.2.zip - UPGRADE_SOURCE_SHA256: 7c789386bf85278f865b8e0d359bf4edb84d5aa408cc3fa54a18c25ca74cd6e7 + UPGRADE_SOURCE_VERSION: 1.0.0-rc.5 + UPGRADE_SOURCE_ASSET: rustfs-linux-x86_64-gnu-v1.0.0-rc.5.zip + UPGRADE_SOURCE_SHA256: 3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d jobs: upgrade: @@ -55,14 +57,27 @@ jobs: fail-fast: false matrix: include: - - name: Direct upgrade from rc.2 + # The two `_from_rc2_` tests keep their names: they assert + # release-independent object contracts and pass unchanged against the + # newer pinned source, so renaming them would only churn history and + # the CI required-check names. UPGRADE_SOURCE_VERSION above is the + # single source of truth for which release they actually run against. + - name: Direct upgrade from the previous release cache_key: e2e-direct-upgrade test: direct_upgrade_from_rc2_preserves_object_contracts artifact: direct-upgrade - - name: Mixed-version rolling upgrade from rc.2 + - name: Mixed-version rolling upgrade from the previous release cache_key: e2e-mixed-version-upgrade test: rolling_upgrade_from_rc2_preserves_mixed_version_contracts artifact: mixed-version-upgrade + - name: Bucket configuration survives the upgrade + cache_key: e2e-bucket-config-upgrade + test: direct_upgrade_from_previous_release_preserves_bucket_configuration + artifact: bucket-config-upgrade + - name: Rollback reads current bucket metadata + cache_key: e2e-bucket-config-rollback + test: rollback_to_previous_release_reads_current_bucket_metadata + artifact: bucket-config-rollback runs-on: ubuntu-latest timeout-minutes: 60 env: diff --git a/.github/workflows/nightly-gnu.yml b/.github/workflows/nightly-gnu.yml index 0dedbf465..1864a9715 100644 --- a/.github/workflows/nightly-gnu.yml +++ b/.github/workflows/nightly-gnu.yml @@ -166,8 +166,9 @@ jobs: # e.g. https://dl.rustfs.com/artifacts/rustfs/packages/nightly/... . # Skipped when the R2 secrets are not configured (artifact-only mode). - name: Upload DEB to Cloudflare R2 - if: env.R2_ACCESS_KEY_ID != '' + id: publish env: + DEB_FILE: ${{ steps.deb.outputs.deb_file }} R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }} @@ -182,28 +183,70 @@ jobs: exit 0 fi - if ! command -v aws >/dev/null 2>&1; then - sudo apt-get update && sudo apt-get install -y -qq awscli - fi - export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID" export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY" export AWS_DEFAULT_REGION="auto" - DEB_FILE="${{ steps.deb.outputs.deb_file }}" + SOURCE_SHA="$(git rev-parse HEAD)" + if [[ "${SOURCE_SHA}" != "${GITHUB_SHA}" ]]; then + echo "Checkout SHA does not match the nightly build run" >&2 + exit 1 + fi + DEB_SHA256="$(sha256sum "${DEB_FILE}" | cut -d ' ' -f 1)" + CANDIDATE_KEY="artifacts/rustfs/packages/nightly/runs/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/${DEB_SHA256}/rustfs.deb" + CANDIDATE_URL="https://dl.rustfs.com/${CANDIDATE_KEY}" + + # Old AWS CLI models lack conditional PutObject support. Never fall + # back to an overwriting upload for a candidate. + AWS_CLI=aws + if ! "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null; then + sudo apt-get update + sudo apt-get install -y -qq python3-venv + AWS_CLI_DIR="$(mktemp -d "${RUNNER_TEMP}/nightly-awscli.XXXXXX")" + trap 'rm -rf "${AWS_CLI_DIR}"' EXIT + python3 -m venv "${AWS_CLI_DIR}" + "${AWS_CLI_DIR}/bin/python" -m pip install --disable-pip-version-check 'awscli==1.44.79' + AWS_CLI="${AWS_CLI_DIR}/bin/aws" + fi + "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null + "${AWS_CLI}" --version + "${AWS_CLI}" s3api put-object --bucket "${R2_BUCKET}" --key "${CANDIDATE_KEY}" \ + --body "${DEB_FILE}" --if-none-match '*' --endpoint-url "${R2_ENDPOINT}" + PUBLISHED_SHA256="$(curl -fsSL --retry 3 --connect-timeout 15 --max-time 300 "${CANDIDATE_URL}" | sha256sum | cut -d ' ' -f 1)" + if [[ "${PUBLISHED_SHA256}" != "${DEB_SHA256}" ]]; then + echo "Published candidate checksum does not match the built package" >&2 + exit 1 + fi + R2_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/nightly/" echo "📤 Uploading ${DEB_FILE} to ${R2_PREFIX}" - aws s3 cp "${DEB_FILE}" "${R2_PREFIX}" --endpoint-url "$R2_ENDPOINT" --only-show-errors + "${AWS_CLI}" s3 cp "${DEB_FILE}" "${R2_PREFIX}" --endpoint-url "$R2_ENDPOINT" --only-show-errors # Stable "latest" alias so tests can fetch the newest nightly # without knowing today's date. echo "📤 Uploading latest alias" - aws s3 cp "${DEB_FILE}" "${R2_PREFIX}rustfs-nightly-latest.deb" \ + "${AWS_CLI}" s3 cp "${DEB_FILE}" "${R2_PREFIX}rustfs-nightly-latest.deb" \ --endpoint-url "$R2_ENDPOINT" --only-show-errors echo "✅ R2 upload complete" + CANDIDATE_FILE="${RUNNER_TEMP}/nightly-candidate-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json" + jq -n --arg source_sha "${SOURCE_SHA}" \ + --argjson build_run_id "${GITHUB_RUN_ID}" --argjson build_run_attempt "${GITHUB_RUN_ATTEMPT}" \ + --arg package_url "${CANDIDATE_URL}" --arg package_sha256 "${DEB_SHA256}" \ + '{schema: 1, source_sha: $source_sha, build_run_id: $build_run_id, build_run_attempt: $build_run_attempt, package_url: $package_url, package_sha256: $package_sha256}' \ + > "${CANDIDATE_FILE}" + echo "candidate_file=${CANDIDATE_FILE}" >> "${GITHUB_OUTPUT}" + + - name: Upload nightly candidate manifest + if: ${{ steps.publish.outputs.candidate_file != '' }} + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: nightly-candidate-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ steps.publish.outputs.candidate_file }} + if-no-files-found: error + # Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774). # # RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and diff --git a/.github/workflows/rustfs-functional-chain.yml b/.github/workflows/rustfs-functional-chain.yml index 6ce828c1a..10b9c67ad 100644 --- a/.github/workflows/rustfs-functional-chain.yml +++ b/.github/workflows/rustfs-functional-chain.yml @@ -14,8 +14,8 @@ # Functional chain driver: runs the ten functional suites in a fixed order # (upgrade -> s3 -> kms -> tier -> storage -> heal -> pool -> security -> -# replication, with performance on its own runner in parallel) and guarantees -# the chain keeps moving even when individual suites fail. +# replication -> performance). Each suite attempts the next handoff even +# when its tests fail. # # Each suite workflow can still be dispatched standalone (workflow_dispatch); # only chain-triggered runs forward to the next suite via repository_dispatch, @@ -59,16 +59,3 @@ jobs: gh api --method POST repos/rustfs/rustfs/dispatches \ -f event_type='rustfs-chain-upgrade' \ -F 'client_payload[from_suite]=nightly-build' - - - name: Dispatch performance suite (parallel, own runner) - env: - GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} - run: | - set -euo pipefail - if [ -z "${GH_TOKEN:-}" ]; then - echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch performance" >&2 - exit 1 - fi - gh api --method POST repos/rustfs/rustfs/dispatches \ - -f event_type='rustfs-chain-performance' \ - -F 'client_payload[from_suite]=nightly-build' diff --git a/.github/workflows/rustfs-heal-test.yml b/.github/workflows/rustfs-heal-test.yml index efbe2a09c..4e6b4e190 100644 --- a/.github/workflows/rustfs-heal-test.yml +++ b/.github/workflows/rustfs-heal-test.yml @@ -59,6 +59,21 @@ jobs: # (storage -> heal -> pool). Pool expansion no longer re-runs heal. if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: + - name: Initialize functional evidence + id: evidence + run: | + set -euo pipefail + umask 077 + FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-heal-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch" + { + printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'RUSTFS_WARP_LOG_FILE=%s/warp.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + } >> "${GITHUB_ENV}" + # auto-testing is private: clone it with the dedicated PF token (not # GITHUB_TOKEN) and retry transient GitHub/network failures. - name: Checkout auto-testing scripts (with retry) @@ -114,7 +129,7 @@ jobs: else ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") fi - ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" + ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}" - name: Preflight checks run: | @@ -124,7 +139,7 @@ jobs: else ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") fi - ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" + ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}" - name: Run heal test (write -> outage -> heal -> verify) id: test @@ -134,13 +149,10 @@ jobs: --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \ --stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \ --warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \ - --log-file /tmp/rustfs-heal-test.log + --log-file "${LOG_FILE}" - name: Generate report - if: always() - env: - LOG_FILE: /tmp/rustfs-heal-test.log - REPORT_FILE: /tmp/rustfs-heal-report.md + if: ${{ always() && steps.evidence.outcome == 'success' }} run: | set -euo pipefail PACKAGE_URL='${{ inputs.package_url }}' @@ -149,8 +161,9 @@ jobs: else PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}" fi - STEPS_TABLE="/tmp/rustfs-heal-steps.md" - python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY' + STEPS_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/steps.md" + CASE_RESULT=success + python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY' || CASE_RESULT=failure import re import sys @@ -162,6 +175,7 @@ jobs: steps = {} order = [] + status_rank = {'SKIP': 0, 'PASS': 1, 'FAIL': 2} version = None version_node = None verdict = None @@ -175,14 +189,15 @@ jobs: n, desc, status = m.group(1), m.group(2), m.group(3) if n not in steps: order.append(n) - steps[n] = (desc, status) # later lines win (fail after pass) + if n not in steps or status_rank[status] > status_rank[steps[n][1]]: + steps[n] = (desc, status) continue m = ver_re.match(line) if m: version, version_node = m.group(1), m.group(2) continue m = result_re.match(line) - if m: + if m and verdict != 'FAIL': verdict, verdict_detail = m.group(1), m.group(2) except FileNotFoundError: pass @@ -202,30 +217,43 @@ jobs: out.write(f'| {n} | {desc} | {status} |\n') if not order: out.write('| - | - | NOT RUN (no step result lines found) |\n') + complete = set(steps) == {str(n) for n in range(1, 8)} + sys.exit(0 if complete and verdict != 'FAIL' and all(status == 'PASS' for _, status in steps.values()) else 1) PY + RESULT=failure + if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then + RESULT=success + fi { echo "# RustFS heal test report" echo "" echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo "- Attempt: ${GITHUB_RUN_ATTEMPT}" + echo "- Workflow Commit: ${GITHUB_SHA}" echo "- Trigger: ${{ github.event_name }}" echo "- Package: ${PACKAGE_SOURCE}" - echo "- Test Step Outcome: ${{ steps.test.outcome }}" + echo "- Test Step Outcome: ${RESULT}" + echo "- Suite Step Outcome: ${{ steps.test.outcome }}" echo "" - cat "${STEPS_TABLE}" || true - echo "" - echo "## Log tail" - echo '```text' - tail -n 200 "${LOG_FILE}" || true - echo '```' + if [ "${RESULT}" = "success" ]; then + cat "${STEPS_TABLE}" + echo "" + echo "## Log tail" + echo '```text' + tail -n 200 "${LOG_FILE}" + echo '```' + else + echo "The suite or evidence validation failed. See this run's artifact for partial step results and suite.log." + fi } | tee "${REPORT_FILE}" cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" + [ "${RESULT}" = "success" ] - name: Upload functional report to dashboard - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} continue-on-error: true env: GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }} - REPORT_FILE: /tmp/rustfs-heal-report.md SUITE: heal run: | set -euo pipefail @@ -257,11 +285,10 @@ jobs: continue-on-error: true env: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} + EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }} SUITE: 'heal' SUITE_LABEL: 'Heal' RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - REPORT_FILE: '/tmp/rustfs-heal-report.md' - LOG_FILE: '/tmp/rustfs-heal-test.log' run: | set -euo pipefail if [ -z "${GH_TOKEN:-}" ]; then @@ -289,14 +316,16 @@ jobs: echo "" echo "- Suite: \`${SUITE}\`" echo "- Run: ${RUN_URL}" + echo "- Attempt: ${GITHUB_RUN_ATTEMPT}" + echo "- Workflow Commit: ${GITHUB_SHA}" echo "- Trigger: ${GITHUB_EVENT_NAME}" echo "- Date: $(date -u +%Y-%m-%d)" echo "" echo "## Report (errors and symptoms)" echo "" - if [ -s "${REPORT_FILE}" ]; then + if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then redact < "${REPORT_FILE}" - elif [ -s "${LOG_FILE:-}" ]; then + elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then echo "(report file missing; log tail below)" echo "" tail -n 200 "${LOG_FILE}" | redact @@ -312,14 +341,16 @@ jobs: echo "filed backlog issue for suite ${SUITE}" - name: Upload test logs - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: - name: rustfs-heal-test-${{ github.run_id }} + name: rustfs-heal-test-${{ github.run_id }}-${{ github.run_attempt }} path: | - /tmp/rustfs-heal-test*.log - /tmp/rustfs-warp.*.log - if-no-files-found: warn + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/warp.log + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/steps.md + if-no-files-found: error - name: Cleanup environment (after) if: ${{ always() && inputs.cleanup_after != 'false' }} diff --git a/.github/workflows/rustfs-kms-test.yml b/.github/workflows/rustfs-kms-test.yml index 5c3a2b6b1..c9eb02d00 100644 --- a/.github/workflows/rustfs-kms-test.yml +++ b/.github/workflows/rustfs-kms-test.yml @@ -52,6 +52,25 @@ jobs: timeout-minutes: 420 if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: + - name: Checkout repository (for report parser) + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + + - name: Initialize functional evidence + id: evidence + run: | + set -euo pipefail + umask 077 + FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-kms-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch" + { + printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + } >> "${GITHUB_ENV}" + # auto-testing is private: clone it with the dedicated PF token (not # GITHUB_TOKEN) and retry transient GitHub/network failures. - name: Checkout auto-testing scripts (with retry) @@ -108,8 +127,6 @@ jobs: - name: Run KMS suite id: test - env: - LOG_FILE: /tmp/rustfs-kms.log run: | set -euo pipefail chmod +x auto-testing/rustfs-kms-test.sh @@ -139,10 +156,7 @@ jobs: ./auto-testing/rustfs-kms-test.sh "${ARGS[@]}" - name: Generate report - if: always() - env: - LOG_FILE: /tmp/rustfs-kms.log - REPORT_FILE: /tmp/rustfs-kms-report.md + if: ${{ always() && steps.evidence.outcome == 'success' }} run: | set -euo pipefail PACKAGE_URL='${{ inputs.package_url }}' @@ -154,79 +168,43 @@ jobs: else PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}" fi - CASE_TABLE="/tmp/rustfs-kms-cases.md" - python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY' - import re - import sys - - log_file, out_file = sys.argv[1], sys.argv[2] - ansi = re.compile(r'\x1b\[[0-9;]*m') - start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$') - done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b') - - rows = [] - index = {} - try: - with open(log_file, 'r', encoding='utf-8', errors='replace') as fh: - for raw in fh: - line = ansi.sub('', raw).strip() - m = start_re.match(line) - if m: - case_id, name = m.group(1), m.group(2) - if case_id not in index: - index[case_id] = len(rows) - rows.append([case_id, name, 'RUNNING']) - continue - m = done_re.match(line) - if m: - status, case_id = m.group(1), m.group(2) - if case_id in index: - rows[index[case_id]][2] = status - else: - rows.append([case_id, case_id, status]) - index[case_id] = len(rows) - 1 - except FileNotFoundError: - rows = [] - - counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0} - for _, _, status in rows: - counts[status] = counts.get(status, 0) + 1 - - with open(out_file, 'w', encoding='utf-8') as out: - out.write('## Case Summary\n\n') - out.write(f"- Total: {len(rows)}\\n") - out.write(f"- PASS: {counts.get('PASS', 0)}\\n") - out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n") - out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n") - out.write('\\n') - out.write('| Case | Name | Status |\\n') - out.write('| --- | --- | --- |\\n') - for case_id, name, status in rows: - out.write(f'| {case_id} | {name} | {status} |\\n') - PY + CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" + CASE_RESULT=success + python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure + RESULT=failure + if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then + RESULT=success + fi { echo "# RustFS KMS test report" echo "" echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo "- Attempt: ${GITHUB_RUN_ATTEMPT}" + echo "- Workflow Commit: ${GITHUB_SHA}" echo "- Trigger: ${{ github.event_name }}" echo "- Package: ${PACKAGE_SOURCE}" - echo "- Test Step Outcome: ${{ steps.test.outcome }}" + echo "- Test Step Outcome: ${RESULT}" + echo "- Suite Step Outcome: ${{ steps.test.outcome }}" echo "" - cat "${CASE_TABLE}" || true - echo "" - echo "## Log tail" - echo '```text' - tail -n 200 "${LOG_FILE}" || true - echo '```' + if [ "${RESULT}" = "success" ]; then + cat "${CASE_TABLE}" + echo "" + echo "## Log tail" + echo '```text' + tail -n 200 "${LOG_FILE}" + echo '```' + else + echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log." + fi } | tee "${REPORT_FILE}" cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" + [ "${RESULT}" = "success" ] - name: Upload functional report to dashboard - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} continue-on-error: true env: GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }} - REPORT_FILE: /tmp/rustfs-kms-report.md SUITE: kms run: | set -euo pipefail @@ -258,11 +236,10 @@ jobs: continue-on-error: true env: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} + EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }} SUITE: 'kms' SUITE_LABEL: 'KMS' RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - REPORT_FILE: '/tmp/rustfs-kms-report.md' - LOG_FILE: '/tmp/rustfs-kms.log' run: | set -euo pipefail if [ -z "${GH_TOKEN:-}" ]; then @@ -290,14 +267,16 @@ jobs: echo "" echo "- Suite: \`${SUITE}\`" echo "- Run: ${RUN_URL}" + echo "- Attempt: ${GITHUB_RUN_ATTEMPT}" + echo "- Workflow Commit: ${GITHUB_SHA}" echo "- Trigger: ${GITHUB_EVENT_NAME}" echo "- Date: $(date -u +%Y-%m-%d)" echo "" echo "## Report (errors and symptoms)" echo "" - if [ -s "${REPORT_FILE}" ]; then + if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then redact < "${REPORT_FILE}" - elif [ -s "${LOG_FILE:-}" ]; then + elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then echo "(report file missing; log tail below)" echo "" tail -n 200 "${LOG_FILE}" | redact @@ -313,14 +292,15 @@ jobs: echo "filed backlog issue for suite ${SUITE}" - name: Upload report and logs - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: - name: rustfs-kms-test-${{ github.run_id }} + name: rustfs-kms-test-${{ github.run_id }}-${{ github.run_attempt }} path: | - /tmp/rustfs-kms.log - /tmp/rustfs-kms-report.md - if-no-files-found: warn + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md + if-no-files-found: error - name: Cleanup environment (after) if: always() diff --git a/.github/workflows/rustfs-performance-test.yml b/.github/workflows/rustfs-performance-test.yml index ff3e2978d..d698b27e0 100644 --- a/.github/workflows/rustfs-performance-test.yml +++ b/.github/workflows/rustfs-performance-test.yml @@ -49,17 +49,16 @@ on: type: boolean default: true repository_dispatch: - # Chain entry: dispatched by rustfs-functional-chain.yml (runs on its own - # pf-testing runner, in parallel with the shared-VM chain). + # Chain handoff: dispatched when the replication suite finishes. types: [rustfs-chain-performance] permissions: contents: read -# Dedicated pf-testing runner/environment: own concurrency group so perf runs -# never block (or are blocked by) the pool-expansion / heal tests. +# The default performance nodes overlap the other suites' remote VMs, even +# though the runner differs. Hold the shared lock through cleanup as well. concurrency: - group: rustfs-performance-test + group: rustfs-shared-functional-tests cancel-in-progress: false defaults: @@ -76,8 +75,6 @@ env: # Package used by the nightly run (workflow_dispatch inputs are empty for # workflow_run events), i.e. the latest nightly deb published by nightly-gnu.yml. RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }} - # Fixed benchmark result directory so later steps can read summary.md - RUSTFS_RESULT_DIR: /tmp/rustfs-perf-results # Cross-repo token for uploading reports to rustfs/dashboard (set in repo settings) PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} @@ -89,6 +86,22 @@ jobs: # Skipped when nightly failed. if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: + - name: Initialize functional evidence + id: evidence + run: | + set -euo pipefail + umask 077 + FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-performance-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch" + { + printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'RUSTFS_RESULT_DIR=%s/results\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'VERSION_FILE=%s/version.txt\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + } >> "${GITHUB_ENV}" + # auto-testing is private: clone it with the dedicated PF token (not # GITHUB_TOKEN) and retry transient GitHub/network failures. - name: Checkout auto-testing scripts (with retry) @@ -120,7 +133,7 @@ jobs: if: ${{ inputs.cleanup_before != 'false' }} run: | chmod +x auto-testing/rustfs_performance_test.sh - ./auto-testing/rustfs_performance_test.sh --step 1 -y + ./auto-testing/rustfs_performance_test.sh --step 1 -y --log-file "${LOG_FILE:-/dev/null}" - name: Install RustFS package & start cluster (4x4) run: | @@ -130,7 +143,7 @@ jobs: else ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") fi - ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" + ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}" - name: Preflight checks run: | @@ -140,7 +153,7 @@ jobs: else ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") fi - ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" + ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}" - name: Run benchmark (GET/PUT/MIXED) id: benchmark @@ -153,17 +166,15 @@ jobs: --step 5 -y \ --warp-duration "${{ inputs.warp_duration || '5m' }}" \ --warp-concurrency "${{ inputs.warp_concurrency || '64' }}" \ - --log-file /tmp/rustfs-perf-test.log + --log-file "${LOG_FILE}" - name: Analyze results if: ${{ steps.benchmark.conclusion == 'success' }} run: | - ./auto-testing/rustfs_performance_test.sh --step 6 -y + ./auto-testing/rustfs_performance_test.sh --step 6 -y --log-file "${LOG_FILE:-/dev/null}" - name: Collect RustFS version info if: ${{ steps.benchmark.conclusion == 'success' }} - env: - VERSION_FILE: /tmp/rustfs-version.txt run: | set -euo pipefail read -r -a NODES <<< "${RUSTFS_NODES}" @@ -183,7 +194,6 @@ jobs: env: GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }} RESULT_DIR: ${{ env.RUSTFS_RESULT_DIR }} - VERSION_FILE: /tmp/rustfs-version.txt run: | set -euo pipefail if [ -z "${GH_TOKEN:-}" ]; then @@ -191,7 +201,7 @@ jobs: exit 0 fi SUMMARY="${RESULT_DIR}/summary.md" - [ -f "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; } + [ -s "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; } DATE="$(date -u +%Y-%m-%d)" REPORT_PATH="reports/${DATE}.md" { @@ -199,6 +209,8 @@ jobs: echo "" echo "- **Date**: ${DATE}" echo "- **Run**: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo "- **Attempt**: ${GITHUB_RUN_ATTEMPT}" + echo "- **Workflow Commit**: ${GITHUB_SHA}" echo "- **Trigger**: ${{ github.event_name }}" echo "- **Package**: ${{ inputs.package_url || 'nightly (R2 latest)' }}" echo "" @@ -208,8 +220,8 @@ jobs: echo '```text' cat "${VERSION_FILE}" echo '```' - } > /tmp/rustfs-perf-report.md - CONTENT="$(python3 -c 'import base64; print(base64.b64encode(open("/tmp/rustfs-perf-report.md","rb").read()).decode())')" + } > "${REPORT_FILE}" + CONTENT="$(python3 -c 'import base64,sys; print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")" SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)" if [ -n "${SHA}" ]; then jq -n --arg msg "report: ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \ @@ -228,11 +240,10 @@ jobs: continue-on-error: true env: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} + EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }} SUITE: 'performance' SUITE_LABEL: 'Performance' RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - REPORT_FILE: '/tmp/rustfs-perf-report.md' - LOG_FILE: '/tmp/rustfs-perf-test.log' run: | set -euo pipefail if [ -z "${GH_TOKEN:-}" ]; then @@ -260,14 +271,16 @@ jobs: echo "" echo "- Suite: \`${SUITE}\`" echo "- Run: ${RUN_URL}" + echo "- Attempt: ${GITHUB_RUN_ATTEMPT}" + echo "- Workflow Commit: ${GITHUB_SHA}" echo "- Trigger: ${GITHUB_EVENT_NAME}" echo "- Date: $(date -u +%Y-%m-%d)" echo "" echo "## Report (errors and symptoms)" echo "" - if [ -s "${REPORT_FILE}" ]; then + if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then redact < "${REPORT_FILE}" - elif [ -s "${LOG_FILE:-}" ]; then + elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then echo "(report file missing; log tail below)" echo "" tail -n 200 "${LOG_FILE}" | redact @@ -283,20 +296,26 @@ jobs: echo "filed backlog issue for suite ${SUITE}" - name: Upload test logs & results - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: - name: rustfs-perf-test-${{ github.run_id }} + name: rustfs-perf-test-${{ github.run_id }}-${{ github.run_attempt }} path: | - /tmp/rustfs-perf-test*.log - /tmp/rustfs-perf-results/** - /tmp/rustfs-version.txt - if-no-files-found: warn + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/version.txt + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/master.log + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/summary.md + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/summary.tsv + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/get_*.txt + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/put_*.txt + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/mixed_*.txt + if-no-files-found: error - name: Reset test environment (after) if: ${{ always() && inputs.cleanup_after != 'false' }} run: | - ./auto-testing/rustfs_performance_test.sh --step 7 -y + ./auto-testing/rustfs_performance_test.sh --step 7 -y --log-file "${LOG_FILE:-/dev/null}" - name: Notify on failure if: failure() diff --git a/.github/workflows/rustfs-replication-test.yml b/.github/workflows/rustfs-replication-test.yml index 839d1de7f..0faaf7809 100644 --- a/.github/workflows/rustfs-replication-test.yml +++ b/.github/workflows/rustfs-replication-test.yml @@ -34,8 +34,7 @@ on: - site default: all repository_dispatch: - # Chain handoff: dispatched when the security suite finishes. This is the - # last link of the functional chain. + # Chain handoff: dispatched when the security suite finishes. types: [rustfs-chain-replication] permissions: @@ -65,6 +64,25 @@ jobs: timeout-minutes: 360 if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: + - name: Checkout repository (for report parser) + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + + - name: Initialize functional evidence + id: evidence + run: | + set -euo pipefail + umask 077 + FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-replication-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch" + { + printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + } >> "${GITHUB_ENV}" + # auto-testing is private: clone it with the dedicated PF token (not # GITHUB_TOKEN) and retry transient GitHub/network failures. - name: Checkout auto-testing scripts (with retry) @@ -113,8 +131,6 @@ jobs: - name: Run replication suite id: test - env: - LOG_FILE: /tmp/rustfs-replication.log run: | set -euo pipefail chmod +x auto-testing/rustfs-replication-test.sh @@ -137,10 +153,7 @@ jobs: ./auto-testing/rustfs-replication-test.sh "${ARGS[@]}" - name: Generate report - if: always() - env: - LOG_FILE: /tmp/rustfs-replication.log - REPORT_FILE: /tmp/rustfs-replication-report.md + if: ${{ always() && steps.evidence.outcome == 'success' }} run: | set -euo pipefail PACKAGE_URL='${{ inputs.package_url }}' @@ -162,80 +175,44 @@ jobs: RUSTFS_VERSION_INFO="${DETECTED_VERSION}" fi fi - CASE_TABLE="/tmp/rustfs-replication-cases.md" - python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY' - import re - import sys - - log_file, out_file = sys.argv[1], sys.argv[2] - ansi = re.compile(r'\x1b\[[0-9;]*m') - start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$') - done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b') - - rows = [] - index = {} - try: - with open(log_file, 'r', encoding='utf-8', errors='replace') as fh: - for raw in fh: - line = ansi.sub('', raw).strip() - m = start_re.match(line) - if m: - case_id, name = m.group(1), m.group(2) - if case_id not in index: - index[case_id] = len(rows) - rows.append([case_id, name, 'RUNNING']) - continue - m = done_re.match(line) - if m: - status, case_id = m.group(1), m.group(2) - if case_id in index: - rows[index[case_id]][2] = status - else: - rows.append([case_id, case_id, status]) - index[case_id] = len(rows) - 1 - except FileNotFoundError: - rows = [] - - counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0} - for _, _, status in rows: - counts[status] = counts.get(status, 0) + 1 - - with open(out_file, 'w', encoding='utf-8') as out: - out.write('## Case Summary\n\n') - out.write(f"- Total: {len(rows)}\\n") - out.write(f"- PASS: {counts.get('PASS', 0)}\\n") - out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n") - out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n") - out.write('\\n') - out.write('| Case | Name | Status |\\n') - out.write('| --- | --- | --- |\\n') - for case_id, name, status in rows: - out.write(f'| {case_id} | {name} | {status} |\\n') - PY + CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" + CASE_RESULT=success + python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure + RESULT=failure + if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then + RESULT=success + fi { echo "# RustFS replication test report" echo "" echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo "- Attempt: ${GITHUB_RUN_ATTEMPT}" + echo "- Workflow Commit: ${GITHUB_SHA}" echo "- Trigger: ${{ github.event_name }}" echo "- Package: ${PACKAGE_SOURCE}" echo "- RustFS Version: ${RUSTFS_VERSION_INFO}" - echo "- Test Step Outcome: ${{ steps.test.outcome }}" + echo "- Test Step Outcome: ${RESULT}" + echo "- Suite Step Outcome: ${{ steps.test.outcome }}" echo "" - cat "${CASE_TABLE}" || true - echo "" - echo "## Log tail" - echo '```text' - tail -n 200 "${LOG_FILE}" || true - echo '```' + if [ "${RESULT}" = "success" ]; then + cat "${CASE_TABLE}" + echo "" + echo "## Log tail" + echo '```text' + tail -n 200 "${LOG_FILE}" + echo '```' + else + echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log." + fi } | tee "${REPORT_FILE}" cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" + [ "${RESULT}" = "success" ] - name: Upload functional report to dashboard - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} continue-on-error: true env: GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }} - REPORT_FILE: /tmp/rustfs-replication-report.md SUITE: replication run: | set -euo pipefail @@ -267,11 +244,10 @@ jobs: continue-on-error: true env: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} + EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }} SUITE: 'replication' SUITE_LABEL: 'Replication (bucket + site)' RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - REPORT_FILE: '/tmp/rustfs-replication-report.md' - LOG_FILE: '/tmp/rustfs-replication.log' run: | set -euo pipefail if [ -z "${GH_TOKEN:-}" ]; then @@ -299,14 +275,16 @@ jobs: echo "" echo "- Suite: \`${SUITE}\`" echo "- Run: ${RUN_URL}" + echo "- Attempt: ${GITHUB_RUN_ATTEMPT}" + echo "- Workflow Commit: ${GITHUB_SHA}" echo "- Trigger: ${GITHUB_EVENT_NAME}" echo "- Date: $(date -u +%Y-%m-%d)" echo "" echo "## Report (errors and symptoms)" echo "" - if [ -s "${REPORT_FILE}" ]; then + if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then redact < "${REPORT_FILE}" - elif [ -s "${LOG_FILE:-}" ]; then + elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then echo "(report file missing; log tail below)" echo "" tail -n 200 "${LOG_FILE}" | redact @@ -322,14 +300,15 @@ jobs: echo "filed backlog issue for suite ${SUITE}" - name: Upload report and logs - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: - name: rustfs-replication-${{ github.run_id }} + name: rustfs-replication-${{ github.run_id }}-${{ github.run_attempt }} path: | - /tmp/rustfs-replication.log - /tmp/rustfs-replication-report.md - if-no-files-found: warn + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md + if-no-files-found: error - name: Cleanup environment (after) if: always() @@ -350,13 +329,50 @@ jobs: ' done - - name: Chain complete - # Replication is the last link of the functional chain: nothing to - # dispatch after it. This step just records that the chain finished. + - name: "Continue functional chain (next: Performance)" if: ${{ always() && github.event_name == 'repository_dispatch' }} + env: + GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} run: | - echo "Functional chain complete: replication (final suite) finished." - echo "from_suite=security trigger=${{ github.event_name }} outcome=${{ steps.test.outcome }}" + set -uo pipefail + if [ -z "${GH_TOKEN:-}" ]; then + echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2 + exit 1 + fi + DISPATCHED=0 + for attempt in 1 2 3; do + if gh api --method POST repos/rustfs/rustfs/dispatches \ + -f event_type='rustfs-chain-performance' \ + -F 'client_payload[from_suite]=replication'; then + echo "dispatched next suite Performance (attempt ${attempt})" + DISPATCHED=1 + break + fi + echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2 + sleep "${attempt}0" + done + if [ "${DISPATCHED:-0}" -ne 1 ]; then + echo "ERROR: functional chain stalled: could not dispatch Performance after 3 attempts" >&2 + TITLE="[functional][chain] stalled after replication (run ${GITHUB_RUN_ID})" + BODY_FILE="$(mktemp)" + trap 'rm -f "${BODY_FILE}"' EXIT + { + echo "The functional chain could not hand off from **replication** to **Performance** after 3 attempts." + echo "" + echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + echo "- Expected next event: 'rustfs-chain-performance'" + echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable." + echo "- Recovery: re-dispatch manually with" + FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}" + echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-performance'" + FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}" + } > "${BODY_FILE}" + gh issue create -R rustfs/backlog --title "${TITLE}" \ + --body-file "${BODY_FILE}" --label functional-test \ + || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \ + || echo "could not file the stall alert issue either; check the token" >&2 + exit 1 + fi - name: Notify on failure if: failure() diff --git a/.github/workflows/rustfs-s3-compat-test.yml b/.github/workflows/rustfs-s3-compat-test.yml index 875db70bb..99828537b 100644 --- a/.github/workflows/rustfs-s3-compat-test.yml +++ b/.github/workflows/rustfs-s3-compat-test.yml @@ -40,6 +40,25 @@ jobs: timeout-minutes: 360 if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: + - name: Checkout repository (for report parser) + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + + - name: Initialize functional evidence + id: evidence + run: | + set -euo pipefail + umask 077 + FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-s3-compat-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch" + { + printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + } >> "${GITHUB_ENV}" + # auto-testing is private: clone it with the dedicated PF token (not # GITHUB_TOKEN) and retry transient GitHub/network failures. - name: Checkout auto-testing scripts (with retry) @@ -87,8 +106,6 @@ jobs: - name: Run S3 compatibility suite id: test - env: - LOG_FILE: /tmp/rustfs-s3-compat.log run: | set -euo pipefail chmod +x auto-testing/rustfs-s3-compat-test.sh @@ -105,10 +122,7 @@ jobs: ./auto-testing/rustfs-s3-compat-test.sh "${ARGS[@]}" - name: Generate report - if: always() - env: - LOG_FILE: /tmp/rustfs-s3-compat.log - REPORT_FILE: /tmp/rustfs-s3-compat-report.md + if: ${{ always() && steps.evidence.outcome == 'success' }} run: | set -euo pipefail PACKAGE_URL='${{ inputs.package_url }}' @@ -130,83 +144,44 @@ jobs: RUSTFS_VERSION_INFO="${DETECTED_VERSION}" fi fi - CASE_TABLE="/tmp/rustfs-s3-compat-cases.md" - python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY' - import re - import sys - - log_file, out_file = sys.argv[1], sys.argv[2] - ansi = re.compile(r'\x1b\[[0-9;]*m') - start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$') - done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b') - - rows = [] - index = {} - current = None - try: - with open(log_file, 'r', encoding='utf-8', errors='replace') as fh: - for raw in fh: - line = ansi.sub('', raw).strip() - m = start_re.match(line) - if m: - case_id, name = m.group(1), m.group(2) - current = case_id - if case_id not in index: - index[case_id] = len(rows) - rows.append([case_id, name, 'RUNNING']) - continue - m = done_re.match(line) - if m: - status, case_id = m.group(1), m.group(2) - if case_id in index: - rows[index[case_id]][2] = status - else: - rows.append([case_id, case_id, status]) - index[case_id] = len(rows) - 1 - current = None - except FileNotFoundError: - rows = [] - - counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0} - for _, _, status in rows: - counts[status] = counts.get(status, 0) + 1 - - with open(out_file, 'w', encoding='utf-8') as out: - out.write('## Case Summary\n\n') - out.write(f"- Total: {len(rows)}\\n") - out.write(f"- PASS: {counts.get('PASS', 0)}\\n") - out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n") - out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n") - out.write('\\n') - out.write('| Case | Name | Status |\\n') - out.write('| --- | --- | --- |\\n') - for case_id, name, status in rows: - out.write(f'| {case_id} | {name} | {status} |\\n') - PY + CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" + CASE_RESULT=success + python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure + RESULT=failure + if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then + RESULT=success + fi { echo "# RustFS S3 compatibility test report" echo "" echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo "- Attempt: ${GITHUB_RUN_ATTEMPT}" + echo "- Workflow Commit: ${GITHUB_SHA}" echo "- Trigger: ${{ github.event_name }}" echo "- Package: ${PACKAGE_SOURCE}" echo "- RustFS Version: ${RUSTFS_VERSION_INFO}" - echo "- Test Step Outcome: ${{ steps.test.outcome }}" + echo "- Test Step Outcome: ${RESULT}" + echo "- Suite Step Outcome: ${{ steps.test.outcome }}" echo "" - cat "${CASE_TABLE}" || true - echo "" - echo "## Log tail" - echo '```text' - tail -n 200 "${LOG_FILE}" || true - echo '```' + if [ "${RESULT}" = "success" ]; then + cat "${CASE_TABLE}" + echo "" + echo "## Log tail" + echo '```text' + tail -n 200 "${LOG_FILE}" + echo '```' + else + echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log." + fi } | tee "${REPORT_FILE}" cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" + [ "${RESULT}" = "success" ] - name: Upload functional report to dashboard - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} continue-on-error: true env: GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }} - REPORT_FILE: /tmp/rustfs-s3-compat-report.md SUITE: s3 run: | set -euo pipefail @@ -238,11 +213,10 @@ jobs: continue-on-error: true env: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} + EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }} SUITE: 's3' SUITE_LABEL: 'S3 compatibility' RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - REPORT_FILE: '/tmp/rustfs-s3-compat-report.md' - LOG_FILE: '/tmp/rustfs-s3-compat.log' run: | set -euo pipefail if [ -z "${GH_TOKEN:-}" ]; then @@ -270,14 +244,16 @@ jobs: echo "" echo "- Suite: \`${SUITE}\`" echo "- Run: ${RUN_URL}" + echo "- Attempt: ${GITHUB_RUN_ATTEMPT}" + echo "- Workflow Commit: ${GITHUB_SHA}" echo "- Trigger: ${GITHUB_EVENT_NAME}" echo "- Date: $(date -u +%Y-%m-%d)" echo "" echo "## Report (errors and symptoms)" echo "" - if [ -s "${REPORT_FILE}" ]; then + if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then redact < "${REPORT_FILE}" - elif [ -s "${LOG_FILE:-}" ]; then + elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then echo "(report file missing; log tail below)" echo "" tail -n 200 "${LOG_FILE}" | redact @@ -293,14 +269,15 @@ jobs: echo "filed backlog issue for suite ${SUITE}" - name: Upload report and logs - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: - name: rustfs-s3-compat-${{ github.run_id }} + name: rustfs-s3-compat-${{ github.run_id }}-${{ github.run_attempt }} path: | - /tmp/rustfs-s3-compat.log - /tmp/rustfs-s3-compat-report.md - if-no-files-found: warn + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md + if-no-files-found: error - name: Cleanup environment (after) if: always() diff --git a/.github/workflows/rustfs-security-test.yml b/.github/workflows/rustfs-security-test.yml index ee37e7d5d..02fe7d662 100644 --- a/.github/workflows/rustfs-security-test.yml +++ b/.github/workflows/rustfs-security-test.yml @@ -92,7 +92,7 @@ jobs: set -euo pipefail umask 077 SECURITY_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-security-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - mkdir -- "${SECURITY_ARTIFACTS_DIR}" + mkdir -- "${SECURITY_ARTIFACTS_DIR}" "${SECURITY_ARTIFACTS_DIR}-scratch" printf 'SECURITY_ARTIFACTS_DIR=%s\n' "${SECURITY_ARTIFACTS_DIR}" >> "${GITHUB_ENV}" # auto-testing is private: clone it with the dedicated PF token (not @@ -148,7 +148,7 @@ jobs: continue-on-error: true env: REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md - TMPDIR: ${{ env.SECURITY_ARTIFACTS_DIR }} + TMPDIR: ${{ env.SECURITY_ARTIFACTS_DIR }}-scratch RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/rustfs-repo/scripts/test/oidc_keycloak_live.sh run: | set -euo pipefail @@ -172,7 +172,7 @@ jobs: else ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}") fi - GITHUB_STEP_SUMMARY=/dev/null ./auto-testing/rustfs-security-test.sh "${ARGS[@]}" + GITHUB_STEP_SUMMARY=/dev/null ./auto-testing/rustfs-security-test.sh "${ARGS[@]}" 2>&1 | tee "${SECURITY_ARTIFACTS_DIR}/suite.log" - name: Generate report id: report @@ -305,7 +305,10 @@ jobs: uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: name: rustfs-security-test-${{ github.run_id }}-${{ github.run_attempt }} - path: ${{ env.SECURITY_ARTIFACTS_DIR }}/ + path: | + ${{ env.SECURITY_ARTIFACTS_DIR }}/report.md + ${{ env.SECURITY_ARTIFACTS_DIR }}/suite.log + ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md if-no-files-found: error retention-days: 3 diff --git a/.github/workflows/rustfs-storage-test.yml b/.github/workflows/rustfs-storage-test.yml index 1e99dce4e..e16fc0058 100644 --- a/.github/workflows/rustfs-storage-test.yml +++ b/.github/workflows/rustfs-storage-test.yml @@ -49,6 +49,25 @@ jobs: timeout-minutes: 360 if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: + - name: Checkout repository (for report parser) + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + + - name: Initialize functional evidence + id: evidence + run: | + set -euo pipefail + umask 077 + FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-storage-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch" + { + printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + } >> "${GITHUB_ENV}" + # auto-testing is private: clone it with the dedicated PF token (not # GITHUB_TOKEN) and retry transient GitHub/network failures. - name: Checkout auto-testing scripts (with retry) @@ -96,8 +115,6 @@ jobs: - name: Run storage engine suite id: test - env: - LOG_FILE: /tmp/rustfs-storage.log run: | set -euo pipefail chmod +x auto-testing/rustfs-storage-test.sh @@ -120,10 +137,7 @@ jobs: ./auto-testing/rustfs-storage-test.sh "${ARGS[@]}" - name: Generate report - if: always() - env: - LOG_FILE: /tmp/rustfs-storage.log - REPORT_FILE: /tmp/rustfs-storage-report.md + if: ${{ always() && steps.evidence.outcome == 'success' }} run: | set -euo pipefail PACKAGE_URL='${{ inputs.package_url }}' @@ -145,83 +159,44 @@ jobs: RUSTFS_VERSION_INFO="${DETECTED_VERSION}" fi fi - CASE_TABLE="/tmp/rustfs-storage-cases.md" - python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY' - import re - import sys - - log_file, out_file = sys.argv[1], sys.argv[2] - ansi = re.compile(r'\x1b\[[0-9;]*m') - start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$') - done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b') - - rows = [] - index = {} - current = None - try: - with open(log_file, 'r', encoding='utf-8', errors='replace') as fh: - for raw in fh: - line = ansi.sub('', raw).strip() - m = start_re.match(line) - if m: - case_id, name = m.group(1), m.group(2) - current = case_id - if case_id not in index: - index[case_id] = len(rows) - rows.append([case_id, name, 'RUNNING']) - continue - m = done_re.match(line) - if m: - status, case_id = m.group(1), m.group(2) - if case_id in index: - rows[index[case_id]][2] = status - else: - rows.append([case_id, case_id, status]) - index[case_id] = len(rows) - 1 - current = None - except FileNotFoundError: - rows = [] - - counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0} - for _, _, status in rows: - counts[status] = counts.get(status, 0) + 1 - - with open(out_file, 'w', encoding='utf-8') as out: - out.write('## Case Summary\n\n') - out.write(f"- Total: {len(rows)}\\n") - out.write(f"- PASS: {counts.get('PASS', 0)}\\n") - out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n") - out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n") - out.write('\\n') - out.write('| Case | Name | Status |\\n') - out.write('| --- | --- | --- |\\n') - for case_id, name, status in rows: - out.write(f'| {case_id} | {name} | {status} |\\n') - PY + CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" + CASE_RESULT=success + python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure + RESULT=failure + if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then + RESULT=success + fi { echo "# RustFS storage engine test report" echo "" echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo "- Attempt: ${GITHUB_RUN_ATTEMPT}" + echo "- Workflow Commit: ${GITHUB_SHA}" echo "- Trigger: ${{ github.event_name }}" echo "- Package: ${PACKAGE_SOURCE}" echo "- RustFS Version: ${RUSTFS_VERSION_INFO}" - echo "- Test Step Outcome: ${{ steps.test.outcome }}" + echo "- Test Step Outcome: ${RESULT}" + echo "- Suite Step Outcome: ${{ steps.test.outcome }}" echo "" - cat "${CASE_TABLE}" || true - echo "" - echo "## Log tail" - echo '```text' - tail -n 200 "${LOG_FILE}" || true - echo '```' + if [ "${RESULT}" = "success" ]; then + cat "${CASE_TABLE}" + echo "" + echo "## Log tail" + echo '```text' + tail -n 200 "${LOG_FILE}" + echo '```' + else + echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log." + fi } | tee "${REPORT_FILE}" cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" + [ "${RESULT}" = "success" ] - name: Upload functional report to dashboard - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} continue-on-error: true env: GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }} - REPORT_FILE: /tmp/rustfs-storage-report.md SUITE: storage run: | set -euo pipefail @@ -253,11 +228,10 @@ jobs: continue-on-error: true env: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} + EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }} SUITE: 'storage' SUITE_LABEL: 'Storage engine' RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - REPORT_FILE: '/tmp/rustfs-storage-report.md' - LOG_FILE: '/tmp/rustfs-storage.log' run: | set -euo pipefail if [ -z "${GH_TOKEN:-}" ]; then @@ -285,14 +259,16 @@ jobs: echo "" echo "- Suite: \`${SUITE}\`" echo "- Run: ${RUN_URL}" + echo "- Attempt: ${GITHUB_RUN_ATTEMPT}" + echo "- Workflow Commit: ${GITHUB_SHA}" echo "- Trigger: ${GITHUB_EVENT_NAME}" echo "- Date: $(date -u +%Y-%m-%d)" echo "" echo "## Report (errors and symptoms)" echo "" - if [ -s "${REPORT_FILE}" ]; then + if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then redact < "${REPORT_FILE}" - elif [ -s "${LOG_FILE:-}" ]; then + elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then echo "(report file missing; log tail below)" echo "" tail -n 200 "${LOG_FILE}" | redact @@ -308,14 +284,15 @@ jobs: echo "filed backlog issue for suite ${SUITE}" - name: Upload report and logs - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: - name: rustfs-storage-${{ github.run_id }} + name: rustfs-storage-${{ github.run_id }}-${{ github.run_attempt }} path: | - /tmp/rustfs-storage.log - /tmp/rustfs-storage-report.md - if-no-files-found: warn + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md + if-no-files-found: error - name: Cleanup environment (after) if: always() diff --git a/.github/workflows/rustfs-upgrade-test.yml b/.github/workflows/rustfs-upgrade-test.yml index 612765874..5a91793a0 100644 --- a/.github/workflows/rustfs-upgrade-test.yml +++ b/.github/workflows/rustfs-upgrade-test.yml @@ -82,6 +82,25 @@ jobs: timeout-minutes: 420 if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: + - name: Checkout repository (for report parser) + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + + - name: Initialize functional evidence + id: evidence + run: | + set -euo pipefail + umask 077 + FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-upgrade-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch" + { + printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + } >> "${GITHUB_ENV}" + # auto-testing is private: clone it with the dedicated PF token (not # GITHUB_TOKEN) and retry transient GitHub/network failures. - name: Checkout auto-testing scripts (with retry) @@ -142,7 +161,6 @@ jobs: - name: Run upgrade compatibility suite id: test env: - LOG_FILE: /tmp/rustfs-upgrade.log GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} run: | set -euo pipefail @@ -200,10 +218,7 @@ jobs: ./auto-testing/rustfs-upgrade-test.sh "${ARGS[@]}" - name: Generate report - if: always() - env: - LOG_FILE: /tmp/rustfs-upgrade.log - REPORT_FILE: /tmp/rustfs-upgrade-report.md + if: ${{ always() && steps.evidence.outcome == 'success' }} run: | set -euo pipefail FROM_URL='${{ inputs.from_url }}' @@ -224,103 +239,47 @@ jobs: else TO_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}" fi - CASE_TABLE="/tmp/rustfs-upgrade-cases.md" - MATRIX_TABLE="/tmp/rustfs-upgrade-matrix.md" - python3 - "${LOG_FILE}" "${CASE_TABLE}" "${MATRIX_TABLE}" <<'PY' - import re - import sys - - log_file, out_file, matrix_file = sys.argv[1], sys.argv[2], sys.argv[3] - ansi = re.compile(r'\x1b\[[0-9;]*m') - start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$') - done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b') - topo_re = re.compile( - r'^\[UPG-TOPO\]\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+PASS=(\d+)\s+FAIL=(\d+)\s*$') - - rows = [] - index = {} - topo_rows = [] - try: - with open(log_file, 'r', encoding='utf-8', errors='replace') as fh: - for raw in fh: - line = ansi.sub('', raw).strip() - m = topo_re.match(line) - if m: - topo_rows.append(m.groups()) - continue - m = start_re.match(line) - if m: - case_id, name = m.group(1), m.group(2) - if case_id not in index: - index[case_id] = len(rows) - rows.append([case_id, name, 'RUNNING']) - continue - m = done_re.match(line) - if m: - status, case_id = m.group(1), m.group(2) - if case_id in index: - rows[index[case_id]][2] = status - else: - rows.append([case_id, case_id, status]) - index[case_id] = len(rows) - 1 - except FileNotFoundError: - rows = [] - - counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0} - for _, _, status in rows: - counts[status] = counts.get(status, 0) + 1 - - with open(out_file, 'w', encoding='utf-8') as out: - out.write('## Case Summary\n\n') - out.write(f"- Total: {len(rows)}\\n") - out.write(f"- PASS: {counts.get('PASS', 0)}\\n") - out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n") - out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n") - out.write('\\n') - out.write('| Case | Name | Status |\\n') - out.write('| --- | --- | --- |\\n') - for case_id, name, status in rows: - out.write(f'| {case_id} | {name} | {status} |\\n') - - # Upgrade matrix: one row per topology/backend with the versions - # captured on the nodes (rustfs --version) and the aggregated - # result. The dashboard renders this table directly. - with open(matrix_file, 'w', encoding='utf-8') as out: - out.write('## Upgrade Matrix\n\n') - out.write('| Topology | KMS Backend | From Version | To Version | Result |\n') - out.write('| --- | --- | --- | --- | --- |\n') - for topo, backend, old_v, new_v, npass, nfail in topo_rows: - result = 'PASS' if nfail == '0' else 'FAIL' - out.write(f'| {topo} | {backend} | {old_v} | {new_v} | {result} (PASS={npass} FAIL={nfail}) |\n') - if not topo_rows: - out.write('| - | - | - | - | NOT RUN (suite failed before upgrade) |\n') - PY + CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" + MATRIX_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/matrix.md" + CASE_RESULT=success + python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" "${MATRIX_TABLE}" || CASE_RESULT=failure + RESULT=failure + if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then + RESULT=success + fi { echo "# RustFS upgrade compatibility report" echo "" echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo "- Attempt: ${GITHUB_RUN_ATTEMPT}" + echo "- Workflow Commit: ${GITHUB_SHA}" echo "- Trigger: ${{ github.event_name }}" echo "- From: ${FROM_SOURCE}" echo "- To: ${TO_SOURCE}" - echo "- Test Step Outcome: ${{ steps.test.outcome }}" + echo "- Test Step Outcome: ${RESULT}" + echo "- Suite Step Outcome: ${{ steps.test.outcome }}" echo "" - cat "${MATRIX_TABLE}" || true - echo "" - cat "${CASE_TABLE}" || true - echo "" - echo "## Log tail" - echo '```text' - tail -n 200 "${LOG_FILE}" || true - echo '```' + if [ "${RESULT}" = "success" ]; then + cat "${MATRIX_TABLE}" + echo "" + cat "${CASE_TABLE}" + echo "" + echo "## Log tail" + echo '```text' + tail -n 200 "${LOG_FILE}" + echo '```' + else + echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log." + fi } | tee "${REPORT_FILE}" cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" + [ "${RESULT}" = "success" ] - name: Upload functional report to dashboard - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} continue-on-error: true env: GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }} - REPORT_FILE: /tmp/rustfs-upgrade-report.md SUITE: upgrade run: | set -euo pipefail @@ -352,11 +311,10 @@ jobs: continue-on-error: true env: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} + EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }} SUITE: 'upgrade' SUITE_LABEL: 'Upgrade compatibility' RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - REPORT_FILE: '/tmp/rustfs-upgrade-report.md' - LOG_FILE: '/tmp/rustfs-upgrade.log' run: | set -euo pipefail if [ -z "${GH_TOKEN:-}" ]; then @@ -384,14 +342,16 @@ jobs: echo "" echo "- Suite: \`${SUITE}\`" echo "- Run: ${RUN_URL}" + echo "- Attempt: ${GITHUB_RUN_ATTEMPT}" + echo "- Workflow Commit: ${GITHUB_SHA}" echo "- Trigger: ${GITHUB_EVENT_NAME}" echo "- Date: $(date -u +%Y-%m-%d)" echo "" echo "## Report (errors and symptoms)" echo "" - if [ -s "${REPORT_FILE}" ]; then + if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then redact < "${REPORT_FILE}" - elif [ -s "${LOG_FILE:-}" ]; then + elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then echo "(report file missing; log tail below)" echo "" tail -n 200 "${LOG_FILE}" | redact @@ -407,14 +367,16 @@ jobs: echo "filed backlog issue for suite ${SUITE}" - name: Upload report and logs - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: - name: rustfs-upgrade-test-${{ github.run_id }} + name: rustfs-upgrade-test-${{ github.run_id }}-${{ github.run_attempt }} path: | - /tmp/rustfs-upgrade-report.md - /tmp/rustfs-upgrade.*/* - if-no-files-found: ignore + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/matrix.md + if-no-files-found: error retention-days: 3 - name: Cleanup environment (after) diff --git a/crates/e2e_test/src/upgrade_compatibility_test.rs b/crates/e2e_test/src/upgrade_compatibility_test.rs index 558c10d4c..9f1871552 100644 --- a/crates/e2e_test/src/upgrade_compatibility_test.rs +++ b/crates/e2e_test/src/upgrade_compatibility_test.rs @@ -12,19 +12,30 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging, rustfs_binary_path}; +use crate::common::{ + RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging, replication_fast_env, rustfs_binary_path, +}; +use crate::fake_s3_target::{FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target}; +use crate::replication_extension_test::{ + LOOPBACK_REPLICATION_TARGET_ENV, ReplicationTargetOptions, put_bucket_replication, set_replication_target_with_options, +}; use aws_sdk_s3::Client; use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{ - BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, ServerSideEncryption, VersioningConfiguration, + BucketLifecycleConfiguration, BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DefaultRetention, + ExpirationStatus, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter, ObjectLockConfiguration, ObjectLockEnabled, + ObjectLockRetentionMode, ObjectLockRule, PublicAccessBlockConfiguration, ServerSideEncryption, ServerSideEncryptionByDefault, + ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, Tagging, VersioningConfiguration, }; +use http::{Method, StatusCode}; use std::path::{Path, PathBuf}; use std::time::Duration; use tokio::task::JoinSet; use tokio::time::{Instant, sleep}; type TestResult = Result<(), Box>; +type BoxError = Box; const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY"; const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY"; @@ -40,6 +51,32 @@ const MULTIPART_UPLOADS_PER_WORKER: usize = 16; // comfortably covers that window plus CI scheduling jitter. const LISTING_CONVERGENCE_TIMEOUT: Duration = Duration::from_secs(30); +// Bucket-configuration upgrade/rollback scenarios (rustfs#7172, #7183, #7089). +const CONFIG_PLAIN_BUCKET: &str = "upgrade-config-plain"; +const CONFIG_ENCRYPTED_BUCKET: &str = "upgrade-config-encrypted"; +const CONFIG_REPLICATED_BUCKET: &str = "upgrade-config-replicated"; +const CONFIG_LOCKED_BUCKET: &str = "upgrade-config-locked"; +const CONFIG_REPLICA_BUCKET: &str = "upgrade-config-replica"; +const ROLLBACK_BUCKET: &str = "rollback-config-data"; +const ROLLBACK_REPLICA_BUCKET: &str = "rollback-config-replica"; +const BUCKET_QUOTA_BYTES: u64 = 64 * 1024 * 1024; +const LIFECYCLE_RULE_ID: &str = "upgrade-expire-logs"; +const LIFECYCLE_PREFIX: &str = "logs/"; +const LIFECYCLE_DAYS: i32 = 30; +const BUCKET_TAG_KEY: &str = "owner"; +const BUCKET_TAG_VALUE: &str = "upgrade-compatibility"; +const OBJECT_LOCK_DAYS: i32 = 1; +// `set-bucket-quota` answers 503 until the scanner has made the bucket's usage +// authoritative; the quota test uses the same 30s budget. +const QUOTA_READINESS_TIMEOUT: Duration = Duration::from_secs(30); +// Quota admission fails closed while a freshly started server has neither +// authoritative usage nor a persisted degraded baseline for the bucket +// (rustfs#5716), so a write to a quota-enabled bucket is retryable-503 for that +// window. It is a restart property, not an upgrade property — the same window +// opens on the very first start — so the write assertions ride it out instead +// of treating it as an upgrade failure. +const QUOTA_ADMISSION_WARMUP_TIMEOUT: Duration = Duration::from_secs(90); + fn source_binary() -> Result> { let path = std::env::var_os(SOURCE_BINARY_ENV) .map(PathBuf::from) @@ -429,3 +466,653 @@ async fn rolling_upgrade_from_rc2_preserves_mixed_version_contracts() -> TestRes Ok(()) } + +/// Child-process environment shared by both bucket-configuration scenarios. +/// +/// The replication target is an in-process fake bound to `127.0.0.1`, which +/// `set-remote-target` rejects as an SSRF risk without the loopback opt-in, and +/// the proxy bypass keeps a developer's `HTTP_PROXY` from intercepting the +/// server's outbound health check. +fn bucket_config_server_env() -> Vec<(&'static str, &'static str)> { + let mut env = vec![ + (SSE_MASTER_KEY_ENV, SSE_MASTER_KEY), + ("NO_PROXY", "127.0.0.1,localhost"), + ("HTTP_PROXY", ""), + ("HTTPS_PROXY", ""), + // Shorten the scanner cycle so the bucket's usage becomes authoritative + // in seconds; both `set-bucket-quota` and quota admission block on it. + ("RUSTFS_SCANNER_CYCLE", "1"), + ("RUSTFS_SCANNER_START_DELAY_SECS", "0"), + ]; + env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV); + env.extend(replication_fast_env()); + env +} + +/// Restart `env` in place on the same data directory using an explicit binary. +/// +/// [`RustFSTestEnvironment::restart_server_preserving_data`] always relaunches +/// the workspace build, which is the upgrade direction only. The rollback +/// scenario needs the reverse: stop the current build and bring the pinned +/// previous release up on the metadata that build just wrote. +async fn restart_from_binary(env: &mut RustFSTestEnvironment, binary: &Path, server_env: &[(&str, &str)]) -> TestResult { + env.stop_server(); + env.start_rustfs_server_from_binary(binary, vec![], server_env).await +} + +async fn set_bucket_quota(env: &RustFSTestEnvironment, bucket: &str, quota_bytes: u64) -> TestResult { + let path = format!("/rustfs/admin/v3/quota/{bucket}"); + let body = serde_json::json!({ "quota": quota_bytes, "quota_type": "HARD" }).to_string(); + let deadline = Instant::now() + QUOTA_READINESS_TIMEOUT; + loop { + let (status, response) = + admin_request(&env.url, Method::PUT, &path, Some(body.clone()), &env.access_key, &env.secret_key).await?; + if status.is_success() { + return Ok(()); + } + if status != StatusCode::SERVICE_UNAVAILABLE || Instant::now() >= deadline { + return Err(format!("setting the quota of {bucket} failed: {status} {response}").into()); + } + sleep(Duration::from_millis(500)).await; + } +} + +/// PUT into a quota-enabled bucket, riding out the post-start quota-admission +/// warm-up described on [`QUOTA_ADMISSION_WARMUP_TIMEOUT`]. +/// +/// Only `ServiceUnavailable` is retried: any other failure, and a warm-up that +/// never ends, is a genuine regression and surfaces as an error. +async fn put_object_through_quota_warmup(client: &Client, bucket: &str, key: &str, body: &'static [u8]) -> TestResult { + let deadline = Instant::now() + QUOTA_ADMISSION_WARMUP_TIMEOUT; + loop { + let result = client + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from_static(body)) + .send() + .await; + let error = match result { + Ok(_) => return Ok(()), + Err(error) => error, + }; + let retryable = error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("ServiceUnavailable"); + if !retryable || Instant::now() >= deadline { + return Err(format!("PUT {bucket}/{key} failed after the quota warm-up window: {error}").into()); + } + sleep(Duration::from_millis(500)).await; + } +} + +async fn get_bucket_quota(env: &RustFSTestEnvironment, bucket: &str) -> Result, BoxError> { + let path = format!("/rustfs/admin/v3/quota/{bucket}"); + let (status, response) = admin_request(&env.url, Method::GET, &path, None, &env.access_key, &env.secret_key).await?; + if status != StatusCode::OK { + return Err(format!("reading the quota of {bucket} failed: {status} {response}").into()); + } + let quota: serde_json::Value = serde_json::from_str(&response)?; + Ok(quota.get("quota").and_then(serde_json::Value::as_u64)) +} + +/// `GET /rustfs/admin/v3/list-remote-targets?bucket=...`. +/// +/// Returns an error for any non-200, because rustfs#7172 made this endpoint +/// fail closed on a `bucket-targets.json` blob the running build cannot parse. +/// An upgrade that misreads a blob written by the previous release therefore +/// shows up here as an error, and a silently dropped target shows up as an +/// empty list — the caller must distinguish the two. +async fn list_remote_targets(env: &RustFSTestEnvironment, bucket: &str) -> Result, BoxError> { + let path = format!("/rustfs/admin/v3/list-remote-targets?bucket={}", urlencoding::encode(bucket)); + let (status, response) = admin_request(&env.url, Method::GET, &path, None, &env.access_key, &env.secret_key).await?; + if status != StatusCode::OK { + return Err(format!("list-remote-targets for {bucket} failed: {status} {response}").into()); + } + Ok(serde_json::from_str(&response)?) +} + +/// Assert that `bucket` still carries exactly the replication target `arn`. +async fn assert_remote_target_preserved(env: &RustFSTestEnvironment, bucket: &str, arn: &str, context: &str) -> TestResult { + let targets = list_remote_targets(env, bucket).await?; + assert_eq!( + targets.len(), + 1, + "{context}: list-remote-targets must still report the single configured target, got {targets:?}" + ); + assert_eq!( + targets[0].get("arn").and_then(serde_json::Value::as_str), + Some(arn), + "{context}: the target ARN changed across the restart: {targets:?}" + ); + Ok(()) +} + +/// Configure a replication target on `bucket` pointing at the in-process fake, +/// then attach an enabled replication rule for it. Returns the target ARN. +async fn configure_replication( + env: &RustFSTestEnvironment, + bucket: &str, + target: &FakeS3Target, + target_bucket: &str, +) -> Result { + let arn = set_replication_target_with_options( + env, + bucket, + ReplicationTargetOptions { + endpoint: &target.address(), + access_key: FAKE_ACCESS_KEY, + secret_key: FAKE_SECRET_KEY, + target_bucket, + secure: false, + skip_tls_verify: false, + ca_cert_pem: None, + }, + ) + .await?; + put_bucket_replication(env, bucket, &arn).await?; + Ok(arn) +} + +async fn put_default_sse_s3_encryption(client: &Client, bucket: &str) -> TestResult { + let configuration = ServerSideEncryptionConfiguration::builder() + .rules( + ServerSideEncryptionRule::builder() + .apply_server_side_encryption_by_default( + ServerSideEncryptionByDefault::builder() + .sse_algorithm(ServerSideEncryption::Aes256) + .build()?, + ) + .build(), + ) + .build()?; + client + .put_bucket_encryption() + .bucket(bucket) + .server_side_encryption_configuration(configuration) + .send() + .await?; + Ok(()) +} + +async fn assert_default_sse_s3_encryption(client: &Client, bucket: &str, context: &str) -> TestResult { + let response = client.get_bucket_encryption().bucket(bucket).send().await?; + let rules = response + .server_side_encryption_configuration() + .ok_or("GetBucketEncryption omitted the configuration")? + .rules(); + assert_eq!(rules.len(), 1, "{context}: expected exactly one encryption rule, got {rules:?}"); + assert_eq!( + rules[0] + .apply_server_side_encryption_by_default() + .map(ServerSideEncryptionByDefault::sse_algorithm), + Some(&ServerSideEncryption::Aes256), + "{context}: the default encryption algorithm changed" + ); + Ok(()) +} + +async fn put_bucket_tag(client: &Client, bucket: &str) -> TestResult { + let tagging = Tagging::builder() + .tag_set(Tag::builder().key(BUCKET_TAG_KEY).value(BUCKET_TAG_VALUE).build()?) + .build()?; + client.put_bucket_tagging().bucket(bucket).tagging(tagging).send().await?; + Ok(()) +} + +async fn assert_bucket_tag(client: &Client, bucket: &str, context: &str) -> TestResult { + let tags = client.get_bucket_tagging().bucket(bucket).send().await?; + let tag_set = tags.tag_set(); + assert_eq!(tag_set.len(), 1, "{context}: expected exactly one bucket tag, got {tag_set:?}"); + assert_eq!(tag_set[0].key(), BUCKET_TAG_KEY, "{context}: bucket tag key changed"); + assert_eq!(tag_set[0].value(), BUCKET_TAG_VALUE, "{context}: bucket tag value changed"); + Ok(()) +} + +async fn assert_versioning_enabled(client: &Client, bucket: &str, context: &str) -> TestResult { + let versioning = client.get_bucket_versioning().bucket(bucket).send().await?; + assert_eq!( + versioning.status(), + Some(&BucketVersioningStatus::Enabled), + "{context}: versioning is no longer Enabled on {bucket}" + ); + Ok(()) +} + +fn bucket_policy_document(bucket: &str) -> serde_json::Value { + serde_json::json!({ + "Version": "2012-10-17", + "Statement": [{ + "Sid": "UpgradePublicRead", + "Effect": "Allow", + "Principal": { "AWS": ["*"] }, + "Action": ["s3:GetObject"], + "Resource": [format!("arn:aws:s3:::{bucket}/public/*")] + }] + }) +} + +/// `GET .../on-demand-migration/{bucket}/status`. +/// +/// The migration module defaults on from rustfs#7089, so a bucket that never +/// configured a source must still answer `configured: false` rather than +/// engaging the migration path. +async fn assert_migration_not_configured(env: &RustFSTestEnvironment, bucket: &str) -> TestResult { + let path = format!("/rustfs/admin/v3/on-demand-migration/{bucket}/status"); + let (status, response) = admin_request(&env.url, Method::GET, &path, None, &env.access_key, &env.secret_key).await?; + assert_eq!( + status, + StatusCode::OK, + "the migration status endpoint must answer for an unconfigured bucket: {status} {response}" + ); + let body: serde_json::Value = serde_json::from_str(&response)?; + assert_eq!( + body.get("configured"), + Some(&serde_json::Value::Bool(false)), + "a bucket upgraded from the previous release must not look migration-configured: {body}" + ); + Ok(()) +} + +/// A GET for a key that was never written must be a plain `NoSuchKey`. +/// +/// With the migration module on by default this is the cheap proof that an +/// unconfigured bucket never consults a source: any migration engagement would +/// surface as a different status or error code here. +async fn assert_missing_key_is_no_such_key(client: &Client, bucket: &str, key: &str) -> TestResult { + let error = client + .get_object() + .bucket(bucket) + .key(key) + .send() + .await + .expect_err("a key that was never written must not be readable"); + assert_eq!( + error.raw_response().map(|response| response.status().as_u16()), + Some(404), + "a missing key must stay a 404 on a bucket with no migration configuration" + ); + assert_eq!( + error.as_service_error().and_then(ProvideErrorMetadata::code), + Some("NoSuchKey"), + "a missing key must stay NoSuchKey on a bucket with no migration configuration" + ); + Ok(()) +} + +/// Bucket configuration written by the pinned previous release must survive an +/// upgrade to the current build unchanged, and must keep working. +/// +/// This pins the three on-disk surfaces the on-demand-migration series moved: +/// +/// * `BucketMetadata` grew two msgpack keys (encoded map length 44 -> 46), so +/// every configuration read below decodes a 44-key blob on 46-key code. +/// * rustfs#7172 made an unreadable `bucket-targets.json` / encryption / +/// public-access-block / quota blob "present but unreadable" instead of +/// silently defaulting, and made `list-remote-targets` fail closed on it. A +/// replication target configured by the old release must therefore still be +/// *listed*, not dropped and not an error. +/// * rustfs#7183 made the object write path refuse a PUT when the bucket's +/// encryption configuration cannot be read, so a misparsed SSE config would +/// turn every PUT to that bucket into a 500. +/// +/// Not covered on purpose: on-demand-migration configuration itself, which the +/// previous release has no public API for — the reverse direction is asserted +/// instead (an upgraded bucket reports `configured: false`). +#[tokio::test] +#[ignore = "requires a pinned previous RustFS release binary"] +async fn direct_upgrade_from_previous_release_preserves_bucket_configuration() -> TestResult { + init_logging(); + let previous_binary = source_binary()?; + + // In-process: the fake target outlives both server processes, so the + // replication target stays reachable across the upgrade. + let replication_target = FakeS3Target::start().await?; + replication_target.create_bucket(CONFIG_REPLICA_BUCKET); + + let mut env = RustFSTestEnvironment::new().await?; + let server_env = bucket_config_server_env(); + env.start_rustfs_server_from_binary(&previous_binary, vec![], &server_env) + .await?; + let old_client = env.create_s3_client(); + + env.create_test_bucket(CONFIG_PLAIN_BUCKET).await?; + env.create_test_bucket(CONFIG_ENCRYPTED_BUCKET).await?; + env.create_test_bucket(CONFIG_REPLICATED_BUCKET).await?; + old_client + .create_bucket() + .bucket(CONFIG_LOCKED_BUCKET) + .object_lock_enabled_for_bucket(true) + .send() + .await?; + + // Plain bucket: policy, tags, lifecycle, quota. + let policy = bucket_policy_document(CONFIG_PLAIN_BUCKET); + old_client + .put_bucket_policy() + .bucket(CONFIG_PLAIN_BUCKET) + .policy(policy.to_string()) + .send() + .await?; + put_bucket_tag(&old_client, CONFIG_PLAIN_BUCKET).await?; + old_client + .put_bucket_lifecycle_configuration() + .bucket(CONFIG_PLAIN_BUCKET) + .lifecycle_configuration( + BucketLifecycleConfiguration::builder() + .rules( + LifecycleRule::builder() + .id(LIFECYCLE_RULE_ID) + .status(ExpirationStatus::Enabled) + .filter(LifecycleRuleFilter::builder().prefix(LIFECYCLE_PREFIX).build()) + .expiration(LifecycleExpiration::builder().days(LIFECYCLE_DAYS).build()) + .build()?, + ) + .build()?, + ) + .send() + .await?; + set_bucket_quota(&env, CONFIG_PLAIN_BUCKET, BUCKET_QUOTA_BYTES).await?; + + // Encrypted bucket: SSE-S3 default encryption plus a fully restrictive + // public access block, both of which rustfs#7172 now fails closed on. + put_default_sse_s3_encryption(&old_client, CONFIG_ENCRYPTED_BUCKET).await?; + old_client + .put_public_access_block() + .bucket(CONFIG_ENCRYPTED_BUCKET) + .public_access_block_configuration( + PublicAccessBlockConfiguration::builder() + .block_public_acls(true) + .ignore_public_acls(true) + .block_public_policy(true) + .restrict_public_buckets(true) + .build(), + ) + .send() + .await?; + + // Replicated bucket: versioning, a validated remote target, a rule. + enable_versioning(&old_client, CONFIG_REPLICATED_BUCKET).await?; + let target_arn = configure_replication(&env, CONFIG_REPLICATED_BUCKET, &replication_target, CONFIG_REPLICA_BUCKET).await?; + assert_remote_target_preserved(&env, CONFIG_REPLICATED_BUCKET, &target_arn, "before the upgrade").await?; + + // Object-lock bucket: a default GOVERNANCE retention on a fresh bucket. + old_client + .put_object_lock_configuration() + .bucket(CONFIG_LOCKED_BUCKET) + .object_lock_configuration( + ObjectLockConfiguration::builder() + .object_lock_enabled(ObjectLockEnabled::Enabled) + .rule( + ObjectLockRule::builder() + .default_retention( + DefaultRetention::builder() + .mode(ObjectLockRetentionMode::Governance) + .days(OBJECT_LOCK_DAYS) + .build(), + ) + .build(), + ) + .build(), + ) + .send() + .await?; + + let plain_key = "plain/written-by-previous"; + let plain_bytes = b"plain object written by the previous RustFS release"; + put_object_through_quota_warmup(&old_client, CONFIG_PLAIN_BUCKET, plain_key, plain_bytes).await?; + + let encrypted_key = "encrypted/written-by-previous"; + let encrypted_bytes = b"default-encrypted object written by the previous RustFS release"; + old_client + .put_object() + .bucket(CONFIG_ENCRYPTED_BUCKET) + .key(encrypted_key) + .body(ByteStream::from_static(encrypted_bytes)) + .send() + .await?; + assert_eq!( + read_object(&old_client, CONFIG_ENCRYPTED_BUCKET, encrypted_key, None) + .await? + .0, + Some(ServerSideEncryption::Aes256), + "the previous release must apply the bucket default encryption it just accepted" + ); + + // The multipart object lives in the default-encrypted bucket so the + // upgraded build has to reassemble parts *and* re-derive the object key. + let multipart_key = "encrypted/multipart-written-by-previous"; + let multipart_parts = vec![vec![b'm'; 5 * 1024 * 1024], b"final multipart bytes".to_vec()]; + let multipart_bytes = multipart_parts.concat(); + write_multipart(&old_client, CONFIG_ENCRYPTED_BUCKET, multipart_key, &multipart_parts).await?; + + let versioned_key = "versioned/written-by-previous"; + let versioned_bytes = b"versioned object written by the previous RustFS release"; + let versioned_id = old_client + .put_object() + .bucket(CONFIG_REPLICATED_BUCKET) + .key(versioned_key) + .body(ByteStream::from_static(versioned_bytes)) + .send() + .await? + .version_id() + .ok_or("versioned PUT omitted version ID")? + .to_string(); + + env.restart_server_preserving_data(vec![], &server_env).await?; + let new_client = env.create_s3_client(); + + // Every configuration must read back unchanged on the upgraded build. + let upgraded_policy = new_client.get_bucket_policy().bucket(CONFIG_PLAIN_BUCKET).send().await?; + let upgraded_policy: serde_json::Value = + serde_json::from_str(upgraded_policy.policy().ok_or("GetBucketPolicy omitted the document")?)?; + assert_eq!(upgraded_policy, policy, "the bucket policy changed across the upgrade"); + assert_bucket_tag(&new_client, CONFIG_PLAIN_BUCKET, "after the upgrade").await?; + + let lifecycle = new_client + .get_bucket_lifecycle_configuration() + .bucket(CONFIG_PLAIN_BUCKET) + .send() + .await?; + let rules = lifecycle.rules(); + assert_eq!(rules.len(), 1, "the lifecycle rule count changed across the upgrade: {rules:?}"); + assert_eq!(rules[0].id(), Some(LIFECYCLE_RULE_ID)); + assert_eq!(rules[0].status(), &ExpirationStatus::Enabled); + assert_eq!( + rules[0].expiration().and_then(LifecycleExpiration::days), + Some(LIFECYCLE_DAYS), + "the lifecycle expiration changed across the upgrade" + ); + + assert_eq!( + get_bucket_quota(&env, CONFIG_PLAIN_BUCKET).await?, + Some(BUCKET_QUOTA_BYTES), + "the bucket quota changed across the upgrade" + ); + + assert_default_sse_s3_encryption(&new_client, CONFIG_ENCRYPTED_BUCKET, "after the upgrade").await?; + let public_access_block = new_client + .get_public_access_block() + .bucket(CONFIG_ENCRYPTED_BUCKET) + .send() + .await?; + let public_access_block = public_access_block + .public_access_block_configuration() + .ok_or("GetPublicAccessBlock omitted the configuration")?; + assert_eq!(public_access_block.block_public_acls(), Some(true)); + assert_eq!(public_access_block.ignore_public_acls(), Some(true)); + assert_eq!(public_access_block.block_public_policy(), Some(true)); + assert_eq!(public_access_block.restrict_public_buckets(), Some(true)); + + assert_versioning_enabled(&new_client, CONFIG_REPLICATED_BUCKET, "after the upgrade").await?; + // rustfs#7172: neither an empty list nor an error is acceptable here. + assert_remote_target_preserved(&env, CONFIG_REPLICATED_BUCKET, &target_arn, "after the upgrade").await?; + let replication = new_client + .get_bucket_replication() + .bucket(CONFIG_REPLICATED_BUCKET) + .send() + .await?; + let replication_rules = replication + .replication_configuration() + .ok_or("GetBucketReplication omitted the configuration")? + .rules(); + assert_eq!( + replication_rules.len(), + 1, + "the replication rule count changed across the upgrade: {replication_rules:?}" + ); + assert_eq!( + replication_rules[0].destination().map(|destination| destination.bucket()), + Some(target_arn.as_str()), + "the replication rule no longer points at the configured target" + ); + + let object_lock = new_client + .get_object_lock_configuration() + .bucket(CONFIG_LOCKED_BUCKET) + .send() + .await?; + let object_lock = object_lock + .object_lock_configuration() + .ok_or("GetObjectLockConfiguration omitted the configuration")?; + assert_eq!(object_lock.object_lock_enabled(), Some(&ObjectLockEnabled::Enabled)); + let retention = object_lock + .rule() + .and_then(ObjectLockRule::default_retention) + .ok_or("the object lock configuration lost its default retention")?; + assert_eq!(retention.mode(), Some(&ObjectLockRetentionMode::Governance)); + assert_eq!(retention.days(), Some(OBJECT_LOCK_DAYS)); + + // rustfs#7183: a PUT into the default-encrypted bucket must still succeed + // and still come back encrypted. + let post_upgrade_encrypted_key = "encrypted/written-after-upgrade"; + let post_upgrade_encrypted_bytes = b"default-encrypted object written by the current RustFS build"; + new_client + .put_object() + .bucket(CONFIG_ENCRYPTED_BUCKET) + .key(post_upgrade_encrypted_key) + .body(ByteStream::from_static(post_upgrade_encrypted_bytes)) + .send() + .await?; + let (encryption, body) = read_object(&new_client, CONFIG_ENCRYPTED_BUCKET, post_upgrade_encrypted_key, None).await?; + assert_eq!( + encryption, + Some(ServerSideEncryption::Aes256), + "a PUT after the upgrade lost the bucket default encryption" + ); + assert_eq!(body, post_upgrade_encrypted_bytes); + + let post_upgrade_plain_key = "plain/written-after-upgrade"; + let post_upgrade_plain_bytes = b"plain object written by the current RustFS build"; + put_object_through_quota_warmup(&new_client, CONFIG_PLAIN_BUCKET, post_upgrade_plain_key, post_upgrade_plain_bytes).await?; + let (encryption, body) = read_object(&new_client, CONFIG_PLAIN_BUCKET, post_upgrade_plain_key, None).await?; + assert_eq!(encryption, None, "a bucket without default encryption must not encrypt a PUT"); + assert_eq!(body, post_upgrade_plain_bytes); + + // Every object written by the previous release reads back byte-identical. + assert_eq!(read_object(&new_client, CONFIG_PLAIN_BUCKET, plain_key, None).await?.1, plain_bytes); + let (encryption, body) = read_object(&new_client, CONFIG_ENCRYPTED_BUCKET, encrypted_key, None).await?; + assert_eq!(encryption, Some(ServerSideEncryption::Aes256)); + assert_eq!(body, encrypted_bytes); + let (encryption, body) = read_object(&new_client, CONFIG_ENCRYPTED_BUCKET, multipart_key, None).await?; + assert_eq!(encryption, Some(ServerSideEncryption::Aes256)); + assert_eq!(body, multipart_bytes, "the multipart object did not survive the upgrade"); + assert_eq!( + read_object(&new_client, CONFIG_REPLICATED_BUCKET, versioned_key, Some(&versioned_id)) + .await? + .1, + versioned_bytes + ); + + // rustfs#7089: the migration module is on by default, but a bucket that + // never configured a source behaves exactly as before. + assert_migration_not_configured(&env, CONFIG_PLAIN_BUCKET).await?; + assert_missing_key_is_no_such_key(&new_client, CONFIG_PLAIN_BUCKET, "plain/never-written").await?; + + replication_target.shutdown().await; + Ok(()) +} + +/// Rolling back to the pinned previous release must still read the bucket +/// metadata the current build wrote. +/// +/// This is the other half of the `BucketMetadata` 44 -> 46 key change: the +/// current build writes a 46-key msgpack map with `OnDemandMigrationConfigJSON` +/// and `OnDemandMigrationConfigUpdatedAt`, and the previous release's decoder +/// has to skip those two unknown keys instead of failing the whole blob. If it +/// did not, every configuration read below would come back empty or error and +/// the rollback would silently discard the bucket's configuration. +#[tokio::test] +#[ignore = "requires a pinned previous RustFS release binary"] +async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestResult { + init_logging(); + let previous_binary = source_binary()?; + + let replication_target = FakeS3Target::start().await?; + replication_target.create_bucket(ROLLBACK_REPLICA_BUCKET); + + let mut env = RustFSTestEnvironment::new().await?; + let server_env = bucket_config_server_env(); + env.start_rustfs_server_with_env(vec![], &server_env).await?; + let new_client = env.create_s3_client(); + + env.create_test_bucket(ROLLBACK_BUCKET).await?; + enable_versioning(&new_client, ROLLBACK_BUCKET).await?; + put_default_sse_s3_encryption(&new_client, ROLLBACK_BUCKET).await?; + put_bucket_tag(&new_client, ROLLBACK_BUCKET).await?; + let target_arn = configure_replication(&env, ROLLBACK_BUCKET, &replication_target, ROLLBACK_REPLICA_BUCKET).await?; + assert_remote_target_preserved(&env, ROLLBACK_BUCKET, &target_arn, "before the rollback").await?; + + let single_key = "rollback/single"; + let single_bytes = b"single-part object written by the current RustFS build"; + let single_version = new_client + .put_object() + .bucket(ROLLBACK_BUCKET) + .key(single_key) + .body(ByteStream::from_static(single_bytes)) + .send() + .await? + .version_id() + .ok_or("versioned PUT omitted version ID")? + .to_string(); + + let multipart_key = "rollback/multipart"; + let multipart_parts = vec![vec![b'r'; 5 * 1024 * 1024], b"final rollback bytes".to_vec()]; + let multipart_bytes = multipart_parts.concat(); + write_multipart(&new_client, ROLLBACK_BUCKET, multipart_key, &multipart_parts).await?; + + restart_from_binary(&mut env, &previous_binary, &server_env).await?; + let old_client = env.create_s3_client(); + + assert_versioning_enabled(&old_client, ROLLBACK_BUCKET, "after the rollback").await?; + assert_default_sse_s3_encryption(&old_client, ROLLBACK_BUCKET, "after the rollback").await?; + assert_bucket_tag(&old_client, ROLLBACK_BUCKET, "after the rollback").await?; + assert_remote_target_preserved(&env, ROLLBACK_BUCKET, &target_arn, "after the rollback").await?; + + let (encryption, body) = read_object(&old_client, ROLLBACK_BUCKET, single_key, Some(&single_version)).await?; + assert_eq!(encryption, Some(ServerSideEncryption::Aes256)); + assert_eq!(body, single_bytes); + let (encryption, body) = read_object(&old_client, ROLLBACK_BUCKET, multipart_key, None).await?; + assert_eq!(encryption, Some(ServerSideEncryption::Aes256)); + assert_eq!(body, multipart_bytes, "the multipart object did not survive the rollback"); + + // A PUT on the rolled-back release must still honour the encryption + // configuration it decoded out of the current build's metadata blob. + let post_rollback_key = "rollback/written-after-rollback"; + let post_rollback_bytes = b"object written by the previous RustFS release after the rollback"; + old_client + .put_object() + .bucket(ROLLBACK_BUCKET) + .key(post_rollback_key) + .body(ByteStream::from_static(post_rollback_bytes)) + .send() + .await?; + let (encryption, body) = read_object(&old_client, ROLLBACK_BUCKET, post_rollback_key, None).await?; + assert_eq!( + encryption, + Some(ServerSideEncryption::Aes256), + "the rolled-back release lost the bucket default encryption" + ); + assert_eq!(body, post_rollback_bytes); + + replication_target.shutdown().await; + Ok(()) +} diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index dd2063129..8dbfec7c5 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -31,6 +31,7 @@ workspace = true [features] default = [] +gcs = ["dep:google-cloud-storage", "dep:google-cloud-auth"] # Compiles the controlled list-objects namespace-journal chaos injector into a # production binary (it is always available to tests). Off by default so the # RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_* env vars cannot rewrite journal @@ -212,8 +213,8 @@ aws-smithy-runtime-api = { workspace = true, features = ["http-1x"] } parking_lot = { workspace = true } base64-simd.workspace = true serde_urlencoded.workspace = true -google-cloud-storage = { workspace = true } -google-cloud-auth = { workspace = true } +google-cloud-storage = { workspace = true, optional = true } +google-cloud-auth = { workspace = true, optional = true } faster-hex = { workspace = true } quick-xml = { workspace = true } ratelimit = { workspace = true } diff --git a/crates/ecstore/src/bucket/on_demand_migration/gcs.rs b/crates/ecstore/src/bucket/on_demand_migration/gcs.rs index 707e1cc82..874ddb0e3 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/gcs.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/gcs.rs @@ -124,6 +124,18 @@ impl GcsNativeSourceBackend { Ok(request) } + async fn send_object(&self, request: reqwest::Request) -> Result { + match self.http.send_object(request, NO_ERROR_CODE_HEADER).await { + Err(SourceError::NotFound) => { + // An XML object URL also returns 404 when its bucket is gone. + // Reuse the read-only listing probe before caching a key miss. + self.probe().await?; + Err(SourceError::NotFound) + } + result => result, + } + } + /// Shared mapping for the XML API's HEAD and GET responses. fn head_from_response(headers: &HeaderMap) -> Result { if header(headers, "x-goog-encryption-key-sha256").is_some() { @@ -164,7 +176,7 @@ impl GcsNativeSourceBackend { impl SourceBackend for GcsNativeSourceBackend { async fn head(&self, key: &str) -> Result { let request = self.request(Method::HEAD, self.object_url(key)?, HeaderMap::new()).await?; - let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?; + let response = self.send_object(request).await?; Self::head_from_response(response.headers()) } @@ -177,7 +189,7 @@ impl SourceBackend for GcsNativeSourceBackend { ); } let request = self.request(Method::GET, self.object_url(key)?, headers).await?; - let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?; + let response = self.send_object(request).await?; let head = Self::head_from_response(response.headers())?; let content_range = header(response.headers(), "content-range").map(str::to_string); Ok(SourceGet { @@ -488,6 +500,7 @@ mod tests { // request; the probe is the next one on the wire. ScriptedResponse::new(200, Vec::new(), "{}".to_string()), ScriptedResponse::new(404, Vec::new(), String::new()), + ScriptedResponse::new(200, Vec::new(), "{}".to_string()), ScriptedResponse::new(403, Vec::new(), String::new()), ]) .await; @@ -503,4 +516,49 @@ mod tests { ) .await; } + + #[tokio::test] + async fn listing_404_is_not_an_object_not_found() { + let (endpoint, _) = scripted_server(vec![ScriptedResponse::new(404, Vec::new(), String::new())]).await; + let err = backend(&endpoint) + .list(&SourceListRequest { + max_keys: 1, + ..Default::default() + }) + .await + .expect_err("a failed bucket listing is not a per-object miss"); + assert_eq!(err.class_label(), "other", "{err:?}"); + } + + #[tokio::test] + async fn object_404_requires_a_readable_source_bucket() { + for method in [Method::HEAD, Method::GET] { + for (probe_status, expected_class) in [ + (200, "not_found"), + (404, "other"), + (403, "access_denied"), + (503, "throttled"), + (500, "server_error"), + ] { + let (endpoint, recorded) = scripted_server(vec![ + ScriptedResponse::new(404, Vec::new(), String::new()), + ScriptedResponse::new(probe_status, Vec::new(), "{}".to_string()), + ]) + .await; + let backend = backend(&endpoint); + let result = if method == Method::HEAD { + backend.head("missing").await.map(|_| ()) + } else { + backend.get("missing", None).await.map(|_| ()) + }; + let error = result.expect_err("the object 404 must remain an error"); + assert_eq!(error.class_label(), expected_class, "{method} with probe HTTP {probe_status}: {error:?}"); + let recorded = recorded.lock().expect("recorder lock"); + assert_eq!(recorded.len(), 2, "one bounded read-only probe per ambiguous object miss"); + assert_eq!(recorded[0].method, method.as_str()); + assert_eq!(recorded[1].method, "GET"); + assert_eq!(recorded[1].target, "/storage/v1/b/legacy/o?maxResults=1"); + } + } + } } diff --git a/crates/ecstore/src/bucket/on_demand_migration/list_through.rs b/crates/ecstore/src/bucket/on_demand_migration/list_through.rs index dd5a6a236..124d82d8f 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/list_through.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/list_through.rs @@ -1089,10 +1089,17 @@ mod tests { #[test] fn progress_tokens_preserve_v1_bytes_and_validate_v2_counts() { + fn framed(payload: &str) -> String { + format!("{LIST_THROUGH_TOKEN_PREFIX}{payload}") + } + let token = progress_token(None, true, false); assert_eq!( token.encode(), - r#"{"t":"odm-list","v":1,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key"}"# + concat!( + "\0odm-list:", + r#"{"t":"odm-list","v":1,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key"}"# + ) ); for count in 1..MAX_LIST_NO_PROGRESS_PAGES { let token = progress_token(Some(count), true, false); @@ -1100,16 +1107,17 @@ mod tests { } for version in [1, 2] { for value in ["null", "0", "16", "-1", "1.5", "256", "18446744073709551616", "\"1\""] { - let encoded = format!(r#"{{"t":"odm-list","v":{version},"no_progress":{value}}}"#); + let encoded = framed(&format!(r#"{{"t":"odm-list","v":{version},"no_progress":{value}}}"#)); assert_eq!(decode_continuation_token(&encoded), Err(ListThroughTokenError::Malformed), "{encoded}"); } } - for encoded in [ + for payload in [ r#"{"t":"odm-list","v":1,"no_progress":1}"#, r#"{"t":"odm-list","v":2}"#, r#"{"t":"odm-list","v":2,"no_progress":1,"extra":true}"#, ] { - assert_eq!(decode_continuation_token(encoded), Err(ListThroughTokenError::Malformed), "{encoded}"); + let encoded = framed(payload); + assert_eq!(decode_continuation_token(&encoded), Err(ListThroughTokenError::Malformed), "{encoded}"); } } diff --git a/crates/ecstore/src/bucket/on_demand_migration/mod.rs b/crates/ecstore/src/bucket/on_demand_migration/mod.rs index 6147f1f94..554dadd60 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/mod.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/mod.rs @@ -30,6 +30,7 @@ mod backend_contract; pub mod backfill; pub mod breaker; pub mod config; +#[cfg(feature = "gcs")] pub mod gcs; pub mod list_through; mod native_http; diff --git a/crates/ecstore/src/bucket/on_demand_migration/native_http.rs b/crates/ecstore/src/bucket/on_demand_migration/native_http.rs index 881db697d..8e5c12dd4 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/native_http.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/native_http.rs @@ -129,6 +129,23 @@ impl NativeHttp { &self, request: reqwest::Request, error_code_header: &str, + ) -> Result { + self.send_classified(request, error_code_header, false).await + } + + pub(super) async fn send_object( + &self, + request: reqwest::Request, + error_code_header: &str, + ) -> Result { + self.send_classified(request, error_code_header, true).await + } + + async fn send_classified( + &self, + request: reqwest::Request, + error_code_header: &str, + not_found_on_404_without_code: bool, ) -> Result { let response = self.client.execute(request).await.map_err(classify_transport_error)?; let status = response.status(); @@ -140,14 +157,14 @@ impl NativeHttp { .get(error_code_header) .and_then(|value| value.to_str().ok()) .map(str::to_string); - Err(classify_status( - status.as_u16(), - None, - match &code { - Some(code) => format!("source returned HTTP {status} ({code})"), - None => format!("source returned HTTP {status}"), - }, - )) + let message = match &code { + Some(code) => format!("source returned HTTP {status} ({code})"), + None => format!("source returned HTTP {status}"), + }; + match classify_status(status.as_u16(), code.as_deref(), message) { + SourceError::Other(_) if not_found_on_404_without_code && status.as_u16() == 404 => Err(SourceError::NotFound), + err => Err(err), + } } } diff --git a/crates/ecstore/src/bucket/on_demand_migration/source_client.rs b/crates/ecstore/src/bucket/on_demand_migration/source_client.rs index 618fff00e..d2fc21c3e 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/source_client.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/source_client.rs @@ -26,6 +26,7 @@ //! forwarded: v1 rejects SSE-C source objects outright. use super::azure::AzureSourceBackend; +#[cfg(feature = "gcs")] use super::gcs::GcsNativeSourceBackend; use super::list_through::{ListPageError, validate_list_page}; use crate::bucket::remote_s3_client::{ @@ -334,8 +335,9 @@ const THROTTLE_CODES: &[&str] = &[ "RequestLimitExceeded", "TooManyRequests", "RequestThrottled", + "ServerBusy", ]; -const NOT_FOUND_CODES: &[&str] = &["NoSuchKey"]; +const NOT_FOUND_CODES: &[&str] = &["NoSuchKey", "BlobNotFound"]; const ACCESS_DENIED_CODES: &[&str] = &[ "AccessDenied", "InvalidAccessKeyId", @@ -343,6 +345,7 @@ const ACCESS_DENIED_CODES: &[&str] = &[ "AllAccessDisabled", "ExpiredToken", "InvalidToken", + "AuthorizationPermissionMismatch", ]; pub(super) fn classify_status(status: u16, code: Option<&str>, message: String) -> SourceError { @@ -720,6 +723,9 @@ impl SourceClient { )?; Ok(Self::from_backend(Box::new(backend), spec)) } + #[cfg(not(feature = "gcs"))] + SourceBackendSpec::Gcs(_) => Err(RemoteS3ClientError::BackendNotCompiled("gcs_native")), + #[cfg(feature = "gcs")] SourceBackendSpec::Gcs(gcs) => { let backend = GcsNativeSourceBackend::new( &spec.endpoint, @@ -1111,6 +1117,28 @@ mod tests { } } + #[cfg(not(feature = "gcs"))] + #[tokio::test] + async fn gcs_backend_not_compiled_keeps_hmac_s3_available() { + let mut native = spec(None); + native.provider = SourceProvider::GcsNative; + native.credentials = None; + native.backend = SourceBackendSpec::Gcs(GcsSourceSpec { + service_account_json: "{}".to_string(), + }); + assert!(matches!( + SourceClient::new(&native).await, + Err(RemoteS3ClientError::BackendNotCompiled("gcs_native")) + )); + + let mut hmac = spec(None); + hmac.provider = SourceProvider::Gcs; + hmac.endpoint = "https://storage.googleapis.com".to_string(); + SourceClient::new(&hmac) + .await + .expect("GCS HMAC uses the always-available S3 backend"); + } + async fn scripted_client(spec: &SourceClientSpec, responses: Vec) -> (SourceClient, Recorded) { let requests: Recorded = Arc::new(Mutex::new(Vec::new())); let connector = SharedHttpConnector::new(ScriptedConnector { @@ -1817,6 +1845,7 @@ mod tests { ok(Vec::new(), CONTRACT_TAGGING), ok(Vec::new(), ""), status(404, ""), + ok(Vec::new(), ""), status(403, ACCESS_DENIED_BODY), ]) .await; diff --git a/crates/ecstore/src/bucket/on_demand_migration/sys.rs b/crates/ecstore/src/bucket/on_demand_migration/sys.rs index 6c348749e..ab42bd602 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/sys.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/sys.rs @@ -85,6 +85,8 @@ pub static GLOBAL_ON_DEMAND_MIGRATION_SYS: OnceLock = Once /// `resolve` as [`OdmLookup::Unavailable`] and through status snapshots. #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] pub enum OdmStateError { + #[error("the {0} backend is not included in this build")] + BackendNotCompiled(&'static str), /// `source.credentials` is `null`; the shared client builder has no /// anonymous mode yet (rustfs/backlog#2149 follow-up). #[error("anonymous source access is not supported yet; configure source credentials")] @@ -310,11 +312,12 @@ impl BucketOdmState { write_back: Option>, ) -> Arc { let spec = source_client_spec(config); - let client = if config.source.credentials.is_none() { + let client = if config.source.credentials.is_none() && !config.source.provider.is_native() { Err(OdmStateError::AnonymousUnsupported) } else { SourceClient::new(&spec).await.map(Arc::new).map_err(|err| match err { RemoteS3ClientError::MissingCredentials => OdmStateError::AnonymousUnsupported, + RemoteS3ClientError::BackendNotCompiled(provider) => OdmStateError::BackendNotCompiled(provider), other => OdmStateError::ClientBuild(other.to_string()), }) }; @@ -746,8 +749,9 @@ impl OnDemandMigrationSys { /// (client construction is async). Requires a Tokio runtime for the /// install path; without one the config is logged and skipped. pub fn publish(&'static self, bucket: &str, config: Option<&OnDemandMigrationConfig>) { - let generation = self.next_generation(); - let Some(config) = self.desired(config) else { + let config = self.desired(config); + let generation = self.reserve_generation(bucket, config.is_some()); + let Some(config) = config else { self.remove_with_generation(bucket, generation); return; }; @@ -777,7 +781,8 @@ impl OnDemandMigrationSys { /// Installs, rebuilds, or removes the bucket state for `config`. /// Idempotent: the same config on an installed bucket is a no-op. pub async fn apply(&self, bucket: &str, config: Option<&OnDemandMigrationConfig>) -> ApplyOutcome { - let generation = self.next_generation(); + let config = self.desired(config); + let generation = self.reserve_generation(bucket, config.is_some()); self.apply_with_generation(bucket, config, generation).await } @@ -838,7 +843,7 @@ impl OnDemandMigrationSys { /// Removes a bucket's state (idempotent), cancelling its token. pub fn remove(&self, bucket: &str) -> ApplyOutcome { - let generation = self.next_generation(); + let generation = self.reserve_generation(bucket, false); self.remove_with_generation(bucket, generation) } @@ -879,8 +884,17 @@ impl OnDemandMigrationSys { snapshots } - fn next_generation(&self) -> u64 { - self.generation.fetch_add(1, Ordering::Relaxed) + 1 + fn reserve_generation(&self, bucket: &str, installing: bool) -> u64 { + // Reserve a desired install before its async client build, under the + // same lock that orders removals. Unconfigured buckets need no slot. + let mut buckets = self.buckets.write(); + let generation = self.generation.fetch_add(1, Ordering::Relaxed) + 1; + if installing { + buckets.entry(bucket.to_string()).or_default().generation = generation; + } else if let Some(slot) = buckets.get_mut(bucket) { + slot.generation = generation; + } + generation } fn desired<'c>(&self, config: Option<&'c OnDemandMigrationConfig>) -> Option<&'c OnDemandMigrationConfig> { @@ -1076,6 +1090,45 @@ mod tests { assert_eq!(sys.apply("b", Some(&cfg)).await, ApplyOutcome::Rebuilt); } + #[tokio::test] + async fn native_azure_uses_provider_credentials_without_s3_credentials() { + let sys = enabled_sys(); + let mut cfg = config(None); + cfg.source.provider = Provider::Azure; + cfg.source.endpoint = None; + cfg.source.credentials = None; + cfg.source.azure = Some(super::super::config::AzureSourceConfig { + account: "legacyaccount".to_string(), + account_key: Some("c2VjcmV0LWtleQ==".to_string()), + sas_token: None, + }); + assert_eq!(sys.apply("b", Some(&cfg)).await, ApplyOutcome::Installed); + let state = ready_state(sys.resolve("b", "k")); + assert!(state.client().is_ok(), "native credentials must not be classified as anonymous S3"); + } + + #[cfg(not(feature = "gcs"))] + #[tokio::test] + async fn gcs_backend_not_compiled_is_unavailable_not_anonymous() { + let sys = enabled_sys(); + let mut cfg = config(None); + cfg.source.provider = Provider::GcsNative; + cfg.source.credentials = None; + cfg.source.gcs = Some(super::super::config::GcsSourceConfig { + service_account_json: "{}".to_string(), + }); + let encoded = cfg.to_json().expect("GCS config is serializable without the backend"); + let restored: OnDemandMigrationConfig = serde_json::from_slice(&encoded).expect("GCS config stays readable"); + assert_eq!(restored, cfg); + assert_eq!(sys.apply("b", Some(&cfg)).await, ApplyOutcome::Installed); + match sys.resolve("b", "k") { + Some(OdmLookup::Unavailable { error, .. }) => { + assert_eq!(error, OdmStateError::BackendNotCompiled("gcs_native")); + } + other => panic!("expected unavailable backend, got {other:?}"), + } + } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn singleflight_admits_one_leader_per_key() { let sys = enabled_sys(); @@ -1291,21 +1344,31 @@ mod tests { assert!(state.is_cancelled()); } + #[tokio::test] + async fn absent_config_updates_do_not_allocate_bucket_slots() { + let sys = enabled_sys(); + for index in 0..1000 { + let bucket = format!("unconfigured-{index}"); + assert_eq!(sys.apply(&bucket, None).await, ApplyOutcome::NotDesired); + assert_eq!(sys.remove(&bucket), ApplyOutcome::NotDesired); + } + assert!(sys.buckets.read().is_empty(), "unconfigured buckets must not accumulate tombstones"); + } + #[tokio::test] async fn stale_install_cannot_overwrite_a_later_removal() { let sys = enabled_sys(); let cfg = config(None); - let older = sys.next_generation(); - let newer = sys.next_generation(); + let older = sys.reserve_generation("b", true); + let newer = sys.reserve_generation("b", false); assert_eq!(sys.remove_with_generation("b", newer), ApplyOutcome::NotDesired); - // The removal above did not create a slot; simulate an install that - // started before it and finishes after. - sys.apply_with_generation("b", Some(&cfg), older).await; - assert!(sys.state("b").is_some(), "no slot yet, so the older install lands"); + assert_eq!(sys.apply_with_generation("b", Some(&cfg), older).await, ApplyOutcome::Superseded); + assert!(sys.state("b").is_none(), "removal must supersede an in-flight first install"); + assert_eq!(sys.apply("b", Some(&cfg)).await, ApplyOutcome::Installed); let installed = sys.state("b").unwrap(); - let older = sys.next_generation(); - let newer = sys.next_generation(); + let older = sys.reserve_generation("b", true); + let newer = sys.reserve_generation("b", false); assert_eq!(sys.remove_with_generation("b", newer), ApplyOutcome::Removed); assert!(installed.is_cancelled()); assert_eq!(sys.apply_with_generation("b", Some(&cfg), older).await, ApplyOutcome::Superseded); diff --git a/crates/ecstore/src/bucket/remote_s3_client.rs b/crates/ecstore/src/bucket/remote_s3_client.rs index a434e09ac..6389aa67e 100644 --- a/crates/ecstore/src/bucket/remote_s3_client.rs +++ b/crates/ecstore/src/bucket/remote_s3_client.rs @@ -180,6 +180,8 @@ impl RemoteS3EndpointSpec { #[derive(Debug, thiserror::Error)] pub enum RemoteS3ClientError { + #[error("the {0} backend is not included in this build")] + BackendNotCompiled(&'static str), #[error("remote endpoint requires credentials")] MissingCredentials, #[error("{0}")] diff --git a/crates/ecstore/src/services/tier/mod.rs b/crates/ecstore/src/services/tier/mod.rs index 8375f9dd0..bc22fb52a 100644 --- a/crates/ecstore/src/services/tier/mod.rs +++ b/crates/ecstore/src/services/tier/mod.rs @@ -25,6 +25,7 @@ pub(crate) mod tier_probe_intent; pub mod warm_backend; pub mod warm_backend_aliyun; pub mod warm_backend_azure; +#[cfg(feature = "gcs")] pub mod warm_backend_gcs; pub mod warm_backend_huaweicloud; pub mod warm_backend_minio; diff --git a/crates/ecstore/src/services/tier/warm_backend.rs b/crates/ecstore/src/services/tier/warm_backend.rs index 22134d744..ca4864622 100644 --- a/crates/ecstore/src/services/tier/warm_backend.rs +++ b/crates/ecstore/src/services/tier/warm_backend.rs @@ -19,13 +19,14 @@ #![allow(clippy::all)] use crate::error::is_err_bucket_not_found; +#[cfg(feature = "gcs")] +use crate::services::tier::warm_backend_gcs::WarmBackendGCS; use crate::services::tier::{ tier::{ERR_TIER_BACKEND_IN_USE, ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED}, tier_config::{TierConfig, TierType}, tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_NOT_FOUND, ERR_TIER_PERM_ERR}, warm_backend_aliyun::WarmBackendAliyun, warm_backend_azure::WarmBackendAzure, - warm_backend_gcs::WarmBackendGCS, warm_backend_huaweicloud::WarmBackendHuaweicloud, warm_backend_minio::WarmBackendMinIO, warm_backend_r2::WarmBackendR2, @@ -912,6 +913,15 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result { + return Err(AdminError { + code: ERR_TIER_TYPE_UNSUPPORTED.code.clone(), + message: "This build does not include the GCS backend; rebuild with the gcs feature".to_string(), + status_code: StatusCode::NOT_IMPLEMENTED, + }); + } + #[cfg(feature = "gcs")] TierType::GCS => { if let Some(gcs_config) = tier.gcs.as_ref() { let dd = WarmBackendGCS::new(gcs_config, &tier.name).await; @@ -1028,6 +1038,27 @@ mod tests { const PROBE_VERSION: &str = "remote-v2"; + #[cfg(not(feature = "gcs"))] + #[tokio::test] + async fn gcs_backend_not_compiled_preserves_config() { + let json = r#"{"name":"ARCHIVE","type":"gcs","gcs":{"bucket":"archive","creds":"secret"}}"#; + let tier: TierConfig = serde_json::from_str(json).expect("GCS config remains readable without the backend"); + assert_eq!(tier.tier_type, TierType::GCS); + let encoded = serde_json::to_vec(&tier).expect("GCS config remains writable"); + let restored: TierConfig = serde_json::from_slice(&encoded).expect("GCS config round trips"); + assert_eq!(restored.tier_type, TierType::GCS); + let restored_gcs = restored.gcs.as_ref().expect("GCS settings preserved"); + assert_eq!(restored_gcs.bucket, "archive"); + assert_eq!(restored_gcs.creds, "secret"); + assert_eq!(tier.redacted().gcs.expect("redacted GCS settings").creds, "REDACTED"); + let error = match new_warm_backend(&tier, false).await { + Ok(_) => panic!("an excluded GCS backend cannot be constructed"), + Err(error) => error, + }; + assert_eq!(error.code, ERR_TIER_TYPE_UNSUPPORTED.code); + assert_eq!(error.status_code, StatusCode::NOT_IMPLEMENTED); + } + struct CountingBackend { put_result: fn() -> Result, removes: Arc, diff --git a/docs/operations/on-demand-migration.md b/docs/operations/on-demand-migration.md index d253e33a2..adf844700 100644 --- a/docs/operations/on-demand-migration.md +++ b/docs/operations/on-demand-migration.md @@ -7,6 +7,12 @@ On-Demand Migration (ODM) attaches an external S3-compatible **source bucket** t The module is on by default (rustfs/backlog#2163); set `RUSTFS_ON_DEMAND_MIGRATION_ENABLED=false` on every node to turn it off (`rustfs/src/module_switches.rs`). With the switch off, the runtime never intervenes on a read and the admin `PUT` route refuses with `OnDemandMigrationDisabled`. Reads of the configuration and of the status endpoint keep working while the switch is off, so a disabled deployment can still be inspected. The switch only decides whether the module may act at all: a bucket with no `on-demand-migration.json` is never resolved by the runtime and makes no source call, so turning the module on changes nothing for buckets you have not configured. +## Optional Google dependencies + +The default and `full` server builds include the `gcs` Cargo feature to preserve native GCS migration and existing GCS tier support. For a server without Google SDK dependencies, build with `cargo build -p rustfs --no-default-features --features ftps,webdav`. Add `gcs` to that feature list to restore native GCS support. The ECStore library has no default Google dependency; library users that need GCS tiers must enable its `gcs` feature. + +Both builds can read, redact and preserve GCS configuration. A build without `gcs` rejects native ODM client construction with `OnDemandMigrationBackendNotCompiled` (HTTP 501); persisted native sources report an unavailable client. GCS tier initialization returns `XRustFSAdminTierTypeUnsupported` (HTTP 501). Do not deploy that build to a cluster with GCS tiers containing transitioned objects: the configuration remains intact, but reading their remote data requires a GCS-capable binary. The `gcs` provider using HMAC credentials and the S3 interoperability API remains available in every build; only `gcs_native` and native GCS tier clients need the feature. + ## List continuation token rollout `RUSTFS_ON_DEMAND_MIGRATION_LIST_V2_TOKENS` defaults to `false`; unset or invalid boolean values also keep it off. It controls only whether a v1 listing may first issue a v2 continuation token after an empty truncated merged page. Every node with this reader support accepts existing v2 tokens and continues their budget even with the switch off. Ordinary pages that consume an object or common prefix retain the original v1 token shape. diff --git a/docs/testing/ci-gates.md b/docs/testing/ci-gates.md index 99d589d64..81418c911 100644 --- a/docs/testing/ci-gates.md +++ b/docs/testing/ci-gates.md @@ -49,7 +49,7 @@ Promotion rule: never promote a report-only lane to required from one green run. | PR touching `paths` in `fuzz.yml` | `Build Fuzz Harness`, `Smoke / ` | `fuzz.yml` `fuzz-build`, `pr-fuzz-smoke` | Report-only | `MAX_TOTAL_TIME=60 ./scripts/fuzz/run.sh` | | PR touching `paths` in `windows-filesystem.yml` | `Rename Safety` | `windows-filesystem.yml` `rename-safety` | Report-only | the `cargo test -p rustfs-ecstore --lib ` commands in the job, on Windows | | PR touching `paths` in `coverage.yml` | `Workspace line coverage` | `coverage.yml` `coverage` | Report-only | `make coverage`; `python3 scripts/check_security_coverage.py target/llvm-cov/coverage.json` | -| PR touching `paths` in `e2e-upgrade.yml` | `Direct upgrade from rc.2` | `e2e-upgrade.yml` `direct-upgrade` | Report-only | the `cargo test --locked -p e2e_test` command in the job with `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous release | +| PR touching `paths` in `e2e-upgrade.yml` | `Direct upgrade from the previous release`, `Mixed-version rolling upgrade from the previous release`, `Bucket configuration survives the upgrade`, `Rollback reads current bucket metadata` | `e2e-upgrade.yml` `upgrade` matrix | Report-only | the `cargo test --locked -p e2e_test` command in the job with `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous release (`UPGRADE_SOURCE_VERSION`) | | PR touching `paths` in `oidc-keycloak.yml` | `OIDC Keycloak live gate` | `oidc-keycloak.yml` `oidc-keycloak-live` | Report-only | `cargo build --locked -p rustfs --bin rustfs`, then `bash scripts/test/oidc_keycloak_live.sh ./target/debug/rustfs` | | PR touching `paths` in `targets-integration.yml` | `PostgreSQL, MySQL, AMQP, and NATS` | `targets-integration.yml` `targets-live` | Report-only | start the containers as in the job, export the `RUSTFS_TEST_*` DSNs, then the job's `cargo test --locked -p rustfs-targets --test -- --ignored --test-threads=1` commands | | PR limited to main-CI-excluded paths | `Quick Checks`, `Test and Lint` | `ci-docs-only.yml` `quick-checks`, `test-and-lint` | Required | `git diff --check`; `make doc-paths-check`; `scripts/check_no_planning_docs.sh` | @@ -84,7 +84,7 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run, sched | `mint.yml` (weekly) | `mint` | report-only by design; per-suite PASS/FAIL/NA and raw `log.json` | yes | pinned Docker sequence in the workflow | | `coverage.yml` (weekly) | `coverage` | report-only trend; lcov and JSON artifact | yes | `make coverage` | | `runner-hygiene.yml` (monthly) | `check-ephemerality` | runner ephemerality | yes | dispatch | -| `e2e-upgrade.yml` (weekly) | `direct-upgrade` | upgrade gate; server logs | no | see the PR row | +| `e2e-upgrade.yml` (weekly) | `upgrade` (4-case matrix) | upgrade and rollback gate; server logs | no | see the PR row | | `oidc-keycloak.yml` (weekly) | `oidc-keycloak-live` | live OIDC gate | no | see the PR row | | `targets-integration.yml` (nightly) | `targets-live` | live target gate; container logs | no | see the PR row | | `scheduled-validation-freshness.yml` (nightly) | `check-freshness` | fails on a never-created or stale schedule | n/a | dispatch | diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 6d716d63b..063ce400e 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -57,7 +57,8 @@ name = "swift_object_integration_test" required-features = ["swift"] [features] -default = ["ftps", "webdav"] +default = ["ftps", "webdav", "gcs"] +gcs = ["rustfs-ecstore/gcs"] metrics-gpu = ["rustfs-obs/gpu"] ftps = ["rustfs-protocols/ftps"] swift = ["rustfs-protocols/swift"] @@ -66,7 +67,7 @@ sftp = ["rustfs-protocols/sftp"] license = [] io-scheduler-debug = [] # Enable debug information in I/O scheduler tracing-chunk-debug = [] # Enable per-chunk tracing in data plane (high noise, for debugging only) -full = ["metrics-gpu", "ftps", "swift", "webdav", "sftp", "pyroscope"] +full = ["metrics-gpu", "ftps", "swift", "webdav", "sftp", "pyroscope", "gcs"] e2e-test-hooks = [] # Shortens Connect credentials only in debug E2E builds. connect-e2e-short-credentials = [] diff --git a/rustfs/src/admin/handlers/on_demand_migration.rs b/rustfs/src/admin/handlers/on_demand_migration.rs index 103e6ee8d..d8bc9d713 100644 --- a/rustfs/src/admin/handlers/on_demand_migration.rs +++ b/rustfs/src/admin/handlers/on_demand_migration.rs @@ -90,6 +90,8 @@ const BACKFILL_OP_CANCEL: &str = "cancel"; pub(crate) const ERR_CODE_MODULE_DISABLED: &str = "OnDemandMigrationDisabled"; /// Error code returned when the source bucket did not answer the probe. pub(crate) const ERR_CODE_SOURCE_UNREACHABLE: &str = "OnDemandMigrationSourceUnreachable"; +/// Error code returned when the configured provider was excluded at build time. +pub(crate) const ERR_CODE_BACKEND_NOT_COMPILED: &str = "OnDemandMigrationBackendNotCompiled"; /// Error code returned by `GET` when the bucket has no configuration. pub(crate) const ERR_CODE_NO_SUCH_CONFIGURATION: &str = "NoSuchConfiguration"; /// Error code (409) returned by `start` while a backfill job holds the lease. @@ -630,12 +632,17 @@ pub(crate) fn source_client_spec(config: &OnDemandMigrationConfig) -> SourceClie } } -/// Builder failures are input errors: the endpoint policy, the CA PEM or the -/// credentials the operator supplied. Anonymous sources are not wired yet +/// Distinguishes excluded backends from invalid endpoint, CA or credentials. +/// Anonymous S3 sources are not wired yet /// (ODM-05 adds the credential-less path), so `MissingCredentials` is a 400 /// naming the field instead of an opaque internal error. fn client_build_error(err: RemoteS3ClientError) -> S3Error { match err { + RemoteS3ClientError::BackendNotCompiled(provider) => custom_error( + ERR_CODE_BACKEND_NOT_COMPILED, + StatusCode::NOT_IMPLEMENTED, + format!("the {provider} backend is not included in this build; rebuild with the gcs feature"), + ), RemoteS3ClientError::MissingCredentials => admin_s3_error( S3ErrorCode::InvalidArgument, "source.credentials is required: anonymous sources are not supported yet", @@ -1288,6 +1295,14 @@ mod tests { assert!(err.message().unwrap_or_default().contains("source.credentials")); } + #[test] + fn backend_not_compiled_is_distinct_from_invalid_credentials() { + let err = client_build_error(RemoteS3ClientError::BackendNotCompiled("gcs_native")); + assert_eq!(err.code(), &S3ErrorCode::Custom(ERR_CODE_BACKEND_NOT_COMPILED.into())); + assert_eq!(err.status_code(), Some(StatusCode::NOT_IMPLEMENTED)); + assert!(err.message().unwrap_or_default().contains("gcs feature")); + } + #[test] fn module_switch_defaults_on_and_reads_the_env() { temp_env::with_var(ENV_ON_DEMAND_MIGRATION_ENABLED, None::<&str>, || assert!(module_enabled())); diff --git a/rustfs/src/app/bucket_list_through.rs b/rustfs/src/app/bucket_list_through.rs index a41b994f9..ce09fc990 100644 --- a/rustfs/src/app/bucket_list_through.rs +++ b/rustfs/src/app/bucket_list_through.rs @@ -451,6 +451,7 @@ mod tests { use crate::app::storage_api::test::StoragePutObjReader; use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; use crate::app::storage_api::test::contract::object::ObjectIO as _; + use s3s::dto::ListObjectsInput; use std::time::Duration; use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -795,6 +796,97 @@ mod tests { (result, requests) } + #[test] + #[serial_test::serial] + fn list_objects_v1_stays_local_with_xml_safe_key_markers() { + run_large_stack_test("list-through-v1-local-markers", || async { + temp_env::async_with_vars( + [ + ("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")), + ("HTTP_PROXY", None), + ("HTTPS_PROXY", None), + ("ALL_PROXY", None), + ("http_proxy", None), + ("https_proxy", None), + ("all_proxy", None), + ("NO_PROXY", Some("*")), + ("no_proxy", Some("*")), + ], + async { + let (endpoint, server, stop) = + list_source(std::iter::repeat(source_xml(None, false, Some("a-source")))).await; + let (_state_guard, source_input) = + source_policy_input(endpoint, SourceErrorPolicy::Propagate, None, None).await; + let store = shared_gating_ecstore().await; + store + .put_object( + &source_input.bucket, + "a&local", + &mut StoragePutObjReader::from_vec(vec![1]), + &StorageObjectOptions::default(), + ) + .await + .expect("seed a second local object"); + + for delimiter in [None, Some("/".to_string())] { + let mut input = ListObjectsInput { + bucket: source_input.bucket.clone(), + max_keys: Some(1), + delimiter, + ..Default::default() + }; + for (index, expected_key) in ["a&local", "z-local"].into_iter().enumerate() { + let request_marker = input.marker.clone().unwrap_or_default(); + let response = tokio::time::timeout( + Duration::from_secs(10), + DefaultBucketUsecase::from_global().execute_list_objects(S3Request { + input: input.clone(), + method: http::Method::GET, + uri: http::Uri::from_static("/"), + headers: HeaderMap::new(), + extensions: http::Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + }), + ) + .await + .expect("v1 pagination must finish") + .expect("list-through must not change v1 listing"); + assert!(!response.headers.contains_key("x-rustfs-on-demand-migration-list")); + let output = response.output; + let contents = output.contents.as_ref().expect("local page contents"); + assert_eq!(contents.len(), 1); + assert_eq!(contents[0].key.as_deref(), Some(expected_key)); + assert_eq!(output.marker.as_deref(), Some(request_marker.as_str())); + assert_eq!(output.is_truncated, Some(index == 0)); + assert_eq!(output.next_marker.as_deref(), (index == 0).then_some(expected_key)); + + let mut xml = Vec::new(); + s3s::xml::Serialize::serialize(&output, &mut s3s::xml::Serializer::new(&mut xml)) + .expect("serialize the real v1 response"); + assert!(!xml.contains(&0), "XML 1.0 forbids NUL in NextMarker"); + let mut reader = quick_xml::Reader::from_reader(xml.as_slice()); + loop { + if reader.read_event().expect("v1 response must be well-formed XML") + == quick_xml::events::Event::Eof + { + break; + } + } + input.marker = output.next_marker; + } + } + stop.cancel(); + let requests = server.await.expect("source server must not panic"); + assert!(requests.is_empty(), "ListObjects v1 must issue no remote LIST requests: {requests:?}"); + }, + ) + .await; + }); + } + #[test] #[serial_test::serial] fn list_through_invalid_source_pagination_obeys_policy_on_the_handler_path() { diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index e1951cd69..fe3b11e45 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -2724,7 +2724,14 @@ impl DefaultBucketUsecase { #[instrument(level = "trace", skip(self, req))] pub async fn execute_list_objects_v2(&self, req: S3Request) -> S3Result> { - // warn!("list_objects_v2 req {:?}", &req.input); + self.execute_list_objects_v2_inner(req, true).await + } + + async fn execute_list_objects_v2_inner( + &self, + req: S3Request, + allow_list_through: bool, + ) -> S3Result> { let ListObjectsV2Input { bucket, continuation_token, @@ -2750,8 +2757,15 @@ impl DefaultBucketUsecase { // The on-demand migration envelope is decoded whether or not this // bucket still merges: a token handed out under `list_through` must keep // paginating after the policy is turned off (rustfs/backlog#2164). - let merged_token = list_through::decode_list_cursor(params.decoded_continuation_token.as_deref())?; - let (object_infos, degraded) = match list_through::list_through_state(&bucket, &req.headers) { + let (merged_token, source_state) = if allow_list_through { + ( + list_through::decode_list_cursor(params.decoded_continuation_token.as_deref())?, + list_through::list_through_state(&bucket, &req.headers), + ) + } else { + (None, None) + }; + let (object_infos, degraded) = match source_state { Some(state) => { let outcome = list_through::merged_list_objects_v2( &store, @@ -2938,7 +2952,9 @@ impl DefaultBucketUsecase { #[instrument(level = "debug", skip(self, req))] pub async fn execute_list_objects(&self, req: S3Request) -> S3Result> { let request_marker = req.input.marker.clone(); - let v2_resp = self.execute_list_objects_v2(req.map_input(Into::into)).await?; + // V1 markers are object keys, so they cannot carry the opaque merged + // pagination state used by V2 list-through. + let v2_resp = self.execute_list_objects_v2_inner(req.map_input(Into::into), false).await?; Ok(v2_resp.map_output(|v2| build_list_objects_output(v2, request_marker))) } diff --git a/rustfs/src/app/object/shared.rs b/rustfs/src/app/object/shared.rs index 454cd9911..8f0808956 100644 --- a/rustfs/src/app/object/shared.rs +++ b/rustfs/src/app/object/shared.rs @@ -992,7 +992,7 @@ pub(crate) fn odm_source_error_response(policy: &PolicyConfig, class: &'static s /// Metrics/message label for a bucket whose source client could not be built. pub(crate) fn odm_state_error_class(error: &OdmStateError) -> &'static str { match error { - OdmStateError::AnonymousUnsupported => "unsupported", + OdmStateError::AnonymousUnsupported | OdmStateError::BackendNotCompiled(_) => "unsupported", OdmStateError::ClientBuild(_) => "client_build", } } @@ -2035,6 +2035,7 @@ mod on_demand_migration_tests { #[test] fn odm_state_error_class_is_stable() { assert_eq!(odm_state_error_class(&OdmStateError::AnonymousUnsupported), "unsupported"); + assert_eq!(odm_state_error_class(&OdmStateError::BackendNotCompiled("gcs_native")), "unsupported"); assert_eq!(odm_state_error_class(&OdmStateError::ClientBuild("tls".to_string())), "client_build"); } diff --git a/scripts/check_test_wiring.py b/scripts/check_test_wiring.py index 63de463d3..ef3f15107 100755 --- a/scripts/check_test_wiring.py +++ b/scripts/check_test_wiring.py @@ -567,7 +567,7 @@ def check_quick_checks(root: Path) -> list[str]: errors.append(f"{relative}: missing composite action") return errors steps = yaml_block(runs, "steps", 2) or [] - for command in ("shellcheck --version && actionlint", "./scripts/check_error_other_format_ratchet.sh"): + for command in ("shellcheck --version && actionlint", "./scripts/check_error_other_format_ratchet.sh", "make script-tests"): step = workflow_step_block(steps, command, key="run", indent=4) if step is None: errors.append(f"{relative}: missing direct execution of {command}") @@ -906,6 +906,7 @@ class SelfTests(unittest.TestCase): " - name: Lint workflows\n shell: bash\n run: shellcheck --version && actionlint\n" " - name: Error format ratchet\n shell: bash\n" " run: ./scripts/check_error_other_format_ratchet.sh\n" + " - name: Script tests\n shell: bash\n run: make script-tests\n" ) sources = { ".github/workflows/ci.yml": caller.replace( @@ -964,6 +965,8 @@ class SelfTests(unittest.TestCase): "only installed actionlint": action.replace("run: shellcheck --version && actionlint", "run: echo actionlint"), "missing shellcheck preflight": action.replace("shellcheck --version && ", ""), "missing ratchet": action.replace("run: ./scripts/check_error_other_format_ratchet.sh", "run: echo skipped"), + "missing script tests": action.replace("run: make script-tests", "run: echo skipped"), + "swallowed script failure": action.replace("run: make script-tests", "run: make script-tests || true"), "swallowed lint failure": action.replace("&& actionlint", "&& actionlint || true"), "swallowed ratchet failure": action.replace("ratchet.sh", "ratchet.sh || true"), "conditional lint": action.replace("run: shellcheck", "if: false\n run: shellcheck"), @@ -973,7 +976,7 @@ class SelfTests(unittest.TestCase): "name: Lint workflows", "name: |\n run: shellcheck --version && actionlint" ).replace("\n run: shellcheck --version && actionlint\n", "\n run: shellcheck --version && actionlint\n || true\n"), } - for command in ("shellcheck --version && actionlint", "./scripts/check_error_other_format_ratchet.sh"): + for command in ("shellcheck --version && actionlint", "./scripts/check_error_other_format_ratchet.sh", "make script-tests"): for key in ("'if' : false", '"if": false', "'continue-on-error': true", '"continue-on-error" : true'): mutations[f"quoted {command} {key}"] = action.replace(f"run: {command}", f"{key}\n run: {command}") for separator in ("", "\n", " # continued command\n"): @@ -994,9 +997,10 @@ class SelfTests(unittest.TestCase): root = Path(tmp) (root / "scripts").mkdir() commands = ("shellcheck", "actionlint", "./scripts/check_error_other_format_ratchet.sh") - for failing in commands: + (root / "Makefile").write_text(".PHONY: script-tests\nscript-tests:\n\texit 17\n") + for failing in (*commands, "make script-tests"): with self.subTest(command=failing): - run = "shellcheck --version && actionlint" if failing != commands[-1] else failing + run = "shellcheck --version && actionlint" if failing in ("shellcheck", "actionlint") else failing step = workflow_step_block(steps, run, key="run", indent=4) self.assertIsNotNone(step) run_index = next(index for index, line in enumerate(step[1]) if line.startswith(" run:")) @@ -1011,7 +1015,7 @@ class SelfTests(unittest.TestCase): cwd=root, env=dict(os.environ, PATH=f"{root}{os.pathsep}{os.environ['PATH']}"), capture_output=True, text=True, ) - self.assertEqual(result.returncode, 17, result.stderr) + self.assertEqual(result.returncode, 2 if failing == "make script-tests" else 17, result.stderr) def test_validate_includes_quick_checks(self) -> None: error = "Quick Checks wiring regression" diff --git a/scripts/functional_case_report.py b/scripts/functional_case_report.py new file mode 100644 index 000000000..7036eaad5 --- /dev/null +++ b/scripts/functional_case_report.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Preserve every functional case execution and its suite context in reports.""" + +from __future__ import annotations + +import argparse +from pathlib import Path +import re + + +def generate_report(log_file: Path, case_file: Path, matrix_file: Path | None = None) -> bool: + ansi = re.compile(r"\x1b\[[0-9;]*m") + start_re = re.compile(r"^---\s+([A-Z][A-Z0-9]*-[0-9]+)\s+(.+?)\s+---$") + done_re = re.compile(r"^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z][A-Z0-9]*-[0-9]+)\b") + context_re = re.compile(r"^(?:\[INFO\]\s+)?==\s+((?:topology|suite):.+?)\s+==$") + topo_re = re.compile(r"^\[UPG-TOPO\]\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+PASS=(\d+)\s+FAIL=(\d+)\s*$") + rows = [] + pending = {} + topo_rows = [] + context = "context not recorded" + complete = True + try: + with log_file.open(encoding="utf-8", errors="replace") as log: + for raw in log: + line = ansi.sub("", raw).strip() + if match := context_re.match(line): + context = match[1] + pending.clear() + elif match := topo_re.match(line): + topo_rows.append(match.groups()) + elif match := start_re.match(line): + case_id, name = match.groups() + pending[case_id] = len(rows) + rows.append([case_id, f"{name} ({context})", "RUNNING"]) + elif match := done_re.match(line): + status, case_id = match.groups() + index = pending.pop(case_id, None) + if index is None: + complete = False + rows.append([case_id, f"{case_id} ({context}; start not recorded)", status]) + else: + rows[index][2] = status + except FileNotFoundError: + pass + + counts = {status: sum(row[2] == status for row in rows) for status in ("PASS", "FAIL", "UNSUPPORTED", "RUNNING")} + with case_file.open("w", encoding="utf-8") as out: + out.write(f"## Case Summary\n\n- Total: {len(rows)}\n") + for status, count in counts.items(): + out.write(f"- {status}: {count}\n") + out.write("\n| Case | Name | Status |\n| --- | --- | --- |\n") + for row in rows: + out.write("| " + " | ".join(value.replace("|", "|") for value in row) + " |\n") + if not rows: + out.write("\nNo case execution was recorded; the log is missing, empty, or stopped before the cases.\n") + + valid = complete and bool(rows) and not counts["FAIL"] and not counts["RUNNING"] + if matrix_file is not None: + with matrix_file.open("w", encoding="utf-8") as out: + out.write("## Upgrade Matrix\n\n| Topology | KMS Backend | From Version | To Version | Result |\n") + out.write("| --- | --- | --- | --- | --- |\n") + for topo, backend, old_v, new_v, npass, nfail in topo_rows: + result = "PASS" if nfail == "0" else "FAIL" + out.write(f"| {topo} | {backend} | {old_v} | {new_v} | {result} (PASS={npass} FAIL={nfail}) |\n") + if not topo_rows: + out.write("| - | - | - | - | NOT RUN (suite failed before upgrade) |\n") + valid = valid and bool(topo_rows) and all(row[-1] == "0" for row in topo_rows) + return valid + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("log_file", type=Path) + parser.add_argument("case_file", type=Path) + parser.add_argument("matrix_file", type=Path, nargs="?") + args = parser.parse_args() + raise SystemExit(0 if generate_report(args.log_file, args.case_file, args.matrix_file) else 1) diff --git a/scripts/test/oidc_keycloak_live.sh b/scripts/test/oidc_keycloak_live.sh index c464f6dcb..73b8eb6da 100755 --- a/scripts/test/oidc_keycloak_live.sh +++ b/scripts/test/oidc_keycloak_live.sh @@ -187,7 +187,7 @@ values = {} for element in root.iter(): values[element.tag.rsplit("}", 1)[-1]] = element.text or "" for field in ("AccessKeyId", "SecretAccessKey", "SessionToken", "Expiration", "SubjectFromWebIdentityToken"): - assert values.get(field), values + assert values.get(field), f"missing required STS field: {field}" print("\t".join(values[field] for field in ("AccessKeyId", "SecretAccessKey", "SessionToken"))) PY ) @@ -218,7 +218,6 @@ TAMPERED_STATUS="$(curl --noproxy '*' -sS \ --data-urlencode DurationSeconds=900 \ --data-urlencode "WebIdentityToken=${TAMPERED_TOKEN}")" [[ "${TAMPERED_STATUS}" == 403 ]] || { - cat "${WORK_DIR}/sts-tampered.xml" >&2 echo "expected tampered token to return HTTP 403, got ${TAMPERED_STATUS}" >&2 exit 1 } @@ -235,7 +234,6 @@ BAD_STATUS="$(curl --noproxy '*' -sS \ --data-urlencode DurationSeconds=900 \ --data-urlencode "WebIdentityToken=${BAD_TOKEN}")" [[ "${BAD_STATUS}" == 403 ]] || { - cat "${WORK_DIR}/sts-bad.xml" >&2 echo "expected wrong-audience token to return HTTP 403, got ${BAD_STATUS}" >&2 exit 1 } diff --git a/scripts/test_nightly_candidate.py b/scripts/test_nightly_candidate.py new file mode 100644 index 000000000..3e5a50401 --- /dev/null +++ b/scripts/test_nightly_candidate.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Exercise the nightly publication step without AWS, network or package builds.""" + +import hashlib +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + +from check_test_wiring import yaml_block + + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOW = ROOT / ".github/workflows/nightly-gnu.yml" + + +class NightlyCandidateTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.package = self.root / "rustfs-nightly-2026-09-06.deb" + self.package.write_bytes(b"built package bytes\x00\xff") + for command in (["git", "init", "-q"], ["git", "add", self.package.name], + ["git", "-c", "user.name=Test", "-c", "user.email=test@example.invalid", "commit", "-qm", "fixture"]): + subprocess.run(command, cwd=self.root, check=True, capture_output=True) + self.sha = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=self.root, text=True).strip() + self.digest = hashlib.sha256(self.package.read_bytes()).hexdigest() + self.output = self.root / "github-output" + self.store = self.root / "store" + self.shims = self.root / "fake-tools.sh" + self.shims.write_text(r'''aws() { + printf '%s\n' "$*" >> "$FAKE_AWS_LOG" + if [[ "$1" == --version ]]; then printf 'aws-cli/1.44.79 fixture\n'; return; fi + if [[ "$*" == *--generate-cli-skeleton* ]]; then + if [[ "$FAKE_MODE" == broken-install || "$FAKE_MODE" =~ ^(old-cli|bootstrap-failure|install-failure)$ && ! -e "$FAKE_INSTALLED" ]]; then + printf '{}\n' + else + printf '{"IfNoneMatch":""}\n' + fi + return + fi + if [[ "$1 $2" == 's3api put-object' ]]; then + [[ "$FAKE_MODE" != upload-failure ]] || return 42 + shift 2 + local key="" body="" condition="" + while [[ $# -gt 0 ]]; do + case "$1" in + --key) key="$2";; + --body) body="$2";; + --if-none-match) condition="$2";; + esac + shift 2 + done + [[ -z "$condition" || "$condition" == '*' ]] || return 43 + if [[ "$condition" == '*' && -e "$FAKE_STORE/$key" ]]; then return 44; fi + mkdir -p "$(dirname "$FAKE_STORE/$key")" + cp "$body" "$FAKE_STORE/$key" + elif [[ "$1 $2" == 's3 cp' ]]; then + [[ "$FAKE_MODE" != alias-failure ]] || return 45 + local destination="${4#s3://test-bucket/}" + [[ "$destination" != */ ]] || destination+="$(basename "$3")" + mkdir -p "$(dirname "$FAKE_STORE/$destination")" + cp "$3" "$FAKE_STORE/$destination" + else + return 46 + fi +} +curl() { + local url="${!#}" + printf '%s\n' "$url" >> "$FAKE_CURL_LOG" + [[ "$url" == https://dl.rustfs.com/artifacts/rustfs/packages/nightly/runs/* ]] || return 22 + [[ "$FAKE_MODE" != missing-public-url ]] || return 22 + if [[ "$FAKE_MODE" == wrong-public-bytes ]]; then printf 'different package'; return; fi + cat "$FAKE_STORE/${url#https://dl.rustfs.com/}" || return 22 + [[ "$FAKE_MODE" != incomplete-download ]] || return 47 +} +sudo() { + [[ "$*" == 'apt-get update' || "$*" == 'apt-get install -y -qq python3-venv' ]] || return 49 + [[ "$FAKE_MODE" != bootstrap-failure ]] || return 48 +} +python3() { + [[ "$1 $2" == '-m venv' ]] || return 50 + mkdir -p "$3/bin" + cat > "$3/bin/python" <<'SH' +#!/usr/bin/env bash +[[ "$*" == '-m pip install --disable-pip-version-check awscli==1.44.79' ]] || exit 51 +[[ "$FAKE_MODE" != install-failure ]] || exit 52 +: > "$FAKE_INSTALLED" +SH + printf '#!/usr/bin/env bash\naws "$@"\n' > "$3/bin/aws" + chmod +x "$3/bin/python" "$3/bin/aws" +} +''') + self.env = dict(os.environ, BASH_ENV=str(self.shims), DEB_FILE=self.package.name, + R2_ACCESS_KEY_ID="fake-access", R2_SECRET_ACCESS_KEY="fake-secret", R2_ENDPOINT="https://r2.example.invalid", R2_BUCKET="test-bucket", + RUNNER_TEMP=str(self.root), GITHUB_SHA=self.sha, GITHUB_RUN_ID="12345", GITHUB_RUN_ATTEMPT="1", GITHUB_OUTPUT=str(self.output), + FAKE_STORE=str(self.store), FAKE_AWS_LOG=str(self.root / "aws.log"), FAKE_CURL_LOG=str(self.root / "curl.log"), FAKE_INSTALLED=str(self.root / "installed"), FAKE_MODE="success") + source = WORKFLOW.read_text() + job = yaml_block(source.splitlines(), "build", 2) + starts = [i for i, line in enumerate(job) if line.startswith(" - name: ")] + self.steps = { + job[start].split(": ", 1)[1]: job[start:end] + for start, end in zip(starts, starts[1:] + [len(job)]) + } + self.publish = self.steps["Upload DEB to Cloudflare R2"] + start = self.publish.index(" run: |") + 1 + self.shell = "\n".join(line[10:] for line in self.publish[start:] if not line.strip() or line.startswith(" ")) + + def run_publish(self, **overrides): + self.output.unlink(missing_ok=True) + return subprocess.run(["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.shell], + cwd=self.root, env=dict(self.env, **overrides), capture_output=True, text=True) + + def manifest(self): + output = self.output.read_text().strip() + self.assertTrue(output.startswith("candidate_file="), output) + return json.loads(Path(output.split("=", 1)[1]).read_text()) + + def test_success_binds_actual_package_checkout_and_attempt(self): + result = self.run_publish() + self.assertEqual(result.returncode, 0, result.stderr) + manifest = self.manifest() + self.assertEqual(manifest, {"schema": 1, "source_sha": self.sha, "build_run_id": 12345, "build_run_attempt": 1, + "package_sha256": self.digest, "package_url": f"https://dl.rustfs.com/artifacts/rustfs/packages/nightly/runs/12345/1/{self.digest}/rustfs.deb"}) + for path in (f"runs/12345/1/{self.digest}/rustfs.deb", self.package.name, "rustfs-nightly-latest.deb"): + self.assertEqual((self.store / "artifacts/rustfs/packages/nightly" / path).read_bytes(), self.package.read_bytes()) + self.assertEqual((self.root / "curl.log").read_text().strip(), manifest["package_url"]) + + def test_missing_credentials_remain_artifact_only(self): + for key in ("R2_ACCESS_KEY_ID", "R2_SECRET_ACCESS_KEY", "R2_ENDPOINT", "R2_BUCKET"): + with self.subTest(missing=key): + result = self.run_publish(**{key: ""}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(self.output.exists()) + self.assertFalse((self.root / "aws.log").exists()) + self.assertEqual(list(self.root.glob("nightly-candidate-*.json")), []) + + def test_publication_failures_never_emit_a_candidate(self): + for index, mode in enumerate(("upload-failure", "missing-public-url", "wrong-public-bytes", "incomplete-download", "alias-failure")): + with self.subTest(mode=mode): + result = self.run_publish(FAKE_MODE=mode, GITHUB_RUN_ID=str(20000 + index)) + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertFalse(self.output.exists()) + self.assertEqual(list(self.root.glob("nightly-candidate-*.json")), []) + + def test_old_cli_is_upgraded_in_an_isolated_temporary_environment(self): + result = self.run_publish(FAKE_MODE="old-cli") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue((self.root / "installed").exists()) + self.assertEqual(self.manifest()["package_sha256"], self.digest) + self.assertEqual(list(self.root.glob("nightly-awscli.*")), []) + + def test_failed_cli_bootstrap_cannot_publish(self): + for mode in ("bootstrap-failure", "install-failure", "broken-install"): + with self.subTest(mode=mode): + (self.root / "installed").unlink(missing_ok=True) + result = self.run_publish(FAKE_MODE=mode) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.output.exists()) + self.assertFalse(self.store.exists()) + self.assertEqual(list(self.root.glob("nightly-awscli.*")), []) + + def test_checkout_sha_mismatch_fails_before_upload(self): + result = self.run_publish(GITHUB_SHA="f" * 40) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Checkout SHA", result.stderr) + self.assertFalse(self.output.exists()) + self.assertFalse((self.root / "aws.log").exists()) + + def test_same_date_builds_and_reruns_keep_distinct_candidates(self): + urls = [] + for run, attempt in (("12345", "1"), ("54321", "1"), ("12345", "2")): + result = self.run_publish(GITHUB_RUN_ID=run, GITHUB_RUN_ATTEMPT=attempt) + self.assertEqual(result.returncode, 0, result.stderr) + urls.append(self.manifest()["package_url"]) + self.assertEqual(len(set(urls)), 3) + self.assertEqual(len(list(self.root.glob("nightly-candidate-*.json"))), 3) + + def test_duplicate_key_is_not_overwritten_or_recertified(self): + result = self.run_publish() + self.assertEqual(result.returncode, 0, result.stderr) + key = self.manifest()["package_url"].removeprefix("https://dl.rustfs.com/") + stored = self.store / key + stored.write_bytes(b"preexisting conflicting object") + (self.root / "nightly-candidate-12345-1.json").unlink() + result = self.run_publish() + self.assertNotEqual(result.returncode, 0) + self.assertEqual(stored.read_bytes(), b"preexisting conflicting object") + self.assertFalse(self.output.exists()) + self.assertEqual(list(self.root.glob("nightly-candidate-*.json")), []) + + def test_manifest_upload_requires_publication_output(self): + upload = self.steps["Upload nightly candidate manifest"] + self.assertIn(" id: publish", self.publish) + self.assertIn(" DEB_FILE: ${{ steps.deb.outputs.deb_file }}", self.publish) + self.assertIn(" if: ${{ steps.publish.outputs.candidate_file != '' }}", upload) + self.assertIn(" name: nightly-candidate-${{ github.run_id }}-${{ github.run_attempt }}", upload) + self.assertIn(" path: ${{ steps.publish.outputs.candidate_file }}", upload) + self.assertIn(" if-no-files-found: error", upload) + self.assertNotIn(" continue-on-error: true", self.publish) + self.assertNotIn(" overwrite: true", upload) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_python_bin.sh b/scripts/test_python_bin.sh index 4c5e149fb..927412fff 100755 --- a/scripts/test_python_bin.sh +++ b/scripts/test_python_bin.sh @@ -62,20 +62,14 @@ exit 1 STUB chmod +x "$TMP_ROOT/bin/python3" -SANDBOX_PATH="$TMP_ROOT/bin:/usr/bin:/bin" -if PATH="$SANDBOX_PATH" command -v uv >/dev/null 2>&1; then - # uv is reachable even from the sandbox PATH, so the resolver would - # legitimately fall back to it instead of failing. Skip this case. - echo "â„šī¸ uv is on the sandbox PATH; skipping the no-interpreter case" -else - if PATH="$SANDBOX_PATH" "$RESOLVER" -c 'pass' \ - >"$TMP_ROOT/none.out" 2>"$TMP_ROOT/none.err"; then - fail "resolver succeeded with no usable interpreter on PATH" - fi - grep -q 'No Python 3.11+ interpreter found' "$TMP_ROOT/none.err" \ - || fail "missing-interpreter failure did not name the requirement" - grep -q 'RUSTFS_PYTHON=' "$TMP_ROOT/none.err" \ - || fail "missing-interpreter failure did not point at the override" +ln -s "$(command -v bash)" "$TMP_ROOT/bin/bash" +if PATH="$TMP_ROOT/bin" RUSTFS_PYTHON="" "$RESOLVER" -c 'pass' \ + >"$TMP_ROOT/none.out" 2>"$TMP_ROOT/none.err"; then + fail "resolver succeeded with no usable interpreter on PATH" fi +grep -q 'No Python 3.11+ interpreter found' "$TMP_ROOT/none.err" \ + || fail "missing-interpreter failure did not name the requirement" +grep -q 'RUSTFS_PYTHON=' "$TMP_ROOT/none.err" \ + || fail "missing-interpreter failure did not point at the override" echo "✅ scripts/python_bin.sh resolver checks passed" diff --git a/scripts/test_security_workflow.py b/scripts/test_security_workflow.py index ea2d75487..23268bf73 100644 --- a/scripts/test_security_workflow.py +++ b/scripts/test_security_workflow.py @@ -1,16 +1,20 @@ #!/usr/bin/env python3 -"""Exercise functional workflow failures and security evidence without remote VMs.""" +"""Exercise functional failures, chain dispatch, and security evidence without remote VMs.""" from __future__ import annotations +import glob +import json import os import re import subprocess +import sys import tempfile import unittest from pathlib import Path from check_test_wiring import yaml_block +from functional_case_report import generate_report ROOT = Path(__file__).resolve().parents[1] @@ -38,7 +42,46 @@ def shell_body(lines: list[str]) -> str: return "\n".join(shell_lines) -class SecurityWorkflowTests(unittest.TestCase): +class WorkflowSteps: + def uploaded_files(self) -> set[Path]: + upload = next(lines for lines in self.steps.values() if any("uses: actions/upload-artifact@" in line for line in lines)) + start = upload.index(" path: |") + 1 + paths = [] + for line in upload[start:]: + if not line.startswith(" "): + break + paths.extend(Path(path) for path in glob.glob(self.render(line.strip()))) + return {file for path in paths for file in (path.rglob("*") if path.is_dir() else [path]) if file.is_file()} + + def render(self, value: str) -> str: + return re.sub(r"\$\{\{\s*(.*?)\s*\}\}", lambda match: self.context[match[1]], value) + + def step_env(self, lines: list[str], indent: int = 8) -> dict[str, str]: + result = {} + for line in yaml_block(lines, "env", indent) or []: + if line.strip() and not line.lstrip().startswith("#"): + key, value = line.strip().split(": ", 1) + result[key] = self.render(value.strip("'\"")) + return result + + def run_step(self, name: str) -> subprocess.CompletedProcess[str]: + lines = self.steps[name] + result = subprocess.run( + ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.render(shell_body(lines))], + cwd=self.directory, env={**self.env, **self.step_env(lines)}, capture_output=True, text=True, + ) + for line in lines: + if line.startswith(" id: "): + self.context[f"steps.{line.split(': ', 1)[1]}.outcome"] = "failure" if result.returncode else "success" + if Path(self.env["GITHUB_ENV"]).exists(): + for line in Path(self.env["GITHUB_ENV"]).read_text().splitlines(): + key, value = line.split("=", 1) + self.env[key] = value + self.context[f"env.{key}"] = value + return result + + +class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase): def setUp(self) -> None: self.source = WORKFLOW.read_text() self.job = yaml_block(self.source.splitlines(), "security-test", 2) @@ -78,6 +121,7 @@ class SecurityWorkflowTests(unittest.TestCase): '#!/usr/bin/env bash\nset -euo pipefail\n' 'log_dir=$(mktemp -d "$TMPDIR/rustfs-security.XXXXXX")\n' 'echo "CURRENT SUITE LOG" > "$log_dir/suite.log"\n' + 'echo "CURRENT SUITE STDOUT"; echo "CURRENT SUITE STDERR" >&2\n' 'case "$FAKE_REPORT" in\n' f' present) printf "%s\\n" "CURRENT SUITE DIAGNOSTIC" "{CASE_ROW}" > "$REPORT_FILE" ;;\n' ' empty) : > "$REPORT_FILE" ;;\n' @@ -86,33 +130,6 @@ class SecurityWorkflowTests(unittest.TestCase): 'exit "$FAKE_EXIT"\n' ) - def render(self, value: str) -> str: - return re.sub(r"\$\{\{\s*(.*?)\s*\}\}", lambda match: self.context[match[1]], value) - - def step_env(self, lines: list[str], indent: int = 8) -> dict[str, str]: - result = {} - for line in yaml_block(lines, "env", indent) or []: - if line.strip() and not line.lstrip().startswith("#"): - key, value = line.strip().split(": ", 1) - result[key] = self.render(value.strip("'\"")) - return result - - def run_step(self, name: str) -> subprocess.CompletedProcess[str]: - lines = self.steps[name] - result = subprocess.run( - ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.render(shell_body(lines))], - cwd=self.directory, env={**self.env, **self.step_env(lines)}, capture_output=True, text=True, - ) - for line in lines: - if line.startswith(" id: "): - self.context[f"steps.{line.split(': ', 1)[1]}.outcome"] = "failure" if result.returncode else "success" - if Path(self.env["GITHUB_ENV"]).exists(): - for line in Path(self.env["GITHUB_ENV"]).read_text().splitlines(): - key, value = line.split("=", 1) - self.env[key] = value - self.context[f"env.{key}"] = value - return result - def test_workflow_wiring(self) -> None: names = list(self.steps) self.assertLess(names.index("Checkout repository (for the OIDC live gate script)"), names.index("Checkout auto-testing scripts (with retry)")) @@ -128,7 +145,7 @@ class SecurityWorkflowTests(unittest.TestCase): for name in ("Upload functional report to dashboard", "Upload report and logs"): self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", self.steps[name]) artifact_settings = yaml_block(self.steps["Upload report and logs"], "with", 8) - self.assertIn(" path: ${{ env.SECURITY_ARTIFACTS_DIR }}/", artifact_settings) + self.assertIn(" path: |", artifact_settings) self.assertIn(" if-no-files-found: error", artifact_settings) def test_suite_report_and_result_matrix(self) -> None: @@ -147,7 +164,7 @@ class SecurityWorkflowTests(unittest.TestCase): if outcome != "skipped" or mode == "present": suite = self.run_step("Run security suite") self.assertEqual(suite.returncode, exit_code, suite.stderr) - logs = list(self.artifacts.glob("rustfs-security.*/suite.log")) + logs = list(Path(str(self.artifacts) + "-scratch").glob("rustfs-security.*/suite.log")) self.assertEqual(len(logs), 1) self.assertEqual(logs[0].read_text(), "CURRENT SUITE LOG\n") self.context["steps.test.outcome"] = outcome @@ -169,37 +186,197 @@ class SecurityWorkflowTests(unittest.TestCase): summary = Path(self.env["GITHUB_STEP_SUMMARY"]).read_text() self.assertEqual(summary, contents) self.assertNotIn("UNWRAPPED SUITE SUMMARY", summary) + expected = {self.artifacts / "report.md"} + if outcome != "skipped" or mode == "present": + expected.add(self.artifacts / "suite.log") + self.assertEqual((self.artifacts / "suite.log").read_text(), "CURRENT SUITE STDOUT\nCURRENT SUITE STDERR\n") + if mode in ("present", "empty"): + expected.add(self.artifacts / "suite-report.md") + (self.artifacts / "unexpected-token.json").write_text("FAKE-SECRET-CANARY") + scratch = Path(str(self.artifacts) + "-scratch") + (scratch / "case.out").write_text("FAKE-SECRET-CANARY") + self.assertEqual(self.uploaded_files(), expected) + + + def test_oidc_negative_responses_never_print_issued_credentials(self): + source = (ROOT / "scripts/test/oidc_keycloak_live.sh").read_text() + for variable, filename in (("TAMPERED_STATUS", "sts-tampered.xml"), ("BAD_STATUS", "sts-bad.xml")): + start = source.index('[[ "${' + variable + '}" == 403 ]]') + end = source.index("\n", source.index("grep -q 'AccessDenied'", start)) + guard = source[start:end] + credential_xml = "FAKE-ACCESS-CANARYFAKE-SECRET-CANARYFAKE-SESSION-CANARY" + for status, body, expected in (("200", credential_xml, 1), ("403", credential_xml, 1), + ("403", "AccessDenied", 0)): + with self.subTest(variable=variable, status=status, expected=expected): + (self.directory / filename).write_text(body) + result = subprocess.run(["bash", "-e", "-o", "pipefail", "-c", guard], + env={**self.env, variable: status, "WORK_DIR": str(self.directory)}, + capture_output=True, text=True) + self.assertEqual(result.returncode, expected, result.stderr) + for canary in ("FAKE-ACCESS-CANARY", "FAKE-SECRET-CANARY", "FAKE-SESSION-CANARY"): + self.assertNotIn(canary, result.stdout + result.stderr) + if status == "200": + self.assertIn("HTTP 403, got 200", result.stderr) + + def test_oidc_incomplete_credentials_report_only_the_missing_field(self): + source = (ROOT / "scripts/test/oidc_keycloak_live.sh").read_text() + start = source.index("IFS=$'\\t' read -r STS_ACCESS_KEY") + end = source.index("\n)\n", start) + 3 + extract = source[start:end] + values = {"AccessKeyId": "FAKE-ACCESS-CANARY", "SecretAccessKey": "FAKE-SECRET-CANARY", + "SessionToken": "FAKE-SESSION-CANARY", "Expiration": "2099-01-01T00:00:00Z", + "SubjectFromWebIdentityToken": "alice"} + for missing in (None, "Expiration", "SubjectFromWebIdentityToken"): + with self.subTest(missing=missing): + xml = "" + "".join(f"<{key}>{value}" for key, value in values.items() if key != missing) + "" + (self.directory / "sts-good.xml").write_text(xml) + result = subprocess.run(["bash", "-e", "-o", "pipefail", "-c", extract], + env={**self.env, "WORK_DIR": str(self.directory)}, capture_output=True, text=True) + self.assertEqual(result.returncode, 1 if missing else 0, result.stderr) + for canary in ("FAKE-ACCESS-CANARY", "FAKE-SECRET-CANARY", "FAKE-SESSION-CANARY"): + self.assertNotIn(canary, result.stdout + result.stderr) + if missing: + self.assertIn(f"missing required STS field: {missing}", result.stderr) def test_existing_evidence_directory_is_rejected(self) -> None: - self.artifacts.mkdir() - stale = self.artifacts / "suite-report.md" - stale.write_text("OLD RUN REPORT") - self.assertNotEqual(self.run_step("Initialize security evidence").returncode, 0) - self.assertEqual(stale.read_text(), "OLD RUN REPORT") - self.assertFalse(Path(self.env["GITHUB_ENV"]).exists()) - (self.artifacts / "report.md").write_text("OLD RUN REPORT") - self.context.update({ - "env.SECURITY_ARTIFACTS_DIR": str(self.artifacts), "secrets.PF_TESTING_GH_TOKEN": "fake-local-token", - }) - fake_bin = self.directory / "bin" - fake_bin.mkdir() - gh = fake_bin / "gh" - gh.write_text( - '#!/usr/bin/env bash\nset -euo pipefail\n' - 'if [ "$1 $2" = "issue create" ]; then\n' - ' while [ "$#" -gt 0 ]; do\n' - ' if [ "$1" = "--body-file" ]; then cat "$2" > "$CAPTURE_BODY"; fi\n' - ' shift\n' - ' done\n' - 'fi\n' - ) - gh.chmod(0o755) - body = self.directory / "issue-body.md" - self.env.update(PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", CAPTURE_BODY=str(body)) - result = self.run_step("File failure issue in rustfs/backlog") - self.assertEqual(result.returncode, 0, result.stderr) - self.assertNotIn("OLD RUN REPORT", body.read_text()) - self.assertIn("https://github.com/rustfs/rustfs/actions/runs/314159", body.read_text()) + for suffix in ("", "-scratch"): + self.setUp() + existing = Path(str(self.artifacts) + suffix) + existing.mkdir() + stale = existing / "suite-report.md" + stale.write_text("OLD RUN REPORT") + self.assertNotEqual(self.run_step("Initialize security evidence").returncode, 0) + self.assertEqual(stale.read_text(), "OLD RUN REPORT") + self.assertFalse(Path(self.env["GITHUB_ENV"]).exists()) + (self.artifacts / "report.md").write_text("OLD RUN REPORT") + self.context.update({ + "env.SECURITY_ARTIFACTS_DIR": str(self.artifacts), "secrets.PF_TESTING_GH_TOKEN": "fake-local-token", + }) + fake_bin = self.directory / "bin" + fake_bin.mkdir() + gh = fake_bin / "gh" + gh.write_text( + '#!/usr/bin/env bash\nset -euo pipefail\n' + 'if [ "$1 $2" = "issue create" ]; then\n' + ' while [ "$#" -gt 0 ]; do\n' + ' if [ "$1" = "--body-file" ]; then cat "$2" > "$CAPTURE_BODY"; fi\n' + ' shift\n' + ' done\n' + 'fi\n' + ) + gh.chmod(0o755) + body = self.directory / "issue-body.md" + self.env.update(PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", CAPTURE_BODY=str(body)) + result = self.run_step("File failure issue in rustfs/backlog") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertNotIn("OLD RUN REPORT", body.read_text()) + self.assertIn("https://github.com/rustfs/rustfs/actions/runs/314159", body.read_text()) + + def test_all_ten_suites_hold_the_shared_lock_for_manual_and_chain_runs(self) -> None: + for suite in ("upgrade", "s3-compat", "kms", "tier", "storage", "heal", "pool-expand", "security", "replication", "performance"): + with self.subTest(suite=suite): + source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text().splitlines() + # Workflow-level concurrency covers every job, including cleanup, + # regardless of trigger or the runner hosting the job. + self.assertEqual([ + line.strip() for line in yaml_block(source, "concurrency", 0) + if line.strip() and not line.lstrip().startswith("#") + ], [ + "group: rustfs-shared-functional-tests", "cancel-in-progress: false", + ]) + self.assertIsNotNone(yaml_block(source, "workflow_dispatch", 2)) + self.assertIsNotNone(yaml_block(source, "repository_dispatch", 2)) + cleanup_name = "Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)" + cleanup = named_steps(yaml_block(source, "jobs", 0))[cleanup_name] + self.assertTrue(any(line.startswith(" if:") and "always()" in line for line in cleanup)) + + def test_root_dispatches_only_upgrade_and_replication_hands_off_after_failure(self) -> None: + for failed_attempts, issue_exit, token in ((0, 0, "fixture"), (2, 0, "fixture"), (3, 0, "fixture"), (3, 7, "fixture"), (0, 0, "")): + with self.subTest(failed_attempts=failed_attempts, issue_exit=issue_exit, token=bool(token)): + self.setUp() + fake_bin = self.directory / "bin" + fake_bin.mkdir() + commands = { + "gh": '''#!/usr/bin/env bash +set -euo pipefail +if [ "$1" = api ]; then + printf '%s\\n' "$*" >> "$DISPATCHES" + attempt=$(wc -l < "$DISPATCHES") + [ "$attempt" -gt "$FAILED_ATTEMPTS" ] +elif [ "$1 $2" = 'issue create' ]; then + printf 'issue\\n' >> "$EXECUTED" + while [ "$#" -gt 0 ]; do + if [ "$1" = --body-file ]; then + cat "$2" > "$CAPTURE_BODY" + printf '%s\\n' "$2" > "$CAPTURE_BODY_PATH" + fi + shift + done + exit "$ISSUE_EXIT" +else + exit 99 +fi +''', + "sleep": '#!/bin/sh\nprintf "sleep %s\\n" "$1" >> "$EXECUTED"\n', + "ssh": '#!/bin/sh\nprintf "cleanup\\n" >> "$EXECUTED"\n', + } + for name, contents in commands.items(): + command = fake_bin / name + command.write_text(contents) + command.chmod(0o755) + dispatches = self.directory / "dispatches" + executed = self.directory / "executed" + body = self.directory / "issue-body.md" + body_path = self.directory / "issue-body-path" + self.env.update( + PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", DISPATCHES=str(dispatches), + EXECUTED=str(executed), CAPTURE_BODY=str(body), CAPTURE_BODY_PATH=str(body_path), + FAILED_ATTEMPTS="0", ISSUE_EXIT=str(issue_exit), + RUSTFS_NODES="fixture-node", RUSTFS_SSH_USER="fixture-user", + RUSTFS_NIGHTLY_PACKAGE_URL="https://example.invalid/package.deb", + ) + self.context.update({"secrets.PF_TESTING_GH_TOKEN": "fixture", "inputs.suite": "all"}) + driver = (ROOT / ".github/workflows/rustfs-functional-chain.yml").read_text() + self.steps = named_steps(yaml_block(driver.splitlines(), "start-chain", 2)) + self.assertEqual(list(self.steps), ["Dispatch first suite (upgrade)"]) + started = self.run_step("Dispatch first suite (upgrade)") + self.assertEqual(started.returncode, 0, started.stderr) + self.assertEqual(dispatches.read_text().splitlines(), [ + "api --method POST repos/rustfs/rustfs/dispatches -f event_type=rustfs-chain-upgrade -F client_payload[from_suite]=nightly-build", + ]) + dispatches.unlink() + + replication = (ROOT / ".github/workflows/rustfs-replication-test.yml").read_text() + job = yaml_block(replication.splitlines(), "replication-test", 2) + self.assertFalse(any(line.startswith(" continue-on-error:") for line in job)) + self.steps = named_steps(job) + handoff = "Continue functional chain (next: Performance)" + self.assertIn(" if: ${{ always() && github.event_name == 'repository_dispatch' }}", self.steps[handoff]) + self.assertFalse(any(line.strip().startswith("continue-on-error:") for line in self.steps[handoff])) + self.assertIn(" if: always()", self.steps["Cleanup environment (after)"]) + self.assertLess(list(self.steps).index("Cleanup environment (after)"), list(self.steps).index(handoff)) + initialized = self.run_step("Initialize functional evidence") + self.assertEqual(initialized.returncode, 0, initialized.stderr) + suite = self.directory / "auto-testing/rustfs-replication-test.sh" + suite.write_text('#!/bin/sh\nprintf "suite failed\\n" >> "$EXECUTED"\nexit 17\n') + failed = self.run_step("Run replication suite") + self.assertEqual(failed.returncode, 17, failed.stderr) + cleaned = self.run_step("Cleanup environment (after)") + self.assertEqual(cleaned.returncode, 0, cleaned.stderr) + self.assertEqual(executed.read_text().splitlines(), ["suite failed", "cleanup"]) + self.env["FAILED_ATTEMPTS"] = str(failed_attempts) + self.context["secrets.PF_TESTING_GH_TOKEN"] = token + forwarded = self.run_step(handoff) + self.assertEqual(forwarded.returncode == 0, bool(token) and failed_attempts < 3, forwarded.stderr) + calls = dispatches.read_text().splitlines() if dispatches.exists() else [] + self.assertEqual(calls, [ + "api --method POST repos/rustfs/rustfs/dispatches -f event_type=rustfs-chain-performance -F client_payload[from_suite]=replication", + ] * (min(failed_attempts + 1, 3) if token else 0)) + if failed_attempts == 3: + self.assertIn("could not hand off from **replication** to **Performance**", body.read_text()) + self.assertIn("rustfs-chain-performance", body.read_text()) + self.assertEqual(executed.read_text().splitlines().count("issue"), 2 if issue_exit else 1) + self.assertFalse(Path(body_path.read_text().strip()).exists()) class FunctionalWorkflowTests(unittest.TestCase): @@ -225,7 +402,7 @@ class FunctionalWorkflowTests(unittest.TestCase): if suite in self.DIRECT_TESTS: test = steps[self.DIRECT_TESTS[suite]] self.assertNotRegex("\n".join(test), r'''(?m)^ ["']?continue-on-error["']?\s*:''') - self.assertIn(" if: always()", steps["Generate report"]) + self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", steps["Generate report"]) cleanup = steps["Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"] condition = next(line.strip() for line in cleanup if line.startswith(" if:")) self.assertIn(condition, ( @@ -234,7 +411,7 @@ class FunctionalWorkflowTests(unittest.TestCase): "if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}", )) if suite != "performance": - handoff = steps["Chain complete"] if suite == "replication" else next( + handoff = next( value for name, value in steps.items() if name.startswith("Continue functional chain") ) self.assertIn(" if: ${{ always() && github.event_name == 'repository_dispatch' }}", handoff) @@ -277,13 +454,387 @@ class FunctionalWorkflowTests(unittest.TestCase): self.assertIn("partial suite diagnostics", failed.stdout) cleanup = execute("Cleanup environment (after)") self.assertEqual(cleanup.returncode, 0, cleanup.stderr) - handoff_name = "Chain complete" if suite == "replication" else next( + handoff_name = next( name for name in steps if name.startswith("Continue functional chain") ) handoff = execute(handoff_name) self.assertEqual(handoff.returncode, 0, handoff.stderr) markers = (root / "executed").read_text().splitlines() - self.assertEqual(markers, ["cleanup"] if suite == "replication" else ["cleanup", "dispatch"]) + self.assertEqual(markers, ["cleanup", "dispatch"]) + + +class FunctionalCaseReportTests(unittest.TestCase): + def report(self, text: str | None, matrix: bool = False) -> tuple[bool, str, str]: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + log = root / "suite.log" + if text is not None: + log.write_text(text) + valid = generate_report(log, root / "cases.md", root / "matrix.md" if matrix else None) + return valid, (root / "cases.md").read_text(), (root / "matrix.md").read_text() if matrix else "" + + def test_repeated_case_executions_preserve_failure_and_context(self): + # log() from rustfs/auto-testing@6120aa0a76de, rustfs-kms-test.sh:131. + log = subprocess.check_output(["bash", "-c", r''' +log() { printf '\033[1;36m[INFO]\033[0m %s\n' "$*"; } +log '== topology: single-single kms-backend: local ==' +printf '\033[32m--- KMS-101 roundtrip ---\033[0m\n[FAIL] KMS-101\n' +log '== topology: single-multi kms-backend: vault-kv2 ==' +printf '%s\n' '--- KMS-101 roundtrip ---' '[PASS] KMS-101' +printf '%s\n' '--- KMS-101 roundtrip ---' '[UNSUPPORTED] KMS-101' +'''], text=True) + valid, cases, _ = self.report(log) + self.assertFalse(valid) + self.assertEqual(cases.count("| KMS-101 |"), 3) + self.assertIn("- Total: 3\n- PASS: 1\n- FAIL: 1\n- UNSUPPORTED: 1\n- RUNNING: 0\n", cases) + self.assertIn("roundtrip (topology: single-single kms-backend: local) | FAIL |", cases) + self.assertIn("roundtrip (topology: single-multi kms-backend: vault-kv2) | PASS |", cases) + self.assertNotIn("\\n", cases) + + def test_missing_empty_unfinished_and_orphan_results_are_not_success(self): + for text in (None, "", "setup failed\n", "--- KMS-101 roundtrip ---\n", "[PASS] KMS-101\n", + "--- KMS-101 first ---\n--- KMS-101 second ---\n[PASS] KMS-101\n", + "--- KMS-101 first ---\n[FAIL] KMS-101\n[PASS] KMS-101\n"): + with self.subTest(log=text): + valid, cases, _ = self.report(text) + self.assertFalse(valid) + self.assertIn("## Case Summary", cases) + valid, cases, _ = self.report("[INFO] == suite: bucket replication (REP-*) ==\n--- REP-101 unsupported ---\n[UNSUPPORTED] REP-101\n") + self.assertTrue(valid) + self.assertIn("suite: bucket replication", cases) + self.assertIn("- UNSUPPORTED: 1\n", cases) + + def test_upgrade_matrix_is_preserved_and_required_for_complete_report(self): + case = "--- UPG-101 upgrade ---\n[PASS] UPG-101\n" + for suffix, expected in (("", False), ("[UPG-TOPO] single-single local v1 v2 PASS=1 FAIL=0\n", True), + ("[UPG-TOPO] single-single local v1 v2 PASS=1 FAIL=1\n", False)): + with self.subTest(matrix=suffix): + valid, _, matrix = self.report(case + suffix, matrix=True) + self.assertEqual(valid, expected) + self.assertIn("| Topology | KMS Backend | From Version | To Version | Result |", matrix) + self.assertIn("| single-single | local | v1 | v2 |" if suffix else "NOT RUN", matrix) + + def test_s3_case_identifiers_include_digits(self): + valid, cases, _ = self.report("--- S3C-101 CreateBucket ---\n[PASS] S3C-101\n") + self.assertTrue(valid) + self.assertIn("| S3C-101 | CreateBucket (context not recorded) | PASS |", cases) + + +class FunctionalEvidenceTests(WorkflowSteps, unittest.TestCase): + SUITES = (*FunctionalWorkflowTests.DIRECT_TESTS, "heal", "performance") + + def prepare(self, suite: str) -> None: + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.directory = Path(self.temp.name) + self.source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text() + self.steps = named_steps(yaml_block(self.source.splitlines(), FunctionalWorkflowTests.JOBS[suite], 2)) + self.context = {expression: "" for expression in re.findall(r"\$\{\{\s*(.*?)\s*\}\}", self.source)} + self.context.update({ + "github.server_url": "https://github.com", "github.repository": "rustfs/rustfs", + "github.run_id": "314159", "github.run_attempt": "2", "github.sha": "0123456789abcdef0123456789abcdef01234567", + "github.event_name": "repository_dispatch", "steps.test.outcome": "success", + "secrets.PF_TESTING_GH_TOKEN": "local-fixture", "env.PF_TESTING_GH_TOKEN": "local-fixture", + }) + self.artifacts = self.directory / f"rustfs-{suite}-314159-2" + self.env = { + **os.environ, "GITHUB_ENV": str(self.directory / "github-env"), "RUNNER_TEMP": self.temp.name, + "GITHUB_STEP_SUMMARY": str(self.directory / "summary.md"), "RUSTFS_NODES": "fixture-node", + "RUSTFS_NIGHTLY_PACKAGE_URL": "https://example.invalid/package.deb", "CAPTURE_BODY": str(self.directory / "issue.md"), + } + for key in ("server_url", "repository", "run_id", "run_attempt", "sha", "event_name"): + self.env[f"GITHUB_{key.upper()}"] = self.context[f"github.{key}"] + (self.directory / "scripts").mkdir() + (self.directory / "scripts/functional_case_report.py").symlink_to(ROOT / "scripts/functional_case_report.py") + fake_bin = self.directory / "bin" + fake_bin.mkdir() + (fake_bin / "python3").symlink_to(sys.executable) + for command, body in ( + ("ssh", 'printf "fixture-version\\n"\n'), + ("gh", 'if [ "$1 $2" = "issue create" ]; then\n' + ' while [ "$#" -gt 0 ]; do\n' + ' if [ "$1" = "--body-file" ]; then cat "$2" > "$CAPTURE_BODY"; fi\n' + ' shift\n' + ' done\n' + 'elif [ "$1 $2" = "api --method" ]; then cat >/dev/null; fi\n'), + ): + script = fake_bin / command + script.write_text("#!/bin/sh\n" + body) + script.chmod(0o755) + self.env["PATH"] = f"{fake_bin}{os.pathsep}{os.environ['PATH']}" + + def test_evidence_wiring_and_failed_initialization_cannot_publish_stale_files(self): + for suite, suffix in ((suite, suffix) for suite in self.SUITES for suffix in ("", "-scratch")): + with self.subTest(suite=suite, collision=suffix or "artifact"): + self.prepare(suite) + self.assertNotIn("/tmp/rustfs-", self.source) + names = list(self.steps) + self.assertLess(names.index("Initialize functional evidence"), names.index("Checkout auto-testing scripts (with retry)")) + if suite in FunctionalWorkflowTests.DIRECT_TESTS: + self.assertLess(names.index("Checkout repository (for report parser)"), names.index("Checkout auto-testing scripts (with retry)")) + for name, lines in self.steps.items(): + if name in ("Generate report", "Upload functional report to dashboard") or any("uses: actions/upload-artifact@" in line for line in lines): + self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", lines) + if any("uses: actions/upload-artifact@" in line for line in lines): + self.assertIn(" path: |", lines) + self.assertIn(" if-no-files-found: error", lines) + existing = Path(str(self.artifacts) + suffix) + existing.mkdir() + for filename in ("report.md", "suite.log"): + (existing / filename).write_text("OLD RUN EVIDENCE") + self.env.update(REPORT_FILE=str(existing / "report.md"), LOG_FILE=str(existing / "suite.log")) + initialized = self.run_step("Initialize functional evidence") + self.assertNotEqual(initialized.returncode, 0) + self.assertFalse(Path(self.env["GITHUB_ENV"]).exists()) + issue = self.run_step("File failure issue in rustfs/backlog") + self.assertEqual(issue.returncode, 0, issue.stderr) + body = Path(self.env["CAPTURE_BODY"]).read_text() + self.assertNotIn("OLD RUN EVIDENCE", body) + self.assertIn("no report or log file was produced", body) + self.assertEqual((existing / "report.md").read_text(), "OLD RUN EVIDENCE") + + def test_reports_use_only_current_complete_suite_evidence(self): + for suite in self.SUITES[:-1]: + good = "--- KMS-101 roundtrip ---\n[PASS] KMS-101\n" + partial = "--- KMS-101 roundtrip ---\n[PASS] KMS-101\n--- KMS-102 unfinished ---\n" + if suite == "s3-compat": + good, partial = good.replace("KMS-", "S3C-"), partial.replace("KMS-", "S3C-") + if suite == "upgrade": + good += "[UPG-TOPO] single-single local v1 v2 PASS=1 FAIL=0\n" + if suite == "heal": + good = "".join(f"[HEAL-STEP] {step} fixture PASS\n" for step in range(1, 8)) + partial = "[HEAL-STEP] 1 fixture PASS\n" + for outcome, log in (("success", good), ("failure", good), ("success", partial), ("success", ""), + ("success", None), ("skipped", None), ("cancelled", good)): + with self.subTest(suite=suite, outcome=outcome, log=log): + self.prepare(suite) + stale = self.directory / "old-suite.log" + stale.write_text("OLD RUN EVIDENCE\n" + good) + self.env.update(LOG_FILE=str(stale), REPORT_FILE=str(stale)) + self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0) + self.assertEqual(self.env["LOG_FILE"], str(self.artifacts / "suite.log")) + self.assertEqual(self.env["TMPDIR"], str(self.artifacts) + "-scratch") + if log is not None: + Path(self.env["LOG_FILE"]).write_text(log) + self.context["steps.test.outcome"] = outcome + report = self.run_step("Generate report") + success = outcome == "success" and log == good + self.assertEqual(report.returncode == 0, success, report.stderr) + contents = Path(self.env["REPORT_FILE"]).read_text() + self.assertNotIn("OLD RUN EVIDENCE", contents) + self.assertEqual("| PASS |" in contents, success) + for value in ("actions/runs/314159", "Attempt: 2", "Workflow Commit: " + self.context["github.sha"], + f"Test Step Outcome: {'success' if success else 'failure'}", f"Suite Step Outcome: {outcome}"): + self.assertIn(value, contents) + self.assertEqual(Path(self.env["GITHUB_STEP_SUMMARY"]).read_text(), contents) + evidence = (self.artifacts / ("steps.md" if suite == "heal" else "cases.md")).read_text() + if log in (good, partial): + self.assertIn("| PASS |", evidence) + self.assertNotIn("OLD RUN EVIDENCE", evidence) + + def test_actual_suite_commands_pass_the_current_log_and_scratch_paths(self): + for suite in self.SUITES: + with self.subTest(suite=suite): + self.prepare(suite) + self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0) + if suite == "heal": + self.assertEqual(self.env["RUSTFS_WARP_LOG_FILE"], str(self.artifacts / "warp.log")) + scripts = self.directory / "auto-testing" + scripts.mkdir() + filename = f"rustfs_{suite}_test.sh" if suite in ("heal", "performance") else f"rustfs-{suite}-test.sh" + script = scripts / filename + script.write_text( + '#!/bin/bash\nset -euo pipefail\nlog=""\n' + 'while [ "$#" -gt 0 ]; do\n' + ' if [ "$1" = "--log-file" ]; then log="$2"; shift; fi\n' + ' shift\n' + 'done\n' + '[ "$log" = "$LOG_FILE" ] || exit 31\n' + 'printf "CURRENT SUITE LOG\\n" > "$log"\n' + 'scratch=$(mktemp -d "$TMPDIR/fixture.XXXXXX")\n' + 'printf "CURRENT SCRATCH\\n" > "$scratch/trace.log"\n' + 'if [ -n "${RUSTFS_RESULT_DIR:-}" ]; then\n' + ' mkdir -p "$RUSTFS_RESULT_DIR"\n' + ' printf "CURRENT RESULTS\\n" > "$RUSTFS_RESULT_DIR/summary.md"\n' + 'fi\n' + ) + script.chmod(0o755) + name = FunctionalWorkflowTests.DIRECT_TESTS.get(suite) or ( + "Run benchmark (GET/PUT/MIXED)" if suite == "performance" else "Run heal test (write -> outage -> heal -> verify)" + ) + result = self.run_step(name) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual((self.artifacts / "suite.log").read_text(), "CURRENT SUITE LOG\n") + self.assertEqual(len(list(Path(self.env["TMPDIR"]).glob("fixture.*/trace.log"))), 1) + self.assertEqual(list(self.artifacts.glob("fixture.*")), []) + if suite == "performance": + self.assertEqual((self.artifacts / "results/summary.md").read_text(), "CURRENT RESULTS\n") + + def test_upload_allowlist_preserves_diagnostics_without_scratch(self): + extra = { + "kms": ["cases.md"], "storage": ["cases.md"], "s3-compat": ["cases.md"], + "upgrade": ["cases.md", "matrix.md"], "replication": ["cases.md"], "heal": ["steps.md", "warp.log"], + "performance": ["version.txt", "results/master.log", "results/summary.md", "results/summary.tsv", + "results/get_1KiB.txt", "results/put_1MiB.txt", "results/mixed_4MiB.txt"], + } + for suite in self.SUITES: + with self.subTest(suite=suite): + self.prepare(suite) + self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0) + expected = {self.artifacts / name for name in ["report.md", "suite.log", *extra[suite]]} + for path in expected: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("PARTIAL FAILURE DIAGNOSTIC") + for directory in (self.artifacts, Path(self.env["TMPDIR"]), self.artifacts / "results"): + directory.mkdir(exist_ok=True) + (directory / "init.json").write_text('{"root_token":"FAKE-SECRET-CANARY"}') + self.assertEqual(self.uploaded_files(), expected) + self.assertTrue(all("FAKE-SECRET-CANARY" not in path.read_text() for path in self.uploaded_files())) + + def test_kms_failure_after_vault_init_keeps_only_failure_evidence(self): + self.prepare("kms") + self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0) + scripts = self.directory / "auto-testing" + scripts.mkdir() + # Vault file writes and EXIT cleanup from auto-testing@06cd3c097350:23-24,57,479-487. + # The Docker boundary returns synthetic credentials; setup fails before vault_stop. + (scripts / "rustfs-kms-test.sh").write_text(r'''#!/bin/bash +set -Eeuo pipefail +TEST_TMP="$(mktemp -d "${TMPDIR:-/tmp}/rustfs-test.XXXXXX")" +trap 'rm -rf "${TEST_TMP}"' EXIT +VAULT_DATA_DIR="${RUSTFS_VAULT_DATA_DIR:-${TMPDIR:-/tmp}/rustfs-vault-data}" +VAULT_CONTAINER="rustfs-vault" +heal_run() { "$@"; } +while [ "$#" -gt 0 ]; do + if [ "$1" = "--log-file" ]; then LOG_FILE="$2"; shift; fi + shift +done +mkdir -p "${VAULT_DATA_DIR}" +tmp_init="${TEST_TMP}/vault-init.json" +tmp_err="${TEST_TMP}/vault-init.stderr" +heal_run docker exec -e "VAULT_ADDR=http://127.0.0.1:8200" "${VAULT_CONTAINER}" \ + vault operator init -key-shares=1 -key-threshold=1 -format=json \ + > "${tmp_init}" 2>"${tmp_err}" +cat "${tmp_init}" | heal_run tee "${VAULT_DATA_DIR}/init.json" >/dev/null +printf '%s\n' 'vault initialized; root token acquired' 'fixture setup failed after init' > "${LOG_FILE}" +exit 42 +''') + docker = self.directory / "bin/docker" + docker.write_text("""#!/bin/sh +[ "$1" = exec ] || exit 99 +printf '%s\\n' '{"root_token":"FAKE-ROOT-CANARY","unseal_keys_b64":["FAKE-UNSEAL-CANARY"]}' +""") + docker.chmod(0o755) + result = self.run_step("Run KMS suite") + self.assertEqual(result.returncode, 42, result.stderr) + vault_init = Path(self.env["TMPDIR"]) / "rustfs-vault-data/init.json" + self.assertEqual(json.loads(vault_init.read_text()), {"root_token": "FAKE-ROOT-CANARY", "unseal_keys_b64": ["FAKE-UNSEAL-CANARY"]}) + self.assertNotIn(self.artifacts, vault_init.parents) + self.assertEqual(list(Path(self.env["TMPDIR"]).glob("rustfs-test.*")), []) + self.assertNotEqual(self.run_step("Generate report").returncode, 0) + self.assertEqual(self.uploaded_files(), {self.artifacts / name for name in ("suite.log", "cases.md", "report.md")}) + self.assertIn("fixture setup failed after init", (self.artifacts / "suite.log").read_text()) + for path in self.uploaded_files(): + self.assertNotIn("FAKE-ROOT-CANARY", path.read_text()) + self.assertNotIn("FAKE-UNSEAL-CANARY", path.read_text()) + + def test_heal_accumulates_actual_staged_steps_without_overwriting_failures(self): + self.prepare("heal") + self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0) + script = self.directory / "auto-testing/rustfs_heal_test.sh" + script.parent.mkdir() + # Result printf and full-run condition from auto-testing@6120aa0a76de:143,1163-1168. + script.write_text(r'''#!/bin/bash +set -euo pipefail +SELECTED_STEPS=() +PREFLIGHT=0 +while [ "$#" -gt 0 ]; do + case "$1" in + --steps) IFS=',' read -ra SELECTED_STEPS <<< "$2"; shift ;; + --log-file) LOG_FILE="$2"; shift ;; + --preflight) PREFLIGHT=1 ;; + esac + shift +done +if [ "$PREFLIGHT" -eq 1 ]; then + printf '\n' >> "$INVOKED_STEPS" + exit 0 +fi +printf '%s\n' "${SELECTED_STEPS[*]}" >> "$INVOKED_STEPS" +emit_step_result() { + local n="$1" desc="$2" status="$3" + printf '[HEAL-STEP] %s %s %s\n' "${n}" "${desc}" "${status}" +} +{ + for step in "${SELECTED_STEPS[@]}"; do + emit_step_result "$step" "fixture step $step" PASS + done + want_all=1 + for s in 1 2 3 4 5 6 7; do + [[ " ${SELECTED_STEPS[*]} " == *" ${s} "* ]] || want_all=0 + done + if [ "${want_all}" -eq 1 ]; then + printf '[HEAL-RESULT] PASS all steps passed\n' + fi +} >> "$LOG_FILE" +''') + script.chmod(0o755) + self.env["INVOKED_STEPS"] = str(self.directory / "invoked-steps") + for name in ("Install RustFS package & start cluster", "Preflight checks", "Run heal test (write -> outage -> heal -> verify)"): + result = self.run_step(name) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(Path(self.env["INVOKED_STEPS"]).read_text().splitlines(), ["1 2", "", "3 4 5 6 7"]) + log = Path(self.env["LOG_FILE"]).read_text() + self.assertNotIn("[HEAL-RESULT]", log) + self.assertEqual(log.count("[HEAL-STEP]"), 7) + report = self.run_step("Generate report") + self.assertEqual(report.returncode, 0, report.stderr) + failed_logs = ["\n".join(line for line in log.splitlines() if not line.startswith(f"[HEAL-STEP] {step} ")) + "\n" + for step in range(1, 8)] + failed_logs += [ + log.replace("[HEAL-STEP] 3", "[HEAL-STEP] 3 original failure FAIL\n[HEAL-STEP] 3"), + log + "[HEAL-STEP] 3 later step failure FAIL\n", + log + "[HEAL-RESULT] FAIL earlier failure\n[HEAL-RESULT] PASS later success\n", + log.replace("[HEAL-STEP] 4 fixture step 4 PASS", "[HEAL-STEP] 4 fixture step 4 SKIP"), + ] + for failed_log in failed_logs: + with self.subTest(log=failed_log): + Path(self.env["LOG_FILE"]).write_text(failed_log) + report = self.run_step("Generate report") + self.assertNotEqual(report.returncode, 0, report.stderr) + contents = Path(self.env["REPORT_FILE"]).read_text() + self.assertIn("Test Step Outcome: failure", contents) + self.assertNotIn("| PASS |", contents) + if "original failure" in failed_log: + self.assertIn("| 3 | original failure | FAIL |", (self.artifacts / "steps.md").read_text()) + if "later step failure" in failed_log: + self.assertIn("| 3 | later step failure | FAIL |", (self.artifacts / "steps.md").read_text()) + + def test_performance_results_version_and_report_are_bound_to_the_run(self): + self.prepare("performance") + initialized = self.run_step("Initialize functional evidence") + self.assertEqual(initialized.returncode, 0, initialized.stderr) + self.assertEqual(self.env["RUSTFS_RESULT_DIR"], str(self.artifacts / "results")) + self.assertEqual(self.env["VERSION_FILE"], str(self.artifacts / "version.txt")) + version = self.run_step("Collect RustFS version info") + self.assertEqual(version.returncode, 0, version.stderr) + self.assertIn("fixture-version", Path(self.env["VERSION_FILE"]).read_text()) + old_summary = self.directory / "old-results/summary.md" + old_summary.parent.mkdir() + old_summary.write_text("OLD RUN EVIDENCE") + upload = "Upload report to dashboard (reports/YYYY-MM-DD.md)" + self.assertNotEqual(self.run_step(upload).returncode, 0) + self.assertFalse(Path(self.env["REPORT_FILE"]).exists()) + results = Path(self.env["RUSTFS_RESULT_DIR"]) + results.mkdir() + (results / "summary.md").write_text("CURRENT PERFORMANCE RESULTS\n") + report = self.run_step(upload) + self.assertEqual(report.returncode, 0, report.stderr) + contents = Path(self.env["REPORT_FILE"]).read_text() + for value in ("actions/runs/314159", "**Attempt**: 2", "**Workflow Commit**: " + self.context["github.sha"], + "CURRENT PERFORMANCE RESULTS", "fixture-version"): + self.assertIn(value, contents) + self.assertNotIn("OLD RUN EVIDENCE", contents) if __name__ == "__main__":