mirror of
https://github.com/rustfs/rustfs.git
synced 2026-10-12 15:48:35 +00:00
ci: reduce repeated PR runner work (#8463)
This commit is contained in:
Chris
@@ -52,6 +52,7 @@ script-tests: ## Run shell script tests
|
||||
./scripts/check_embedded_secrets.sh --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_upgrade_workflow.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_migration_gate_evidence.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/ci_gate.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
|
||||
|
||||
@@ -390,7 +390,9 @@ jobs:
|
||||
|
||||
test-and-lint-rio-v2:
|
||||
name: Test and Lint (rio-v2)
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
# Alternate feature configurations run on main, merge groups, and scheduled
|
||||
# or manual validation; the default workspace remains the PR gate.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && github.event_name != 'pull_request'
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
@@ -506,7 +508,7 @@ jobs:
|
||||
|
||||
test-and-lint-protocols:
|
||||
name: "Test and Lint (${{ matrix.features.name }})"
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
if: needs.classify-changes.outputs.mode == 'full' && github.event_name != 'pull_request'
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
|
||||
@@ -96,38 +96,8 @@ jobs:
|
||||
overwrite: true
|
||||
|
||||
upgrade:
|
||||
name: ${{ matrix.name }}
|
||||
name: Upgrade compatibility
|
||||
needs: build
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
# The two `_from_rc2_` tests keep their names: they assert
|
||||
# release-independent object contracts and pass unchanged against the
|
||||
# newer pinned source, so renaming them would only churn history and
|
||||
# the CI required-check names. UPGRADE_SOURCE_VERSION above is the
|
||||
# single source of truth for which release they actually run against.
|
||||
- name: Direct upgrade from the previous release
|
||||
test: direct_upgrade_from_rc2_preserves_object_contracts
|
||||
artifact: direct-upgrade
|
||||
- name: Mixed-version rolling upgrade from the previous release
|
||||
test: rolling_upgrade_from_rc2_preserves_mixed_version_contracts
|
||||
artifact: mixed-version-upgrade
|
||||
- name: Bucket configuration survives the upgrade
|
||||
test: direct_upgrade_from_previous_release_preserves_bucket_configuration
|
||||
artifact: bucket-config-upgrade
|
||||
- name: Rollback reads current bucket metadata
|
||||
test: rollback_to_previous_release_reads_current_bucket_metadata
|
||||
artifact: bucket-config-rollback
|
||||
- name: ODM configuration recovery after rc.5 rollback
|
||||
test: rc5_rollback_requires_restoring_odm_configuration
|
||||
artifact: odm-config-rollback
|
||||
- name: Multipart layouts survive the rc.5 upgrade
|
||||
test: direct_upgrade_from_rc5_preserves_multipart_layouts
|
||||
artifact: multipart-layout-upgrade
|
||||
- name: rc.5 multipart replication baseline
|
||||
test: rc5_baseline_replicates_multipart_layouts
|
||||
artifact: multipart-layout-baseline
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
@@ -142,8 +112,7 @@ jobs:
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
cache-shared-key: e2e-upgrade-tests
|
||||
# One main-branch consumer warms the shared test-harness dependencies.
|
||||
cache-save-if: ${{ github.ref == 'refs/heads/main' && matrix.artifact == 'direct-upgrade' }}
|
||||
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
install-build-packaging-tools: "false"
|
||||
install-test-tools: "false"
|
||||
|
||||
@@ -170,22 +139,49 @@ jobs:
|
||||
echo "RUSTFS_UPGRADE_SOURCE_BINARY=$SOURCE_DIR/rustfs" >> "$GITHUB_ENV"
|
||||
echo "RUSTFS_E2E_LOG_DIR=$RUNNER_TEMP/rustfs-upgrade-logs" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Run upgrade compatibility test
|
||||
# Reuse one compiled harness, while keeping each case in its own process.
|
||||
# Continue after a failure to retain the former matrix's complete verdict.
|
||||
- name: Run upgrade compatibility tests
|
||||
shell: bash
|
||||
env:
|
||||
RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR: ${{ runner.temp }}/rustfs-upgrade-g09-evidence/${{ matrix.artifact }}
|
||||
NO_PROXY: 127.0.0.1,localhost,::1
|
||||
run: |
|
||||
export no_proxy="$NO_PROXY"
|
||||
python3 scripts/e2e_binary.py run -- cargo test --locked -p e2e_test \
|
||||
"upgrade_compatibility_test::${{ matrix.test }}" \
|
||||
-- --ignored --exact --nocapture
|
||||
cases=(
|
||||
direct-upgrade:direct_upgrade_from_rc2_preserves_object_contracts
|
||||
mixed-version-upgrade:rolling_upgrade_from_rc2_preserves_mixed_version_contracts
|
||||
bucket-config-upgrade:direct_upgrade_from_previous_release_preserves_bucket_configuration
|
||||
bucket-config-rollback:rollback_to_previous_release_reads_current_bucket_metadata
|
||||
odm-config-rollback:rc5_rollback_requires_restoring_odm_configuration
|
||||
multipart-layout-upgrade:direct_upgrade_from_rc5_preserves_multipart_layouts
|
||||
multipart-layout-baseline:rc5_baseline_replicates_multipart_layouts
|
||||
)
|
||||
status=0
|
||||
for entry in "${cases[@]}"; do
|
||||
artifact="${entry%%:*}"
|
||||
test="${entry#*:}"
|
||||
export RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR="$RUNNER_TEMP/rustfs-upgrade-g09-evidence/$artifact"
|
||||
export RUSTFS_E2E_LOG_DIR="$RUNNER_TEMP/rustfs-upgrade-logs/$artifact"
|
||||
mkdir -p "$RUSTFS_E2E_LOG_DIR"
|
||||
echo "::group::$test"
|
||||
if python3 scripts/e2e_binary.py run -- cargo test --locked -p e2e_test \
|
||||
"upgrade_compatibility_test::$test" -- --ignored --exact --nocapture \
|
||||
2>&1 | tee "$RUSTFS_E2E_LOG_DIR/test.log"; then
|
||||
echo "PASS: $test"
|
||||
else
|
||||
echo "::error::Upgrade compatibility failed: $test"
|
||||
status=1
|
||||
fi
|
||||
echo "::endgroup::"
|
||||
done
|
||||
exit "$status"
|
||||
|
||||
- name: Upload scanner/heal G09 evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: ${{ matrix.artifact }}-g09-evidence-${{ github.run_number }}
|
||||
path: ${{ runner.temp }}/rustfs-upgrade-g09-evidence/${{ matrix.artifact }}
|
||||
name: upgrade-g09-evidence-${{ github.run_number }}
|
||||
path: ${{ runner.temp }}/rustfs-upgrade-g09-evidence
|
||||
if-no-files-found: ignore
|
||||
retention-days: 14
|
||||
|
||||
@@ -193,7 +189,7 @@ jobs:
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: ${{ matrix.artifact }}-server-logs-${{ github.run_number }}
|
||||
name: upgrade-server-logs-${{ github.run_number }}
|
||||
path: ${{ runner.temp }}/rustfs-upgrade-logs
|
||||
if-no-files-found: warn
|
||||
retention-days: 14
|
||||
|
||||
@@ -15,10 +15,12 @@ The `main` ruleset (`6436880`) requires exactly these contexts, with `strict_req
|
||||
| `Quick Checks` | `ci.yml` job `quick-checks` | Formatting and repository guard scripts |
|
||||
| `Test and Lint` | `ci.yml` job `required-checks` | Exact expected results for every CI validation job, including workspace checks, critical E2E, feature lanes, and event-specific full suites |
|
||||
|
||||
Every PR enters `ci.yml`. The `classify-changes` job uses the base revision of `scripts/ci_gate.py` to select a conservative documentation-only path: root Markdown/licenses, `AGENTS.md`, Markdown under `docs/` or `.agents/skills/`, and documentation images. Unknown paths, unavailable Git history, an empty diff, or a missing base policy select the full matrix. Renames include their deleted source path. Documentation-only PRs still run Quick Checks and Typos; the aggregate requires the expensive jobs to be skipped exactly as selected.
|
||||
Every PR enters `ci.yml`. The `classify-changes` job uses the base revision of `scripts/ci_gate.py` to select a conservative documentation-only path: root Markdown/licenses, `AGENTS.md`, Markdown under `docs/` or `.agents/skills/`, and documentation images. Unknown paths, unavailable Git history, an empty diff, or a missing base policy select the full scope for that event. Renames include their deleted source path. Documentation-only PRs still run Quick Checks and Typos; the aggregate requires the expensive jobs to be skipped exactly as selected.
|
||||
|
||||
PRs changing only Rust sources under `crates/e2e_test/src/` and known E2E selection digests (optionally with those documentation paths) use the `e2e` scope. They keep workspace validation, the server build, E2E smoke, and both S3 lanes; independent workflows retain their path selection. Connect boundary, serial ILM, optional protocol/rio-v2, and real io_uring jobs are skipped because their sources and test inputs are unchanged. Manifests, fixtures, shared nextest configuration, scripts, production sources, and unknown paths still select `full`. A dependency on `e2e_test` from another workspace member or reachable local dependency disables this shortcut. All non-PR events retain their full selection.
|
||||
|
||||
Alternate-feature unit tests and Clippy for rio-v2, Swift, SFTP, and HTTP/3 run on main pushes, merge groups, and scheduled/manual validation. PRs run the default workspace and retain ILM, S3, real io_uring, and Connect security boundaries in their applicable scopes. The aggregate requires the alternate-feature jobs to be skipped on PRs and successful on non-PR events. This reduces repeated builds across PR revisions; feature-specific regressions can be detected after merging when the merge queue is not used. For changes to these features, dispatch `ci.yml` against the candidate branch before merging when that signal is needed, including PRs targeting `gateway/integration`, which has no automatic push trigger in this workflow. Scheduled results supplement the main baseline and do not replace it.
|
||||
|
||||
`required-checks` runs even after failed or skipped dependencies. `scripts/ci_gate.py verify` rejects missing jobs, unexpected jobs, failure, cancellation, and unexpected skips; optional lanes are required only on their declared events. `Workspace Test and Lint` is the ordinary Rust job, while `Test and Lint` uniquely names the aggregate. New validation jobs must update both its direct dependencies and the script contract. Test this wiring and its failure cases with `python3 scripts/ci_gate.py --self-test`.
|
||||
|
||||
The `gateway/integration` ruleset (`24663286`) requires the same three contexts with the same review rules for the s3s to gateway integration branch (rustfs/backlog#2734). It omits `required_linear_history` because `gateway-integration-sync.yml` lands `--no-ff` merge commits from `main` on that branch every weekday, and it names the `rustfs-cla-assistant` app as the bypass actor that pushes them. `ci.yml` runs on pull requests targeting either branch.
|
||||
@@ -48,7 +50,7 @@ The four site-replication state-writer concurrency proofs reserve the available
|
||||
| PR, non-doc change | `Workspace Test and Lint` | `ci.yml` `test-and-lint` | Via aggregate | `cargo clippy --all-targets -- -D warnings`; `cargo nextest run --profile ci --all --exclude e2e_test`; `cargo test --all --doc`; migration evidence check described below |
|
||||
| PR, non-doc change | `Typos` | `ci.yml` `typos` | Via aggregate | `typos` |
|
||||
| PR, full selection | `ILM Integration (serial)` | `ci.yml` `test-ilm-integration-serial` | Via aggregate | exact command in the job |
|
||||
| PR, full selection | `Test and Lint (rio-v2)`, `Test and Lint (swift)`, `Test and Lint (sftp)` | `ci.yml` `test-and-lint-rio-v2`, `test-and-lint-protocols` | Via aggregate | `cargo nextest run` with the job's `--features` |
|
||||
| Main push, merge group, schedule, manual dispatch | `Test and Lint (rio-v2)`, `Test and Lint (swift)`, `Test and Lint (sftp)`, `Test and Lint (http3)` | `ci.yml` `test-and-lint-rio-v2`, `test-and-lint-protocols` | Via aggregate on these events; skipped on PRs | `cargo nextest run` with the job's `--features` |
|
||||
| PR, full selection | `Connect Short Credential Boundary` | `ci.yml` `connect-short-credential-boundary` | Via aggregate | `cargo test -p rustfs --test connect_registration --features connect-e2e-short-credentials`; `cargo check -p rustfs --release --features connect-e2e-short-credentials` must fail |
|
||||
| PR, full selection | `Offline Enrollment E2E Root Boundary` | `ci.yml` `offline-enrollment-e2e` | Via aggregate | `scripts/check_offline_enrollment_e2e.sh` |
|
||||
| PR, non-doc change | `Build RustFS Debug Binary` | `ci.yml` `build-rustfs-debug-binary` | Via aggregate; prerequisite for black-box jobs | `python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks` (binary plus its `rustfs.e2e.json` sidecar) |
|
||||
@@ -61,7 +63,7 @@ The four site-replication state-writer concurrency proofs reserve the available
|
||||
| PR touching `paths` in `fuzz.yml` | `Build Fuzz Harness`, `Smoke / <target>` | `fuzz.yml` `fuzz-build`, `pr-fuzz-smoke` | Report-only | `MAX_TOTAL_TIME=60 ./scripts/fuzz/run.sh` |
|
||||
| PR touching `paths` in `windows-filesystem.yml` | `Rename Safety` | `windows-filesystem.yml` `rename-safety` | Report-only | the `cargo test -p rustfs-ecstore --lib <filter>` commands in the job, on Windows |
|
||||
| PR touching `paths` in `coverage.yml` | `Workspace line coverage` | `coverage.yml` `coverage` | Report-only | `make coverage`; `python3 scripts/check_security_coverage.py target/llvm-cov/coverage.json` |
|
||||
| PR touching `paths` in `e2e-upgrade.yml` | `Direct upgrade from the previous release`, `Mixed-version rolling upgrade from the previous release`, `Bucket configuration survives the upgrade`, `Rollback reads current bucket metadata` | `e2e-upgrade.yml` `upgrade` matrix | Report-only | `python3 scripts/e2e_binary.py build`, then the job's `python3 scripts/e2e_binary.py run -- cargo test --locked -p e2e_test` command with `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous release (`UPGRADE_SOURCE_VERSION`) |
|
||||
| PR touching `paths` in `e2e-upgrade.yml` | `Upgrade compatibility` | `e2e-upgrade.yml` `upgrade` | Report-only | `python3 scripts/e2e_binary.py build`, then the job's `python3 scripts/e2e_binary.py run -- cargo test --locked -p e2e_test` commands with `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous release (`UPGRADE_SOURCE_VERSION`) |
|
||||
| PR touching `paths` in `oidc-keycloak.yml` | `OIDC Keycloak live gate` | `oidc-keycloak.yml` `oidc-keycloak-live` | Report-only | `cargo build --locked -p rustfs --bin rustfs`, then `bash scripts/test/oidc_keycloak_live.sh ./target/debug/rustfs` |
|
||||
| PR touching `paths` in `targets-integration.yml` | `PostgreSQL, MySQL, AMQP, and NATS` | `targets-integration.yml` `targets-live` | Report-only | start the containers as in the job, export the `RUSTFS_TEST_*` DSNs, then the job's `cargo test --locked -p rustfs-targets --test <name> -- --ignored --test-threads=1` commands |
|
||||
| PR, documentation-only selection | `Quick Checks`, `Typos`, `Test and Lint` | `ci.yml` `quick-checks`, `typos`, `required-checks` | Required directly or via aggregate | Quick Checks commands; `python3 scripts/ci_gate.py --self-test` |
|
||||
@@ -69,7 +71,7 @@ The four site-replication state-writer concurrency proofs reserve the available
|
||||
|
||||
e2e filters live in `.config/nextest.toml`; extend a profile instead of adding a second selector. Before a profile runs, `scripts/check_test_wiring.py` compares its listing to the committed digest in `.config/e2e-<profile>-selection.txt`, so a silent test drop fails closed.
|
||||
|
||||
Architecture migration rules run once in the required Quick Checks job. Migration proofs reuse the successful workspace run: `scripts/check_migration_gate_count.sh evidence <core-listing.json> <junit.xml>` checks the unchanged name selection and committed floor, and requires exactly one successful execution without retries for each proof. Missing, filtered, skipped, failed, or duplicate evidence fails the gate. Local `check` and `run` modes remain available. Upgrade compatibility builds one current server and shares its binary and provenance sidecar across the seven cases; each case independently verifies the pinned previous release checksum and runs its existing test.
|
||||
Architecture migration rules run once in the required Quick Checks job. Migration proofs reuse the successful workspace run: `scripts/check_migration_gate_count.sh evidence <core-listing.json> <junit.xml>` checks the unchanged name selection and committed floor, and requires exactly one successful execution without retries for each proof. Missing, filtered, skipped, failed, or duplicate evidence fails the gate. Local `check` and `run` modes remain available. Upgrade compatibility builds one current server and shares its binary and provenance sidecar with one test runner. That runner verifies the pinned previous release checksum once, reuses the compiled harness, and executes every existing upgrade case in a separate process. Failed cases do not stop the remaining cases; any failure fails the job. Logs and G09 evidence retain per-case subdirectories in the aggregate artifacts. `scripts/test_upgrade_workflow.py` checks case membership and failure propagation.
|
||||
|
||||
Scanner usage and heal rebuild coverage are intentionally split by risk and
|
||||
cost. `data_usage_test` runs in the PR `e2e-smoke` lane so changes that affect
|
||||
@@ -99,7 +101,7 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run, sched
|
||||
| `mint.yml` (weekly) | `mint` | report-only by design; per-suite PASS/FAIL/NA and raw `log.json` | yes | pinned Docker sequence in the workflow |
|
||||
| `coverage.yml` (weekly) | `coverage` | report-only trend; lcov and JSON artifact | yes | `make coverage` |
|
||||
| `runner-hygiene.yml` (monthly) | `check-ephemerality` | runner ephemerality | yes | dispatch |
|
||||
| `e2e-upgrade.yml` (weekly) | `upgrade` (4-case matrix) | upgrade and rollback gate; server logs | no | see the PR row |
|
||||
| `e2e-upgrade.yml` (weekly) | `upgrade` (serial cases sharing one harness build) | upgrade and rollback gate; server logs | no | see the PR row |
|
||||
| `oidc-keycloak.yml` (weekly) | `oidc-keycloak-live` | live OIDC gate | no | see the PR row |
|
||||
| `targets-integration.yml` (nightly) | `targets-live` | live target gate; container logs | no | see the PR row |
|
||||
| `gateway-integration-sync.yml` (weekdays) | `sync` | merges `main` into `gateway/integration` as a `--no-ff` merge commit; a conflicting merge is aborted and opens a `sync/<source>-<date>` pull request listing the conflicting files, which keeps the run green; a rejected push or missing pull request fails it | no | dispatch, optionally with `source_ref` |
|
||||
|
||||
+18
-2
@@ -21,6 +21,9 @@ CODE_JOBS = (
|
||||
"build-rustfs-debug-binary", "uring-integration", "e2e-tests",
|
||||
"s3-implemented-tests", "s3-lifecycle-behavior-tests",
|
||||
)
|
||||
# Full alternate-feature tests remain mandatory on main, merge groups, and
|
||||
# scheduled/manual runs, without repeating four builds on every PR revision.
|
||||
EXTENDED_FEATURE_JOBS = ("test-and-lint-rio-v2", "test-and-lint-protocols")
|
||||
# These jobs still exercise the server and its black-box harness when only E2E
|
||||
# Rust sources change. Production code, manifests and shared test configuration
|
||||
# always select the full matrix. No production package depends on e2e_test.
|
||||
@@ -127,6 +130,8 @@ def expected_results(mode: str, event: str, ref: str) -> dict[str, str]:
|
||||
raise ValueError(f"invalid CI selection: {mode!r} for {event!r}")
|
||||
expected = {job: "success" for job in ALWAYS_JOBS}
|
||||
expected.update({job: "success" if mode == "full" or (mode == "e2e" and job in E2E_JOBS) else "skipped" for job in CODE_JOBS})
|
||||
if event == "pull_request":
|
||||
expected.update({job: "skipped" for job in EXTENDED_FEATURE_JOBS})
|
||||
rio = mode == "full" and event in ("schedule", "workflow_dispatch")
|
||||
expected.update({job: "success" if rio else "skipped" for job in OPTIONAL_JOBS[:2]})
|
||||
full = mode == "full" and (event in ("merge_group", "workflow_dispatch") or (event == "push" and ref == "refs/heads/main"))
|
||||
@@ -215,6 +220,10 @@ def check_workflow(root: Path) -> list[str]:
|
||||
errors.append("Test and Lint must uniquely name the aggregate gate")
|
||||
if scalar(gate, "if", 4) != "always() && (github.event_name != 'pull_request' || github.event.action != 'closed')":
|
||||
errors.append("required-checks must run after failed or skipped dependencies")
|
||||
for job in EXTENDED_FEATURE_JOBS:
|
||||
block = yaml_block(jobs, job, 2) or []
|
||||
if scalar(block, "if", 4) != "needs.classify-changes.outputs.mode == 'full' && github.event_name != 'pull_request'":
|
||||
errors.append(f"{job} must retain full validation on every non-PR event")
|
||||
if scalar(gate, "shell", 8) != "bash" or scalar(gate, "run", 8) != "python3 scripts/ci_gate.py verify" or scalar(gate, "CI_NEEDS", 10) != "${{ toJSON(needs) }}":
|
||||
errors.append("required-checks must verify the actual needs results")
|
||||
if any(re.match(r'''\s+(?:- )?(?:["']?continue-on-error["']?\s*:|["']?if["']?\s*:)''', line) and not line.startswith(" if:") for line in gate):
|
||||
@@ -315,13 +324,14 @@ class SelfTests(unittest.TestCase):
|
||||
|
||||
def test_event_contract_requires_complete_candidate_and_optional_lanes(self):
|
||||
ordinary = expected_results("full", "pull_request", "refs/pull/1/merge")
|
||||
self.assertEqual({job for job, state in ordinary.items() if state == "skipped"}, set(OPTIONAL_JOBS))
|
||||
self.assertEqual({job for job, state in ordinary.items() if state == "skipped"}, set(OPTIONAL_JOBS + EXTENDED_FEATURE_JOBS))
|
||||
docs = expected_results("docs", "pull_request", "refs/pull/1/merge")
|
||||
self.assertEqual({job for job, state in docs.items() if state == "success"}, set(ALWAYS_JOBS))
|
||||
e2e = expected_results("e2e", "pull_request", "refs/pull/1/merge")
|
||||
self.assertEqual({job for job, state in e2e.items() if state == "success"}, set(ALWAYS_JOBS + E2E_JOBS))
|
||||
for event in ("schedule", "workflow_dispatch", "merge_group", "push"):
|
||||
result = expected_results("full", event, "refs/heads/main")
|
||||
self.assertTrue(all(result[job] == "success" for job in CODE_JOBS))
|
||||
self.assertEqual(result["e2e-full"], "skipped" if event == "schedule" else "success")
|
||||
self.assertEqual(result["e2e-tests-rio-v2"], "success" if event in ("schedule", "workflow_dispatch") else "skipped")
|
||||
with self.assertRaises(ValueError):
|
||||
@@ -330,7 +340,7 @@ class SelfTests(unittest.TestCase):
|
||||
expected_results("e2e", event, "refs/heads/main")
|
||||
|
||||
def test_every_wrong_result_missing_job_or_selection_fails_closed(self):
|
||||
for mode, event in (("full", "pull_request"), ("docs", "pull_request"), ("e2e", "pull_request"), ("full", "schedule"), ("full", "workflow_dispatch"), ("full", "merge_group")):
|
||||
for mode, event in (("full", "pull_request"), ("docs", "pull_request"), ("e2e", "pull_request"), ("full", "push"), ("full", "schedule"), ("full", "workflow_dispatch"), ("full", "merge_group")):
|
||||
good = {job: {"result": value} for job, value in expected_results(mode, event, "refs/heads/main").items()}
|
||||
good["classify-changes"]["outputs"] = {"mode": mode}
|
||||
self.assertEqual(verify_results(good, event, "refs/heads/main"), [])
|
||||
@@ -377,6 +387,12 @@ class SelfTests(unittest.TestCase):
|
||||
(root / ".github/workflows").mkdir(parents=True)
|
||||
source = (ROOT / ".github/workflows/ci.yml").read_text()
|
||||
path = root / ".github/workflows/ci.yml"
|
||||
for job in EXTENDED_FEATURE_JOBS:
|
||||
before, block = source.split(f" {job}:\n", 1)
|
||||
for condition in ("false", "github.event_name == 'schedule'", "github.event_name != 'pull_request'"):
|
||||
changed = re.sub(r"^ if: .*", f" if: {condition}", block, count=1, flags=re.MULTILINE)
|
||||
path.write_text(before + f" {job}:\n" + changed)
|
||||
self.assertIn(f"{job} must retain full validation on every non-PR event", check_workflow(root))
|
||||
for job in ALWAYS_JOBS + CODE_JOBS + OPTIONAL_JOBS:
|
||||
before, gate = source.split(" required-checks:\n", 1)
|
||||
path.write_text(before + " required-checks:\n" + gate.replace(f" - {job}\n", "", 1))
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Exercise the consolidated upgrade lane without building RustFS."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from check_test_wiring import yaml_block
|
||||
from test_security_workflow import named_steps, shell_body
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
class UpgradeWorkflowTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
source = (ROOT / ".github/workflows/e2e-upgrade.yml").read_text().splitlines()
|
||||
self.job = yaml_block(source, "upgrade", 2)
|
||||
self.steps = named_steps(self.job)
|
||||
self.step = self.steps["Run upgrade compatibility tests"]
|
||||
# Explicit bash enables pipefail in Actions, so tee cannot hide a failure.
|
||||
self.assertIn(" shell: bash", self.step)
|
||||
self.assertNotIn(" strategy:", self.job)
|
||||
rust = (ROOT / "crates/e2e_test/src/upgrade_compatibility_test.rs").read_text()
|
||||
self.expected = set(re.findall(r'#\[ignore[^\n]*\]\s*async fn (\w+)\(', rust))
|
||||
self.assertTrue(self.expected)
|
||||
|
||||
def run_cases(self, failed_test="", fail_tee=False):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
commands = root / "commands.jsonl"
|
||||
fake_python = root / "python3"
|
||||
fake_python.write_text(f"#!{sys.executable}\n" + '''import json, os, sys
|
||||
from pathlib import Path
|
||||
args = sys.argv[1:]
|
||||
assert args[:8] == ["scripts/e2e_binary.py", "run", "--", "cargo", "test", "--locked", "-p", "e2e_test"]
|
||||
assert args[9:] == ["--", "--ignored", "--exact", "--nocapture"]
|
||||
test = args[8].removeprefix("upgrade_compatibility_test::")
|
||||
assert args[8] != test
|
||||
assert os.environ["no_proxy"] == "127.0.0.1,localhost,::1"
|
||||
with open(os.environ["COMMANDS"], "a") as output:
|
||||
output.write(json.dumps({"test": test, "logs": os.environ["RUSTFS_E2E_LOG_DIR"],
|
||||
"evidence": os.environ["RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR"]}) + "\\n")
|
||||
Path(os.environ["RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR"]).mkdir(parents=True)
|
||||
print("executed " + test)
|
||||
sys.exit(17 if test == os.environ["FAILED_TEST"] else 0)
|
||||
''')
|
||||
fake_python.chmod(0o755)
|
||||
if fail_tee:
|
||||
tee = root / "tee"
|
||||
tee.write_text("#!/bin/sh\ncat >/dev/null\nexit 23\n")
|
||||
tee.chmod(0o755)
|
||||
env = dict(os.environ, PATH=f"{root}{os.pathsep}{os.environ['PATH']}",
|
||||
RUNNER_TEMP=str(root), COMMANDS=str(commands), FAILED_TEST=failed_test,
|
||||
NO_PROXY="127.0.0.1,localhost,::1")
|
||||
result = subprocess.run(["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", shell_body(self.step)],
|
||||
cwd=root, env=env, capture_output=True, text=True)
|
||||
calls = [json.loads(line) for line in commands.read_text().splitlines()]
|
||||
self.assertEqual({call["test"] for call in calls}, self.expected)
|
||||
self.assertEqual(len(calls), len(self.expected))
|
||||
for field in ("logs", "evidence"):
|
||||
self.assertEqual(len({call[field] for call in calls}), len(calls))
|
||||
self.assertTrue(all(Path(call[field]).is_dir() for call in calls))
|
||||
if not fail_tee:
|
||||
for call in calls:
|
||||
self.assertIn(call["test"], (Path(call["logs"]) / "test.log").read_text())
|
||||
return result
|
||||
|
||||
def test_all_ignored_upgrade_cases_run_once_with_separate_evidence(self):
|
||||
result = self.run_cases()
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
def test_early_or_late_failure_does_not_hide_remaining_cases(self):
|
||||
for name in ("direct_upgrade_from_rc2_preserves_object_contracts", "rc5_baseline_replicates_multipart_layouts"):
|
||||
with self.subTest(name=name):
|
||||
result = self.run_cases(failed_test=name)
|
||||
self.assertNotEqual(result.returncode, 0, result.stdout)
|
||||
self.assertIn(f"::error::Upgrade compatibility failed: {name}", result.stdout)
|
||||
|
||||
def test_log_write_failure_fails_the_job(self):
|
||||
self.assertNotEqual(self.run_cases(fail_tee=True).returncode, 0)
|
||||
|
||||
def test_reports_are_uploaded_even_after_failure(self):
|
||||
for name, directory in (("Upload scanner/heal G09 evidence", "rustfs-upgrade-g09-evidence"),
|
||||
("Upload server logs", "rustfs-upgrade-logs")):
|
||||
step = self.steps[name]
|
||||
self.assertIn(" if: always()", step)
|
||||
self.assertIn(" path: ${{ runner.temp }}/" + directory, step)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user