diff --git a/.config/nextest.toml b/.config/nextest.toml index fef2d38f2..2e64bffb9 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -97,6 +97,76 @@ retries = 2 # # Each e2e test spawns its own rustfs server on a random port with an isolated # temp dir (crates/e2e_test/src/common.rs), so the subset is parallel-safe. +# +# Replication PR subset (backlog#1147 repl-1): the second clause admits the 20 +# FAST bucket-replication tests from replication_extension_test — the +# target-registration / replication-check / list / remove / delete admin paths +# that validate config synchronously and never wait for asynchronous +# replication convergence. Each spawns its own single-node rustfs server(s) on +# random ports (source, plus an independent single-node target for the pair +# checks — NOT a cluster), so the subset stays parallel-safe and single-digit +# seconds. The SLOW data-plane tests (which poll for convergence at production +# background-loop intervals) and all `_real_dual_node` / `_real_single_node` +# site-replication tests run in the [profile.e2e-repl-nightly] lane below, NOT +# here. This allowlist is the single source of truth for the PR/nightly split: +# the nightly profile derives its set as "the replication module MINUS this +# allowlist", so any new replication test lands in nightly by default (never +# silently unrun) until it is explicitly blessed as fast here. Keep the two +# regexes byte-identical. Count invariant: 20 here + 16 nightly = 36 total +# (authority: `cargo nextest list`; docs/testing/e2e-suite-inventory.md). [profile.e2e-smoke] -default-filter = 'package(e2e_test) & test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_source_invalid_date|content_encoding|anonymous_access|bucket_policy_check)_test::/)' +default-filter = """ + package(e2e_test) & ( + test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_source_invalid_date|content_encoding|anonymous_access|bucket_policy_check)_test::/) + | test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/) + ) +""" fail-fast = false + +# --------------------------------------------------------------------------- +# e2e-repl-nightly profile — scheduled full replication e2e lane (repl-1) +# --------------------------------------------------------------------------- +# backlog#1147 repl-1 (deps: ci-4). Runs the SLOW / cross-process replication +# tests that are unfit for the per-PR e2e-smoke gate: +# +# * 6 slow bucket-replication data-plane tests — they PUT an object and poll +# until it converges on the target at production background-loop intervals +# (health check 5s / MRF flush 10s), so each takes tens of seconds; two of +# them replicate over HTTPS. +# * 9 `_real_dual_node` site-replication tests — each spawns TWO full rustfs +# servers and drives the cross-process site-replication control plane. +# * 1 `_real_single_node` service-account round-trip test. +# +# The set is defined as "everything in replication_extension_test that is NOT +# in the e2e-smoke PR allowlist above" (the negated clause is byte-identical to +# the allowlist), so a newly added replication test automatically runs here +# until it is explicitly promoted to the fast PR subset — no replication test +# is ever silently left out of CI. +# +# #[serial] does NOT serialize under nextest (process-per-test; see the file +# header). These tests need no cross-test serialization: each spawns its own +# server(s) on random ports with isolated temp dirs, so they are parallel-safe +# by construction — the same property the e2e-smoke subset relies on. If load +# on the runner surfaces a real flake, quarantine the specific test with an +# OPEN issue link (ci-10 / backlog#937 policy), never blanket-retry or exclude. +# +# Wired by .github/workflows/e2e-replication-nightly.yml (schedule + +# workflow_dispatch), which builds the rustfs binary once, installs awscurl so +# the STS dual-node test actually exercises its path (it skips gracefully with +# a visible log line when awscurl is absent), and routes scheduled failures +# through .github/actions/schedule-failure-issue (ci-8). Explicit division of +# labor with ci-5's future e2e-full merge gate: these tests run ONLY here, not +# double-run there. TODO(ci-7): fold this interim repl-owned lane into the ci +# domain's consolidated scheduled e2e workflow once it exists. +[profile.e2e-repl-nightly] +default-filter = """ + package(e2e_test) + & test(/^replication_extension_test::/) + & !test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/) +""" +fail-fast = false + +[profile.e2e-repl-nightly.junit] +# Emitted to target/nextest/e2e-repl-nightly/junit.xml; uploaded by the nightly +# workflow as the failure-triage artifact. +path = "junit.xml" diff --git a/.github/workflows/e2e-replication-nightly.yml b/.github/workflows/e2e-replication-nightly.yml new file mode 100644 index 000000000..ebe5bb9f6 --- /dev/null +++ b/.github/workflows/e2e-replication-nightly.yml @@ -0,0 +1,121 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Nightly full replication e2e lane (backlog#1147 repl-1, deps: ci-4). +# +# The per-PR gate (ci.yml `e2e-tests` job, `--profile e2e-smoke`) runs the 20 +# FAST bucket-replication tests. This scheduled lane runs the remaining 16 +# heavier replication e2e tests that are unfit for a per-PR gate: +# +# * 6 slow bucket-replication data-plane tests (PUT + poll for convergence at +# production background-loop intervals; two replicate over HTTPS). +# * 9 `_real_dual_node` site-replication tests (each spawns TWO rustfs +# servers and drives the cross-process site-replication control plane). +# * 1 `_real_single_node` service-account round-trip test. +# +# The selection is the [profile.e2e-repl-nightly] default-filter in +# .config/nextest.toml — the single wiring mechanism (repl-1 / ci-4). Do NOT +# add ad-hoc cargo-test steps here; change the filterset instead. +# +# Explicit division of labor: these 16 tests run ONLY here, never double-run +# in ci-5's future e2e-full merge gate. TODO(ci-7): once the ci domain's +# consolidated scheduled e2e workflow exists, fold this interim repl-owned lane +# into it rather than growing a second scheduled entrypoint. + +name: e2e-replication-nightly + +on: + workflow_dispatch: + schedule: + # 04:00 UTC nightly — staggered clear of fuzz/e2e-s3tests (02:00), + # stale (01:30) and performance-ab (06:00). + - cron: "0 4 * * *" + +# Only alert-on-failure needs more than read access; it declares its own +# job-level `issues: write`. +permissions: + contents: read + +jobs: + repl-nightly: + name: Replication e2e (nightly) + # dual-node tests spawn two full rustfs servers each; the extra headroom + # over the per-PR sm-standard-2 e2e job keeps the parallel process fan-out + # from starving the runner. Mirrors the ILM serial lane's runner class. + runs-on: sm-standard-4 + timeout-minutes: 45 + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + + - name: Setup Rust environment + uses: ./.github/actions/setup + with: + rust-version: stable + cache-shared-key: ci-e2e-repl + github-token: ${{ secrets.GITHUB_TOKEN }} + cache-save-if: ${{ github.ref == 'refs/heads/main' }} + + # awscurl lets the STS dual-node test actually exercise its path. Without + # it the test skips gracefully with a visible log line + # (`awscurl_available()` in crates/e2e_test/src/common.rs), so the lane + # still passes — installing it just upgrades that one test from skip to + # real coverage. + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: "3.12" + + - name: Install awscurl + run: python3 -m pip install --user --upgrade pip awscurl + + # Build the rustfs binary once up front. The e2e tests spawn it as a + # child process (crates/e2e_test/src/common.rs) and will build it on + # demand otherwise, but a single explicit build avoids several parallel + # nextest test processes racing to build it at once. + - name: Build rustfs binary + run: cargo build -p rustfs --bins + + - name: Run replication e2e nightly suite + run: cargo nextest run --profile e2e-repl-nightly -p e2e_test + + - name: Upload nextest junit report + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: e2e-replication-nightly-junit-${{ github.run_number }} + path: target/nextest/e2e-repl-nightly/junit.xml + retention-days: 7 + if-no-files-found: ignore + + alert-on-failure: + name: Alert on scheduled failure + needs: [repl-nightly] + # Only scheduled runs open/append the tracking issue (backlog#1149 ci-8); + # manual workflow_dispatch runs stay quiet so a debugging run never files a + # spurious alert. + if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure') + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + - name: Open or update failure-tracking issue + uses: ./.github/actions/schedule-failure-issue + with: + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/crates/e2e_test/README.md b/crates/e2e_test/README.md index f2c3507ab..a48ff6434 100644 --- a/crates/e2e_test/README.md +++ b/crates/e2e_test/README.md @@ -170,7 +170,8 @@ join CI through the nextest profile system only (never as ad-hoc jobs): | ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) | | KMS suite | — | Not in CI yet (backlog#1149 ci-5) | | Protocols (FTPS/WebDAV/SFTP) | — | Not in CI yet (backlog#1149 ci-7) | -| Cluster / replication e2e | — | Not in CI yet (backlog#1147 repl-1) | +| Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) | +| Replication (slow + dual-node) | `e2e-repl-nightly` profile, scheduled workflow | **Active** (backlog#1147 repl-1) | | `reliant/*` (pre-started server) | — | Manual only | Links: [`ci.yml`](../../.github/workflows/ci.yml) `e2e-tests` (line 347), @@ -185,8 +186,11 @@ and may add lanes; keep the table above easy to extend. **Reproduce a CI failure locally** — run the exact profile/lane: ```bash -# Smoke (e2e-tests job) +# Smoke (e2e-tests job) — includes the 20 fast replication tests cargo nextest run --profile e2e-smoke -p e2e_test +# Replication nightly lane (16 slow + dual-node tests; install awscurl for the +# STS dual-node test, else it skips gracefully) +cargo nextest run --profile e2e-repl-nightly -p e2e_test # ILM serial lane cargo nextest run -j1 --run-ignored ignored-only -p rustfs-scanner -p rustfs \ -E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))' diff --git a/docs/testing/e2e-suite-inventory.md b/docs/testing/e2e-suite-inventory.md index cd3ea2de6..8dd6f37c7 100644 --- a/docs/testing/e2e-suite-inventory.md +++ b/docs/testing/e2e-suite-inventory.md @@ -7,6 +7,10 @@ > ``` > Modules marked ✅ are in the PR smoke profile `e2e-smoke` > (`.config/nextest.toml`); admission criteria: `crates/e2e_test/README.md`. +> 🌙 marks tests in the scheduled `e2e-repl-nightly` profile (backlog#1147 +> repl-1): `replication_extension_test` splits 20 fast tests into the PR smoke +> lane and 16 slow / `_real_dual_node` / `_real_single_node` tests into the +> nightly lane (`.github/workflows/e2e-replication-nightly.yml`). > Note: counts exclude `#[ignore]`d tests (nextest lists them separately). | module | tests | PR smoke | @@ -50,7 +54,7 @@ | quota_test | 13 | | | reliability_disk_fault_test | 3 | | | reliant | 6 | | -| replication_extension_test | 36 | | +| replication_extension_test | 36 | 20 ✅ +16 🌙 | | security_boundary_test | 4 | | | server_startup_failfast_test | 1 | | | snowball_auto_extract_test | 6 | | @@ -59,4 +63,4 @@ | tls_gen | 3 | | | version_id_regression_test | 10 | ✅ | -**Total listed: 394 tests across 47 modules · PR smoke subset: 63 tests / 17 modules** · generated 2026-07-10. +**Total listed: 394 tests across 47 modules · PR smoke subset: 83 tests / 18 modules** (17 full modules + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 16 tests** · generated 2026-07-10.