mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-11 07:36:53 +00:00
Zhengchao An
5c7c757a30
Activate the 36 dormant replication e2e tests in crates/e2e_test/src/replication_extension_test.rs (zero ran anywhere before). Split via the ci-4 nextest profile mechanism, no hand-rolled cargo-test lane: - PR smoke (profile.e2e-smoke, existing e2e-tests job): the 20 fast bucket-replication tests (target-registration / replication-check / list / remove / delete admin paths) that validate config synchronously and never wait for async convergence. Each spawns its own single-node rustfs server(s) on random ports with isolated temp dirs, so parallel-safe by construction (serial_test's #[serial] is a no-op under nextest's process-per-test model; no test-group needed). - Nightly (profile.e2e-repl-nightly + .github/workflows/e2e-replication-nightly.yml): the remaining 16 = 6 slow data-plane tests + 9 _real_dual_node + 1 _real_single_node. Defined as 'replication module MINUS the PR allowlist' so new replication tests default to nightly and are never silently unrun. The nightly workflow builds the binary once, installs awscurl so the STS dual-node test runs (skips gracefully with a visible log line otherwise), and routes scheduled failures through .github/actions/schedule-failure-issue (ci-8). Explicit division of labor with ci-5 e2e-full: these run only here. Counts (cargo nextest list): e2e-smoke 83 (63 + 20), e2e-repl-nightly 16. Docs updated: e2e-suite-inventory.md, e2e_test/README.md. Refs backlog#1147 repl-1, backlog#1155.
173 lines
12 KiB
TOML
173 lines
12 KiB
TOML
# nextest configuration for RustFS.
|
|
#
|
|
# Serialize two known load-sensitive / global-state-sharing ecstore test groups
|
|
# so the full parallel nextest suite stops producing spurious failures
|
|
# (backlog #937). These tests pass in isolation but flake under the loaded
|
|
# parallel run for two distinct reasons:
|
|
#
|
|
# * store::bucket::tests::bucket_delete_* share process/global state (disk
|
|
# registry, lock client) and race make_bucket into InsufficientWriteQuorum
|
|
# when run concurrently with other ecstore tests.
|
|
# * bucket_lifecycle_ops::tests::concurrent_resend_same_part_commits_one_generation
|
|
# asserts a lock-acquire correctness property whose serialized cross-disk
|
|
# commits exceed the (already max'd, 60s) acquire deadline only when the
|
|
# suite saturates disk I/O.
|
|
#
|
|
# serial_test's #[serial] attribute does NOT serialize these across runs:
|
|
# nextest executes each test in its own process, where the in-process
|
|
# serial_test mutex has no effect. A nextest test-group with max-threads = 1 is
|
|
# the mechanism that actually serializes across nextest's process boundary.
|
|
#
|
|
# ---------------------------------------------------------------------------
|
|
# Profiles
|
|
# ---------------------------------------------------------------------------
|
|
# The `default` profile is what local `cargo nextest run` uses. It NEVER
|
|
# retries: a red test locally means a real failure to investigate, not noise to
|
|
# paper over. The `ci` profile (below) is the strict CI gate: global
|
|
# retries = 0 so a new race's first occurrence is never masked, plus a
|
|
# narrowly-scoped quarantine list (retries = 2) for tests with a tracked OPEN
|
|
# flake issue. Flake policy lives in docs/testing/README.md.
|
|
|
|
[test-groups]
|
|
ecstore-serial-flaky = { max-threads = 1 }
|
|
|
|
# --- default profile (local): serialize the flaky groups, never retry --------
|
|
[[profile.default.overrides]]
|
|
filter = 'package(rustfs-ecstore) & (test(concurrent_resend_same_part_commits_one_generation) | test(/^store::bucket::tests::bucket_delete_(mark_delete_marks|purge_removes|default_s3_delete)/))'
|
|
test-group = 'ecstore-serial-flaky'
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ci profile — the strict CI gate (ci.yml `cargo nextest run --profile ci`)
|
|
# ---------------------------------------------------------------------------
|
|
[profile.ci]
|
|
# Strict: a new race must fail on its first occurrence, never be retried away.
|
|
retries = 0
|
|
# Report every failure in one run instead of bailing on the first.
|
|
fail-fast = false
|
|
|
|
[profile.ci.junit]
|
|
# Emitted to target/nextest/ci/junit.xml; uploaded as a CI artifact.
|
|
# Tests that pass only after a quarantine retry are marked `flaky` here — that
|
|
# marker is the observable signal the flake policy is built around.
|
|
path = "junit.xml"
|
|
|
|
# ===========================================================================
|
|
# QUARANTINE — flaky tests granted retries = 2 under the ci profile ONLY.
|
|
#
|
|
# RULES (enforced by review, see docs/testing/README.md):
|
|
# * Every entry MUST link exactly one OPEN issue tracking the flake.
|
|
# * An entry stays until the issue is fixed (test made robust) or the test is
|
|
# deleted — 30-day policy. No entry may exist without a live issue link.
|
|
#
|
|
# Each entry also re-declares the `ecstore-serial-flaky` test-group so the
|
|
# serialization holds under the ci profile (nextest evaluates a named
|
|
# profile's own overrides list, not the default profile's).
|
|
# ===========================================================================
|
|
|
|
# QUARANTINE: OPEN backlog#937 — concurrent_resend lock-acquire deadline flakes
|
|
# under saturated disk I/O in the full parallel suite.
|
|
[[profile.ci.overrides]]
|
|
filter = 'package(rustfs-ecstore) & test(concurrent_resend_same_part_commits_one_generation)'
|
|
test-group = 'ecstore-serial-flaky'
|
|
retries = 2
|
|
|
|
# QUARANTINE: OPEN backlog#937 — store::bucket::tests::bucket_delete_* race
|
|
# make_bucket into InsufficientWriteQuorum via shared global state under load.
|
|
[[profile.ci.overrides]]
|
|
filter = 'package(rustfs-ecstore) & test(/^store::bucket::tests::bucket_delete_(mark_delete_marks|purge_removes|default_s3_delete)/)'
|
|
test-group = 'ecstore-serial-flaky'
|
|
retries = 2
|
|
|
|
# QUARANTINE: OPEN rustfs#4690 — walk_dir stall-budget accounting test depends
|
|
# on producer/consumer timing windows that stretch past the budget on loaded
|
|
# CI runners (regression test for rustfs#4644; failed on a zero-Rust-diff PR).
|
|
[[profile.ci.overrides]]
|
|
filter = 'package(rustfs-ecstore) & test(walk_dir_does_not_charge_consumer_backpressure_to_the_stall_budget)'
|
|
retries = 2
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# e2e-smoke profile — PR smoke subset of the e2e_test crate (backlog#1149 ci-4)
|
|
# ---------------------------------------------------------------------------
|
|
# PR smoke subset of the e2e_test crate (backlog#1149 ci-4). This profile is
|
|
# the single wiring mechanism for e2e tests in CI: other suites join by
|
|
# extending this filter (or a sibling profile), never by adding ad-hoc e2e
|
|
# jobs to ci.yml. Admission criteria (see crates/e2e_test/README.md): fast,
|
|
# single-node topology, no external dependencies (no awscurl / Vault / fixed
|
|
# ports / pre-started server), no #[ignore].
|
|
#
|
|
# Each e2e test spawns its own rustfs server on a random port with an isolated
|
|
# temp dir (crates/e2e_test/src/common.rs), so the subset is parallel-safe.
|
|
#
|
|
# Replication PR subset (backlog#1147 repl-1): the second clause admits the 20
|
|
# FAST bucket-replication tests from replication_extension_test — the
|
|
# target-registration / replication-check / list / remove / delete admin paths
|
|
# that validate config synchronously and never wait for asynchronous
|
|
# replication convergence. Each spawns its own single-node rustfs server(s) on
|
|
# random ports (source, plus an independent single-node target for the pair
|
|
# checks — NOT a cluster), so the subset stays parallel-safe and single-digit
|
|
# seconds. The SLOW data-plane tests (which poll for convergence at production
|
|
# background-loop intervals) and all `_real_dual_node` / `_real_single_node`
|
|
# site-replication tests run in the [profile.e2e-repl-nightly] lane below, NOT
|
|
# here. This allowlist is the single source of truth for the PR/nightly split:
|
|
# the nightly profile derives its set as "the replication module MINUS this
|
|
# allowlist", so any new replication test lands in nightly by default (never
|
|
# silently unrun) until it is explicitly blessed as fast here. Keep the two
|
|
# regexes byte-identical. Count invariant: 20 here + 16 nightly = 36 total
|
|
# (authority: `cargo nextest list`; docs/testing/e2e-suite-inventory.md).
|
|
[profile.e2e-smoke]
|
|
default-filter = """
|
|
package(e2e_test) & (
|
|
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_source_invalid_date|content_encoding|anonymous_access|bucket_policy_check)_test::/)
|
|
| test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
|
|
)
|
|
"""
|
|
fail-fast = false
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# e2e-repl-nightly profile — scheduled full replication e2e lane (repl-1)
|
|
# ---------------------------------------------------------------------------
|
|
# backlog#1147 repl-1 (deps: ci-4). Runs the SLOW / cross-process replication
|
|
# tests that are unfit for the per-PR e2e-smoke gate:
|
|
#
|
|
# * 6 slow bucket-replication data-plane tests — they PUT an object and poll
|
|
# until it converges on the target at production background-loop intervals
|
|
# (health check 5s / MRF flush 10s), so each takes tens of seconds; two of
|
|
# them replicate over HTTPS.
|
|
# * 9 `_real_dual_node` site-replication tests — each spawns TWO full rustfs
|
|
# servers and drives the cross-process site-replication control plane.
|
|
# * 1 `_real_single_node` service-account round-trip test.
|
|
#
|
|
# The set is defined as "everything in replication_extension_test that is NOT
|
|
# in the e2e-smoke PR allowlist above" (the negated clause is byte-identical to
|
|
# the allowlist), so a newly added replication test automatically runs here
|
|
# until it is explicitly promoted to the fast PR subset — no replication test
|
|
# is ever silently left out of CI.
|
|
#
|
|
# #[serial] does NOT serialize under nextest (process-per-test; see the file
|
|
# header). These tests need no cross-test serialization: each spawns its own
|
|
# server(s) on random ports with isolated temp dirs, so they are parallel-safe
|
|
# by construction — the same property the e2e-smoke subset relies on. If load
|
|
# on the runner surfaces a real flake, quarantine the specific test with an
|
|
# OPEN issue link (ci-10 / backlog#937 policy), never blanket-retry or exclude.
|
|
#
|
|
# Wired by .github/workflows/e2e-replication-nightly.yml (schedule +
|
|
# workflow_dispatch), which builds the rustfs binary once, installs awscurl so
|
|
# the STS dual-node test actually exercises its path (it skips gracefully with
|
|
# a visible log line when awscurl is absent), and routes scheduled failures
|
|
# through .github/actions/schedule-failure-issue (ci-8). Explicit division of
|
|
# labor with ci-5's future e2e-full merge gate: these tests run ONLY here, not
|
|
# double-run there. TODO(ci-7): fold this interim repl-owned lane into the ci
|
|
# domain's consolidated scheduled e2e workflow once it exists.
|
|
[profile.e2e-repl-nightly]
|
|
default-filter = """
|
|
package(e2e_test)
|
|
& test(/^replication_extension_test::/)
|
|
& !test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
|
|
"""
|
|
fail-fast = false
|
|
|
|
[profile.e2e-repl-nightly.junit]
|
|
# Emitted to target/nextest/e2e-repl-nightly/junit.xml; uploaded by the nightly
|
|
# workflow as the failure-triage artifact.
|
|
path = "junit.xml"
|