From 2477e31059c3ddb496b50f4bf2eef934e3c20422 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 5 Sep 2026 14:22:48 +0800 Subject: [PATCH] test(ecstore): require core regressions in the existing CI lane (#7162) * test(ecstore): require core invariant tests in existing CI lane * test(ci): require a fresh core JUnit report * test(ecstore): match sealed context fixture map type --- .config/ecstore-required-tests.json | 72 ++++++++++++++++++++++++++ .github/workflows/ci.yml | 7 +++ docs/testing/ci-gates.md | 10 ++++ scripts/check_test_wiring.py | 79 ++++++++++++++++++++++++++++- 4 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 .config/ecstore-required-tests.json diff --git a/.config/ecstore-required-tests.json b/.config/ecstore-required-tests.json new file mode 100644 index 000000000..6cadc7815 --- /dev/null +++ b/.config/ecstore-required-tests.json @@ -0,0 +1,72 @@ +{ + "lane": "ci/test-and-lint", + "tests": [ + { + "invariant": "write-quorum", + "suite": "rustfs-ecstore", + "name": "set_disk::ops::object::inline_put_commit_path_tests::inline_put_direct_commit_accepts_exact_quorum_and_rejects_quorum_minus_one" + }, + { + "invariant": "metadata-rollback", + "suite": "rustfs-ecstore", + "name": "set_disk::core::io_primitives::tests::write_unique_file_info_reverts_metadata_when_write_quorum_fails" + }, + { + "invariant": "stale-writer", + "suite": "rustfs-ecstore", + "name": "set_disk::ops::object::put_object_tmp_cleanup_tests::put_object_no_lock_aborts_after_outer_namespace_lock_loss" + }, + { + "invariant": "range-body", + "suite": "rustfs-ecstore", + "name": "set_disk::ops::object::transition_upload_integrity_tests::transitioned_compressed_object_range_get_returns_plaintext_slice" + }, + { + "invariant": "multipart-cancellation", + "suite": "rustfs-ecstore", + "name": "set_disk::ops::multipart::tests::cancelled_complete_keeps_upload_lock_through_tail_cleanup" + }, + { + "invariant": "list-uncommitted-version", + "suite": "rustfs-filemeta", + "name": "metacache::tests::resolve_with_write_quorum_slack_keeps_partial_latest_hidden_during_merge" + }, + { + "invariant": "minio-object-fixture", + "suite": "rustfs-filemeta", + "name": "filemeta::test::parses_real_minio_object_xlmeta" + }, + { + "invariant": "corrupt-part-arrays", + "suite": "rustfs-filemeta", + "name": "filemeta::test::crc_valid_but_part_arrays_corrupt_into_fileinfo_errors_not_panics" + } + ], + "fixtures": [ + { + "path": "crates/filemeta/tests/fixtures/minio/object_large_bin.xlmeta.hex", + "sha256": "e8093767806d701e639b48d023190e858fbc4cde69bcfd83c22af8cba8452ce5", + "source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md" + }, + { + "path": "crates/filemeta/tests/fixtures/minio/object_small_txt.xlmeta.hex", + "sha256": "2a415ad3a3be5a9440035d4026ff880e0e8c1ec1701be9f4e077734e8dce03da", + "source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md" + }, + { + "path": "crates/filemeta/tests/fixtures/minio/object_versioned_txt.xlmeta.hex", + "sha256": "7f21f50c326dd8b0228deb6dbdb7052b3d0a3f8ee6c85d43486f0e6bb7a97261", + "source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md" + }, + { + "path": "crates/ecstore/tests/fixtures/minio/bucket_metadata.blob.hex", + "sha256": "f2b6e260aff106adf6039feb1c645686e84e75404ff725491fb18668be5db203", + "source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md" + }, + { + "path": "crates/ecstore/tests/fixtures/minio/bucket_metadata_full.xlmeta.hex", + "sha256": "3b6de589519c08a1614c8bd409bb8199c17d42043861b07bce513075e6fbfc12", + "source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md" + } + ] +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4dc00bf5f..98fc6cc30 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -269,6 +269,7 @@ jobs: CARGO_BUILD_JOBS: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && '3' || '2' }} run: | mkdir -p artifacts/test-and-lint + rm -f target/nextest/ci/junit.xml ./scripts/ci/resource_sampler.sh start nextest trap './scripts/ci/resource_sampler.sh stop' EXIT set +e @@ -277,6 +278,12 @@ jobs: --status-level all --final-status-level all \ 2>&1 | tee artifacts/test-and-lint/nextest.log status=${PIPESTATUS[0]} + if [[ "${status}" -eq 0 ]]; then + cargo nextest list --profile ci --all --exclude e2e_test --message-format json \ + > artifacts/test-and-lint/core-test-listing.json \ + && python3 scripts/check_test_wiring.py --check-core artifacts/test-and-lint/core-test-listing.json \ + && test -s target/nextest/ci/junit.xml || status=$? + fi { echo "command=cargo nextest run --profile ci --all --exclude e2e_test" echo "exit_status=${status}" diff --git a/docs/testing/ci-gates.md b/docs/testing/ci-gates.md index 256b41bd4..93770b74d 100644 --- a/docs/testing/ci-gates.md +++ b/docs/testing/ci-gates.md @@ -109,3 +109,13 @@ Use an exact preview tag for an end-to-end release rehearsal. Manual dispatches ## Change checklist Update this file in the same PR when a job or check name changes, a workflow gains or loses a `pull_request` or `schedule` trigger, required contexts or strict/merge-queue policy change, report-only vs gating semantics change, or `.github/scheduled-validations.json` membership changes. Do not copy timeouts, crons, or test counts here. + +## ECStore invariant selection + +The existing `ci.yml` test-and-lint job runs the ordinary ECStore and filemeta tests. After that run, `scripts/check_test_wiring.py --check-core` checks the same nextest profile and package selection against `.config/ecstore-required-tests.json`. Every named test must exist, match the filter, and be non-ignored; the job also requires a nonempty JUnit report. This checks membership without running the tests twice. `core-test-listing.json`, JUnit, and the run log are retained in the existing test-and-lint artifact. + +The manifest records a minimum set of invariants: write quorum, metadata rollback, stale-writer lock loss, plaintext Range content, multipart cancellation, hiding uncommitted LIST versions, real MinIO metadata, and corrupt part arrays. Renaming or moving a required test must update the manifest in the same change after checking the compiled listing. Extend this list as new deterministic regressions land; it is not a claim that all storage invariants are covered. + +The checked-in MinIO corpus is pinned by file SHA256 and its documented source release. The static wiring guard and the CI selection check both reject missing or changed fixtures. These are metadata fixtures, not a legacy shard-body corpus or proof of crash durability. Optional `legacy_bitrot_read_test` runs may still skip when their external corpus is absent; they do not satisfy a required compatibility lane. Real encrypted fixture reads remain in `minio-interop.yml`, and multi-node fault schedules remain in the existing nightly cluster lane. In-process reopen tests do not establish power-loss durability. + +Run `python3 scripts/check_test_wiring.py --self-test` to exercise the negative cases: removed/ignored/filtered tests, malformed listing, absent fixtures, and wrong fixture hashes. Do not update hashes merely to silence the guard; a fixture change needs source/provenance and compatibility review. diff --git a/scripts/check_test_wiring.py b/scripts/check_test_wiring.py index 1df46c234..8b1cf1246 100755 --- a/scripts/check_test_wiring.py +++ b/scripts/check_test_wiring.py @@ -764,8 +764,53 @@ def check_profile_listing(root: Path, profile: str, listing: Path) -> list[str]: return [] +def core_requirements(root: Path) -> dict: + data = json.loads((root / ".config/ecstore-required-tests.json").read_text()) + if not data["tests"] or not data["fixtures"]: + raise ValueError("core test and fixture requirements must not be empty") + identities = [(test["suite"], test["name"]) for test in data["tests"]] + if len(set(identities)) != len(identities): + raise ValueError("duplicate core test requirement") + return data + + +def check_core_fixtures(root: Path) -> list[str]: + try: + fixtures = core_requirements(root)["fixtures"] + errors = [] + for fixture in fixtures: + path = (root / fixture["path"]).resolve() + if not path.is_relative_to(root.resolve()): + raise ValueError("core fixture path escapes repository") + if not path.is_file(): + errors.append(f"{fixture['path']}: required core fixture missing") + elif hashlib.sha256(path.read_bytes()).hexdigest() != fixture["sha256"]: + errors.append(f"{fixture['path']}: core fixture sha256 mismatch") + return errors + except (OSError, KeyError, TypeError, ValueError) as error: + return [f"cannot validate core fixtures: {error}"] + + +def check_core_listing(root: Path, listing: Path) -> list[str]: + """Check the existing CI run's selection, not a second filtered test run.""" + try: + required = core_requirements(root)["tests"] + suites = json.loads(listing.read_text())["rust-suites"] + if not isinstance(suites, dict): + raise ValueError("rust-suites must be an object") + errors = check_core_fixtures(root) + for test in required: + testcase = suites.get(test["suite"], {}).get("testcases", {}).get(test["name"], {}) + if testcase.get("ignored") is not False or testcase.get("filter-match", {}).get("status") != "matches": + errors.append(f"{test['invariant']}: required test not selected: {test['suite']}::{test['name']}") + return errors + except (OSError, KeyError, TypeError, ValueError) as error: + return [f"cannot read core nextest listing: {error}"] + + def validate(root: Path) -> list[str]: errors: list[str] = [] + errors.extend(check_core_fixtures(root)) errors.extend(check_e2e_modules(root)) errors.extend(check_vault_test_groups(root)) errors.extend(check_ilm_build_budget(root)) @@ -779,6 +824,32 @@ def validate(root: Path) -> list[str]: class SelfTests(unittest.TestCase): + def test_core_gate_rejects_missing_ignored_filtered_and_corrupt_inputs(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / ".config").mkdir() + fixture = root / "fixture.hex" + fixture.write_text("4142") + requirements = { + "tests": [{"invariant": "commit", "suite": "store", "name": "commit_test"}], + "fixtures": [{"path": "fixture.hex", "sha256": hashlib.sha256(fixture.read_bytes()).hexdigest()}], + } + (root / ".config/ecstore-required-tests.json").write_text(json.dumps(requirements)) + listing = root / "listing.json" + good = {"ignored": False, "filter-match": {"status": "matches"}} + for case, testcase in (("selected", good), ("missing", {}), ("ignored", dict(good, ignored=True)), + ("filtered", dict(good, **{"filter-match": {"status": "mismatch"}}))): + with self.subTest(case=case): + listing.write_text(json.dumps({"rust-suites": {"store": {"testcases": {"commit_test": testcase}}}})) + self.assertEqual(bool(check_core_listing(root, listing)), case != "selected") + listing.write_text(json.dumps({"rust-suites": {"store": {"testcases": {"commit_test": good}}}})) + fixture.write_text("4143") + self.assertIn("sha256 mismatch", check_core_listing(root, listing)[0]) + fixture.unlink() + self.assertIn("fixture missing", check_core_listing(root, listing)[0]) + listing.write_text("not json") + self.assertIn("cannot read", check_core_listing(root, listing)[0]) + def test_ilm_lane_keeps_the_measured_cargo_build_budget(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -981,6 +1052,7 @@ class SelfTests(unittest.TestCase): mock.patch(__name__ + ".check_e2e_modules", return_value=[]), mock.patch(__name__ + ".check_vault_test_groups", return_value=[]), mock.patch(__name__ + ".check_fuzz_targets", return_value=[]), + mock.patch(__name__ + ".check_core_fixtures", return_value=[]), mock.patch(__name__ + ".check_runner_selection", return_value=[]), mock.patch(__name__ + ".check_workflow_readiness", return_value=[]), mock.patch(__name__ + ".check_profile_definitions", return_value=[]), @@ -1393,6 +1465,11 @@ def main() -> int: if sys.argv[1:] == ["--self-test"]: suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTests) return 0 if unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful() else 1 + if len(sys.argv) == 3 and sys.argv[1] == "--check-core": + errors = check_core_listing(ROOT, Path(sys.argv[2])) + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + return 1 if errors else 0 if len(sys.argv) == 4 and sys.argv[1] == "--check-profile": errors = check_profile_listing(ROOT, sys.argv[2], Path(sys.argv[3])) if errors: @@ -1410,7 +1487,7 @@ def main() -> int: return 0 if sys.argv[1:]: print( - "usage: check_test_wiring.py [--self-test | --check-profile PROFILE LISTING | " + "usage: check_test_wiring.py [--self-test | --check-core LISTING | --check-profile PROFILE LISTING | " "--update-profile PROFILE LISTING PLATFORM]", file=sys.stderr, )