gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-22 04:16:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

ci: harden test selection and nightly coverage (#6341)

Browse Source
This commit is contained in:
Zhengchao An
2026-08-21 23:04:08 +08:00
committed by GitHub
parent bce5922aef
commit bc07cfd115
31 changed files with 1146 additions and 259 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/check_test_wiring.py
Executable
+419
View File
@@ -0,0 +1,419 @@
#!/usr/bin/env python3
"""Fail when committed tests silently fall out of their execution wiring."""
from __future__ import annotations
import hashlib
import json
import re
import sys
import tempfile
import tomllib
import unittest
from unittest import mock
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def words(value: str) -> set[str]:
return {item.strip() for item in value.split(",") if item.strip()}
def rust_code_only(source: str) -> str:
"""Blank Rust comments and literals while preserving byte positions."""
def quoted_end(quote_index: int, delimiter: str) -> int:
end = quote_index + 1
while end < len(source):
if source[end] == "\\":
end += 2
elif source[end] == delimiter:
return end + 1
else:
end += 1
return end
code = list(source)
index = 0
while index < len(source):
if source.startswith("//", index):
end = source.find("\n", index)
end = len(source) if end < 0 else end
elif source.startswith("/*", index):
depth = 1
end = index + 2
while end < len(source) and depth:
if source.startswith("/*", end):
depth += 1
end += 2
elif source.startswith("*/", end):
depth -= 1
end += 2
else:
end += 1
else:
raw = re.match(r'(?:br|cr|r)(?P<hashes>#{0,})"', source[index:])
if raw:
marker = '"' + raw.group("hashes")
end = source.find(marker, index + raw.end())
end = len(source) if end < 0 else end + len(marker)
elif source[index] == '"' or source.startswith(('b"', 'c"'), index):
start_quote = index if source[index] == '"' else index + 1
end = quoted_end(start_quote, '"')
elif source[index] == "'" and index + 2 < len(source) and (
source[index + 1] == "\\" or source[index + 2] == "'"
):
end = quoted_end(index, "'")
elif source.startswith("b'", index):
end = quoted_end(index + 1, "'")
else:
index += 1
continue
for offset in range(index, end):
if code[offset] != "\n":
code[offset] = " "
index = end
return "".join(code)
def declared(parent: Path, module: str) -> bool:
source = parent.read_text()
code = rust_code_only(source)
pattern = re.compile(rf"^\s*(?:pub(?:\([^)]*\))?\s+)?mod\s+{re.escape(module)}\s*;", re.MULTILINE)
allowed_preambles = {
"#[cfg(test)]": "#[cfg(test)]",
"#[cfg(all(test,target_os=))]": '#[cfg(all(test,target_os="linux"))]',
}
for match in pattern.finditer(code):
prefix = code[: match.start()]
depths = {"(": 0, "[": 0, "{": 0}
pairs = {")": "(", "]": "[", "}": "{"}
for char in prefix:
if char in depths:
depths[char] += 1
elif char in pairs:
depths[pairs[char]] -= 1
if any(depths.values()):
continue
boundary = max(prefix.rfind(";"), prefix.rfind("{"), prefix.rfind("}"))
code_preamble = re.sub(r"\s+", "", prefix[boundary + 1 :])
if not code_preamble:
return True
if code_preamble in allowed_preambles:
attr_start = prefix.rfind("#[cfg", boundary + 1)
if attr_start >= 0 and re.sub(r"\s+", "", source[attr_start : match.start()]) == allowed_preambles[code_preamble]:
return True
return False
def module_source(src: Path, directory: Path) -> Path | None:
if not directory.parts:
return src / "lib.rs"
mod_file = src / directory / "mod.rs"
if mod_file.is_file():
return mod_file
sibling = src.joinpath(*directory.parts[:-1], f"{directory.name}.rs")
return sibling if sibling.is_file() else None
def check_e2e_modules(root: Path) -> list[str]:
src = root / "crates/e2e_test/src"
errors: list[str] = []
for test_file in sorted(src.rglob("*_test.rs")):
relative = test_file.relative_to(root).as_posix()
directory = test_file.relative_to(src).parent
parent = module_source(src, directory)
if parent is None:
errors.append(f"{relative}: no canonical parent module")
continue
if not declared(parent, test_file.stem):
errors.append(f"{relative}: not declared by {parent.relative_to(root).as_posix()}")
while directory.parts:
module = directory.name
directory = directory.parent
parent = module_source(src, directory)
if parent is None:
errors.append(f"{relative}: module {module} has no canonical parent")
break
if not declared(parent, module):
errors.append(f"{relative}: module {module} not declared by {parent.relative_to(root).as_posix()}")
return errors
def check_fuzz_targets(root: Path) -> list[str]:
manifest = tomllib.loads((root / "fuzz/Cargo.toml").read_text())
expected = {item["name"] for item in manifest.get("bin", []) if "name" in item}
errors: list[str] = []
if not expected:
return ["fuzz/Cargo.toml: no [[bin]] fuzz targets found"]
runner = (root / "scripts/fuzz/run.sh").read_text()
match = re.search(r'^targets="([^"]+)"', runner, re.MULTILINE)
runner_targets = set(match.group(1).split()) if match else set()
if runner_targets != expected:
errors.append(f"scripts/fuzz/run.sh targets {sorted(runner_targets)} != manifest {sorted(expected)}")
workflow = (root / ".github/workflows/fuzz.yml").read_text()
matrices = [words(value) for value in re.findall(r"^\s*target:\s*\[([^]]+)]", workflow, re.MULTILINE)]
if len(matrices) != 2:
errors.append(f".github/workflows/fuzz.yml: expected smoke and nightly target matrices, found {len(matrices)}")
for index, matrix in enumerate(matrices, start=1):
if matrix != expected:
errors.append(f".github/workflows/fuzz.yml matrix {index} {sorted(matrix)} != manifest {sorted(expected)}")
runtime_targets = re.findall(r"^\s*FUZZ_TARGET:\s*(\S.*?)\s*$", workflow, re.MULTILINE)
if runtime_targets != ["${{ matrix.target }}", "${{ matrix.target }}"]:
errors.append(".github/workflows/fuzz.yml: smoke and nightly jobs must pass matrix.target to FUZZ_TARGET")
dependency_paths = {
f"{path.removeprefix('../')}/**"
for dependency in manifest.get("dependencies", {}).values()
if isinstance(dependency, dict)
and isinstance(path := dependency.get("path"), str)
and path.startswith("../crates/")
}
missing_paths = sorted(path for path in dependency_paths if f'"{path}"' not in workflow)
if missing_paths:
errors.append(f".github/workflows/fuzz.yml missing direct dependency paths: {', '.join(missing_paths)}")
staged_matches = re.findall(r"^\s*for target in ([^;]+); do", workflow, re.MULTILINE)
staged = set(staged_matches[0].split()) if staged_matches else set()
if staged != expected:
errors.append(f".github/workflows/fuzz.yml staged binaries {sorted(staged)} != manifest {sorted(expected)}")
return errors
def check_runner_selection(root: Path) -> list[str]:
runner = (root / "scripts/run_e2e_tests.sh").read_text()
errors: list[str] = []
if "--include-ignored" not in runner:
errors.append("scripts/run_e2e_tests.sh: runner must include default and ignored tests")
if "--test-threads=1" not in runner:
errors.append("scripts/run_e2e_tests.sh: runner must serialize fixed-port protocol tests")
if re.search(r"(?<!include-)--ignored\b", runner):
errors.append("scripts/run_e2e_tests.sh: bare --ignored silently excludes default tests")
if "--exact" in runner:
errors.append("scripts/run_e2e_tests.sh: --test is documented as a pattern and must not force exact matching")
if 'eval "$test_cmd"' in runner:
errors.append("scripts/run_e2e_tests.sh: command construction must not use eval")
return errors
def profile_selection(root: Path, profile: str) -> str:
if not re.fullmatch(r"e2e-[a-z0-9-]+", profile):
raise ValueError(f"invalid e2e profile name: {profile}")
path = root / f".config/{profile}-selection.txt"
lines = [line for line in path.read_text().splitlines() if line.strip()]
values = dict(line.split("=", 1) for line in lines if "=" in line)
if len(values) != len(lines) or any(not re.fullmatch(r"sha256(?:-[a-z0-9]+)?", key) for key in values):
raise ValueError(f"{path.relative_to(root).as_posix()}: invalid sha256 entry")
key = f"sha256-{sys.platform}"
digest = values.get(key, values.get("sha256", ""))
if not re.fullmatch(r"[0-9a-f]{64}", digest):
raise ValueError(f"{path.relative_to(root).as_posix()}: missing sha256 for {sys.platform}")
return digest
def check_profile_definitions(root: Path) -> list[str]:
config = tomllib.loads((root / ".config/nextest.toml").read_text())
profiles = {
profile
for profile in config.get("profile", {})
if profile.startswith("e2e-")
}
selection_profiles = {
path.name.removesuffix("-selection.txt") for path in (root / ".config").glob("e2e-*-selection.txt")
}
errors: list[str] = []
for profile in sorted(profiles | selection_profiles):
if profile not in profiles:
errors.append(f".config/nextest.toml: missing profile.{profile}")
if profile not in selection_profiles:
errors.append(f".config/{profile}-selection.txt: missing expected profile selection")
continue
try:
profile_selection(root, profile)
except (FileNotFoundError, ValueError) as error:
errors.append(str(error))
return errors
def check_profile_listing(root: Path, profile: str, listing: Path) -> list[str]:
try:
expected_digest = profile_selection(root, profile)
data = json.loads(listing.read_text())
selected = sorted(
f"{suite_id}::{test_name}"
for suite_id, suite in data["rust-suites"].items()
for test_name, testcase in suite["testcases"].items()
if testcase.get("filter-match", {}).get("status") == "matches"
)
digest = hashlib.sha256(("\n".join(selected) + "\n").encode()).hexdigest()
except (FileNotFoundError, KeyError, TypeError, ValueError, json.JSONDecodeError) as error:
return [f"cannot read {profile} nextest listing: {error}"]
if digest != expected_digest:
return [
f"{profile} selection changed: count={len(selected)} sha256={digest}; "
f"expected sha256={expected_digest}"
]
print(f"{profile} selection OK: {len(selected)} tests, sha256={digest}")
return []
def validate(root: Path) -> list[str]:
errors: list[str] = []
errors.extend(check_e2e_modules(root))
errors.extend(check_fuzz_targets(root))
errors.extend(check_runner_selection(root))
errors.extend(check_profile_definitions(root))
return errors
class SelfTests(unittest.TestCase):
def test_e2e_requires_registration(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
src = root / "crates/e2e_test/src"
src.mkdir(parents=True)
(src / "lib.rs").write_text("")
test_file = src / "boundary_test.rs"
test_file.write_text("#[test]\nfn boundary() {}\n")
self.assertEqual(len(check_e2e_modules(root)), 1)
(src / "lib.rs").write_text("mod boundary_test;\n")
self.assertEqual(check_e2e_modules(root), [])
nested = src / "protocols"
nested.mkdir()
(nested / "mod.rs").write_text("mod fixed_port_test;\n")
(nested / "fixed_port_test.rs").write_text("#[test]\nfn fixed_port() {}\n")
self.assertEqual(len(check_e2e_modules(root)), 1)
(src / "lib.rs").write_text("mod boundary_test;\nmod protocols;\n")
self.assertEqual(check_e2e_modules(root), [])
(src / "lib.rs").write_text("#[cfg(any())]\nmod boundary_test;\nmod protocols;\n")
self.assertEqual(len(check_e2e_modules(root)), 1)
(src / "lib.rs").write_text(
"#[cfg(any())]\n/// hidden module\nmod boundary_test;\n#[cfg_attr(test, cfg(any()))]\nmod protocols;\n"
)
self.assertEqual(len(check_e2e_modules(root)), 2)
(src / "lib.rs").write_text(
'const PHANTOM: &str = r#"{\nmod boundary_test;\n"#;\ndiscard! { mod protocols; }\n'
)
self.assertEqual(len(check_e2e_modules(root)), 2)
(src / "lib.rs").write_text(
'#[cfg(all(test, target_os = r"windows" /* target_os = "linux" */))]\n'
"mod boundary_test;\nmod protocols;\n"
)
self.assertEqual(len(check_e2e_modules(root)), 1)
(src / "lib.rs").write_text(
'#[cfg(all(test, target_os = r"windows"))] // #[cfg(all(test, target_os = "linux"))]\n'
"mod boundary_test;\nmod protocols;\n"
)
self.assertEqual(len(check_e2e_modules(root)), 1)
def test_fuzz_runtime_uses_matrix_target(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
(root / "fuzz").mkdir()
(root / "scripts/fuzz").mkdir(parents=True)
(root / ".github/workflows").mkdir(parents=True)
(root / "fuzz/Cargo.toml").write_text(
'dep = { path = "../crates/dep" }\n[[bin]]\nname = "one"\n'
)
(root / "scripts/fuzz/run.sh").write_text('targets="one"\n')
(root / ".github/workflows/fuzz.yml").write_text(
'paths:\n - "crates/dep/**"\n'
"target: [one]\nFUZZ_TARGET: fixed\n"
"target: [one]\nFUZZ_TARGET: ${{ matrix.target }}\n"
"for target in one; do\n"
" fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/one\n"
)
self.assertEqual(len(check_fuzz_targets(root)), 1)
def test_profile_listing_enforces_selection(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
(root / ".config").mkdir()
digest = hashlib.sha256(b"suite::two\n").hexdigest()
(root / ".config/e2e-smoke-selection.txt").write_text(f"sha256={digest}\n")
listing = root / "listing.json"
listing.write_text(
json.dumps(
{
"rust-suites": {
"suite": {
"testcases": {
"one": {"filter-match": {"status": "matches"}},
"two": {"filter-match": {"status": "mismatch"}},
}
}
}
}
)
)
self.assertEqual(len(check_profile_listing(root, "e2e-smoke", listing)), 1)
def test_profile_listing_binds_platform_digest(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
(root / ".config").mkdir()
darwin_digest = hashlib.sha256(b"suite::darwin\n").hexdigest()
linux_digest = hashlib.sha256(b"suite::linux\n").hexdigest()
(root / ".config/e2e-full-selection.txt").write_text(
f"sha256-darwin={darwin_digest}\nsha256-linux={linux_digest}\n"
)
listing = root / "listing.json"
listing.write_text(
json.dumps(
{
"rust-suites": {
"suite": {
"testcases": {"darwin": {"filter-match": {"status": "matches"}}}
}
}
}
)
)
with mock.patch.object(sys, "platform", "linux"):
self.assertEqual(len(check_profile_listing(root, "e2e-full", listing)), 1)
def main() -> int:
if sys.argv[1:] == ["--self-test"]:
suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTests)
return 0 if unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful() else 1
if len(sys.argv) == 4 and sys.argv[1] == "--check-profile":
errors = check_profile_listing(ROOT, sys.argv[2], Path(sys.argv[3]))
if errors:
for error in errors:
print(f"ERROR: {error}", file=sys.stderr)
return 1
return 0
if sys.argv[1:]:
print(
"usage: check_test_wiring.py [--self-test | --check-profile PROFILE LISTING]",
file=sys.stderr,
)
return 2
errors = validate(ROOT)
if errors:
for error in errors:
print(f"ERROR: {error}", file=sys.stderr)
return 1
print("OK: e2e modules, runner selection, fuzz matrices, and profile guards are wired")
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/run_e2e_tests.sh
+14 -11
View File
@@ -20,6 +20,8 @@ DATA_DIR="$TARGET_DIR/rustfs_test_data"
RUSTFS_PID=""
TEST_FILTER=""
TEST_TYPE="all"
RUSTFS_BUILD_FEATURES="${RUSTFS_BUILD_FEATURES:-ftps,webdav,sftp}"
export RUSTFS_BUILD_FEATURES
# Function to print colored output
print_info() {
@@ -92,7 +94,7 @@ build_rustfs() {
print_info "Building RustFS..."
cd "$PROJECT_ROOT"
if ! cargo build --bin rustfs; then
if ! cargo build --bin rustfs --features "$RUSTFS_BUILD_FEATURES"; then
print_error "Failed to build RustFS"
exit 1
fi
@@ -219,27 +221,28 @@ start_rustfs() {
run_tests() {
print_info "Running e2e tests..."
cd "$PROJECT_ROOT"
local test_cmd="cargo test --package e2e_test --lib"
local test_cmd=(cargo test --package e2e_test --lib)
case "$TEST_TYPE" in
"specific")
test_cmd="$test_cmd -- $TEST_FILTER --exact --show-output --ignored"
test_cmd+=(-- "$TEST_FILTER")
print_info "Running specific test: $TEST_FILTER"
;;
"file")
test_cmd="$test_cmd -- $TEST_FILTER --show-output --ignored"
test_cmd+=(-- "$TEST_FILTER")
print_info "Running tests in file/module: $TEST_FILTER"
;;
"all")
test_cmd="$test_cmd -- --show-output --ignored"
test_cmd+=(--)
print_info "Running all e2e tests"
;;
esac
print_info "Test command: $test_cmd"
if eval "$test_cmd"; then
test_cmd+=(--show-output --include-ignored --test-threads=1)
print_info "Test command: ${test_cmd[*]}"
if "${test_cmd[@]}"; then
print_success "All tests passed!"
return 0
else
scripts/s3-tests/README.md
+11 -5
View File
@@ -253,6 +253,9 @@ Test results are saved in the `artifacts/s3tests-${TEST_MODE}/` directory (defau
- `junit.xml`: Test results in JUnit format (compatible with CI/CD systems)
- `pytest.log`: Detailed pytest logs with full test output
- `all-collected-nodeids.txt`: Exact node IDs in the pinned upstream suite
- `selected-nodeids.txt`: Exact node IDs expected in this run
- `unsharded-selected-nodeids.txt`: Exact node IDs before deterministic sharding
- `compat-report.md`: Classification report generated by `report_compat.py` —
regressions against `implemented_tests.txt`, promotion candidates (tests
that pass but are still listed as unimplemented/excluded), and tests missing
@@ -449,9 +452,11 @@ RustFS. Two GitHub Actions workflows delegate to it:
- **Full sweep** (`.github/workflows/e2e-s3tests.yml`): weekly scheduled (and
manually dispatchable) run of the ENTIRE upstream suite (`TEST_SCOPE=all`)
against a Docker deployment — single node or a real 4-node distributed
cluster behind HAProxy. The sweep fails only on regressions in the
implemented whitelist; everything else is reported by `report_compat.py`
as promotion candidates or unclassified tests.
cluster behind HAProxy. Regressions, unclassified tests, incomplete
execution, and infrastructure errors fail the sweep; classified unsupported
behavior remains informational. Scheduled topology runs are split into four
deterministic exact-node-ID shards, and every case has a five-minute timeout,
so one stalled case cannot erase the entire sweep's evidence.
Keeping both workflows on this script means local runs, the PR gate, and the
scheduled sweep always execute tests the same way (same pinned s3-tests
@@ -466,8 +471,9 @@ pass/fail table in the job summary.
## Companion Tools
- `report_compat.py` — diffs a junit.xml result against the classification
lists; run automatically at the end of `run.sh`, and used by the weekly
sweep to gate on whitelist regressions only (`--fail-on-regression`).
lists and the exact pytest collection; run before execution to reject stale
or missing classifications, then after execution to detect regressions and
incomplete parameterized cases.
- `api_coverage.py` — quantifies S3 API surface coverage by comparing the
s3s `S3` trait (at the revision pinned in Cargo.toml) against the methods
RustFS overrides in `impl S3 for FS`:
scripts/s3-tests/excluded_tests.txt
+7
View File
@@ -307,3 +307,10 @@ test_object_acl_write
test_object_acl_writeacp
test_put_bucket_acl_grant_group_read
test_object_raw_get_bucket_acl
# Require upstream cloud-storage or IAM account services
test_bucket_logging_requester_assumed_role
test_lifecycle_cloud_transition_target_by_bucket
test_lifecycle_cloud_transition_target_by_bucket_multiple_buckets
test_list_object_versions_restore_status
test_list_objects_restore_status
scripts/s3-tests/implemented_tests.txt
+4
View File
@@ -521,9 +521,13 @@ test_atomic_dual_conditional_write_1mb
test_atomic_write_bucket_gone
test_bucket_acl_canned_private_to_private
test_bucket_concurrent_set_canned_acl
test_bucket_create_delete
test_bucket_policy
test_bucket_policy_acl
test_bucket_policy_put_obj_acl
test_bucketv2_policy_acl
test_copy_enc
test_copy_part_enc
test_copy_object_ifmatch_failed
test_copy_object_ifnonematch_good
test_cors_presigned_put_object_tenant_with_acl
scripts/s3-tests/report_compat.py
+137 -19
View File
@@ -21,14 +21,17 @@ Classifies every executed test into:
- unclassified passes: passed but not present in any list (new upstream tests)
- unclassified failures: failed and not present in any list (new upstream tests)
Writes a markdown report and prints a summary to stdout. Exit code is 0 unless
--fail-on-regression is given and at least one regression was found.
Writes a markdown report and prints a summary to stdout. Optional gates reject
regressions, unclassified tests, stale classifications, and incomplete node-ID
execution.
"""
from __future__ import annotations
import argparse
from collections import Counter
import pathlib
import re
import sys
import xml.etree.ElementTree as ET
@@ -45,33 +48,47 @@ LIST_FILES = {
}
def load_list(path: pathlib.Path) -> set[str]:
names: set[str] = set()
def load_entries(path: pathlib.Path) -> list[str]:
names: list[str] = []
if not path.is_file():
return names
for line in path.read_text(encoding="utf-8").splitlines():
line = line.strip()
if line and not line.startswith("#"):
names.add(line)
names.append(line)
return names
def classification_errors(entries: dict[str, list[str]]) -> list[str]:
errors: list[str] = []
lists = {key: set(names) for key, names in entries.items()}
for key, names in entries.items():
duplicates = sorted(name for name, count in Counter(names).items() if count > 1)
if duplicates:
errors.append(f"{LIST_FILES[key]} has duplicates: {', '.join(duplicates)}")
keys = tuple(lists)
for index, left in enumerate(keys):
for right in keys[index + 1 :]:
overlap = sorted(lists[left] & lists[right])
if overlap:
errors.append(f"{left}/{right} classifications overlap: {', '.join(overlap)}")
return errors
def base_name(testcase_name: str) -> str:
"""Strip pytest parametrization (test_foo[param]) to match list entries."""
return testcase_name.split("[", 1)[0]
def parse_junit(path: pathlib.Path) -> dict[str, str]:
"""Return {test name: status} with status in passed/failed/error/skipped.
Parametrized cases collapse onto their base name; any failing variant marks
the whole test failed.
"""
def parse_junit(path: pathlib.Path) -> tuple[dict[str, str], list[str], list[tuple[str, str, str, str]]]:
"""Return exact statuses, pytest-timeout cases, and failure summaries."""
results: dict[str, str] = {}
timed_out: list[str] = []
failures: list[tuple[str, str, str, str]] = []
severity = {"skipped": 0, "passed": 1, "failed": 2, "error": 2}
root = ET.parse(path).getroot()
for case in root.iter("testcase"):
name = base_name(case.get("name", ""))
name = case.get("name", "")
if not name:
continue
if case.find("failure") is not None:
@@ -85,7 +102,35 @@ def parse_junit(path: pathlib.Path) -> dict[str, str]:
prev = results.get(name)
if prev is None or severity[status] > severity[prev]:
results[name] = status
return results
node = case.find("failure") if status == "failed" else case.find("error")
if node is not None:
details = " ".join(filter(None, [node.get("message", ""), node.text or ""]))
message = node.get("message") or next(iter((node.text or "").strip().splitlines()), "")
failures.append((case.get("classname", ""), name, case.get("time", "0"), message))
if re.search(r"\bTimeout\s*(?:>|\()", details, re.IGNORECASE):
timed_out.append(name)
return results, timed_out, failures
def collapse_results(results: dict[str, str]) -> dict[str, str]:
"""Collapse parametrized cases for classification-level reporting."""
collapsed: dict[str, str] = {}
severity = {"skipped": 0, "passed": 1, "failed": 2, "error": 2}
for exact_name, status in results.items():
name = base_name(exact_name)
previous = collapsed.get(name)
if previous is None or severity[status] > severity[previous]:
collapsed[name] = status
return collapsed
def load_collected_nodeids(path: pathlib.Path) -> set[str]:
names: set[str] = set()
for line in path.read_text(encoding="utf-8").splitlines():
nodeid = line.strip()
if nodeid:
names.add(nodeid.rsplit("::", 1)[-1])
return names
def render_section(title: str, rows: list[str], hint: str = "") -> list[str]:
@@ -102,7 +147,7 @@ def render_section(title: str, rows: list[str], hint: str = "") -> list[str]:
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--junit", required=True, type=pathlib.Path, help="junit.xml produced by pytest")
parser.add_argument("--junit", type=pathlib.Path, help="junit.xml produced by pytest")
parser.add_argument(
"--lists-dir",
type=pathlib.Path,
@@ -115,14 +160,60 @@ def main() -> int:
action="store_true",
help="exit non-zero when a test from implemented_tests.txt failed",
)
parser.add_argument(
"--fail-on-unclassified",
action="store_true",
help="exit non-zero when an executed test is absent from every classification",
)
parser.add_argument(
"--collected-nodeids",
type=pathlib.Path,
help="exact pytest node IDs from the pinned suite's collect-only pass",
)
parser.add_argument(
"--check-classifications-only",
action="store_true",
help="validate classification names against collected node IDs without reading JUnit",
)
args = parser.parse_args()
if not args.junit.is_file():
entries = {key: load_entries(args.lists_dir / fname) for key, fname in LIST_FILES.items()}
lists = {key: set(names) for key, names in entries.items()}
invalid_classifications = classification_errors(entries)
collected: set[str] = set()
if args.collected_nodeids:
collected = load_collected_nodeids(args.collected_nodeids)
collected_base = {base_name(name) for name in collected}
classified = set().union(*lists.values())
missing_classifications = sorted(collected_base - classified)
stale_classifications = sorted(classified - collected_base)
else:
missing_classifications = []
stale_classifications = []
if args.check_classifications_only:
if not args.collected_nodeids:
parser.error("--check-classifications-only requires --collected-nodeids")
for error in invalid_classifications:
print(f"[INVALID] {error}")
for name in missing_classifications:
print(f"[UNCLASSIFIED] {name}")
for name in stale_classifications:
print(f"[STALE] {name}")
return 1 if invalid_classifications or missing_classifications or stale_classifications else 0
if invalid_classifications:
for error in invalid_classifications:
print(f"[ERROR] {error}", file=sys.stderr)
return 2
if not args.junit or not args.junit.is_file():
print(f"[ERROR] junit file not found: {args.junit}", file=sys.stderr)
return 2
lists = {key: load_list(args.lists_dir / fname) for key, fname in LIST_FILES.items()}
results = parse_junit(args.junit)
exact_results, timed_out, failures = parse_junit(args.junit)
results = collapse_results(exact_results)
missing_results = sorted(collected - exact_results.keys()) if collected else []
regressions: list[str] = []
promotions: dict[str, list[str]] = {"unimplemented": [], "excluded": []}
@@ -155,7 +246,9 @@ def main() -> int:
lines = [
"# S3 compatibility report",
"",
f"Executed: {len(results)} tests — "
f"Executed: {len(exact_results)} exact cases across {len(results)} classified tests.",
"",
"Classification status — "
f"{counts['passed']} passed, {counts['failed']} failed, "
f"{counts['error']} errored, {counts['skipped']} skipped.",
"",
@@ -191,6 +284,16 @@ def main() -> int:
unclassified_failed,
"Failing and absent from every list — triage into `unimplemented_tests.txt` or `excluded_tests.txt`.",
)
lines += render_section(
"Missing results",
missing_results,
"Present in the pinned upstream suite but absent from JUnit — the sweep was incomplete.",
)
lines += render_section(
"Timed out",
timed_out,
"Per-test timeout is an infrastructure failure regardless of compatibility classification.",
)
report = "\n".join(lines)
if args.output:
@@ -201,13 +304,28 @@ def main() -> int:
print(
f"[INFO] {len(regressions)} regression(s), "
f"{len(promotions['unimplemented']) + len(promotions['excluded']) + len(unclassified_passed)} promotion candidate(s), "
f"{len(unclassified_failed)} unclassified failure(s)"
f"{len(unclassified_failed)} unclassified failure(s), "
f"{len(missing_results)} missing result(s), "
f"{len(timed_out)} timeout(s)"
)
for name in sorted(regressions):
print(f"[REGRESSION] {name}")
if failures:
print("[ERROR] s3-tests failed testcase summary:")
for classname, name, duration, message in failures[:20]:
nodeid = f"{classname}::{name}" if classname else name
print(f"[ERROR] - {nodeid} ({duration}s): {message}")
if len(failures) > 20:
print(f"[ERROR] - ... {len(failures) - 20} additional failed testcases omitted")
if args.fail_on_regression and regressions:
return 1
if args.fail_on_unclassified and (unclassified_passed or unclassified_failed):
return 1
if args.collected_nodeids and missing_results:
return 1
if timed_out:
return 1
return 0
scripts/s3-tests/run.sh
+96 -62
View File
@@ -58,6 +58,19 @@ if [[ "${TEST_SCOPE}" != "implemented" && "${TEST_SCOPE}" != "all" ]]; then
echo "[ERROR] Invalid TEST_SCOPE: ${TEST_SCOPE} (must be \"implemented\" or \"all\")" >&2
exit 1
fi
S3_SHARD_COUNT="${S3_SHARD_COUNT:-1}"
S3_SHARD_INDEX="${S3_SHARD_INDEX:-0}"
TEST_TIMEOUT="${TEST_TIMEOUT:-300}"
if [[ ! "${S3_SHARD_COUNT}" =~ ^[1-9][0-9]*$ ]] \
|| [[ ! "${S3_SHARD_INDEX}" =~ ^[0-9]+$ ]] \
|| (( S3_SHARD_INDEX >= S3_SHARD_COUNT )); then
echo "[ERROR] Invalid S3 shard ${S3_SHARD_INDEX}/${S3_SHARD_COUNT}" >&2
exit 1
fi
if [[ ! "${TEST_TIMEOUT}" =~ ^[1-9][0-9]*$ ]]; then
echo "[ERROR] Invalid TEST_TIMEOUT: ${TEST_TIMEOUT}" >&2
exit 1
fi
# Upstream ceph/s3-tests suite, pinned for reproducible runs.
# Bump S3TESTS_REV deliberately: upstream changes can rename tests or change
@@ -96,55 +109,6 @@ log_error() {
echo -e "${RED}[ERROR]${NC} $*"
}
summarize_junit_failures() {
local junit_path="$1"
if [ ! -f "${junit_path}" ]; then
log_warn "JUnit report not found: ${junit_path}"
return 0
fi
python3 - "${junit_path}" <<'PY'
import sys
import xml.etree.ElementTree as ET
junit_path = sys.argv[1]
try:
root = ET.parse(junit_path).getroot()
except Exception as exc:
print(f"[WARN] Failed to parse JUnit report {junit_path}: {exc}")
raise SystemExit(0)
failures = []
for case in root.iter("testcase"):
failure = case.find("failure")
error = case.find("error")
node = failure if failure is not None else error
if node is None:
continue
classname = case.attrib.get("classname", "")
name = case.attrib.get("name", "")
duration = case.attrib.get("time", "0")
message = node.attrib.get("message") or (node.text or "").strip().splitlines()[0:1]
if isinstance(message, list):
message = message[0] if message else ""
failures.append((classname, name, duration, message))
if not failures:
print("[INFO] No failed testcases found in JUnit report")
raise SystemExit(0)
print("[ERROR] s3-tests failed testcase summary:")
for classname, name, duration, message in failures[:20]:
nodeid = f"{classname}::{name}" if classname else name
print(f"[ERROR] - {nodeid} ({duration}s): {message}")
if len(failures) > 20:
print(f"[ERROR] - ... {len(failures) - 20} additional failed testcases omitted")
PY
}
# =============================================================================
# Test Classification Files
# =============================================================================
@@ -322,6 +286,9 @@ Environment Variables:
MAXFAIL - Stop after N failures, 0 = never stop (default: 1)
XDIST - Enable parallel execution with N workers (default: 0)
TEST_SCOPE - "implemented" (whitelist, default) or "all" (entire upstream suite)
S3_SHARD_COUNT - Number of deterministic exact-node-ID shards (default: 1)
S3_SHARD_INDEX - Zero-based shard index (default: 0)
TEST_TIMEOUT - Per-test timeout in seconds (default: 300)
S3TESTS_REPO - s3-tests repository URL (default: https://github.com/ceph/s3-tests.git)
S3TESTS_REV - Pinned s3-tests commit; bump deliberately and reclassify test lists
MARKEXPR - pytest marker expression (default: no marker filtering)
@@ -982,9 +949,10 @@ mkdir -p "${ARTIFACTS_DIR}"
XDIST_ARGS=""
if [ "${XDIST}" != "0" ]; then
# Add pytest-xdist to requirements.txt so tox installs it inside its virtualenv
echo "pytest-xdist" >> requirements.txt
grep -qxF "pytest-xdist" requirements.txt || echo "pytest-xdist" >> requirements.txt
XDIST_ARGS="-n ${XDIST} --dist=loadgroup"
fi
grep -qxF "pytest-timeout" requirements.txt || echo "pytest-timeout" >> requirements.txt
# Resolve config path (absolute path for tox)
if [[ "${S3TESTS_CONF}" = /* ]]; then
@@ -1003,12 +971,69 @@ else
PYTEST_SELECTION_ARGS=("${S3_TEST_FILE}")
fi
collect_nodeids() {
local output_path="$1"
shift
local collect_log="${output_path%.txt}.log"
local collect_rc=0
local node_prefix="${S3_TEST_FILE//./\\.}::"
set +e
S3TEST_CONF="${CONF_OUTPUT_PATH}" tox -- -q --collect-only "$@" 2>&1 | tee "${collect_log}"
collect_rc=${PIPESTATUS[0]}
set -e
if [ "${collect_rc}" -ne 0 ]; then
log_error "pytest collection failed with exit code ${collect_rc}"
return "${collect_rc}"
fi
grep -E "^${node_prefix}" "${collect_log}" > "${output_path}" || true
if [ ! -s "${output_path}" ]; then
log_error "pytest collection produced no S3 test node IDs"
return 1
fi
}
ALL_COLLECTED_NODEIDS="${ARTIFACTS_DIR}/all-collected-nodeids.txt"
UNSHARDED_SELECTED_NODEIDS="${ARTIFACTS_DIR}/unsharded-selected-nodeids.txt"
SELECTED_NODEIDS="${ARTIFACTS_DIR}/selected-nodeids.txt"
collect_nodeids "${ALL_COLLECTED_NODEIDS}" "${S3_TEST_FILE}" -m "not rustfs_never_marker"
python3 "${SCRIPT_DIR}/report_compat.py" \
--lists-dir "${SCRIPT_DIR}" \
--collected-nodeids "${ALL_COLLECTED_NODEIDS}" \
--check-classifications-only || {
log_error "S3 test classifications do not match pinned revision ${S3TESTS_REV}"
exit 1
}
if [[ "${TEST_SCOPE}" == "all" && -z "${TESTEXPR}" && "${MARKEXPR}" == "not rustfs_never_marker" ]]; then
cp "${ALL_COLLECTED_NODEIDS}" "${UNSHARDED_SELECTED_NODEIDS}"
else
collect_nodeids "${UNSHARDED_SELECTED_NODEIDS}" "${PYTEST_SELECTION_ARGS[@]}" -m "${MARKEXPR}"
fi
if (( S3_SHARD_COUNT > 1 )); then
awk -v count="${S3_SHARD_COUNT}" -v shard_index="${S3_SHARD_INDEX}" \
'((NR - 1) % count) == shard_index' \
"${UNSHARDED_SELECTED_NODEIDS}" > "${SELECTED_NODEIDS}"
if [[ ! -s "${SELECTED_NODEIDS}" ]]; then
log_error "Shard ${S3_SHARD_INDEX}/${S3_SHARD_COUNT} selected no tests"
exit 1
fi
PYTEST_SELECTION_ARGS=()
while IFS= read -r nodeid; do
PYTEST_SELECTION_ARGS+=("${nodeid}")
done < "${SELECTED_NODEIDS}"
log_info "Selected shard ${S3_SHARD_INDEX}/${S3_SHARD_COUNT}: ${#PYTEST_SELECTION_ARGS[@]} exact cases"
else
cp "${UNSHARDED_SELECTED_NODEIDS}" "${SELECTED_NODEIDS}"
fi
# Run tests from s3tests/functional
set +e
S3TEST_CONF="${CONF_OUTPUT_PATH}" \
tox -- \
-vv -ra --showlocals --tb=long \
--maxfail="${MAXFAIL}" \
--timeout="${TEST_TIMEOUT}" \
--junitxml="${ARTIFACTS_DIR}/junit.xml" \
${XDIST_ARGS} \
"${PYTEST_SELECTION_ARGS[@]}" \
@@ -1033,19 +1058,22 @@ elif [ "${DEPLOY_MODE}" = "existing" ]; then
echo "{\"host\": \"${S3_HOST}\", \"port\": ${S3_PORT}, \"mode\": \"existing\"}" > "${ARTIFACTS_DIR}/rustfs-${TEST_MODE}/inspect.json" || true
fi
# Step 11: Classification report (informational, never fails the run)
# Step 11: Classification report and gate
REPORT_SCRIPT="${SCRIPT_DIR}/report_compat.py"
if [ -f "${REPORT_SCRIPT}" ] && [ -f "${ARTIFACTS_DIR}/junit.xml" ]; then
python3 "${REPORT_SCRIPT}" \
--junit "${ARTIFACTS_DIR}/junit.xml" \
--lists-dir "${SCRIPT_DIR}" \
--output "${ARTIFACTS_DIR}/compat-report.md" \
|| log_warn "Compatibility report generation failed"
fi
if [ ${TEST_EXIT_CODE} -ne 0 ]; then
summarize_junit_failures "${ARTIFACTS_DIR}/junit.xml"
REPORT_ARGS=(
--junit "${ARTIFACTS_DIR}/junit.xml"
--lists-dir "${SCRIPT_DIR}"
--collected-nodeids "${SELECTED_NODEIDS}"
--output "${ARTIFACTS_DIR}/compat-report.md"
--fail-on-regression
)
if [[ "${TEST_SCOPE}" == "all" ]]; then
REPORT_ARGS+=(--fail-on-unclassified)
fi
set +e
python3 "${REPORT_SCRIPT}" "${REPORT_ARGS[@]}"
REPORT_EXIT_CODE=$?
set -e
# Summary
if [ ${TEST_EXIT_CODE} -eq 0 ]; then
@@ -1059,4 +1087,10 @@ else
log_info "Check RustFS logs: ${ARTIFACTS_DIR}/rustfs-${TEST_MODE}/rustfs.log"
fi
exit ${TEST_EXIT_CODE}
if [[ "${TEST_EXIT_CODE}" -ne 0 && "${TEST_EXIT_CODE}" -ne 1 ]]; then
exit "${TEST_EXIT_CODE}"
fi
if [[ "${TEST_SCOPE}" == "implemented" && "${TEST_EXIT_CODE}" -ne 0 ]]; then
exit "${TEST_EXIT_CODE}"
fi
exit "${REPORT_EXIT_CODE}"
scripts/s3-tests/test_report_compat.py
+119
View File
@@ -0,0 +1,119 @@
#!/usr/bin/env python3
"""Regression tests for the S3 compatibility report."""
from __future__ import annotations
import importlib.util
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
REPORT_PATH = Path(__file__).with_name("report_compat.py")
SPEC = importlib.util.spec_from_file_location("report_compat", REPORT_PATH)
assert SPEC and SPEC.loader
REPORT = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(REPORT)
class ReportCompatTests(unittest.TestCase):
def test_upstream_names_expose_incomplete_junit(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
directory = Path(tmp)
collected = directory / "collected.txt"
collected.write_text("s3tests/functional/test_s3.py::test_one[a]\ns3tests/functional/test_s3.py::test_one[b]\n")
junit = directory / "junit.xml"
junit.write_text('<testsuite><testcase name="test_one[a]" /></testsuite>')
expected = REPORT.load_collected_nodeids(collected)
results, _, _ = REPORT.parse_junit(junit)
self.assertEqual(expected - results.keys(), {"test_one[b]"})
def test_cli_fails_an_incomplete_sweep(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
directory = Path(tmp)
collected = directory / "collected.txt"
collected.write_text("s3tests/functional/test_s3.py::test_one\ns3tests/functional/test_s3.py::test_two\n")
junit = directory / "junit.xml"
junit.write_text('<testsuite><testcase name="test_one" /></testsuite>')
for filename in REPORT.LIST_FILES.values():
(directory / filename).write_text("")
(directory / "implemented_tests.txt").write_text("test_one\n")
result = subprocess.run(
[
sys.executable,
str(REPORT_PATH),
"--junit",
str(junit),
"--lists-dir",
str(directory),
"--collected-nodeids",
str(collected),
"--fail-on-regression",
"--fail-on-unclassified",
],
check=False,
capture_output=True,
text=True,
)
self.assertEqual(result.returncode, 1)
self.assertIn("1 missing result(s)", result.stdout)
def test_preflight_rejects_missing_and_stale_classifications(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
directory = Path(tmp)
collected = directory / "collected.txt"
collected.write_text("s3tests/functional/test_s3.py::test_known[a]\ntest_new\n")
for filename in REPORT.LIST_FILES.values():
(directory / filename).write_text("")
(directory / "implemented_tests.txt").write_text("test_known\ntest_stale\ntest_stale\n")
result = subprocess.run(
[
sys.executable,
str(REPORT_PATH),
"--lists-dir",
str(directory),
"--collected-nodeids",
str(collected),
"--check-classifications-only",
],
check=False,
capture_output=True,
text=True,
)
self.assertEqual(result.returncode, 1)
self.assertIn("[UNCLASSIFIED] test_new", result.stdout)
self.assertIn("[STALE] test_stale", result.stdout)
self.assertIn("[INVALID] implemented_tests.txt has duplicates: test_stale", result.stdout)
def test_timeout_fails_even_when_test_is_excluded(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
directory = Path(tmp)
junit = directory / "junit.xml"
junit.write_text(
'<testsuite><testcase name="test_slow"><failure message="Failed: Timeout (&gt;300.0s)" /></testcase></testsuite>'
)
for filename in REPORT.LIST_FILES.values():
(directory / filename).write_text("")
(directory / "excluded_tests.txt").write_text("test_slow\n")
result = subprocess.run(
[sys.executable, str(REPORT_PATH), "--junit", str(junit), "--lists-dir", str(directory)],
check=False,
capture_output=True,
text=True,
)
self.assertEqual(result.returncode, 1)
self.assertIn("1 timeout(s)", result.stdout)
if __name__ == "__main__":
unittest.main()
scripts/s3-tests/unimplemented_tests.txt
+4
View File
@@ -11,8 +11,12 @@
# Failed tests
test_bucket_create_delete_bucket_ownership
test_bucket_logging_request_id
test_create_bucket_no_ownership_controls
test_bucket_logging_owner
test_head_object_404_with_policy_prefix
test_multipart_reupload_checksum_and_etag
test_multipart_upload_complete_without_create
test_object_copy_not_owned_bucket
test_bucket_policy_multipart
test_post_object_upload_checksum