ci: harden test selection and nightly coverage (#6341)

2026-08-22 12:26:37 +00:00 · 2026-08-21 23:04:08 +08:00
parent bce5922aef
commit bc07cfd115
31 changed files with 1146 additions and 259 deletions
@@ -253,6 +253,9 @@ Test results are saved in the `artifacts/s3tests-${TEST_MODE}/` directory (defau

 - `junit.xml`: Test results in JUnit format (compatible with CI/CD systems)
 - `pytest.log`: Detailed pytest logs with full test output
+- `all-collected-nodeids.txt`: Exact node IDs in the pinned upstream suite
+- `selected-nodeids.txt`: Exact node IDs expected in this run
+- `unsharded-selected-nodeids.txt`: Exact node IDs before deterministic sharding
 - `compat-report.md`: Classification report generated by `report_compat.py` —
  regressions against `implemented_tests.txt`, promotion candidates (tests
  that pass but are still listed as unimplemented/excluded), and tests missing
@@ -449,9 +452,11 @@ RustFS. Two GitHub Actions workflows delegate to it:
 - **Full sweep** (`.github/workflows/e2e-s3tests.yml`): weekly scheduled (and
  manually dispatchable) run of the ENTIRE upstream suite (`TEST_SCOPE=all`)
  against a Docker deployment — single node or a real 4-node distributed
-  cluster behind HAProxy. The sweep fails only on regressions in the
-  implemented whitelist; everything else is reported by `report_compat.py`
-  as promotion candidates or unclassified tests.
+  cluster behind HAProxy. Regressions, unclassified tests, incomplete
+  execution, and infrastructure errors fail the sweep; classified unsupported
+  behavior remains informational. Scheduled topology runs are split into four
+  deterministic exact-node-ID shards, and every case has a five-minute timeout,
+  so one stalled case cannot erase the entire sweep's evidence.

 Keeping both workflows on this script means local runs, the PR gate, and the
 scheduled sweep always execute tests the same way (same pinned s3-tests
@@ -466,8 +471,9 @@ pass/fail table in the job summary.
 ## Companion Tools

 - `report_compat.py` — diffs a junit.xml result against the classification
-  lists; run automatically at the end of `run.sh`, and used by the weekly
-  sweep to gate on whitelist regressions only (`--fail-on-regression`).
+  lists and the exact pytest collection; run before execution to reject stale
+  or missing classifications, then after execution to detect regressions and
+  incomplete parameterized cases.
 - `api_coverage.py` — quantifies S3 API surface coverage by comparing the
  s3s `S3` trait (at the revision pinned in Cargo.toml) against the methods
  RustFS overrides in `impl S3 for FS`:
@@ -307,3 +307,10 @@ test_object_acl_write
 test_object_acl_writeacp
 test_put_bucket_acl_grant_group_read
 test_object_raw_get_bucket_acl
+
+# Require upstream cloud-storage or IAM account services
+test_bucket_logging_requester_assumed_role
+test_lifecycle_cloud_transition_target_by_bucket
+test_lifecycle_cloud_transition_target_by_bucket_multiple_buckets
+test_list_object_versions_restore_status
+test_list_objects_restore_status
@@ -521,9 +521,13 @@ test_atomic_dual_conditional_write_1mb
 test_atomic_write_bucket_gone
 test_bucket_acl_canned_private_to_private
 test_bucket_concurrent_set_canned_acl
+test_bucket_create_delete
+test_bucket_policy
 test_bucket_policy_acl
 test_bucket_policy_put_obj_acl
 test_bucketv2_policy_acl
+test_copy_enc
+test_copy_part_enc
 test_copy_object_ifmatch_failed
 test_copy_object_ifnonematch_good
 test_cors_presigned_put_object_tenant_with_acl
@@ -21,14 +21,17 @@ Classifies every executed test into:
  - unclassified passes:    passed but not present in any list (new upstream tests)
  - unclassified failures:  failed and not present in any list (new upstream tests)

-Writes a markdown report and prints a summary to stdout. Exit code is 0 unless
--fail-on-regression is given and at least one regression was found.
+Writes a markdown report and prints a summary to stdout. Optional gates reject
+regressions, unclassified tests, stale classifications, and incomplete node-ID
+execution.
 """

 from __future__ import annotations

 import argparse
+from collections import Counter
 import pathlib
+import re
 import sys
 import xml.etree.ElementTree as ET

@@ -45,33 +48,47 @@ LIST_FILES = {
 }


-def load_list(path: pathlib.Path) -> set[str]:
-    names: set[str] = set()
+def load_entries(path: pathlib.Path) -> list[str]:
+    names: list[str] = []
    if not path.is_file():
        return names
    for line in path.read_text(encoding="utf-8").splitlines():
        line = line.strip()
        if line and not line.startswith("#"):
-            names.add(line)
+            names.append(line)
    return names


+def classification_errors(entries: dict[str, list[str]]) -> list[str]:
+    errors: list[str] = []
+    lists = {key: set(names) for key, names in entries.items()}
+    for key, names in entries.items():
+        duplicates = sorted(name for name, count in Counter(names).items() if count > 1)
+        if duplicates:
+            errors.append(f"{LIST_FILES[key]} has duplicates: {', '.join(duplicates)}")
+    keys = tuple(lists)
+    for index, left in enumerate(keys):
+        for right in keys[index + 1 :]:
+            overlap = sorted(lists[left] & lists[right])
+            if overlap:
+                errors.append(f"{left}/{right} classifications overlap: {', '.join(overlap)}")
+    return errors
+
+
 def base_name(testcase_name: str) -> str:
    """Strip pytest parametrization (test_foo[param]) to match list entries."""
    return testcase_name.split("[", 1)[0]


-def parse_junit(path: pathlib.Path) -> dict[str, str]:
-    """Return {test name: status} with status in passed/failed/error/skipped.
-
-    Parametrized cases collapse onto their base name; any failing variant marks
-    the whole test failed.
-    """
+def parse_junit(path: pathlib.Path) -> tuple[dict[str, str], list[str], list[tuple[str, str, str, str]]]:
+    """Return exact statuses, pytest-timeout cases, and failure summaries."""
    results: dict[str, str] = {}
+    timed_out: list[str] = []
+    failures: list[tuple[str, str, str, str]] = []
    severity = {"skipped": 0, "passed": 1, "failed": 2, "error": 2}
    root = ET.parse(path).getroot()
    for case in root.iter("testcase"):
-        name = base_name(case.get("name", ""))
+        name = case.get("name", "")
        if not name:
            continue
        if case.find("failure") is not None:
@@ -85,7 +102,35 @@ def parse_junit(path: pathlib.Path) -> dict[str, str]:
        prev = results.get(name)
        if prev is None or severity[status] > severity[prev]:
            results[name] = status
-    return results
+        node = case.find("failure") if status == "failed" else case.find("error")
+        if node is not None:
+            details = " ".join(filter(None, [node.get("message", ""), node.text or ""]))
+            message = node.get("message") or next(iter((node.text or "").strip().splitlines()), "")
+            failures.append((case.get("classname", ""), name, case.get("time", "0"), message))
+            if re.search(r"\bTimeout\s*(?:>|\()", details, re.IGNORECASE):
+                timed_out.append(name)
+    return results, timed_out, failures
+
+
+def collapse_results(results: dict[str, str]) -> dict[str, str]:
+    """Collapse parametrized cases for classification-level reporting."""
+    collapsed: dict[str, str] = {}
+    severity = {"skipped": 0, "passed": 1, "failed": 2, "error": 2}
+    for exact_name, status in results.items():
+        name = base_name(exact_name)
+        previous = collapsed.get(name)
+        if previous is None or severity[status] > severity[previous]:
+            collapsed[name] = status
+    return collapsed
+
+
+def load_collected_nodeids(path: pathlib.Path) -> set[str]:
+    names: set[str] = set()
+    for line in path.read_text(encoding="utf-8").splitlines():
+        nodeid = line.strip()
+        if nodeid:
+            names.add(nodeid.rsplit("::", 1)[-1])
+    return names


 def render_section(title: str, rows: list[str], hint: str = "") -> list[str]:
@@ -102,7 +147,7 @@ def render_section(title: str, rows: list[str], hint: str = "") -> list[str]:

 def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--junit", required=True, type=pathlib.Path, help="junit.xml produced by pytest")
+    parser.add_argument("--junit", type=pathlib.Path, help="junit.xml produced by pytest")
    parser.add_argument(
        "--lists-dir",
        type=pathlib.Path,
@@ -115,14 +160,60 @@ def main() -> int:
        action="store_true",
        help="exit non-zero when a test from implemented_tests.txt failed",
    )
+    parser.add_argument(
+        "--fail-on-unclassified",
+        action="store_true",
+        help="exit non-zero when an executed test is absent from every classification",
+    )
+    parser.add_argument(
+        "--collected-nodeids",
+        type=pathlib.Path,
+        help="exact pytest node IDs from the pinned suite's collect-only pass",
+    )
+    parser.add_argument(
+        "--check-classifications-only",
+        action="store_true",
+        help="validate classification names against collected node IDs without reading JUnit",
+    )
    args = parser.parse_args()

-    if not args.junit.is_file():
+    entries = {key: load_entries(args.lists_dir / fname) for key, fname in LIST_FILES.items()}
+    lists = {key: set(names) for key, names in entries.items()}
+    invalid_classifications = classification_errors(entries)
+    collected: set[str] = set()
+    if args.collected_nodeids:
+        collected = load_collected_nodeids(args.collected_nodeids)
+        collected_base = {base_name(name) for name in collected}
+        classified = set().union(*lists.values())
+        missing_classifications = sorted(collected_base - classified)
+        stale_classifications = sorted(classified - collected_base)
+    else:
+        missing_classifications = []
+        stale_classifications = []
+
+    if args.check_classifications_only:
+        if not args.collected_nodeids:
+            parser.error("--check-classifications-only requires --collected-nodeids")
+        for error in invalid_classifications:
+            print(f"[INVALID] {error}")
+        for name in missing_classifications:
+            print(f"[UNCLASSIFIED] {name}")
+        for name in stale_classifications:
+            print(f"[STALE] {name}")
+        return 1 if invalid_classifications or missing_classifications or stale_classifications else 0
+
+    if invalid_classifications:
+        for error in invalid_classifications:
+            print(f"[ERROR] {error}", file=sys.stderr)
+        return 2
+
+    if not args.junit or not args.junit.is_file():
        print(f"[ERROR] junit file not found: {args.junit}", file=sys.stderr)
        return 2

-    lists = {key: load_list(args.lists_dir / fname) for key, fname in LIST_FILES.items()}
-    results = parse_junit(args.junit)
+    exact_results, timed_out, failures = parse_junit(args.junit)
+    results = collapse_results(exact_results)
+    missing_results = sorted(collected - exact_results.keys()) if collected else []

    regressions: list[str] = []
    promotions: dict[str, list[str]] = {"unimplemented": [], "excluded": []}
@@ -155,7 +246,9 @@ def main() -> int:
    lines = [
        "# S3 compatibility report",
        "",
-        f"Executed: {len(results)} tests — "
+        f"Executed: {len(exact_results)} exact cases across {len(results)} classified tests.",
+        "",
+        "Classification status — "
        f"{counts['passed']} passed, {counts['failed']} failed, "
        f"{counts['error']} errored, {counts['skipped']} skipped.",
        "",
@@ -191,6 +284,16 @@ def main() -> int:
        unclassified_failed,
        "Failing and absent from every list — triage into `unimplemented_tests.txt` or `excluded_tests.txt`.",
    )
+    lines += render_section(
+        "Missing results",
+        missing_results,
+        "Present in the pinned upstream suite but absent from JUnit — the sweep was incomplete.",
+    )
+    lines += render_section(
+        "Timed out",
+        timed_out,
+        "Per-test timeout is an infrastructure failure regardless of compatibility classification.",
+    )

    report = "\n".join(lines)
    if args.output:
@@ -201,13 +304,28 @@ def main() -> int:
    print(
        f"[INFO] {len(regressions)} regression(s), "
        f"{len(promotions['unimplemented']) + len(promotions['excluded']) + len(unclassified_passed)} promotion candidate(s), "
-        f"{len(unclassified_failed)} unclassified failure(s)"
+        f"{len(unclassified_failed)} unclassified failure(s), "
+        f"{len(missing_results)} missing result(s), "
+        f"{len(timed_out)} timeout(s)"
    )
    for name in sorted(regressions):
        print(f"[REGRESSION] {name}")
+    if failures:
+        print("[ERROR] s3-tests failed testcase summary:")
+        for classname, name, duration, message in failures[:20]:
+            nodeid = f"{classname}::{name}" if classname else name
+            print(f"[ERROR] - {nodeid} ({duration}s): {message}")
+        if len(failures) > 20:
+            print(f"[ERROR] - ... {len(failures) - 20} additional failed testcases omitted")

    if args.fail_on_regression and regressions:
        return 1
+    if args.fail_on_unclassified and (unclassified_passed or unclassified_failed):
+        return 1
+    if args.collected_nodeids and missing_results:
+        return 1
+    if timed_out:
+        return 1
    return 0


@@ -58,6 +58,19 @@ if [[ "${TEST_SCOPE}" != "implemented" && "${TEST_SCOPE}" != "all" ]]; then
    echo "[ERROR] Invalid TEST_SCOPE: ${TEST_SCOPE} (must be \"implemented\" or \"all\")" >&2
    exit 1
 fi
+S3_SHARD_COUNT="${S3_SHARD_COUNT:-1}"
+S3_SHARD_INDEX="${S3_SHARD_INDEX:-0}"
+TEST_TIMEOUT="${TEST_TIMEOUT:-300}"
+if [[ ! "${S3_SHARD_COUNT}" =~ ^[1-9][0-9]*$ ]] \
+    || [[ ! "${S3_SHARD_INDEX}" =~ ^[0-9]+$ ]] \
+    || (( S3_SHARD_INDEX >= S3_SHARD_COUNT )); then
+    echo "[ERROR] Invalid S3 shard ${S3_SHARD_INDEX}/${S3_SHARD_COUNT}" >&2
+    exit 1
+fi
+if [[ ! "${TEST_TIMEOUT}" =~ ^[1-9][0-9]*$ ]]; then
+    echo "[ERROR] Invalid TEST_TIMEOUT: ${TEST_TIMEOUT}" >&2
+    exit 1
+fi

 # Upstream ceph/s3-tests suite, pinned for reproducible runs.
 # Bump S3TESTS_REV deliberately: upstream changes can rename tests or change
@@ -96,55 +109,6 @@ log_error() {
    echo -e "${RED}[ERROR]${NC} $*"
 }

-summarize_junit_failures() {
-    local junit_path="$1"
-
-    if [ ! -f "${junit_path}" ]; then
-        log_warn "JUnit report not found: ${junit_path}"
-        return 0
-    fi
-
-    python3 - "${junit_path}" <<'PY'
-import sys
-import xml.etree.ElementTree as ET
-
-junit_path = sys.argv[1]
-try:
-    root = ET.parse(junit_path).getroot()
-except Exception as exc:
-    print(f"[WARN] Failed to parse JUnit report {junit_path}: {exc}")
-    raise SystemExit(0)
-
-failures = []
-for case in root.iter("testcase"):
-    failure = case.find("failure")
-    error = case.find("error")
-    node = failure if failure is not None else error
-    if node is None:
-        continue
-
-    classname = case.attrib.get("classname", "")
-    name = case.attrib.get("name", "")
-    duration = case.attrib.get("time", "0")
-    message = node.attrib.get("message") or (node.text or "").strip().splitlines()[0:1]
-    if isinstance(message, list):
-        message = message[0] if message else ""
-    failures.append((classname, name, duration, message))
-
-if not failures:
-    print("[INFO] No failed testcases found in JUnit report")
-    raise SystemExit(0)
-
-print("[ERROR] s3-tests failed testcase summary:")
-for classname, name, duration, message in failures[:20]:
-    nodeid = f"{classname}::{name}" if classname else name
-    print(f"[ERROR] - {nodeid} ({duration}s): {message}")
-
-if len(failures) > 20:
-    print(f"[ERROR] - ... {len(failures) - 20} additional failed testcases omitted")
-PY
-}
-
 # =============================================================================
 # Test Classification Files
 # =============================================================================
@@ -322,6 +286,9 @@ Environment Variables:
  MAXFAIL                - Stop after N failures, 0 = never stop (default: 1)
  XDIST                  - Enable parallel execution with N workers (default: 0)
  TEST_SCOPE             - "implemented" (whitelist, default) or "all" (entire upstream suite)
+  S3_SHARD_COUNT         - Number of deterministic exact-node-ID shards (default: 1)
+  S3_SHARD_INDEX         - Zero-based shard index (default: 0)
+  TEST_TIMEOUT           - Per-test timeout in seconds (default: 300)
  S3TESTS_REPO           - s3-tests repository URL (default: https://github.com/ceph/s3-tests.git)
  S3TESTS_REV            - Pinned s3-tests commit; bump deliberately and reclassify test lists
  MARKEXPR               - pytest marker expression (default: no marker filtering)
@@ -982,9 +949,10 @@ mkdir -p "${ARTIFACTS_DIR}"
 XDIST_ARGS=""
 if [ "${XDIST}" != "0" ]; then
    # Add pytest-xdist to requirements.txt so tox installs it inside its virtualenv
-    echo "pytest-xdist" >> requirements.txt
+    grep -qxF "pytest-xdist" requirements.txt || echo "pytest-xdist" >> requirements.txt
    XDIST_ARGS="-n ${XDIST} --dist=loadgroup"
 fi
+grep -qxF "pytest-timeout" requirements.txt || echo "pytest-timeout" >> requirements.txt

 # Resolve config path (absolute path for tox)
 if [[ "${S3TESTS_CONF}" = /* ]]; then
@@ -1003,12 +971,69 @@ else
    PYTEST_SELECTION_ARGS=("${S3_TEST_FILE}")
 fi

+collect_nodeids() {
+    local output_path="$1"
+    shift
+    local collect_log="${output_path%.txt}.log"
+    local collect_rc=0
+    local node_prefix="${S3_TEST_FILE//./\\.}::"
+
+    set +e
+    S3TEST_CONF="${CONF_OUTPUT_PATH}" tox -- -q --collect-only "$@" 2>&1 | tee "${collect_log}"
+    collect_rc=${PIPESTATUS[0]}
+    set -e
+    if [ "${collect_rc}" -ne 0 ]; then
+        log_error "pytest collection failed with exit code ${collect_rc}"
+        return "${collect_rc}"
+    fi
+    grep -E "^${node_prefix}" "${collect_log}" > "${output_path}" || true
+    if [ ! -s "${output_path}" ]; then
+        log_error "pytest collection produced no S3 test node IDs"
+        return 1
+    fi
+}
+
+ALL_COLLECTED_NODEIDS="${ARTIFACTS_DIR}/all-collected-nodeids.txt"
+UNSHARDED_SELECTED_NODEIDS="${ARTIFACTS_DIR}/unsharded-selected-nodeids.txt"
+SELECTED_NODEIDS="${ARTIFACTS_DIR}/selected-nodeids.txt"
+collect_nodeids "${ALL_COLLECTED_NODEIDS}" "${S3_TEST_FILE}" -m "not rustfs_never_marker"
+python3 "${SCRIPT_DIR}/report_compat.py" \
+    --lists-dir "${SCRIPT_DIR}" \
+    --collected-nodeids "${ALL_COLLECTED_NODEIDS}" \
+    --check-classifications-only || {
+    log_error "S3 test classifications do not match pinned revision ${S3TESTS_REV}"
+    exit 1
+}
+if [[ "${TEST_SCOPE}" == "all" && -z "${TESTEXPR}" && "${MARKEXPR}" == "not rustfs_never_marker" ]]; then
+    cp "${ALL_COLLECTED_NODEIDS}" "${UNSHARDED_SELECTED_NODEIDS}"
+else
+    collect_nodeids "${UNSHARDED_SELECTED_NODEIDS}" "${PYTEST_SELECTION_ARGS[@]}" -m "${MARKEXPR}"
+fi
+
+if (( S3_SHARD_COUNT > 1 )); then
+    awk -v count="${S3_SHARD_COUNT}" -v shard_index="${S3_SHARD_INDEX}" \
+        '((NR - 1) % count) == shard_index' \
+        "${UNSHARDED_SELECTED_NODEIDS}" > "${SELECTED_NODEIDS}"
+    if [[ ! -s "${SELECTED_NODEIDS}" ]]; then
+        log_error "Shard ${S3_SHARD_INDEX}/${S3_SHARD_COUNT} selected no tests"
+        exit 1
+    fi
+    PYTEST_SELECTION_ARGS=()
+    while IFS= read -r nodeid; do
+        PYTEST_SELECTION_ARGS+=("${nodeid}")
+    done < "${SELECTED_NODEIDS}"
+    log_info "Selected shard ${S3_SHARD_INDEX}/${S3_SHARD_COUNT}: ${#PYTEST_SELECTION_ARGS[@]} exact cases"
+else
+    cp "${UNSHARDED_SELECTED_NODEIDS}" "${SELECTED_NODEIDS}"
+fi
+
 # Run tests from s3tests/functional
 set +e
 S3TEST_CONF="${CONF_OUTPUT_PATH}" \
    tox -- \
    -vv -ra --showlocals --tb=long \
    --maxfail="${MAXFAIL}" \
+    --timeout="${TEST_TIMEOUT}" \
    --junitxml="${ARTIFACTS_DIR}/junit.xml" \
    ${XDIST_ARGS} \
    "${PYTEST_SELECTION_ARGS[@]}" \
@@ -1033,19 +1058,22 @@ elif [ "${DEPLOY_MODE}" = "existing" ]; then
    echo "{\"host\": \"${S3_HOST}\", \"port\": ${S3_PORT}, \"mode\": \"existing\"}" > "${ARTIFACTS_DIR}/rustfs-${TEST_MODE}/inspect.json" || true
 fi

-# Step 11: Classification report (informational, never fails the run)
+# Step 11: Classification report and gate
 REPORT_SCRIPT="${SCRIPT_DIR}/report_compat.py"
-if [ -f "${REPORT_SCRIPT}" ] && [ -f "${ARTIFACTS_DIR}/junit.xml" ]; then
-    python3 "${REPORT_SCRIPT}" \
-        --junit "${ARTIFACTS_DIR}/junit.xml" \
-        --lists-dir "${SCRIPT_DIR}" \
-        --output "${ARTIFACTS_DIR}/compat-report.md" \
-        || log_warn "Compatibility report generation failed"
-fi
-
-if [ ${TEST_EXIT_CODE} -ne 0 ]; then
-    summarize_junit_failures "${ARTIFACTS_DIR}/junit.xml"
+REPORT_ARGS=(
+    --junit "${ARTIFACTS_DIR}/junit.xml"
+    --lists-dir "${SCRIPT_DIR}"
+    --collected-nodeids "${SELECTED_NODEIDS}"
+    --output "${ARTIFACTS_DIR}/compat-report.md"
+    --fail-on-regression
+)
+if [[ "${TEST_SCOPE}" == "all" ]]; then
+    REPORT_ARGS+=(--fail-on-unclassified)
 fi
+set +e
+python3 "${REPORT_SCRIPT}" "${REPORT_ARGS[@]}"
+REPORT_EXIT_CODE=$?
+set -e

 # Summary
 if [ ${TEST_EXIT_CODE} -eq 0 ]; then
@@ -1059,4 +1087,10 @@ else
    log_info "Check RustFS logs: ${ARTIFACTS_DIR}/rustfs-${TEST_MODE}/rustfs.log"
 fi

-exit ${TEST_EXIT_CODE}
+if [[ "${TEST_EXIT_CODE}" -ne 0 && "${TEST_EXIT_CODE}" -ne 1 ]]; then
+    exit "${TEST_EXIT_CODE}"
+fi
+if [[ "${TEST_SCOPE}" == "implemented" && "${TEST_EXIT_CODE}" -ne 0 ]]; then
+    exit "${TEST_EXIT_CODE}"
+fi
+exit "${REPORT_EXIT_CODE}"
@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+"""Regression tests for the S3 compatibility report."""
+
+from __future__ import annotations
+
+import importlib.util
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+
+REPORT_PATH = Path(__file__).with_name("report_compat.py")
+SPEC = importlib.util.spec_from_file_location("report_compat", REPORT_PATH)
+assert SPEC and SPEC.loader
+REPORT = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(REPORT)
+
+
+class ReportCompatTests(unittest.TestCase):
+    def test_upstream_names_expose_incomplete_junit(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            directory = Path(tmp)
+            collected = directory / "collected.txt"
+            collected.write_text("s3tests/functional/test_s3.py::test_one[a]\ns3tests/functional/test_s3.py::test_one[b]\n")
+            junit = directory / "junit.xml"
+            junit.write_text('<testsuite><testcase name="test_one[a]" /></testsuite>')
+
+            expected = REPORT.load_collected_nodeids(collected)
+            results, _, _ = REPORT.parse_junit(junit)
+
+            self.assertEqual(expected - results.keys(), {"test_one[b]"})
+
+    def test_cli_fails_an_incomplete_sweep(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            directory = Path(tmp)
+            collected = directory / "collected.txt"
+            collected.write_text("s3tests/functional/test_s3.py::test_one\ns3tests/functional/test_s3.py::test_two\n")
+            junit = directory / "junit.xml"
+            junit.write_text('<testsuite><testcase name="test_one" /></testsuite>')
+            for filename in REPORT.LIST_FILES.values():
+                (directory / filename).write_text("")
+            (directory / "implemented_tests.txt").write_text("test_one\n")
+
+            result = subprocess.run(
+                [
+                    sys.executable,
+                    str(REPORT_PATH),
+                    "--junit",
+                    str(junit),
+                    "--lists-dir",
+                    str(directory),
+                    "--collected-nodeids",
+                    str(collected),
+                    "--fail-on-regression",
+                    "--fail-on-unclassified",
+                ],
+                check=False,
+                capture_output=True,
+                text=True,
+            )
+
+            self.assertEqual(result.returncode, 1)
+            self.assertIn("1 missing result(s)", result.stdout)
+
+    def test_preflight_rejects_missing_and_stale_classifications(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            directory = Path(tmp)
+            collected = directory / "collected.txt"
+            collected.write_text("s3tests/functional/test_s3.py::test_known[a]\ntest_new\n")
+            for filename in REPORT.LIST_FILES.values():
+                (directory / filename).write_text("")
+            (directory / "implemented_tests.txt").write_text("test_known\ntest_stale\ntest_stale\n")
+
+            result = subprocess.run(
+                [
+                    sys.executable,
+                    str(REPORT_PATH),
+                    "--lists-dir",
+                    str(directory),
+                    "--collected-nodeids",
+                    str(collected),
+                    "--check-classifications-only",
+                ],
+                check=False,
+                capture_output=True,
+                text=True,
+            )
+
+            self.assertEqual(result.returncode, 1)
+            self.assertIn("[UNCLASSIFIED] test_new", result.stdout)
+            self.assertIn("[STALE] test_stale", result.stdout)
+            self.assertIn("[INVALID] implemented_tests.txt has duplicates: test_stale", result.stdout)
+
+    def test_timeout_fails_even_when_test_is_excluded(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            directory = Path(tmp)
+            junit = directory / "junit.xml"
+            junit.write_text(
+                '<testsuite><testcase name="test_slow"><failure message="Failed: Timeout (&gt;300.0s)" /></testcase></testsuite>'
+            )
+            for filename in REPORT.LIST_FILES.values():
+                (directory / filename).write_text("")
+            (directory / "excluded_tests.txt").write_text("test_slow\n")
+
+            result = subprocess.run(
+                [sys.executable, str(REPORT_PATH), "--junit", str(junit), "--lists-dir", str(directory)],
+                check=False,
+                capture_output=True,
+                text=True,
+            )
+
+            self.assertEqual(result.returncode, 1)
+            self.assertIn("1 timeout(s)", result.stdout)
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -11,8 +11,12 @@

 # Failed tests
 test_bucket_create_delete_bucket_ownership
+test_bucket_logging_request_id
 test_create_bucket_no_ownership_controls
 test_bucket_logging_owner
+test_head_object_404_with_policy_prefix
+test_multipart_reupload_checksum_and_etag
+test_multipart_upload_complete_without_create
 test_object_copy_not_owned_bucket
 test_bucket_policy_multipart
 test_post_object_upload_checksum