diff --git a/.config/coverage-baselines.toml b/.config/coverage-baselines.toml new file mode 100644 index 000000000..77409d9d2 --- /dev/null +++ b/.config/coverage-baselines.toml @@ -0,0 +1,20 @@ +# Report-only calibration baseline from https://github.com/rustfs/rustfs/actions/runs/29394996173. +# Update counts only with a linked coverage run and a reviewed explanation. +phase = "report-only" +allowed_drop_percentage_points = 1.0 + +[crates."crates/iam"] +covered = 5149 +count = 8131 + +[crates."crates/kms"] +covered = 2950 +count = 4200 + +[crates."crates/policy"] +covered = 4636 +count = 5464 + +[crates."crates/crypto"] +covered = 469 +count = 494 diff --git a/.config/make/tests.mak b/.config/make/tests.mak index 626df9b84..993f27347 100644 --- a/.config/make/tests.mak +++ b/.config/make/tests.mak @@ -36,6 +36,7 @@ script-tests: ## Run shell script tests ./scripts/test_manual_transition_runbooks.sh ./scripts/check_embedded_secrets.sh --self-test python3 ./scripts/check_test_wiring.py --self-test + python3 ./scripts/check_security_coverage.py --self-test python3 ./scripts/check_scheduled_validation_freshness.py --self-test python3 ./scripts/s3-tests/test_report_compat.py bash -n ./scripts/validate_object_data_cache_cold_stampede.sh diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index ece00c2eb..8a843fdd5 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -12,14 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Weekly workspace line-coverage baseline (backlog#1153 infra-5). +# Workspace line-coverage baseline and security-crate calibration +# (backlog#1153 infra-5/infra-6). # -# NON-BLOCKING by design: this workflow only runs on schedule and manual -# dispatch, so it never attaches a status to a PR and must never be made a -# required check. It exists to give coverage a visible baseline and trend -# (per-crate table in the job summary, lcov artifact kept 90 days) — the -# per-crate ratchet for the security-critical crates builds on it later -# (backlog#1153 infra-6, report-only first per the ci-11 ladder). +# NON-BLOCKING by design: the weekly job gives coverage a visible baseline and +# trend, while relevant pull requests run a report-only security-crate +# comparison. Neither job is a required check during calibration. # # Measurement scope matches the PR test gate (ci.yml "Run tests"): # `--workspace --exclude e2e_test` with the `ci` nextest profile. Doctests are @@ -31,6 +29,17 @@ name: coverage on: + pull_request: + branches: [main] + paths: + - "crates/iam/**" + - "crates/kms/**" + - "crates/policy/**" + - "crates/crypto/**" + - ".config/coverage-baselines.toml" + - "scripts/coverage_per_crate.py" + - "scripts/check_security_coverage.py" + - ".github/workflows/coverage.yml" workflow_dispatch: schedule: # 07:00 UTC Sunday — staggered clear of the other Sunday crons: ci (00:00), @@ -39,6 +48,10 @@ on: # e2e-replication-nightly (04:00) and performance-ab (06:00) lanes. - cron: "43 7 * * 0" +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name != 'schedule' }} + # Only alert-on-failure needs more than read access; it declares its own # job-level `issues: write`. permissions: @@ -46,12 +59,14 @@ permissions: jobs: coverage: - name: Workspace coverage (weekly) + name: Workspace line coverage runs-on: sm-standard-4 # The instrumented build cannot reuse the regular CI cache (different - # RUSTFLAGS), so a cold week rebuilds the workspace before running the - # full suite; give it double the test job's 60-minute budget. - timeout-minutes: 120 + # RUSTFLAGS), so a cold run rebuilds the workspace before running the + # full suite. Two later exact-head runs exhausted 150 minutes before the + # report steps, so allow one additional 90-minute cold-run margin while + # keeping the calibration job bounded. + timeout-minutes: 240 env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" # Match the PR gate's nextest semantics (ci.yml runs `--profile ci`): @@ -91,7 +106,9 @@ jobs: cargo llvm-cov report --json --output-path target/llvm-cov/coverage.json - name: Write per-crate summary - run: python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY" + run: | + python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY" + python3 scripts/check_security_coverage.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY" - name: Upload coverage artifact if: always() diff --git a/docs/testing/README.md b/docs/testing/README.md index e38c0d1a2..702c2c4c9 100644 --- a/docs/testing/README.md +++ b/docs/testing/README.md @@ -158,10 +158,11 @@ added by backlog#1153 infra-4. ## Coverage -Line coverage is measured **weekly, not per-PR**, and is non-blocking: it -exists for visibility and trend, never as a required check. Per-crate ratchets -for the security-critical crates (iam / kms / policy / crypto) build on this -baseline later (backlog#1153 infra-6, report-only first). +Workspace line coverage is measured weekly. Pull requests that touch iam, kms, +policy, or crypto also run a non-required, report-only comparison against +`.config/coverage-baselines.toml`. During calibration, a regression is recorded +in the job summary without failing the job; missing or malformed coverage +evidence still fails closed (backlog#1153 infra-6). - **CI**: `.github/workflows/coverage.yml` runs every Sunday and on manual dispatch: `cargo llvm-cov nextest --workspace --exclude e2e_test` under the @@ -174,6 +175,12 @@ baseline later (backlog#1153 infra-6, report-only first). plus the full suite). It prints the same per-crate table via `scripts/coverage_per_crate.py` and writes `target/llvm-cov/lcov.info` and `coverage.json`. +- **Security-critical ratchet**: relevant pull requests compare iam / kms / + policy / crypto line coverage with the versioned baseline. Drops greater than + the configured one-percentage-point calibration threshold are marked + `REGRESSION (report-only)`. The weekly summary runs the same comparison so + calibration continues even when no relevant pull request is open. Baseline + changes require a linked coverage run and a reviewed explanation. - **Trend comparison**: each run's job summary is the weekly per-crate snapshot — open two runs from the Actions history (workflow "coverage") and compare their tables. For line-level diffs, download the two runs' diff --git a/scripts/check_security_coverage.py b/scripts/check_security_coverage.py new file mode 100644 index 000000000..710421589 --- /dev/null +++ b/scripts/check_security_coverage.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python3 +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compare security-critical crate line coverage with the report-only baseline.""" + +import argparse +import json +import math +import os +import sys +import tempfile +import tomllib +from pathlib import Path + +from coverage_per_crate import fmt_pct, load_coverage + + +SECURITY_CRATES = ("crates/iam", "crates/kms", "crates/policy", "crates/crypto") + + +def load_baselines(path: str) -> tuple[float, dict[str, tuple[int, int]]]: + with open(path, "rb") as fh: + config = tomllib.load(fh) + + if config.get("phase") != "report-only": + raise ValueError("coverage baseline phase must be report-only") + + raw_allowed_drop = config["allowed_drop_percentage_points"] + if isinstance(raw_allowed_drop, bool) or not isinstance(raw_allowed_drop, (int, float)): + raise ValueError("allowed_drop_percentage_points must be a number") + allowed_drop = float(raw_allowed_drop) + if not math.isfinite(allowed_drop) or allowed_drop < 0: + raise ValueError("allowed_drop_percentage_points must be finite and non-negative") + + baselines: dict[str, tuple[int, int]] = {} + for crate, values in config["crates"].items(): + covered = values["covered"] + count = values["count"] + if type(covered) is not int or type(count) is not int: + raise ValueError(f"invalid baseline for {crate}: covered and count must be integers") + if covered < 0 or count <= 0 or covered > count: + raise ValueError(f"invalid baseline for {crate}: {covered}/{count}") + baselines[crate] = (covered, count) + missing = [crate for crate in SECURITY_CRATES if crate not in baselines] + unexpected = sorted(set(baselines).difference(SECURITY_CRATES)) + if missing or unexpected: + raise ValueError(f"coverage baseline crate set mismatch: missing={missing}, unexpected={unexpected}") + return allowed_drop, baselines + + +def compare( + current: dict[str, list[int]], + baselines: dict[str, tuple[int, int]], + allowed_drop: float, +) -> list[tuple[str, int, int, int, int, float, bool]]: + rows = [] + for crate, (baseline_covered, baseline_count) in baselines.items(): + if crate not in current: + raise ValueError(f"coverage report is missing {crate}") + covered, count = current[crate] + if type(covered) is not int or type(count) is not int: + raise ValueError(f"invalid coverage for {crate}: covered and count must be integers") + if covered < 0 or count <= 0 or covered > count: + raise ValueError(f"invalid coverage for {crate}: {covered}/{count}") + current_pct = 100.0 * covered / count + baseline_pct = 100.0 * baseline_covered / baseline_count + delta = current_pct - baseline_pct + rows.append((crate, covered, count, baseline_covered, baseline_count, delta, delta < -allowed_drop)) + return rows + + +def print_report(rows: list[tuple[str, int, int, int, int, float, bool]], allowed_drop: float) -> None: + print("## Security-critical coverage ratchet (report-only)") + print() + print(f"Calibration threshold: a drop greater than {allowed_drop:.2f} percentage points is reported as a regression.") + print() + print("| Crate | Current | Baseline | Delta | Status |") + print("|---|---:|---:|---:|---|") + for crate, covered, count, baseline_covered, baseline_count, delta, regressed in rows: + status = "REGRESSION (report-only)" if regressed else "OK" + print( + f"| `{crate}` | {fmt_pct(covered, count)} ({covered}/{count}) " + f"| {fmt_pct(baseline_covered, baseline_count)} ({baseline_covered}/{baseline_count}) " + f"| {delta:+.2f} pp | {status} |" + ) + print() + print("This calibration phase records regressions without failing the job; malformed or incomplete evidence still fails closed.") + + +def self_test() -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + coverage = root / "coverage.json" + baseline = root / "baseline.toml" + coverage_data = { + "data": [ + { + "files": [ + { + "filename": str(root / "crates/iam/src/lib.rs"), + "summary": {"lines": {"covered": 80, "count": 100}}, + }, + { + "filename": str(root / "crates/kms/src/lib.rs"), + "summary": {"lines": {"covered": 90, "count": 100}}, + }, + { + "filename": str(root / "crates/policy/src/lib.rs"), + "summary": {"lines": {"covered": 90, "count": 100}}, + }, + { + "filename": str(root / "crates/crypto/src/lib.rs"), + "summary": {"lines": {"covered": 90, "count": 100}}, + }, + ], + "totals": {"lines": {"covered": 350, "count": 400}}, + } + ] + } + coverage.write_text(json.dumps(coverage_data), encoding="utf-8") + baseline_text = """phase = "report-only" +allowed_drop_percentage_points = 1.0 +[crates."crates/iam"] +covered = 90 +count = 100 +[crates."crates/kms"] +covered = 85 +count = 100 +[crates."crates/policy"] +covered = 90 +count = 100 +[crates."crates/crypto"] +covered = 90 +count = 100 +""" + baseline.write_text(baseline_text, encoding="utf-8") + current, _ = load_coverage(str(coverage), str(root)) + allowed_drop, baselines = load_baselines(str(baseline)) + rows = compare(current, baselines, allowed_drop) + assert [row[-1] for row in rows] == [True, False, False, False] + try: + compare({"crates/iam": current["crates/iam"]}, baselines, allowed_drop) + except ValueError as error: + assert str(error) == "coverage report is missing crates/kms" + else: + raise AssertionError("missing crate must fail closed") + try: + compare({**current, "crates/iam": [101, 100]}, baselines, allowed_drop) + except ValueError as error: + assert str(error) == "invalid coverage for crates/iam: 101/100" + else: + raise AssertionError("invalid coverage must fail closed") + for invalid_threshold in ("true", '"1.0"', "nan", "inf", "-inf"): + baseline.write_text( + baseline_text.replace("allowed_drop_percentage_points = 1.0", f"allowed_drop_percentage_points = {invalid_threshold}"), + encoding="utf-8", + ) + try: + load_baselines(str(baseline)) + except ValueError: + pass + else: + raise AssertionError(f"non-finite threshold {invalid_threshold} must fail closed") + for field, invalid_values in ( + ("covered", ("true", '"90"', "90.0", "90.5")), + ("count", ("true", '"100"', "100.0", "100.5")), + ): + for invalid_value in invalid_values: + baseline.write_text( + baseline_text.replace(f"{field} = {90 if field == 'covered' else 100}", f"{field} = {invalid_value}", 1), + encoding="utf-8", + ) + try: + load_baselines(str(baseline)) + except ValueError: + pass + else: + raise AssertionError(f"non-integer baseline {field} {invalid_value} must fail closed") + for covered, count in ( + (True, 100), + (80, True), + (80.0, 100), + (80, 100.0), + (float("nan"), 100), + (80, float("inf")), + ): + try: + compare({**current, "crates/iam": [covered, count]}, baselines, allowed_drop) + except ValueError: + pass + else: + raise AssertionError(f"invalid aggregate coverage {covered}/{count} must fail closed") + lines = coverage_data["data"][0]["files"][0]["summary"]["lines"] + for field, invalid_values in ( + ("covered", (True, "80", 80.0, 80.5, float("nan"), float("inf"), float("-inf"))), + ("count", (True, "100", 100.0, 100.5, float("nan"), float("inf"), float("-inf"))), + ): + original = lines[field] + for invalid_value in invalid_values: + lines[field] = invalid_value + coverage.write_text(json.dumps(coverage_data), encoding="utf-8") + try: + load_coverage(str(coverage), str(root)) + except ValueError: + pass + else: + raise AssertionError(f"invalid raw coverage {field} {invalid_value} must fail closed") + lines[field] = original + baseline.write_text( + baseline_text.replace( + '[crates."crates/crypto"]\ncovered = 90\ncount = 100\n', + "", + ), + encoding="utf-8", + ) + try: + load_baselines(str(baseline)) + except ValueError: + pass + else: + raise AssertionError("missing security-crate baseline must fail closed") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("coverage_json", nargs="?") + parser.add_argument("--baseline", default=".config/coverage-baselines.toml") + parser.add_argument("--repo-root", default=os.getcwd()) + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args() + + if args.self_test: + self_test() + print("security coverage self-test passed") + return 0 + if not args.coverage_json: + parser.error("coverage_json is required unless --self-test is used") + + try: + current, _ = load_coverage(args.coverage_json, os.path.abspath(args.repo_root)) + allowed_drop, baselines = load_baselines(args.baseline) + rows = compare(current, baselines, allowed_drop) + except (OSError, ValueError, KeyError, IndexError, json.JSONDecodeError, tomllib.TOMLDecodeError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + + print_report(rows, allowed_drop) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/coverage_per_crate.py b/scripts/coverage_per_crate.py index 6c6622022..4a0a37b3e 100755 --- a/scripts/coverage_per_crate.py +++ b/scripts/coverage_per_crate.py @@ -47,6 +47,31 @@ def fmt_pct(covered: int, count: int) -> str: return f"{100.0 * covered / count:.2f}%" if count else "—" +def _line_counts(lines: dict[str, int], source: str) -> tuple[int, int]: + covered = lines["covered"] + count = lines["count"] + if type(covered) is not int or type(count) is not int or covered < 0 or count < 0 or covered > count: + raise ValueError(f"invalid line coverage for {source}: {covered}/{count}") + return covered, count + + +def load_coverage(path: str, root: str) -> tuple[dict[str, list[int]], dict[str, int]]: + with open(path, encoding="utf-8") as fh: + export = json.load(fh) + + data = export["data"][0] + files = data["files"] + total_covered, total_count = _line_counts(data["totals"]["lines"], "totals") + + crates: dict[str, list[int]] = {} + for f in files: + covered, count = _line_counts(f["summary"]["lines"], f["filename"]) + acc = crates.setdefault(crate_label(f["filename"], root), [0, 0]) + acc[0] += covered + acc[1] += count + return crates, {"covered": total_covered, "count": total_count} + + def main() -> int: if len(sys.argv) < 2 or len(sys.argv) > 3: print(__doc__.strip(), file=sys.stderr) @@ -54,24 +79,12 @@ def main() -> int: path = sys.argv[1] root = os.path.abspath(sys.argv[2] if len(sys.argv) == 3 else os.getcwd()) - with open(path, encoding="utf-8") as fh: - export = json.load(fh) - try: - data = export["data"][0] - files = data["files"] - totals = data["totals"]["lines"] - except (KeyError, IndexError) as exc: + crates, totals = load_coverage(path, root) + except (KeyError, IndexError, ValueError) as exc: print(f"error: unexpected llvm-cov JSON shape ({exc})", file=sys.stderr) return 1 - crates: dict[str, list[int]] = {} - for f in files: - lines = f["summary"]["lines"] - acc = crates.setdefault(crate_label(f["filename"], root), [0, 0]) - acc[0] += lines["covered"] - acc[1] += lines["count"] - rows = sorted( crates.items(), key=lambda kv: (100.0 * kv[1][0] / kv[1][1]) if kv[1][1] else 101.0,