ci(coverage): add security ratchet calibration (#6388)

scripts/check_security_coverage.py

@@ -0,0 +1,264 @@
+#!/usr/bin/env python3
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Compare security-critical crate line coverage with the report-only baseline."""
+
+import argparse
+import json
+import math
+import os
+import sys
+import tempfile
+import tomllib
+from pathlib import Path
+
+from coverage_per_crate import fmt_pct, load_coverage
+
+
+SECURITY_CRATES = ("crates/iam", "crates/kms", "crates/policy", "crates/crypto")
+
+
+def load_baselines(path: str) -> tuple[float, dict[str, tuple[int, int]]]:
+    with open(path, "rb") as fh:
+        config = tomllib.load(fh)
+
+    if config.get("phase") != "report-only":
+        raise ValueError("coverage baseline phase must be report-only")
+
+    raw_allowed_drop = config["allowed_drop_percentage_points"]
+    if isinstance(raw_allowed_drop, bool) or not isinstance(raw_allowed_drop, (int, float)):
+        raise ValueError("allowed_drop_percentage_points must be a number")
+    allowed_drop = float(raw_allowed_drop)
+    if not math.isfinite(allowed_drop) or allowed_drop < 0:
+        raise ValueError("allowed_drop_percentage_points must be finite and non-negative")
+
+    baselines: dict[str, tuple[int, int]] = {}
+    for crate, values in config["crates"].items():
+        covered = values["covered"]
+        count = values["count"]
+        if type(covered) is not int or type(count) is not int:
+            raise ValueError(f"invalid baseline for {crate}: covered and count must be integers")
+        if covered < 0 or count <= 0 or covered > count:
+            raise ValueError(f"invalid baseline for {crate}: {covered}/{count}")
+        baselines[crate] = (covered, count)
+    missing = [crate for crate in SECURITY_CRATES if crate not in baselines]
+    unexpected = sorted(set(baselines).difference(SECURITY_CRATES))
+    if missing or unexpected:
+        raise ValueError(f"coverage baseline crate set mismatch: missing={missing}, unexpected={unexpected}")
+    return allowed_drop, baselines
+
+
+def compare(
+    current: dict[str, list[int]],
+    baselines: dict[str, tuple[int, int]],
+    allowed_drop: float,
+) -> list[tuple[str, int, int, int, int, float, bool]]:
+    rows = []
+    for crate, (baseline_covered, baseline_count) in baselines.items():
+        if crate not in current:
+            raise ValueError(f"coverage report is missing {crate}")
+        covered, count = current[crate]
+        if type(covered) is not int or type(count) is not int:
+            raise ValueError(f"invalid coverage for {crate}: covered and count must be integers")
+        if covered < 0 or count <= 0 or covered > count:
+            raise ValueError(f"invalid coverage for {crate}: {covered}/{count}")
+        current_pct = 100.0 * covered / count
+        baseline_pct = 100.0 * baseline_covered / baseline_count
+        delta = current_pct - baseline_pct
+        rows.append((crate, covered, count, baseline_covered, baseline_count, delta, delta < -allowed_drop))
+    return rows
+
+
+def print_report(rows: list[tuple[str, int, int, int, int, float, bool]], allowed_drop: float) -> None:
+    print("## Security-critical coverage ratchet (report-only)")
+    print()
+    print(f"Calibration threshold: a drop greater than {allowed_drop:.2f} percentage points is reported as a regression.")
+    print()
+    print("| Crate | Current | Baseline | Delta | Status |")
+    print("|---|---:|---:|---:|---|")
+    for crate, covered, count, baseline_covered, baseline_count, delta, regressed in rows:
+        status = "REGRESSION (report-only)" if regressed else "OK"
+        print(
+            f"| `{crate}` | {fmt_pct(covered, count)} ({covered}/{count}) "
+            f"| {fmt_pct(baseline_covered, baseline_count)} ({baseline_covered}/{baseline_count}) "
+            f"| {delta:+.2f} pp | {status} |"
+        )
+    print()
+    print("This calibration phase records regressions without failing the job; malformed or incomplete evidence still fails closed.")
+
+
+def self_test() -> None:
+    with tempfile.TemporaryDirectory() as tmp:
+        root = Path(tmp)
+        coverage = root / "coverage.json"
+        baseline = root / "baseline.toml"
+        coverage_data = {
+            "data": [
+                {
+                    "files": [
+                        {
+                            "filename": str(root / "crates/iam/src/lib.rs"),
+                            "summary": {"lines": {"covered": 80, "count": 100}},
+                        },
+                        {
+                            "filename": str(root / "crates/kms/src/lib.rs"),
+                            "summary": {"lines": {"covered": 90, "count": 100}},
+                        },
+                        {
+                            "filename": str(root / "crates/policy/src/lib.rs"),
+                            "summary": {"lines": {"covered": 90, "count": 100}},
+                        },
+                        {
+                            "filename": str(root / "crates/crypto/src/lib.rs"),
+                            "summary": {"lines": {"covered": 90, "count": 100}},
+                        },
+                    ],
+                    "totals": {"lines": {"covered": 350, "count": 400}},
+                }
+            ]
+        }
+        coverage.write_text(json.dumps(coverage_data), encoding="utf-8")
+        baseline_text = """phase = "report-only"
+allowed_drop_percentage_points = 1.0
+[crates."crates/iam"]
+covered = 90
+count = 100
+[crates."crates/kms"]
+covered = 85
+count = 100
+[crates."crates/policy"]
+covered = 90
+count = 100
+[crates."crates/crypto"]
+covered = 90
+count = 100
+"""
+        baseline.write_text(baseline_text, encoding="utf-8")
+        current, _ = load_coverage(str(coverage), str(root))
+        allowed_drop, baselines = load_baselines(str(baseline))
+        rows = compare(current, baselines, allowed_drop)
+        assert [row[-1] for row in rows] == [True, False, False, False]
+        try:
+            compare({"crates/iam": current["crates/iam"]}, baselines, allowed_drop)
+        except ValueError as error:
+            assert str(error) == "coverage report is missing crates/kms"
+        else:
+            raise AssertionError("missing crate must fail closed")
+        try:
+            compare({**current, "crates/iam": [101, 100]}, baselines, allowed_drop)
+        except ValueError as error:
+            assert str(error) == "invalid coverage for crates/iam: 101/100"
+        else:
+            raise AssertionError("invalid coverage must fail closed")
+        for invalid_threshold in ("true", '"1.0"', "nan", "inf", "-inf"):
+            baseline.write_text(
+                baseline_text.replace("allowed_drop_percentage_points = 1.0", f"allowed_drop_percentage_points = {invalid_threshold}"),
+                encoding="utf-8",
+            )
+            try:
+                load_baselines(str(baseline))
+            except ValueError:
+                pass
+            else:
+                raise AssertionError(f"non-finite threshold {invalid_threshold} must fail closed")
+        for field, invalid_values in (
+            ("covered", ("true", '"90"', "90.0", "90.5")),
+            ("count", ("true", '"100"', "100.0", "100.5")),
+        ):
+            for invalid_value in invalid_values:
+                baseline.write_text(
+                    baseline_text.replace(f"{field} = {90 if field == 'covered' else 100}", f"{field} = {invalid_value}", 1),
+                    encoding="utf-8",
+                )
+                try:
+                    load_baselines(str(baseline))
+                except ValueError:
+                    pass
+                else:
+                    raise AssertionError(f"non-integer baseline {field} {invalid_value} must fail closed")
+        for covered, count in (
+            (True, 100),
+            (80, True),
+            (80.0, 100),
+            (80, 100.0),
+            (float("nan"), 100),
+            (80, float("inf")),
+        ):
+            try:
+                compare({**current, "crates/iam": [covered, count]}, baselines, allowed_drop)
+            except ValueError:
+                pass
+            else:
+                raise AssertionError(f"invalid aggregate coverage {covered}/{count} must fail closed")
+        lines = coverage_data["data"][0]["files"][0]["summary"]["lines"]
+        for field, invalid_values in (
+            ("covered", (True, "80", 80.0, 80.5, float("nan"), float("inf"), float("-inf"))),
+            ("count", (True, "100", 100.0, 100.5, float("nan"), float("inf"), float("-inf"))),
+        ):
+            original = lines[field]
+            for invalid_value in invalid_values:
+                lines[field] = invalid_value
+                coverage.write_text(json.dumps(coverage_data), encoding="utf-8")
+                try:
+                    load_coverage(str(coverage), str(root))
+                except ValueError:
+                    pass
+                else:
+                    raise AssertionError(f"invalid raw coverage {field} {invalid_value} must fail closed")
+            lines[field] = original
+        baseline.write_text(
+            baseline_text.replace(
+                '[crates."crates/crypto"]\ncovered = 90\ncount = 100\n',
+                "",
+            ),
+            encoding="utf-8",
+        )
+        try:
+            load_baselines(str(baseline))
+        except ValueError:
+            pass
+        else:
+            raise AssertionError("missing security-crate baseline must fail closed")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("coverage_json", nargs="?")
+    parser.add_argument("--baseline", default=".config/coverage-baselines.toml")
+    parser.add_argument("--repo-root", default=os.getcwd())
+    parser.add_argument("--self-test", action="store_true")
+    args = parser.parse_args()
+
+    if args.self_test:
+        self_test()
+        print("security coverage self-test passed")
+        return 0
+    if not args.coverage_json:
+        parser.error("coverage_json is required unless --self-test is used")
+
+    try:
+        current, _ = load_coverage(args.coverage_json, os.path.abspath(args.repo_root))
+        allowed_drop, baselines = load_baselines(args.baseline)
+        rows = compare(current, baselines, allowed_drop)
+    except (OSError, ValueError, KeyError, IndexError, json.JSONDecodeError, tomllib.TOMLDecodeError) as error:
+        print(f"error: {error}", file=sys.stderr)
+        return 1
+
+    print_report(rows, allowed_drop)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/coverage_per_crate.py

@@ -47,6 +47,31 @@ def fmt_pct(covered: int, count: int) -> str:
     return f"{100.0 * covered / count:.2f}%" if count else "—"
 
 
+def _line_counts(lines: dict[str, int], source: str) -> tuple[int, int]:
+    covered = lines["covered"]
+    count = lines["count"]
+    if type(covered) is not int or type(count) is not int or covered < 0 or count < 0 or covered > count:
+        raise ValueError(f"invalid line coverage for {source}: {covered}/{count}")
+    return covered, count
+
+
+def load_coverage(path: str, root: str) -> tuple[dict[str, list[int]], dict[str, int]]:
+    with open(path, encoding="utf-8") as fh:
+        export = json.load(fh)
+
+    data = export["data"][0]
+    files = data["files"]
+    total_covered, total_count = _line_counts(data["totals"]["lines"], "totals")
+
+    crates: dict[str, list[int]] = {}
+    for f in files:
+        covered, count = _line_counts(f["summary"]["lines"], f["filename"])
+        acc = crates.setdefault(crate_label(f["filename"], root), [0, 0])
+        acc[0] += covered
+        acc[1] += count
+    return crates, {"covered": total_covered, "count": total_count}
+
+
 def main() -> int:
     if len(sys.argv) < 2 or len(sys.argv) > 3:
         print(__doc__.strip(), file=sys.stderr)
@@ -54,24 +79,12 @@ def main() -> int:
     path = sys.argv[1]
     root = os.path.abspath(sys.argv[2] if len(sys.argv) == 3 else os.getcwd())
 
-    with open(path, encoding="utf-8") as fh:
-        export = json.load(fh)
-
     try:
-        data = export["data"][0]
-        files = data["files"]
-        totals = data["totals"]["lines"]
-    except (KeyError, IndexError) as exc:
+        crates, totals = load_coverage(path, root)
+    except (KeyError, IndexError, ValueError) as exc:
         print(f"error: unexpected llvm-cov JSON shape ({exc})", file=sys.stderr)
         return 1
 
-    crates: dict[str, list[int]] = {}
-    for f in files:
-        lines = f["summary"]["lines"]
-        acc = crates.setdefault(crate_label(f["filename"], root), [0, 0])
-        acc[0] += lines["covered"]
-        acc[1] += lines["count"]
-
     rows = sorted(
         crates.items(),
         key=lambda kv: (100.0 * kv[1][0] / kv[1][1]) if kv[1][1] else 101.0,