From 5fd2e8b6a1054dd86c6a56da5d250131cc171f3b Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Wed, 15 Jul 2026 11:10:22 +0800 Subject: [PATCH] ci(coverage): add weekly cargo-llvm-cov baseline workflow (#4820) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ci(coverage): weekly cargo-llvm-cov workspace baseline (non-blocking) Add the weekly line-coverage report (backlog#1153 infra-5): - .github/workflows/coverage.yml — Sunday 07:00 UTC + workflow_dispatch; runs `cargo llvm-cov nextest --workspace --exclude e2e_test` under NEXTEST_PROFILE=ci (same scope and profile as the ci.yml test gate), writes a per-crate line-coverage table to the job summary, uploads lcov + JSON as a 90-day artifact, and routes scheduled failures through the shared schedule-failure-issue action (ci-8). Runs only on schedule/dispatch, so it can never become a required PR check. - scripts/coverage_per_crate.py — stdlib-only aggregation of the llvm-cov JSON export into the per-crate markdown table (worst-first, TOTAL row), shared by the workflow and the local target. - make coverage (.config/make/coverage.mak) — local equivalent with the same command sequence; fails with install hints when cargo-llvm-cov or cargo-nextest are missing. Listed in make help. - docs/testing/README.md — Coverage section: cadence, where the table and artifacts live, the trend-comparison method, and what is not measured (doctests, e2e_test). --- .config/make/coverage.mak | 26 +++++++ .github/workflows/coverage.yml | 122 +++++++++++++++++++++++++++++++++ docs/testing/README.md | 27 ++++++++ scripts/coverage_per_crate.py | 94 +++++++++++++++++++++++++ 4 files changed, 269 insertions(+) create mode 100644 .config/make/coverage.mak create mode 100644 .github/workflows/coverage.yml create mode 100755 scripts/coverage_per_crate.py diff --git a/.config/make/coverage.mak b/.config/make/coverage.mak new file mode 100644 index 000000000..366535d85 --- /dev/null +++ b/.config/make/coverage.mak @@ -0,0 +1,26 @@ +## —— Coverage -------------------------------------------------------------------------------------- + +# Local equivalent of the weekly coverage workflow (.github/workflows/coverage.yml, +# backlog#1153 infra-5): same measurement scope (--workspace --exclude e2e_test, +# nextest `ci` profile) and the same per-crate table. Slow — the instrumented +# build cannot reuse your normal target cache and then runs the whole suite. +# Doctests are not measured (needs nightly). Outputs land in target/llvm-cov/. +.PHONY: coverage +coverage: core-deps ## Workspace line coverage (cargo-llvm-cov + nextest; slow, writes target/llvm-cov/) + @if ! command -v cargo-llvm-cov >/dev/null 2>&1; then \ + echo >&2 "❌ cargo-llvm-cov is required for 'make coverage' but was not found."; \ + echo >&2 " Install it with:"; \ + echo >&2 " cargo install cargo-llvm-cov --locked"; \ + echo >&2 " rustup component add llvm-tools-preview"; \ + exit 1; \ + fi + @if ! command -v cargo-nextest >/dev/null 2>&1; then \ + echo >&2 "❌ cargo-nextest is required for 'make coverage' (see 'make test')."; \ + echo >&2 " Install it with: cargo install cargo-nextest --locked"; \ + exit 1; \ + fi + NEXTEST_PROFILE=ci cargo llvm-cov nextest --workspace --exclude e2e_test --no-report + @mkdir -p target/llvm-cov + cargo llvm-cov report --lcov --output-path target/llvm-cov/lcov.info + cargo llvm-cov report --json --output-path target/llvm-cov/coverage.json + python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 000000000..b7e1fb25d --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,122 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Weekly workspace line-coverage baseline (backlog#1153 infra-5). +# +# NON-BLOCKING by design: this workflow only runs on schedule and manual +# dispatch, so it never attaches a status to a PR and must never be made a +# required check. It exists to give coverage a visible baseline and trend +# (per-crate table in the job summary, lcov artifact kept 90 days) — the +# per-crate ratchet for the security-critical crates builds on it later +# (backlog#1153 infra-6, report-only first per the ci-11 ladder). +# +# Measurement scope matches the PR test gate (ci.yml "Run tests"): +# `--workspace --exclude e2e_test` with the `ci` nextest profile. Doctests are +# NOT measured (ci.yml runs them uninstrumented; `cargo llvm-cov` needs a +# nightly toolchain to cover doctests). Trend-comparison workflow: +# docs/testing/README.md "Coverage" section. `make coverage` is the local +# equivalent. Scheduled failures alert via the ci-8 composite action. + +name: coverage + +on: + workflow_dispatch: + schedule: + # 07:00 UTC Sunday — staggered clear of the other Sunday crons: ci (00:00), + # build (01:00), e2e-s3tests (02:00), audit (03:00), nix-flake-update + # (05:00), mint (06:00), and the daily fuzz (02:00), minio-interop (03:17), + # e2e-replication-nightly (04:00) and performance-ab (06:00) lanes. + - cron: "0 7 * * 0" + +# Only alert-on-failure needs more than read access; it declares its own +# job-level `issues: write`. +permissions: + contents: read + +jobs: + coverage: + name: Workspace coverage (weekly) + runs-on: sm-standard-4 + # The instrumented build cannot reuse the regular CI cache (different + # RUSTFLAGS), so a cold week rebuilds the workspace before running the + # full suite; give it double the test job's 60-minute budget. + timeout-minutes: 120 + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + # Match the PR gate's nextest semantics (ci.yml runs `--profile ci`): + # retries=0 plus the quarantine list and the ecstore-serial-flaky + # serialization. Set via env because `cargo llvm-cov`'s own --profile + # flag selects the *cargo build* profile, not the nextest profile. + NEXTEST_PROFILE: ci + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + + - name: Setup Rust environment + uses: ./.github/actions/setup + with: + rust-version: stable + cache-shared-key: ci-coverage + github-token: ${{ secrets.GITHUB_TOKEN }} + cache-save-if: ${{ github.ref == 'refs/heads/main' }} + + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2 + with: + tool: cargo-llvm-cov + + - name: Install llvm-tools component + run: rustup component add llvm-tools-preview + + - name: Run instrumented test suite + run: cargo llvm-cov nextest --workspace --exclude e2e_test --no-report + + - name: Generate lcov and JSON reports + run: | + mkdir -p target/llvm-cov + cargo llvm-cov report --lcov --output-path target/llvm-cov/lcov.info + cargo llvm-cov report --json --output-path target/llvm-cov/coverage.json + + - name: Write per-crate summary + run: python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY" + + - name: Upload coverage artifact + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: coverage-lcov-${{ github.run_number }} + path: | + target/llvm-cov/lcov.info + target/llvm-cov/coverage.json + retention-days: 90 + if-no-files-found: ignore + + alert-on-failure: + name: Alert on scheduled failure + needs: [coverage] + # Only scheduled runs open/append the tracking issue (backlog#1149 ci-8); + # manual workflow_dispatch runs stay quiet so a debugging run never files a + # spurious alert. + if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure') + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + - name: Open or update failure-tracking issue + uses: ./.github/actions/schedule-failure-issue + with: + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/docs/testing/README.md b/docs/testing/README.md index 22882b81d..e17559d28 100644 --- a/docs/testing/README.md +++ b/docs/testing/README.md @@ -152,6 +152,33 @@ and `advance`) or explicit event synchronization over fixed `sleep` race windows. The written convention and the `docs/testing/time-control.md` guide are added by backlog#1153 infra-4. +## Coverage + +Line coverage is measured **weekly, not per-PR**, and is non-blocking: it +exists for visibility and trend, never as a required check. Per-crate ratchets +for the security-critical crates (iam / kms / policy / crypto) build on this +baseline later (backlog#1153 infra-6, report-only first). + +- **CI**: `.github/workflows/coverage.yml` runs every Sunday and on manual + dispatch: `cargo llvm-cov nextest --workspace --exclude e2e_test` under the + `ci` nextest profile — the same scope and profile as the PR test gate. The + per-crate line-coverage table lands in the run's job summary; the lcov + + JSON exports are uploaded as a `coverage-lcov-` artifact kept for + 90 days. Scheduled failures open/append the `[scheduled-failure] coverage` + issue via the shared alert action (ci-8). +- **Local**: `make coverage` is the equivalent (slow — instrumented rebuild + plus the full suite). It prints the same per-crate table via + `scripts/coverage_per_crate.py` and writes `target/llvm-cov/lcov.info` and + `coverage.json`. +- **Trend comparison**: each run's job summary is the weekly per-crate + snapshot — open two runs from the Actions history (workflow "coverage") and + compare their tables. For line-level diffs, download the two runs' + `coverage-lcov-*` artifacts and compare the `lcov.info` files with your lcov + tooling of choice. +- **Not measured**: doctests (ci.yml runs them uninstrumented; covering them + would require a nightly toolchain) and the `e2e_test` crate (excluded from + the unit gate; its lanes are described in the taxonomy above). + ## Flake policy A flaky test is one that fails non-deterministically without a corresponding diff --git a/scripts/coverage_per_crate.py b/scripts/coverage_per_crate.py new file mode 100755 index 000000000..6c6622022 --- /dev/null +++ b/scripts/coverage_per_crate.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Aggregate a cargo-llvm-cov JSON export into a per-crate line-coverage table. + +Usage: + coverage_per_crate.py [repo-root] + +Reads the JSON export written by ``cargo llvm-cov report --json`` and prints a +GitHub-flavoured markdown table with one row per workspace crate directory +(``crates/`` or ``rustfs``), sorted worst-first, followed by the +workspace TOTAL row. Consumed by ``.github/workflows/coverage.yml`` (job +summary) and ``make coverage`` (backlog#1153 infra-5). Stdlib only. +""" + +import json +import os +import sys + + +def crate_label(filename: str, root: str) -> str: + """Map an absolute source path to its workspace crate directory.""" + rel = os.path.relpath(filename, root) + parts = rel.split(os.sep) + if parts[0] == "..": + return "(external)" + if parts[0] == "crates" and len(parts) > 1: + return f"crates/{parts[1]}" + # Top-level crate directories (the `rustfs` binary crate) and anything + # else that sneaks into the export (e.g. generated code at the root). + return parts[0] + + +def fmt_pct(covered: int, count: int) -> str: + return f"{100.0 * covered / count:.2f}%" if count else "—" + + +def main() -> int: + if len(sys.argv) < 2 or len(sys.argv) > 3: + print(__doc__.strip(), file=sys.stderr) + return 2 + path = sys.argv[1] + root = os.path.abspath(sys.argv[2] if len(sys.argv) == 3 else os.getcwd()) + + with open(path, encoding="utf-8") as fh: + export = json.load(fh) + + try: + data = export["data"][0] + files = data["files"] + totals = data["totals"]["lines"] + except (KeyError, IndexError) as exc: + print(f"error: unexpected llvm-cov JSON shape ({exc})", file=sys.stderr) + return 1 + + crates: dict[str, list[int]] = {} + for f in files: + lines = f["summary"]["lines"] + acc = crates.setdefault(crate_label(f["filename"], root), [0, 0]) + acc[0] += lines["covered"] + acc[1] += lines["count"] + + rows = sorted( + crates.items(), + key=lambda kv: (100.0 * kv[1][0] / kv[1][1]) if kv[1][1] else 101.0, + ) + + print("## Per-crate line coverage") + print() + print("| Crate | Line coverage | Lines covered / total |") + print("|---|---:|---:|") + for name, (covered, count) in rows: + print(f"| `{name}` | {fmt_pct(covered, count)} | {covered} / {count} |") + print( + f"| **TOTAL** | **{fmt_pct(totals['covered'], totals['count'])}** " + f"| {totals['covered']} / {totals['count']} |" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main())