ci(coverage): add weekly cargo-llvm-cov baseline workflow (#4820)

ci(coverage): weekly cargo-llvm-cov workspace baseline (non-blocking)

Add the weekly line-coverage report (backlog#1153 infra-5):

- .github/workflows/coverage.yml — Sunday 07:00 UTC + workflow_dispatch;
  runs `cargo llvm-cov nextest --workspace --exclude e2e_test` under
  NEXTEST_PROFILE=ci (same scope and profile as the ci.yml test gate),
  writes a per-crate line-coverage table to the job summary, uploads
  lcov + JSON as a 90-day artifact, and routes scheduled failures
  through the shared schedule-failure-issue action (ci-8). Runs only on
  schedule/dispatch, so it can never become a required PR check.
- scripts/coverage_per_crate.py — stdlib-only aggregation of the
  llvm-cov JSON export into the per-crate markdown table (worst-first,
  TOTAL row), shared by the workflow and the local target.
- make coverage (.config/make/coverage.mak) — local equivalent with the
  same command sequence; fails with install hints when cargo-llvm-cov
  or cargo-nextest are missing. Listed in make help.
- docs/testing/README.md — Coverage section: cadence, where the table
  and artifacts live, the trend-comparison method, and what is not
  measured (doctests, e2e_test).

.config/make/coverage.mak

@@ -0,0 +1,26 @@
+## —— Coverage --------------------------------------------------------------------------------------
+
+# Local equivalent of the weekly coverage workflow (.github/workflows/coverage.yml,
+# backlog#1153 infra-5): same measurement scope (--workspace --exclude e2e_test,
+# nextest `ci` profile) and the same per-crate table. Slow — the instrumented
+# build cannot reuse your normal target cache and then runs the whole suite.
+# Doctests are not measured (needs nightly). Outputs land in target/llvm-cov/.
+.PHONY: coverage
+coverage: core-deps ## Workspace line coverage (cargo-llvm-cov + nextest; slow, writes target/llvm-cov/)
+	@if ! command -v cargo-llvm-cov >/dev/null 2>&1; then \
+		echo >&2 "❌ cargo-llvm-cov is required for 'make coverage' but was not found."; \
+		echo >&2 "   Install it with:"; \
+		echo >&2 "     cargo install cargo-llvm-cov --locked"; \
+		echo >&2 "     rustup component add llvm-tools-preview"; \
+		exit 1; \
+	fi
+	@if ! command -v cargo-nextest >/dev/null 2>&1; then \
+		echo >&2 "❌ cargo-nextest is required for 'make coverage' (see 'make test')."; \
+		echo >&2 "   Install it with: cargo install cargo-nextest --locked"; \
+		exit 1; \
+	fi
+	NEXTEST_PROFILE=ci cargo llvm-cov nextest --workspace --exclude e2e_test --no-report
+	@mkdir -p target/llvm-cov
+	cargo llvm-cov report --lcov --output-path target/llvm-cov/lcov.info
+	cargo llvm-cov report --json --output-path target/llvm-cov/coverage.json
+	python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json

.github/workflows/coverage.yml

@@ -0,0 +1,122 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Weekly workspace line-coverage baseline (backlog#1153 infra-5).
+#
+# NON-BLOCKING by design: this workflow only runs on schedule and manual
+# dispatch, so it never attaches a status to a PR and must never be made a
+# required check. It exists to give coverage a visible baseline and trend
+# (per-crate table in the job summary, lcov artifact kept 90 days) — the
+# per-crate ratchet for the security-critical crates builds on it later
+# (backlog#1153 infra-6, report-only first per the ci-11 ladder).
+#
+# Measurement scope matches the PR test gate (ci.yml "Run tests"):
+# `--workspace --exclude e2e_test` with the `ci` nextest profile. Doctests are
+# NOT measured (ci.yml runs them uninstrumented; `cargo llvm-cov` needs a
+# nightly toolchain to cover doctests). Trend-comparison workflow:
+# docs/testing/README.md "Coverage" section. `make coverage` is the local
+# equivalent. Scheduled failures alert via the ci-8 composite action.
+
+name: coverage
+
+on:
+  workflow_dispatch:
+  schedule:
+    # 07:00 UTC Sunday — staggered clear of the other Sunday crons: ci (00:00),
+    # build (01:00), e2e-s3tests (02:00), audit (03:00), nix-flake-update
+    # (05:00), mint (06:00), and the daily fuzz (02:00), minio-interop (03:17),
+    # e2e-replication-nightly (04:00) and performance-ab (06:00) lanes.
+    - cron: "0 7 * * 0"
+
+# Only alert-on-failure needs more than read access; it declares its own
+# job-level `issues: write`.
+permissions:
+  contents: read
+
+jobs:
+  coverage:
+    name: Workspace coverage (weekly)
+    runs-on: sm-standard-4
+    # The instrumented build cannot reuse the regular CI cache (different
+    # RUSTFLAGS), so a cold week rebuilds the workspace before running the
+    # full suite; give it double the test job's 60-minute budget.
+    timeout-minutes: 120
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+      # Match the PR gate's nextest semantics (ci.yml runs `--profile ci`):
+      # retries=0 plus the quarantine list and the ecstore-serial-flaky
+      # serialization. Set via env because `cargo llvm-cov`'s own --profile
+      # flag selects the *cargo build* profile, not the nextest profile.
+      NEXTEST_PROFILE: ci
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-coverage
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+
+      - name: Install cargo-llvm-cov
+        uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
+        with:
+          tool: cargo-llvm-cov
+
+      - name: Install llvm-tools component
+        run: rustup component add llvm-tools-preview
+
+      - name: Run instrumented test suite
+        run: cargo llvm-cov nextest --workspace --exclude e2e_test --no-report
+
+      - name: Generate lcov and JSON reports
+        run: |
+          mkdir -p target/llvm-cov
+          cargo llvm-cov report --lcov --output-path target/llvm-cov/lcov.info
+          cargo llvm-cov report --json --output-path target/llvm-cov/coverage.json
+
+      - name: Write per-crate summary
+        run: python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload coverage artifact
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: coverage-lcov-${{ github.run_number }}
+          path: |
+            target/llvm-cov/lcov.info
+            target/llvm-cov/coverage.json
+          retention-days: 90
+          if-no-files-found: ignore
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs: [coverage]
+    # Only scheduled runs open/append the tracking issue (backlog#1149 ci-8);
+    # manual workflow_dispatch runs stay quiet so a debugging run never files a
+    # spurious alert.
+    if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}

docs/testing/README.md

@@ -152,6 +152,33 @@ and `advance`) or explicit event synchronization over fixed `sleep` race
 windows. The written convention and the `docs/testing/time-control.md` guide are
 added by backlog#1153 infra-4.
 
+## Coverage
+
+Line coverage is measured **weekly, not per-PR**, and is non-blocking: it
+exists for visibility and trend, never as a required check. Per-crate ratchets
+for the security-critical crates (iam / kms / policy / crypto) build on this
+baseline later (backlog#1153 infra-6, report-only first).
+
+- **CI**: `.github/workflows/coverage.yml` runs every Sunday and on manual
+  dispatch: `cargo llvm-cov nextest --workspace --exclude e2e_test` under the
+  `ci` nextest profile — the same scope and profile as the PR test gate. The
+  per-crate line-coverage table lands in the run's job summary; the lcov +
+  JSON exports are uploaded as a `coverage-lcov-<run>` artifact kept for
+  90 days. Scheduled failures open/append the `[scheduled-failure] coverage`
+  issue via the shared alert action (ci-8).
+- **Local**: `make coverage` is the equivalent (slow — instrumented rebuild
+  plus the full suite). It prints the same per-crate table via
+  `scripts/coverage_per_crate.py` and writes `target/llvm-cov/lcov.info` and
+  `coverage.json`.
+- **Trend comparison**: each run's job summary is the weekly per-crate
+  snapshot — open two runs from the Actions history (workflow "coverage") and
+  compare their tables. For line-level diffs, download the two runs'
+  `coverage-lcov-*` artifacts and compare the `lcov.info` files with your lcov
+  tooling of choice.
+- **Not measured**: doctests (ci.yml runs them uninstrumented; covering them
+  would require a nightly toolchain) and the `e2e_test` crate (excluded from
+  the unit gate; its lanes are described in the taxonomy above).
+
 ## Flake policy
 
 A flaky test is one that fails non-deterministically without a corresponding

scripts/coverage_per_crate.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Aggregate a cargo-llvm-cov JSON export into a per-crate line-coverage table.
+
+Usage:
+    coverage_per_crate.py <coverage.json> [repo-root]
+
+Reads the JSON export written by ``cargo llvm-cov report --json`` and prints a
+GitHub-flavoured markdown table with one row per workspace crate directory
+(``crates/<name>`` or ``rustfs``), sorted worst-first, followed by the
+workspace TOTAL row. Consumed by ``.github/workflows/coverage.yml`` (job
+summary) and ``make coverage`` (backlog#1153 infra-5). Stdlib only.
+"""
+
+import json
+import os
+import sys
+
+
+def crate_label(filename: str, root: str) -> str:
+    """Map an absolute source path to its workspace crate directory."""
+    rel = os.path.relpath(filename, root)
+    parts = rel.split(os.sep)
+    if parts[0] == "..":
+        return "(external)"
+    if parts[0] == "crates" and len(parts) > 1:
+        return f"crates/{parts[1]}"
+    # Top-level crate directories (the `rustfs` binary crate) and anything
+    # else that sneaks into the export (e.g. generated code at the root).
+    return parts[0]
+
+
+def fmt_pct(covered: int, count: int) -> str:
+    return f"{100.0 * covered / count:.2f}%" if count else "—"
+
+
+def main() -> int:
+    if len(sys.argv) < 2 or len(sys.argv) > 3:
+        print(__doc__.strip(), file=sys.stderr)
+        return 2
+    path = sys.argv[1]
+    root = os.path.abspath(sys.argv[2] if len(sys.argv) == 3 else os.getcwd())
+
+    with open(path, encoding="utf-8") as fh:
+        export = json.load(fh)
+
+    try:
+        data = export["data"][0]
+        files = data["files"]
+        totals = data["totals"]["lines"]
+    except (KeyError, IndexError) as exc:
+        print(f"error: unexpected llvm-cov JSON shape ({exc})", file=sys.stderr)
+        return 1
+
+    crates: dict[str, list[int]] = {}
+    for f in files:
+        lines = f["summary"]["lines"]
+        acc = crates.setdefault(crate_label(f["filename"], root), [0, 0])
+        acc[0] += lines["covered"]
+        acc[1] += lines["count"]
+
+    rows = sorted(
+        crates.items(),
+        key=lambda kv: (100.0 * kv[1][0] / kv[1][1]) if kv[1][1] else 101.0,
+    )
+
+    print("## Per-crate line coverage")
+    print()
+    print("| Crate | Line coverage | Lines covered / total |")
+    print("|---|---:|---:|")
+    for name, (covered, count) in rows:
+        print(f"| `{name}` | {fmt_pct(covered, count)} | {covered} / {count} |")
+    print(
+        f"| **TOTAL** | **{fmt_pct(totals['covered'], totals['count'])}** "
+        f"| {totals['covered']} / {totals['count']} |"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())