From c9c16efbb6ecb368fec568c1aa37dec1d2f6b7d2 Mon Sep 17 00:00:00 2001 From: Chris Date: Thu, 8 Oct 2026 06:25:42 -0700 Subject: [PATCH] ci: add the per-crate test-count ratchet (#8404) Quick Checks now fails any PR that lowers the number of test attributes in a workspace crate or adds #[ignore], measured per crate against the exact rows of .config/test-count-baseline.txt (rustfs/backlog#2734 section 1 rule 3). The counter widens the spec's literal #[(tokio::)?test] to the tokio attributes that carry arguments and both spellings of #[test_case(...)], and anchors at line start so commented-out attributes do not count. The initial baseline covers 52 crates, 21077 tests and 164 ignored. Refs rustfs/backlog#2738 Refs rustfs/backlog#2734 --- .config/make/lint-fmt.mak | 5 + .config/make/pre-commit.mak | 4 +- .config/test-count-baseline.txt | 63 +++++++ .github/actions/quick-checks/action.yml | 4 + scripts/README.md | 1 + scripts/check_test_count_ratchet.sh | 212 ++++++++++++++++++++++++ 6 files changed, 287 insertions(+), 2 deletions(-) create mode 100644 .config/test-count-baseline.txt create mode 100755 scripts/check_test_count_ratchet.sh diff --git a/.config/make/lint-fmt.mak b/.config/make/lint-fmt.mak index b9bf210d4..69cf7545c 100644 --- a/.config/make/lint-fmt.mak +++ b/.config/make/lint-fmt.mak @@ -70,6 +70,11 @@ s3s-footprint-check: ## Check the s3s dependency footprint ratchet stays frozen @echo "๐Ÿ“ฆ Checking s3s footprint ratchet..." ./scripts/check_s3s_footprint.sh +.PHONY: test-count-ratchet-check +test-count-ratchet-check: ## Check no workspace crate lost tests or gained #[ignore] against the committed baseline + @echo "๐Ÿงช Checking per-crate test-count ratchet..." + ./scripts/check_test_count_ratchet.sh + .PHONY: fips-wording-check fips-wording-check: ## Check docs and crates/kms do not over-claim crypto capabilities @echo "๐Ÿ“ฃ Checking cryptographic capability wording guard..." diff --git a/.config/make/pre-commit.mak b/.config/make/pre-commit.mak index c23adc230..b19d74cac 100644 --- a/.config/make/pre-commit.mak +++ b/.config/make/pre-commit.mak @@ -20,11 +20,11 @@ planning-docs-check: ## Check that no planning-type documents are committed ./scripts/check_no_planning_docs.sh .PHONY: pre-commit -pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests +pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests @echo "โœ… All pre-commit checks passed!" .PHONY: pre-pr -pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests +pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests @echo "โœ… All pre-PR checks passed!" .PHONY: dev-check diff --git a/.config/test-count-baseline.txt b/.config/test-count-baseline.txt new file mode 100644 index 000000000..c08e57a5b --- /dev/null +++ b/.config/test-count-baseline.txt @@ -0,0 +1,63 @@ +# Per-crate test-count baseline consumed by scripts/check_test_count_ratchet.sh +# (rustfs/backlog#2734 section 1 rule 3; T0.3 in rustfs/backlog#2738). +# +# One row per workspace member: , tests, ignored, tab-separated. +# tests counts #[test], #[tokio::test], #[tokio::test(...)] and #[test_case(...)] +# attribute lines under the package directory; ignored counts #[ignore] and +# #[ignore = "..."]. Both columns are exact: CI fails when a crate's tests fall +# below or rise above its row, or when its ignored count changes. Raise a row in +# the PR that adds tests; lowering one is a reviewed statement that tests were +# intentionally removed. Regenerate with: +# scripts/check_test_count_ratchet.sh --update +e2e_test 870 18 +rustfs 5475 62 +rustfs-audit 70 0 +rustfs-checksums 26 0 +rustfs-common 23 0 +rustfs-concurrency 27 0 +rustfs-config 43 0 +rustfs-connect-symbol-catalog 6 0 +rustfs-credentials 24 0 +rustfs-crypto 67 0 +rustfs-data-usage 54 0 +rustfs-ecstore 6203 1 +rustfs-extension-schema 23 0 +rustfs-filemeta 306 0 +rustfs-heal 836 3 +rustfs-heal-contracts 6 0 +rustfs-iam 357 0 +rustfs-io-core 49 0 +rustfs-io-metrics 123 0 +rustfs-keystone 29 0 +rustfs-kms 715 25 +rustfs-license 2 0 +rustfs-lifecycle 152 0 +rustfs-lock 135 0 +rustfs-log-analyzer 87 0 +rustfs-madmin 209 0 +rustfs-notify 136 0 +rustfs-object-capacity 92 0 +rustfs-object-data-cache 140 0 +rustfs-obs 362 0 +rustfs-policy 663 0 +rustfs-protocols 733 1 +rustfs-protos 52 0 +rustfs-replication 200 0 +rustfs-rio 198 0 +rustfs-rio-v2 34 6 +rustfs-s3-client 65 0 +rustfs-s3-ops 8 0 +rustfs-s3-types 21 0 +rustfs-s3select-api 206 0 +rustfs-s3select-query 188 2 +rustfs-scanner 968 20 +rustfs-scanner-metrics 75 0 +rustfs-security-governance 20 0 +rustfs-signer 30 0 +rustfs-storage-api 31 0 +rustfs-targets 603 26 +rustfs-test-utils 1 0 +rustfs-tls-runtime 20 0 +rustfs-trusted-proxies 68 0 +rustfs-utils 216 0 +rustfs-zip 30 0 diff --git a/.github/actions/quick-checks/action.yml b/.github/actions/quick-checks/action.yml index 176778f49..5815ea63e 100644 --- a/.github/actions/quick-checks/action.yml +++ b/.github/actions/quick-checks/action.yml @@ -86,6 +86,10 @@ runs: shell: bash run: ./scripts/check_s3s_footprint.sh + - name: Check per-crate test-count ratchet + shell: bash + run: ./scripts/check_test_count_ratchet.sh + - name: Check cryptographic capability wording shell: bash run: ./scripts/check_fips_wording.sh diff --git a/scripts/README.md b/scripts/README.md index 1ea493f2c..ffa67c455 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -36,6 +36,7 @@ their issue closes. | `check_no_planning_docs.sh` | ci-gate | Blocks committed planning-type documents | ci.yml Quick Checks; `make pre-commit` | | `check_no_tokio_io_uring.sh` | ci-gate | Keeps tokio's io-uring backend disabled | ci.yml Quick Checks | | `check_s3s_footprint.sh` | ci-gate | Lower-only ratchet freezing the direct s3s surface ahead of the s3gate migration | ci.yml Quick Checks; `make pre-commit` | +| `check_test_count_ratchet.sh` | ci-gate | Exact per-crate test-count and `#[ignore]` ratchet against `.config/test-count-baseline.txt`; no crate may lose tests or gain ignores | ci.yml Quick Checks; `make pre-commit` | | `check_unsafe_code_allowances.sh` | ci-gate | Unsafe-code allowance ledger guard | ci.yml Quick Checks | | `layer-dependency-baseline.txt` | ci-gate (data) | Committed baseline consumed by `check_layer_dependencies.sh` | [Architecture guard troubleshooting](../docs/operations/architecture-guard-troubleshooting.md) | | `static.sh` | ci-gate | Static-build helper executed inside image builds | `Dockerfile.source`, `Dockerfile.decommission-local` | diff --git a/scripts/check_test_count_ratchet.sh b/scripts/check_test_count_ratchet.sh new file mode 100755 index 000000000..7e33b3af9 --- /dev/null +++ b/scripts/check_test_count_ratchet.sh @@ -0,0 +1,212 @@ +#!/usr/bin/env bash +# Per-crate test-count ratchet for the gateway migration (rustfs/backlog#2734 +# section 1 rule 3; task T0.3 in rustfs/backlog#2738). +# +# No PR may lower the number of tests in any workspace crate, and no PR may +# add #[ignore]: "delete the test to get green" is rejected in CI. The counter +# is deliberately per crate, never a workspace total โ€” a drop in one crate +# must not hide behind an addition in another. +# +# Workspace members come from `cargo metadata --no-deps` (the same source that +# `Cargo.toml` [workspace].members feeds), so a renamed or added crate is seen +# without a hand-kept list. For each member two counters are taken over every +# .rs file under its package directory (src/, tests/, benches/, examples/): +# +# tests lines matching TEST_PATTERN: #[test], #[tokio::test], +# #[tokio::test(flavor = ...)], and each #[test_case(...)] or +# #[test_case::test_case(...)] case. The spec's literal +# `#\[(tokio::)?test\]` misses the tokio attributes that carry +# arguments and every test_case case (792 and 477 lines when this +# ratchet landed), so the pattern is widened here on purpose. +# ignored lines matching IGNORE_PATTERN: #[ignore] and #[ignore = "..."]. +# +# Both patterns are anchored at line start. rustfmt (enforced by Quick Checks) +# keeps attributes on their own line, so the anchor costs nothing and keeps +# doc comments that mention `#[test]` out of the count. +# +# The counts are compared row by row with .config/test-count-baseline.txt +# (`\t\t`, one row per member). The baseline is an +# EXACT value, not a floor, in both columns: +# +# - tests below the row fails: `crate : < baseline `. +# Restore the tests. If the removal is intentional, lower the row in the +# same PR and justify it in the PR description, so the reduction is visible +# in the diff instead of disappearing into a green check. +# - tests above the row fails as well, telling the author to raise the row in +# the same PR. An exact baseline cannot drift upward silently, so the next +# deletion is always measured against the true count. +# - ignored above the row fails (the line contains `ignored`); adding +# #[ignore] is not accepted on gateway/integration. Ignored below the row +# fails too, asking for the row to be lowered so that ratchet stays tight. +# - a row whose crate is no longer a member fails (deleting a crate deletes +# its tests), and a member without a row fails. +# +# `--update` rewrites the baseline from the current tree. Review the diff it +# produces before committing it; it is the only sanctioned way to change the +# file other than editing the single row a PR touched. +# +# Usage: +# scripts/check_test_count_ratchet.sh # compare against the baseline +# scripts/check_test_count_ratchet.sh --update # regenerate the baseline + +set -euo pipefail + +cd "$(dirname "$0")/.." + +BASELINE_FILE=".config/test-count-baseline.txt" +TEST_PATTERN='^[[:space:]]*#\[((tokio::)?test([]]|\()|(test_case::)?test_case\()' +IGNORE_PATTERN='^[[:space:]]*#\[ignore([]]|[[:space:]]*=)' + +mode="check" +case "${1:-}" in + "") ;; + --update) mode="update" ;; + *) + echo "usage: $0 [--update]" >&2 + exit 2 + ;; +esac + +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT + +# One `\t` line per workspace member, sorted. python3 is already +# a Quick Checks dependency (check_test_wiring.py); jq is not. +cargo metadata --no-deps --format-version 1 --offline | + python3 -c ' +import json, os, sys +meta = json.load(sys.stdin) +members = set(meta["workspace_members"]) +root = meta["workspace_root"] +for pkg in meta["packages"]: + if pkg["id"] in members: + pkg_dir = os.path.relpath(os.path.dirname(pkg["manifest_path"]), root) + print(pkg["name"] + "\t" + pkg_dir) +' | LC_ALL=C sort >"$TMP_DIR/packages" + +if ! grep -q . "$TMP_DIR/packages"; then + echo "error: cargo metadata reported no workspace members" >&2 + exit 1 +fi + +# grep exits 1 on zero matches (a legitimate count of 0 for a crate without +# tests) and >1 on real errors; only the latter may abort the check. +count_matching_lines() { + local pattern="$1" dir="$2" status=0 + grep -rE --include='*.rs' -- "$pattern" "$dir" >"$TMP_DIR/matches" || status=$? + if ((status > 1)); then + echo "error: 'grep -rE $pattern $dir' failed with status $status" >&2 + exit 1 + fi + grep -c . "$TMP_DIR/matches" || true +} + +while IFS=$'\t' read -r name dir; do + if [[ ! -d "$dir" ]]; then + echo "error: package $name has no directory at $dir" >&2 + exit 1 + fi + tests="$(count_matching_lines "$TEST_PATTERN" "$dir")" + ignored="$(count_matching_lines "$IGNORE_PATTERN" "$dir")" + printf '%s\t%s\t%s\n' "$name" "$tests" "$ignored" +done <"$TMP_DIR/packages" >"$TMP_DIR/actual" + +crate_count="$(grep -c . "$TMP_DIR/actual")" +total_tests="$(awk -F'\t' '{sum += $2} END {print sum + 0}' "$TMP_DIR/actual")" +total_ignored="$(awk -F'\t' '{sum += $3} END {print sum + 0}' "$TMP_DIR/actual")" + +# Sanity assertion: every crate counting 0 means the counter broke (a grep +# that searched the wrong tree or a pattern no rustfmt-formatted attribute +# matches), not that the workspace has no tests. Fail hard rather than +# writing or comparing against an all-zero table. +if ((total_tests == 0)); then + echo "error: counted 0 tests across $crate_count crates; the counter itself is broken" >&2 + exit 1 +fi + +if [[ "$mode" == "update" ]]; then + { + echo "# Per-crate test-count baseline consumed by scripts/check_test_count_ratchet.sh" + echo "# (rustfs/backlog#2734 section 1 rule 3; T0.3 in rustfs/backlog#2738)." + echo "#" + echo "# One row per workspace member: , tests, ignored, tab-separated." + echo "# tests counts #[test], #[tokio::test], #[tokio::test(...)] and #[test_case(...)]" + echo "# attribute lines under the package directory; ignored counts #[ignore] and" + echo "# #[ignore = \"...\"]. Both columns are exact: CI fails when a crate's tests fall" + echo "# below or rise above its row, or when its ignored count changes. Raise a row in" + echo "# the PR that adds tests; lowering one is a reviewed statement that tests were" + echo "# intentionally removed. Regenerate with:" + echo "# scripts/check_test_count_ratchet.sh --update" + cat "$TMP_DIR/actual" + } >"$BASELINE_FILE" + echo "wrote $BASELINE_FILE: $crate_count crates, $total_tests tests, $total_ignored ignored" + exit 0 +fi + +if [[ ! -f "$BASELINE_FILE" ]]; then + echo "error: $BASELINE_FILE is missing; create it with '$0 --update' and commit it" >&2 + exit 1 +fi + +grep -Ev '^[[:space:]]*(#|$)' "$BASELINE_FILE" >"$TMP_DIR/baseline" || true + +if ! grep -q . "$TMP_DIR/baseline"; then + echo "error: $BASELINE_FILE has no baseline rows" >&2 + exit 1 +fi +if grep -nEv $'^[^\t]+\t[0-9]+\t[0-9]+$' "$TMP_DIR/baseline" >"$TMP_DIR/malformed"; then + printf 'error: malformed rows in %s (expected \\t\\t):\n' "$BASELINE_FILE" >&2 + sed 's/^/ line /' "$TMP_DIR/malformed" >&2 + exit 1 +fi +if cut -f1 "$TMP_DIR/baseline" | LC_ALL=C sort | uniq -d | grep . >"$TMP_DIR/duplicates"; then + echo "error: duplicate rows in $BASELINE_FILE for:" >&2 + sed 's/^/ /' "$TMP_DIR/duplicates" >&2 + exit 1 +fi + +# Every violation is one line. The ` < baseline ` and `ignored` +# shapes are the acceptance contract of rustfs/backlog#2738; keep them. +awk -F'\t' ' +NR == FNR { + baseline_tests[$1] = $2 + baseline_ignored[$1] = $3 + next +} +{ + seen[$1] = 1 + if (!($1 in baseline_tests)) { + printf "crate %s: no baseline row (%d tests, %d ignored); add it in this PR\n", $1, $2, $3 + next + } + if ($2 + 0 < baseline_tests[$1] + 0) { + printf "crate %s: %d < baseline %d\n", $1, $2, baseline_tests[$1] + } else if ($2 + 0 > baseline_tests[$1] + 0) { + printf "crate %s: %d > baseline %d; raise the baseline row to %d in this PR\n", $1, $2, baseline_tests[$1], $2 + } + if ($3 + 0 > baseline_ignored[$1] + 0) { + printf "crate %s: %d ignored > baseline %d; adding #[ignore] is not accepted\n", $1, $3, baseline_ignored[$1] + } else if ($3 + 0 < baseline_ignored[$1] + 0) { + printf "crate %s: %d ignored < baseline %d; lower the baseline row to %d in this PR\n", $1, $3, baseline_ignored[$1], $3 + } +} +END { + for (name in baseline_tests) { + if (!(name in seen)) { + printf "crate %s: not a workspace member (baseline %d tests); remove the row in this PR if the crate was intentionally deleted\n", name, baseline_tests[name] + } + } +} +' "$TMP_DIR/baseline" "$TMP_DIR/actual" | LC_ALL=C sort >"$TMP_DIR/violations" + +if grep -q . "$TMP_DIR/violations"; then + cat "$TMP_DIR/violations" >&2 + echo "error: test-count ratchet violated (rustfs/backlog#2734 section 1 rule 3)." >&2 + echo " A crate below its baseline lost tests: restore them. If the removal is" >&2 + echo " intentional, lower that row in $BASELINE_FILE in the same PR and" >&2 + echo " justify it in the PR description. A crate above its baseline raises its" >&2 + echo " row in the same PR ('$0 --update' rewrites the file; review the diff)." >&2 + exit 1 +fi + +echo "test-count ratchet OK: $crate_count crates, $total_tests tests, $total_ignored ignored match $BASELINE_FILE"