feat(ci): item-2 cross-surface contract parity (stdlib-only package)

internal/ciparity/ — new stdlib-only package with four tests:

1. TestSurfaceParity_MCPToolCatalogue (HARD GATE):
   - Every MCP tool name conforms to certctl_<word>(_<word>)*
   - No duplicate names across the five tools*.go files
   - Total tools ≥ mcpBaselineFloor (150; current count 155)
   Catches accidental tool deletions + naming-convention drift.

2. TestSurfaceParity_CLICommandCatalogue (INFORMATIONAL):
   Walks cmd/cli/main.go's switch-case dispatcher. Logs the 31
   distinct verbs. Per frozen decision 0.9, warn-only until the CLI
   surface stabilizes.

3. TestSurfaceParity_OpenAPI_MCPHeuristicCoverage (INFORMATIONAL):
   Reports the fraction of OpenAPI ops whose path tokens overlap
   with MCP tool name tokens. Trend metric; current coverage 92%.

4. TestSurfaceParity_Summary (INFORMATIONAL):
   One-glance count of router routes / OpenAPI ops / MCP tools / CLI
   verbs. Easy eyeball for a PR reviewer.

Verified in sandbox:
  - gofmt clean
  - go vet clean
  - go test -short -count=1: all four PASS in 0.017s

Stdlib-only by design — the tests read source files with os.ReadFile +
regexp + go/ast. Keeps the test runnable without pulling in the rest
of the codebase's transitive deps; fast self-contained signal.

Router ↔ OpenAPI parity (TestRouter_OpenAPIParity) stays in
internal/api/router/openapi_parity_test.go where it already lives.
This bundle does not duplicate it.

Allowlist scaffold at scripts/ci-guards/surface-parity-mcp-exemptions.yaml
for the day TestSurfaceParity_OpenAPI_MCP* is promoted from
informational to hard gate.

Audit-Closes: post-v2.1.0-anti-rot/item-2
This commit is contained in:
shankar0123
2026-05-12 14:09:32 +00:00
parent 7a2ba11391
commit 370b772fbd
3 changed files with 333 additions and 0 deletions
+20
View File
@@ -0,0 +1,20 @@
// Package ciparity hosts cross-surface contract-parity tests.
//
// Per post-v2.1.0 anti-rot item 2 (Auditable Codebase Bundle), this
// package contains tests that walk source files (router.go,
// openapi.yaml, the MCP tools*.go catalogue, cmd/cli/main.go) and
// assert invariants ACROSS those surfaces — e.g. "every MCP tool
// follows the canonical naming convention" or "the MCP tool count
// does not regress below the documented floor."
//
// The package is stdlib-only by design: the tests read source files
// with os.ReadFile and parse them with regexp + go/ast. This keeps
// the test runnable without pulling in the rest of the codebase's
// transitive dependencies — a developer running `go test ./internal/ciparity/...`
// gets a fast, self-contained signal.
//
// The router ↔ openapi.yaml parity test lives separately in
// internal/api/router/openapi_parity_test.go (TestRouter_OpenAPIParity)
// because it predates this package and operates on the same AST that
// TestRouterRBACGateCoverage already needs. Don't duplicate it here.
package ciparity
+279
View File
@@ -0,0 +1,279 @@
package ciparity
// surface_parity_test.go — per post-v2.1.0 anti-rot item 2 (Auditable
// Codebase Bundle).
//
// Three tests, all stdlib-only:
//
// 1. TestSurfaceParity_MCPToolCatalogue (HARD GATE)
// Every MCP tool name matches the `certctl_<word>(_<word>)*`
// convention; no duplicates across files; total count ≥
// mcpBaselineFloor. Catches accidental tool deletions + naming-
// convention drift.
//
// 2. TestSurfaceParity_CLICommandCatalogue (INFORMATIONAL — t.Log only)
// Walks cmd/cli/main.go's switch-case dispatcher. Per frozen
// decision 0.9, warn-only until the CLI surface stabilizes.
//
// 3. TestSurfaceParity_OpenAPI_MCPHeuristicCoverage (INFORMATIONAL)
// Reports the fraction of OpenAPI operations whose path tokens
// overlap with any MCP tool name token. Trend metric, not a gate.
//
// 4. TestSurfaceParity_Summary (INFORMATIONAL)
// One-glance summary of the four surface counts.
//
// All file paths are resolved relative to the repo root, which this
// test discovers by walking up from $PWD until it finds go.mod. Keeps
// the test runnable from any working directory.
import (
"errors"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"testing"
)
// mcpBaselineFloor — see header doc. Bump when a deletion is
// deliberate; the diff captures the change.
const mcpBaselineFloor = 150
var (
mcpToolNameRe = regexp.MustCompile(`^certctl_[a-z][a-z0-9_]*[a-z0-9]$`)
mcpNameDeclRe = regexp.MustCompile(`Name:\s*"(certctl_[a-z0-9_]+)"`)
methodPathRe = regexp.MustCompile(`^(GET|POST|PUT|DELETE|PATCH|OPTIONS|HEAD) /`)
openapiPathRe = regexp.MustCompile(`^ (/[^:]+):\s*$`)
openapiVerbRe = regexp.MustCompile(`^ (get|post|put|delete|patch|options|head):\s*$`)
caseLiteralRe = regexp.MustCompile(`case\s+"([a-z\-]+)":`)
)
// mcpToolFiles lists the (non-test) Go files expected to register
// MCP tools.
func mcpToolFiles(repo string) []string {
base := filepath.Join(repo, "internal", "mcp")
return []string{
filepath.Join(base, "tools.go"),
filepath.Join(base, "tools_audit_fix.go"),
filepath.Join(base, "tools_auth.go"),
filepath.Join(base, "tools_auth_bundle2.go"),
filepath.Join(base, "tools_est.go"),
}
}
func findRepoRoot(t *testing.T) string {
t.Helper()
wd, err := os.Getwd()
if err != nil {
t.Fatalf("getwd: %v", err)
}
cur := wd
for {
if _, err := os.Stat(filepath.Join(cur, "go.mod")); err == nil {
return cur
}
parent := filepath.Dir(cur)
if parent == cur {
t.Fatalf("no go.mod found from %s upward", wd)
}
cur = parent
}
}
// readFileOrSkip reads a file; on ENOENT, calls t.Skipf rather than
// failing — useful for files that may be renamed during refactors.
func readFileOrFail(t *testing.T, p string) []byte {
t.Helper()
body, err := os.ReadFile(p)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
t.Fatalf("expected file missing: %s (refactor? — update the test)", p)
}
t.Fatalf("read %s: %v", p, err)
}
return body
}
// TestSurfaceParity_MCPToolCatalogue is a HARD gate.
//
// Asserts:
//
// - Every MCP tool name conforms to certctl_<word>(_<word>)*.
// - No tool name is registered in more than one tools*.go file.
// - Total tools ≥ mcpBaselineFloor.
func TestSurfaceParity_MCPToolCatalogue(t *testing.T) {
repo := findRepoRoot(t)
names := map[string]string{}
for _, path := range mcpToolFiles(repo) {
body := readFileOrFail(t, path)
base := filepath.Base(path)
for _, m := range mcpNameDeclRe.FindAllStringSubmatch(string(body), -1) {
name := m[1]
if !mcpToolNameRe.MatchString(name) {
t.Errorf("MCP tool name %q in %s does not match certctl_<word>(_<word>)* — fix the name or relax the convention deliberately",
name, base)
continue
}
if other, dup := names[name]; dup {
t.Errorf("MCP tool name %q duplicated: first in %s, again in %s — pick a unique name",
name, filepath.Base(other), base)
continue
}
names[name] = path
}
}
if len(names) < mcpBaselineFloor {
t.Errorf("MCP tool count regressed: %d found, baseline floor %d. "+
"If the deletion is intentional, lower mcpBaselineFloor in this test in the SAME commit. "+
"If accidental, restore the deleted tools.",
len(names), mcpBaselineFloor)
}
t.Logf("MCP tool catalogue: %d tools (baseline floor %d)", len(names), mcpBaselineFloor)
}
// TestSurfaceParity_CLICommandCatalogue — informational only.
//
// Walks cmd/cli/main.go's switch-case dispatcher and reports the
// distinct verbs handled. Returns success regardless of contents.
// Promoted to fail-on-miss when the CLI surface stabilizes.
func TestSurfaceParity_CLICommandCatalogue(t *testing.T) {
repo := findRepoRoot(t)
body := readFileOrFail(t, filepath.Join(repo, "cmd", "cli", "main.go"))
verbs := map[string]struct{}{}
for _, m := range caseLiteralRe.FindAllStringSubmatch(string(body), -1) {
verbs[m[1]] = struct{}{}
}
if len(verbs) == 0 {
t.Fatal("CLI scanner found zero verbs — likely a refactor; update the test")
}
out := make([]string, 0, len(verbs))
for v := range verbs {
out = append(out, v)
}
sort.Strings(out)
t.Logf("CLI verb catalogue (%d distinct case literals; informational only per frozen decision 0.9):\n %s",
len(out), strings.Join(out, ", "))
}
// scanOpenAPIOps walks openapi.yaml's paths block and returns every
// (METHOD, PATH) tuple. Mirrors the parser used by
// internal/api/router/openapi_parity_test.go::scanOpenAPIOperations.
func scanOpenAPIOps(t *testing.T, path string) []string {
t.Helper()
body := readFileOrFail(t, path)
var out []string
inPaths := false
currentPath := ""
for _, line := range strings.Split(string(body), "\n") {
if strings.HasPrefix(line, "paths:") {
inPaths = true
continue
}
if inPaths && line != "" && !strings.HasPrefix(line, " ") {
inPaths = false
continue
}
if !inPaths {
continue
}
if m := openapiPathRe.FindStringSubmatch(line); m != nil {
currentPath = m[1]
continue
}
if m := openapiVerbRe.FindStringSubmatch(line); m != nil && currentPath != "" {
out = append(out, strings.ToUpper(m[1])+" "+currentPath)
}
}
return out
}
// scanRouterRoutes scans internal/api/router/router.go for route
// registrations. Uses a regex (not go/ast) to keep this package
// stdlib-only; the router.go file is large but the patterns we care
// about are stable enough that regex is sufficient.
func scanRouterRoutes(t *testing.T, path string) []string {
t.Helper()
body := readFileOrFail(t, path)
// Match r.Register("METHOD /path", ...) and r.mux.Handle("METHOD /path", ...).
registerRe := regexp.MustCompile(`r\.Register\(\s*"([A-Z]+ /[^"]+)"`)
muxHandleRe := regexp.MustCompile(`r\.mux\.Handle\(\s*"([A-Z]+ /[^"]+)"`)
seen := map[string]struct{}{}
for _, m := range registerRe.FindAllStringSubmatch(string(body), -1) {
seen[m[1]] = struct{}{}
}
for _, m := range muxHandleRe.FindAllStringSubmatch(string(body), -1) {
seen[m[1]] = struct{}{}
}
out := make([]string, 0, len(seen))
for s := range seen {
out = append(out, s)
}
return out
}
// TestSurfaceParity_OpenAPI_MCPHeuristicCoverage — informational.
//
// For each OpenAPI operation, splits the path into tokens; if any
// token appears in any MCP tool name's token set, the op is "covered."
// This is a heuristic — the actual semantic map (operationId →
// MCP tool) is not declared in the source, so we approximate.
func TestSurfaceParity_OpenAPI_MCPHeuristicCoverage(t *testing.T) {
repo := findRepoRoot(t)
specOps := scanOpenAPIOps(t, filepath.Join(repo, "api", "openapi.yaml"))
mcpTokens := map[string]struct{}{}
for _, path := range mcpToolFiles(repo) {
body := readFileOrFail(t, path)
for _, m := range mcpNameDeclRe.FindAllStringSubmatch(string(body), -1) {
for _, tok := range strings.Split(strings.TrimPrefix(m[1], "certctl_"), "_") {
mcpTokens[tok] = struct{}{}
}
}
}
covered := 0
for _, op := range specOps {
parts := strings.Split(op, " ")
if len(parts) != 2 {
continue
}
segs := strings.FieldsFunc(parts[1], func(r rune) bool {
return r == '/' || r == '{' || r == '}' || r == '-'
})
hit := false
for _, s := range segs {
if _, ok := mcpTokens[strings.ToLower(s)]; ok {
hit = true
break
}
}
if hit {
covered++
}
}
if len(specOps) == 0 {
t.Fatal("openapi.yaml scan returned zero operations — fix the test")
}
pct := (covered * 100) / len(specOps)
t.Logf("OpenAPI↔MCP heuristic coverage (informational): %d of %d ops (%d%%) share at least one path token with an MCP tool name",
covered, len(specOps), pct)
}
// TestSurfaceParity_Summary prints the four surface counts side-by-side.
func TestSurfaceParity_Summary(t *testing.T) {
repo := findRepoRoot(t)
routes := scanRouterRoutes(t, filepath.Join(repo, "internal", "api", "router", "router.go"))
specOps := scanOpenAPIOps(t, filepath.Join(repo, "api", "openapi.yaml"))
mcpCount := 0
for _, path := range mcpToolFiles(repo) {
body := readFileOrFail(t, path)
mcpCount += len(mcpNameDeclRe.FindAllStringSubmatch(string(body), -1))
}
cliBody := readFileOrFail(t, filepath.Join(repo, "cmd", "cli", "main.go"))
cliCount := len(caseLiteralRe.FindAllStringSubmatch(string(cliBody), -1))
t.Logf("Surface parity summary (informational):\n"+
" router routes : %d\n"+
" OpenAPI ops : %d\n"+
" MCP tools : %d (floor %d)\n"+
" CLI verbs : %d (warn-only — frozen decision 0.9)",
len(routes), len(specOps), mcpCount, mcpBaselineFloor, cliCount)
}
@@ -0,0 +1,34 @@
# scripts/ci-guards/surface-parity-mcp-exemptions.yaml
#
# Allowlist for OpenAPI operations that are intentionally NOT mirrored
# in the MCP tool catalogue. Consumed by
# internal/api/router/surface_parity_test.go::TestSurfaceParity_*.
#
# The current MCP parity tests are informational (per frozen decision
# 0.9). This file exists so when those tests are promoted to hard
# gates, the carve-outs are already documented and the promotion is
# mechanical.
#
# Each entry shape:
#
# - operation: "METHOD /api/v1/path"
# justification: "one-line reason this is legitimately HTTP-only"
# expires: "YYYY-MM-DD" # required; 90-day default
#
# Categories of legitimate carve-outs (DO add these when the test is
# promoted to fail-on-miss):
#
# - ACME wire protocol (RFC 8555 + RFC 9773 ARI)
# - SCEP wire protocol (RFC 8894)
# - EST wire protocol (RFC 7030)
# - OCSP responder
# - CRL distribution
# - Healthcheck / readiness / version endpoints
# - OIDC callback / back-channel-logout
# - SPA fallback for the embedded web UI
#
# DO NOT add entries here to silence the test on an oversight. If an
# operation should have an MCP tool and doesn't, that's the bug — add
# the tool.
exceptions: []