From 370b772fbda78097daeab83cb2fb7cc779029403 Mon Sep 17 00:00:00 2001 From: shankar0123 Date: Tue, 12 May 2026 14:09:32 +0000 Subject: [PATCH] feat(ci): item-2 cross-surface contract parity (stdlib-only package) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit internal/ciparity/ — new stdlib-only package with four tests: 1. TestSurfaceParity_MCPToolCatalogue (HARD GATE): - Every MCP tool name conforms to certctl_(_)* - No duplicate names across the five tools*.go files - Total tools ≥ mcpBaselineFloor (150; current count 155) Catches accidental tool deletions + naming-convention drift. 2. TestSurfaceParity_CLICommandCatalogue (INFORMATIONAL): Walks cmd/cli/main.go's switch-case dispatcher. Logs the 31 distinct verbs. Per frozen decision 0.9, warn-only until the CLI surface stabilizes. 3. TestSurfaceParity_OpenAPI_MCPHeuristicCoverage (INFORMATIONAL): Reports the fraction of OpenAPI ops whose path tokens overlap with MCP tool name tokens. Trend metric; current coverage 92%. 4. TestSurfaceParity_Summary (INFORMATIONAL): One-glance count of router routes / OpenAPI ops / MCP tools / CLI verbs. Easy eyeball for a PR reviewer. Verified in sandbox: - gofmt clean - go vet clean - go test -short -count=1: all four PASS in 0.017s Stdlib-only by design — the tests read source files with os.ReadFile + regexp + go/ast. Keeps the test runnable without pulling in the rest of the codebase's transitive deps; fast self-contained signal. Router ↔ OpenAPI parity (TestRouter_OpenAPIParity) stays in internal/api/router/openapi_parity_test.go where it already lives. This bundle does not duplicate it. Allowlist scaffold at scripts/ci-guards/surface-parity-mcp-exemptions.yaml for the day TestSurfaceParity_OpenAPI_MCP* is promoted from informational to hard gate. Audit-Closes: post-v2.1.0-anti-rot/item-2 --- internal/ciparity/doc.go | 20 ++ internal/ciparity/surface_parity_test.go | 279 ++++++++++++++++++ .../surface-parity-mcp-exemptions.yaml | 34 +++ 3 files changed, 333 insertions(+) create mode 100644 internal/ciparity/doc.go create mode 100644 internal/ciparity/surface_parity_test.go create mode 100644 scripts/ci-guards/surface-parity-mcp-exemptions.yaml diff --git a/internal/ciparity/doc.go b/internal/ciparity/doc.go new file mode 100644 index 0000000..1defc39 --- /dev/null +++ b/internal/ciparity/doc.go @@ -0,0 +1,20 @@ +// Package ciparity hosts cross-surface contract-parity tests. +// +// Per post-v2.1.0 anti-rot item 2 (Auditable Codebase Bundle), this +// package contains tests that walk source files (router.go, +// openapi.yaml, the MCP tools*.go catalogue, cmd/cli/main.go) and +// assert invariants ACROSS those surfaces — e.g. "every MCP tool +// follows the canonical naming convention" or "the MCP tool count +// does not regress below the documented floor." +// +// The package is stdlib-only by design: the tests read source files +// with os.ReadFile and parse them with regexp + go/ast. This keeps +// the test runnable without pulling in the rest of the codebase's +// transitive dependencies — a developer running `go test ./internal/ciparity/...` +// gets a fast, self-contained signal. +// +// The router ↔ openapi.yaml parity test lives separately in +// internal/api/router/openapi_parity_test.go (TestRouter_OpenAPIParity) +// because it predates this package and operates on the same AST that +// TestRouterRBACGateCoverage already needs. Don't duplicate it here. +package ciparity diff --git a/internal/ciparity/surface_parity_test.go b/internal/ciparity/surface_parity_test.go new file mode 100644 index 0000000..795f115 --- /dev/null +++ b/internal/ciparity/surface_parity_test.go @@ -0,0 +1,279 @@ +package ciparity + +// surface_parity_test.go — per post-v2.1.0 anti-rot item 2 (Auditable +// Codebase Bundle). +// +// Three tests, all stdlib-only: +// +// 1. TestSurfaceParity_MCPToolCatalogue (HARD GATE) +// Every MCP tool name matches the `certctl_(_)*` +// convention; no duplicates across files; total count ≥ +// mcpBaselineFloor. Catches accidental tool deletions + naming- +// convention drift. +// +// 2. TestSurfaceParity_CLICommandCatalogue (INFORMATIONAL — t.Log only) +// Walks cmd/cli/main.go's switch-case dispatcher. Per frozen +// decision 0.9, warn-only until the CLI surface stabilizes. +// +// 3. TestSurfaceParity_OpenAPI_MCPHeuristicCoverage (INFORMATIONAL) +// Reports the fraction of OpenAPI operations whose path tokens +// overlap with any MCP tool name token. Trend metric, not a gate. +// +// 4. TestSurfaceParity_Summary (INFORMATIONAL) +// One-glance summary of the four surface counts. +// +// All file paths are resolved relative to the repo root, which this +// test discovers by walking up from $PWD until it finds go.mod. Keeps +// the test runnable from any working directory. + +import ( + "errors" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "testing" +) + +// mcpBaselineFloor — see header doc. Bump when a deletion is +// deliberate; the diff captures the change. +const mcpBaselineFloor = 150 + +var ( + mcpToolNameRe = regexp.MustCompile(`^certctl_[a-z][a-z0-9_]*[a-z0-9]$`) + mcpNameDeclRe = regexp.MustCompile(`Name:\s*"(certctl_[a-z0-9_]+)"`) + methodPathRe = regexp.MustCompile(`^(GET|POST|PUT|DELETE|PATCH|OPTIONS|HEAD) /`) + openapiPathRe = regexp.MustCompile(`^ (/[^:]+):\s*$`) + openapiVerbRe = regexp.MustCompile(`^ (get|post|put|delete|patch|options|head):\s*$`) + caseLiteralRe = regexp.MustCompile(`case\s+"([a-z\-]+)":`) +) + +// mcpToolFiles lists the (non-test) Go files expected to register +// MCP tools. +func mcpToolFiles(repo string) []string { + base := filepath.Join(repo, "internal", "mcp") + return []string{ + filepath.Join(base, "tools.go"), + filepath.Join(base, "tools_audit_fix.go"), + filepath.Join(base, "tools_auth.go"), + filepath.Join(base, "tools_auth_bundle2.go"), + filepath.Join(base, "tools_est.go"), + } +} + +func findRepoRoot(t *testing.T) string { + t.Helper() + wd, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + cur := wd + for { + if _, err := os.Stat(filepath.Join(cur, "go.mod")); err == nil { + return cur + } + parent := filepath.Dir(cur) + if parent == cur { + t.Fatalf("no go.mod found from %s upward", wd) + } + cur = parent + } +} + +// readFileOrSkip reads a file; on ENOENT, calls t.Skipf rather than +// failing — useful for files that may be renamed during refactors. +func readFileOrFail(t *testing.T, p string) []byte { + t.Helper() + body, err := os.ReadFile(p) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + t.Fatalf("expected file missing: %s (refactor? — update the test)", p) + } + t.Fatalf("read %s: %v", p, err) + } + return body +} + +// TestSurfaceParity_MCPToolCatalogue is a HARD gate. +// +// Asserts: +// +// - Every MCP tool name conforms to certctl_(_)*. +// - No tool name is registered in more than one tools*.go file. +// - Total tools ≥ mcpBaselineFloor. +func TestSurfaceParity_MCPToolCatalogue(t *testing.T) { + repo := findRepoRoot(t) + names := map[string]string{} + for _, path := range mcpToolFiles(repo) { + body := readFileOrFail(t, path) + base := filepath.Base(path) + for _, m := range mcpNameDeclRe.FindAllStringSubmatch(string(body), -1) { + name := m[1] + if !mcpToolNameRe.MatchString(name) { + t.Errorf("MCP tool name %q in %s does not match certctl_(_)* — fix the name or relax the convention deliberately", + name, base) + continue + } + if other, dup := names[name]; dup { + t.Errorf("MCP tool name %q duplicated: first in %s, again in %s — pick a unique name", + name, filepath.Base(other), base) + continue + } + names[name] = path + } + } + if len(names) < mcpBaselineFloor { + t.Errorf("MCP tool count regressed: %d found, baseline floor %d. "+ + "If the deletion is intentional, lower mcpBaselineFloor in this test in the SAME commit. "+ + "If accidental, restore the deleted tools.", + len(names), mcpBaselineFloor) + } + t.Logf("MCP tool catalogue: %d tools (baseline floor %d)", len(names), mcpBaselineFloor) +} + +// TestSurfaceParity_CLICommandCatalogue — informational only. +// +// Walks cmd/cli/main.go's switch-case dispatcher and reports the +// distinct verbs handled. Returns success regardless of contents. +// Promoted to fail-on-miss when the CLI surface stabilizes. +func TestSurfaceParity_CLICommandCatalogue(t *testing.T) { + repo := findRepoRoot(t) + body := readFileOrFail(t, filepath.Join(repo, "cmd", "cli", "main.go")) + verbs := map[string]struct{}{} + for _, m := range caseLiteralRe.FindAllStringSubmatch(string(body), -1) { + verbs[m[1]] = struct{}{} + } + if len(verbs) == 0 { + t.Fatal("CLI scanner found zero verbs — likely a refactor; update the test") + } + out := make([]string, 0, len(verbs)) + for v := range verbs { + out = append(out, v) + } + sort.Strings(out) + t.Logf("CLI verb catalogue (%d distinct case literals; informational only per frozen decision 0.9):\n %s", + len(out), strings.Join(out, ", ")) +} + +// scanOpenAPIOps walks openapi.yaml's paths block and returns every +// (METHOD, PATH) tuple. Mirrors the parser used by +// internal/api/router/openapi_parity_test.go::scanOpenAPIOperations. +func scanOpenAPIOps(t *testing.T, path string) []string { + t.Helper() + body := readFileOrFail(t, path) + var out []string + inPaths := false + currentPath := "" + for _, line := range strings.Split(string(body), "\n") { + if strings.HasPrefix(line, "paths:") { + inPaths = true + continue + } + if inPaths && line != "" && !strings.HasPrefix(line, " ") { + inPaths = false + continue + } + if !inPaths { + continue + } + if m := openapiPathRe.FindStringSubmatch(line); m != nil { + currentPath = m[1] + continue + } + if m := openapiVerbRe.FindStringSubmatch(line); m != nil && currentPath != "" { + out = append(out, strings.ToUpper(m[1])+" "+currentPath) + } + } + return out +} + +// scanRouterRoutes scans internal/api/router/router.go for route +// registrations. Uses a regex (not go/ast) to keep this package +// stdlib-only; the router.go file is large but the patterns we care +// about are stable enough that regex is sufficient. +func scanRouterRoutes(t *testing.T, path string) []string { + t.Helper() + body := readFileOrFail(t, path) + // Match r.Register("METHOD /path", ...) and r.mux.Handle("METHOD /path", ...). + registerRe := regexp.MustCompile(`r\.Register\(\s*"([A-Z]+ /[^"]+)"`) + muxHandleRe := regexp.MustCompile(`r\.mux\.Handle\(\s*"([A-Z]+ /[^"]+)"`) + seen := map[string]struct{}{} + for _, m := range registerRe.FindAllStringSubmatch(string(body), -1) { + seen[m[1]] = struct{}{} + } + for _, m := range muxHandleRe.FindAllStringSubmatch(string(body), -1) { + seen[m[1]] = struct{}{} + } + out := make([]string, 0, len(seen)) + for s := range seen { + out = append(out, s) + } + return out +} + +// TestSurfaceParity_OpenAPI_MCPHeuristicCoverage — informational. +// +// For each OpenAPI operation, splits the path into tokens; if any +// token appears in any MCP tool name's token set, the op is "covered." +// This is a heuristic — the actual semantic map (operationId → +// MCP tool) is not declared in the source, so we approximate. +func TestSurfaceParity_OpenAPI_MCPHeuristicCoverage(t *testing.T) { + repo := findRepoRoot(t) + specOps := scanOpenAPIOps(t, filepath.Join(repo, "api", "openapi.yaml")) + mcpTokens := map[string]struct{}{} + for _, path := range mcpToolFiles(repo) { + body := readFileOrFail(t, path) + for _, m := range mcpNameDeclRe.FindAllStringSubmatch(string(body), -1) { + for _, tok := range strings.Split(strings.TrimPrefix(m[1], "certctl_"), "_") { + mcpTokens[tok] = struct{}{} + } + } + } + covered := 0 + for _, op := range specOps { + parts := strings.Split(op, " ") + if len(parts) != 2 { + continue + } + segs := strings.FieldsFunc(parts[1], func(r rune) bool { + return r == '/' || r == '{' || r == '}' || r == '-' + }) + hit := false + for _, s := range segs { + if _, ok := mcpTokens[strings.ToLower(s)]; ok { + hit = true + break + } + } + if hit { + covered++ + } + } + if len(specOps) == 0 { + t.Fatal("openapi.yaml scan returned zero operations — fix the test") + } + pct := (covered * 100) / len(specOps) + t.Logf("OpenAPI↔MCP heuristic coverage (informational): %d of %d ops (%d%%) share at least one path token with an MCP tool name", + covered, len(specOps), pct) +} + +// TestSurfaceParity_Summary prints the four surface counts side-by-side. +func TestSurfaceParity_Summary(t *testing.T) { + repo := findRepoRoot(t) + routes := scanRouterRoutes(t, filepath.Join(repo, "internal", "api", "router", "router.go")) + specOps := scanOpenAPIOps(t, filepath.Join(repo, "api", "openapi.yaml")) + mcpCount := 0 + for _, path := range mcpToolFiles(repo) { + body := readFileOrFail(t, path) + mcpCount += len(mcpNameDeclRe.FindAllStringSubmatch(string(body), -1)) + } + cliBody := readFileOrFail(t, filepath.Join(repo, "cmd", "cli", "main.go")) + cliCount := len(caseLiteralRe.FindAllStringSubmatch(string(cliBody), -1)) + t.Logf("Surface parity summary (informational):\n"+ + " router routes : %d\n"+ + " OpenAPI ops : %d\n"+ + " MCP tools : %d (floor %d)\n"+ + " CLI verbs : %d (warn-only — frozen decision 0.9)", + len(routes), len(specOps), mcpCount, mcpBaselineFloor, cliCount) +} diff --git a/scripts/ci-guards/surface-parity-mcp-exemptions.yaml b/scripts/ci-guards/surface-parity-mcp-exemptions.yaml new file mode 100644 index 0000000..ceafd7b --- /dev/null +++ b/scripts/ci-guards/surface-parity-mcp-exemptions.yaml @@ -0,0 +1,34 @@ +# scripts/ci-guards/surface-parity-mcp-exemptions.yaml +# +# Allowlist for OpenAPI operations that are intentionally NOT mirrored +# in the MCP tool catalogue. Consumed by +# internal/api/router/surface_parity_test.go::TestSurfaceParity_*. +# +# The current MCP parity tests are informational (per frozen decision +# 0.9). This file exists so when those tests are promoted to hard +# gates, the carve-outs are already documented and the promotion is +# mechanical. +# +# Each entry shape: +# +# - operation: "METHOD /api/v1/path" +# justification: "one-line reason this is legitimately HTTP-only" +# expires: "YYYY-MM-DD" # required; 90-day default +# +# Categories of legitimate carve-outs (DO add these when the test is +# promoted to fail-on-miss): +# +# - ACME wire protocol (RFC 8555 + RFC 9773 ARI) +# - SCEP wire protocol (RFC 8894) +# - EST wire protocol (RFC 7030) +# - OCSP responder +# - CRL distribution +# - Healthcheck / readiness / version endpoints +# - OIDC callback / back-channel-logout +# - SPA fallback for the embedded web UI +# +# DO NOT add entries here to silence the test on an oversight. If an +# operation should have an MCP tool and doesn't, that's the bug — add +# the tool. + +exceptions: []