mirror of
https://github.com/shankar0123/certctl.git
synced 2026-06-07 22:21:30 +00:00
feat(ci): item-2 cross-surface contract parity (stdlib-only package)
internal/ciparity/ — new stdlib-only package with four tests: 1. TestSurfaceParity_MCPToolCatalogue (HARD GATE): - Every MCP tool name conforms to certctl_<word>(_<word>)* - No duplicate names across the five tools*.go files - Total tools ≥ mcpBaselineFloor (150; current count 155) Catches accidental tool deletions + naming-convention drift. 2. TestSurfaceParity_CLICommandCatalogue (INFORMATIONAL): Walks cmd/cli/main.go's switch-case dispatcher. Logs the 31 distinct verbs. Per frozen decision 0.9, warn-only until the CLI surface stabilizes. 3. TestSurfaceParity_OpenAPI_MCPHeuristicCoverage (INFORMATIONAL): Reports the fraction of OpenAPI ops whose path tokens overlap with MCP tool name tokens. Trend metric; current coverage 92%. 4. TestSurfaceParity_Summary (INFORMATIONAL): One-glance count of router routes / OpenAPI ops / MCP tools / CLI verbs. Easy eyeball for a PR reviewer. Verified in sandbox: - gofmt clean - go vet clean - go test -short -count=1: all four PASS in 0.017s Stdlib-only by design — the tests read source files with os.ReadFile + regexp + go/ast. Keeps the test runnable without pulling in the rest of the codebase's transitive deps; fast self-contained signal. Router ↔ OpenAPI parity (TestRouter_OpenAPIParity) stays in internal/api/router/openapi_parity_test.go where it already lives. This bundle does not duplicate it. Allowlist scaffold at scripts/ci-guards/surface-parity-mcp-exemptions.yaml for the day TestSurfaceParity_OpenAPI_MCP* is promoted from informational to hard gate. Audit-Closes: post-v2.1.0-anti-rot/item-2
This commit is contained in:
@@ -0,0 +1,20 @@
|
|||||||
|
// Package ciparity hosts cross-surface contract-parity tests.
|
||||||
|
//
|
||||||
|
// Per post-v2.1.0 anti-rot item 2 (Auditable Codebase Bundle), this
|
||||||
|
// package contains tests that walk source files (router.go,
|
||||||
|
// openapi.yaml, the MCP tools*.go catalogue, cmd/cli/main.go) and
|
||||||
|
// assert invariants ACROSS those surfaces — e.g. "every MCP tool
|
||||||
|
// follows the canonical naming convention" or "the MCP tool count
|
||||||
|
// does not regress below the documented floor."
|
||||||
|
//
|
||||||
|
// The package is stdlib-only by design: the tests read source files
|
||||||
|
// with os.ReadFile and parse them with regexp + go/ast. This keeps
|
||||||
|
// the test runnable without pulling in the rest of the codebase's
|
||||||
|
// transitive dependencies — a developer running `go test ./internal/ciparity/...`
|
||||||
|
// gets a fast, self-contained signal.
|
||||||
|
//
|
||||||
|
// The router ↔ openapi.yaml parity test lives separately in
|
||||||
|
// internal/api/router/openapi_parity_test.go (TestRouter_OpenAPIParity)
|
||||||
|
// because it predates this package and operates on the same AST that
|
||||||
|
// TestRouterRBACGateCoverage already needs. Don't duplicate it here.
|
||||||
|
package ciparity
|
||||||
@@ -0,0 +1,279 @@
|
|||||||
|
package ciparity
|
||||||
|
|
||||||
|
// surface_parity_test.go — per post-v2.1.0 anti-rot item 2 (Auditable
|
||||||
|
// Codebase Bundle).
|
||||||
|
//
|
||||||
|
// Three tests, all stdlib-only:
|
||||||
|
//
|
||||||
|
// 1. TestSurfaceParity_MCPToolCatalogue (HARD GATE)
|
||||||
|
// Every MCP tool name matches the `certctl_<word>(_<word>)*`
|
||||||
|
// convention; no duplicates across files; total count ≥
|
||||||
|
// mcpBaselineFloor. Catches accidental tool deletions + naming-
|
||||||
|
// convention drift.
|
||||||
|
//
|
||||||
|
// 2. TestSurfaceParity_CLICommandCatalogue (INFORMATIONAL — t.Log only)
|
||||||
|
// Walks cmd/cli/main.go's switch-case dispatcher. Per frozen
|
||||||
|
// decision 0.9, warn-only until the CLI surface stabilizes.
|
||||||
|
//
|
||||||
|
// 3. TestSurfaceParity_OpenAPI_MCPHeuristicCoverage (INFORMATIONAL)
|
||||||
|
// Reports the fraction of OpenAPI operations whose path tokens
|
||||||
|
// overlap with any MCP tool name token. Trend metric, not a gate.
|
||||||
|
//
|
||||||
|
// 4. TestSurfaceParity_Summary (INFORMATIONAL)
|
||||||
|
// One-glance summary of the four surface counts.
|
||||||
|
//
|
||||||
|
// All file paths are resolved relative to the repo root, which this
|
||||||
|
// test discovers by walking up from $PWD until it finds go.mod. Keeps
|
||||||
|
// the test runnable from any working directory.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// mcpBaselineFloor — see header doc. Bump when a deletion is
|
||||||
|
// deliberate; the diff captures the change.
|
||||||
|
const mcpBaselineFloor = 150
|
||||||
|
|
||||||
|
var (
|
||||||
|
mcpToolNameRe = regexp.MustCompile(`^certctl_[a-z][a-z0-9_]*[a-z0-9]$`)
|
||||||
|
mcpNameDeclRe = regexp.MustCompile(`Name:\s*"(certctl_[a-z0-9_]+)"`)
|
||||||
|
methodPathRe = regexp.MustCompile(`^(GET|POST|PUT|DELETE|PATCH|OPTIONS|HEAD) /`)
|
||||||
|
openapiPathRe = regexp.MustCompile(`^ (/[^:]+):\s*$`)
|
||||||
|
openapiVerbRe = regexp.MustCompile(`^ (get|post|put|delete|patch|options|head):\s*$`)
|
||||||
|
caseLiteralRe = regexp.MustCompile(`case\s+"([a-z\-]+)":`)
|
||||||
|
)
|
||||||
|
|
||||||
|
// mcpToolFiles lists the (non-test) Go files expected to register
|
||||||
|
// MCP tools.
|
||||||
|
func mcpToolFiles(repo string) []string {
|
||||||
|
base := filepath.Join(repo, "internal", "mcp")
|
||||||
|
return []string{
|
||||||
|
filepath.Join(base, "tools.go"),
|
||||||
|
filepath.Join(base, "tools_audit_fix.go"),
|
||||||
|
filepath.Join(base, "tools_auth.go"),
|
||||||
|
filepath.Join(base, "tools_auth_bundle2.go"),
|
||||||
|
filepath.Join(base, "tools_est.go"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func findRepoRoot(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
wd, err := os.Getwd()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("getwd: %v", err)
|
||||||
|
}
|
||||||
|
cur := wd
|
||||||
|
for {
|
||||||
|
if _, err := os.Stat(filepath.Join(cur, "go.mod")); err == nil {
|
||||||
|
return cur
|
||||||
|
}
|
||||||
|
parent := filepath.Dir(cur)
|
||||||
|
if parent == cur {
|
||||||
|
t.Fatalf("no go.mod found from %s upward", wd)
|
||||||
|
}
|
||||||
|
cur = parent
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// readFileOrSkip reads a file; on ENOENT, calls t.Skipf rather than
|
||||||
|
// failing — useful for files that may be renamed during refactors.
|
||||||
|
func readFileOrFail(t *testing.T, p string) []byte {
|
||||||
|
t.Helper()
|
||||||
|
body, err := os.ReadFile(p)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
t.Fatalf("expected file missing: %s (refactor? — update the test)", p)
|
||||||
|
}
|
||||||
|
t.Fatalf("read %s: %v", p, err)
|
||||||
|
}
|
||||||
|
return body
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSurfaceParity_MCPToolCatalogue is a HARD gate.
|
||||||
|
//
|
||||||
|
// Asserts:
|
||||||
|
//
|
||||||
|
// - Every MCP tool name conforms to certctl_<word>(_<word>)*.
|
||||||
|
// - No tool name is registered in more than one tools*.go file.
|
||||||
|
// - Total tools ≥ mcpBaselineFloor.
|
||||||
|
func TestSurfaceParity_MCPToolCatalogue(t *testing.T) {
|
||||||
|
repo := findRepoRoot(t)
|
||||||
|
names := map[string]string{}
|
||||||
|
for _, path := range mcpToolFiles(repo) {
|
||||||
|
body := readFileOrFail(t, path)
|
||||||
|
base := filepath.Base(path)
|
||||||
|
for _, m := range mcpNameDeclRe.FindAllStringSubmatch(string(body), -1) {
|
||||||
|
name := m[1]
|
||||||
|
if !mcpToolNameRe.MatchString(name) {
|
||||||
|
t.Errorf("MCP tool name %q in %s does not match certctl_<word>(_<word>)* — fix the name or relax the convention deliberately",
|
||||||
|
name, base)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if other, dup := names[name]; dup {
|
||||||
|
t.Errorf("MCP tool name %q duplicated: first in %s, again in %s — pick a unique name",
|
||||||
|
name, filepath.Base(other), base)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
names[name] = path
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(names) < mcpBaselineFloor {
|
||||||
|
t.Errorf("MCP tool count regressed: %d found, baseline floor %d. "+
|
||||||
|
"If the deletion is intentional, lower mcpBaselineFloor in this test in the SAME commit. "+
|
||||||
|
"If accidental, restore the deleted tools.",
|
||||||
|
len(names), mcpBaselineFloor)
|
||||||
|
}
|
||||||
|
t.Logf("MCP tool catalogue: %d tools (baseline floor %d)", len(names), mcpBaselineFloor)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSurfaceParity_CLICommandCatalogue — informational only.
|
||||||
|
//
|
||||||
|
// Walks cmd/cli/main.go's switch-case dispatcher and reports the
|
||||||
|
// distinct verbs handled. Returns success regardless of contents.
|
||||||
|
// Promoted to fail-on-miss when the CLI surface stabilizes.
|
||||||
|
func TestSurfaceParity_CLICommandCatalogue(t *testing.T) {
|
||||||
|
repo := findRepoRoot(t)
|
||||||
|
body := readFileOrFail(t, filepath.Join(repo, "cmd", "cli", "main.go"))
|
||||||
|
verbs := map[string]struct{}{}
|
||||||
|
for _, m := range caseLiteralRe.FindAllStringSubmatch(string(body), -1) {
|
||||||
|
verbs[m[1]] = struct{}{}
|
||||||
|
}
|
||||||
|
if len(verbs) == 0 {
|
||||||
|
t.Fatal("CLI scanner found zero verbs — likely a refactor; update the test")
|
||||||
|
}
|
||||||
|
out := make([]string, 0, len(verbs))
|
||||||
|
for v := range verbs {
|
||||||
|
out = append(out, v)
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
t.Logf("CLI verb catalogue (%d distinct case literals; informational only per frozen decision 0.9):\n %s",
|
||||||
|
len(out), strings.Join(out, ", "))
|
||||||
|
}
|
||||||
|
|
||||||
|
// scanOpenAPIOps walks openapi.yaml's paths block and returns every
|
||||||
|
// (METHOD, PATH) tuple. Mirrors the parser used by
|
||||||
|
// internal/api/router/openapi_parity_test.go::scanOpenAPIOperations.
|
||||||
|
func scanOpenAPIOps(t *testing.T, path string) []string {
|
||||||
|
t.Helper()
|
||||||
|
body := readFileOrFail(t, path)
|
||||||
|
var out []string
|
||||||
|
inPaths := false
|
||||||
|
currentPath := ""
|
||||||
|
for _, line := range strings.Split(string(body), "\n") {
|
||||||
|
if strings.HasPrefix(line, "paths:") {
|
||||||
|
inPaths = true
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if inPaths && line != "" && !strings.HasPrefix(line, " ") {
|
||||||
|
inPaths = false
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !inPaths {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if m := openapiPathRe.FindStringSubmatch(line); m != nil {
|
||||||
|
currentPath = m[1]
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if m := openapiVerbRe.FindStringSubmatch(line); m != nil && currentPath != "" {
|
||||||
|
out = append(out, strings.ToUpper(m[1])+" "+currentPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// scanRouterRoutes scans internal/api/router/router.go for route
|
||||||
|
// registrations. Uses a regex (not go/ast) to keep this package
|
||||||
|
// stdlib-only; the router.go file is large but the patterns we care
|
||||||
|
// about are stable enough that regex is sufficient.
|
||||||
|
func scanRouterRoutes(t *testing.T, path string) []string {
|
||||||
|
t.Helper()
|
||||||
|
body := readFileOrFail(t, path)
|
||||||
|
// Match r.Register("METHOD /path", ...) and r.mux.Handle("METHOD /path", ...).
|
||||||
|
registerRe := regexp.MustCompile(`r\.Register\(\s*"([A-Z]+ /[^"]+)"`)
|
||||||
|
muxHandleRe := regexp.MustCompile(`r\.mux\.Handle\(\s*"([A-Z]+ /[^"]+)"`)
|
||||||
|
seen := map[string]struct{}{}
|
||||||
|
for _, m := range registerRe.FindAllStringSubmatch(string(body), -1) {
|
||||||
|
seen[m[1]] = struct{}{}
|
||||||
|
}
|
||||||
|
for _, m := range muxHandleRe.FindAllStringSubmatch(string(body), -1) {
|
||||||
|
seen[m[1]] = struct{}{}
|
||||||
|
}
|
||||||
|
out := make([]string, 0, len(seen))
|
||||||
|
for s := range seen {
|
||||||
|
out = append(out, s)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSurfaceParity_OpenAPI_MCPHeuristicCoverage — informational.
|
||||||
|
//
|
||||||
|
// For each OpenAPI operation, splits the path into tokens; if any
|
||||||
|
// token appears in any MCP tool name's token set, the op is "covered."
|
||||||
|
// This is a heuristic — the actual semantic map (operationId →
|
||||||
|
// MCP tool) is not declared in the source, so we approximate.
|
||||||
|
func TestSurfaceParity_OpenAPI_MCPHeuristicCoverage(t *testing.T) {
|
||||||
|
repo := findRepoRoot(t)
|
||||||
|
specOps := scanOpenAPIOps(t, filepath.Join(repo, "api", "openapi.yaml"))
|
||||||
|
mcpTokens := map[string]struct{}{}
|
||||||
|
for _, path := range mcpToolFiles(repo) {
|
||||||
|
body := readFileOrFail(t, path)
|
||||||
|
for _, m := range mcpNameDeclRe.FindAllStringSubmatch(string(body), -1) {
|
||||||
|
for _, tok := range strings.Split(strings.TrimPrefix(m[1], "certctl_"), "_") {
|
||||||
|
mcpTokens[tok] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
covered := 0
|
||||||
|
for _, op := range specOps {
|
||||||
|
parts := strings.Split(op, " ")
|
||||||
|
if len(parts) != 2 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
segs := strings.FieldsFunc(parts[1], func(r rune) bool {
|
||||||
|
return r == '/' || r == '{' || r == '}' || r == '-'
|
||||||
|
})
|
||||||
|
hit := false
|
||||||
|
for _, s := range segs {
|
||||||
|
if _, ok := mcpTokens[strings.ToLower(s)]; ok {
|
||||||
|
hit = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if hit {
|
||||||
|
covered++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(specOps) == 0 {
|
||||||
|
t.Fatal("openapi.yaml scan returned zero operations — fix the test")
|
||||||
|
}
|
||||||
|
pct := (covered * 100) / len(specOps)
|
||||||
|
t.Logf("OpenAPI↔MCP heuristic coverage (informational): %d of %d ops (%d%%) share at least one path token with an MCP tool name",
|
||||||
|
covered, len(specOps), pct)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSurfaceParity_Summary prints the four surface counts side-by-side.
|
||||||
|
func TestSurfaceParity_Summary(t *testing.T) {
|
||||||
|
repo := findRepoRoot(t)
|
||||||
|
routes := scanRouterRoutes(t, filepath.Join(repo, "internal", "api", "router", "router.go"))
|
||||||
|
specOps := scanOpenAPIOps(t, filepath.Join(repo, "api", "openapi.yaml"))
|
||||||
|
mcpCount := 0
|
||||||
|
for _, path := range mcpToolFiles(repo) {
|
||||||
|
body := readFileOrFail(t, path)
|
||||||
|
mcpCount += len(mcpNameDeclRe.FindAllStringSubmatch(string(body), -1))
|
||||||
|
}
|
||||||
|
cliBody := readFileOrFail(t, filepath.Join(repo, "cmd", "cli", "main.go"))
|
||||||
|
cliCount := len(caseLiteralRe.FindAllStringSubmatch(string(cliBody), -1))
|
||||||
|
t.Logf("Surface parity summary (informational):\n"+
|
||||||
|
" router routes : %d\n"+
|
||||||
|
" OpenAPI ops : %d\n"+
|
||||||
|
" MCP tools : %d (floor %d)\n"+
|
||||||
|
" CLI verbs : %d (warn-only — frozen decision 0.9)",
|
||||||
|
len(routes), len(specOps), mcpCount, mcpBaselineFloor, cliCount)
|
||||||
|
}
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
# scripts/ci-guards/surface-parity-mcp-exemptions.yaml
|
||||||
|
#
|
||||||
|
# Allowlist for OpenAPI operations that are intentionally NOT mirrored
|
||||||
|
# in the MCP tool catalogue. Consumed by
|
||||||
|
# internal/api/router/surface_parity_test.go::TestSurfaceParity_*.
|
||||||
|
#
|
||||||
|
# The current MCP parity tests are informational (per frozen decision
|
||||||
|
# 0.9). This file exists so when those tests are promoted to hard
|
||||||
|
# gates, the carve-outs are already documented and the promotion is
|
||||||
|
# mechanical.
|
||||||
|
#
|
||||||
|
# Each entry shape:
|
||||||
|
#
|
||||||
|
# - operation: "METHOD /api/v1/path"
|
||||||
|
# justification: "one-line reason this is legitimately HTTP-only"
|
||||||
|
# expires: "YYYY-MM-DD" # required; 90-day default
|
||||||
|
#
|
||||||
|
# Categories of legitimate carve-outs (DO add these when the test is
|
||||||
|
# promoted to fail-on-miss):
|
||||||
|
#
|
||||||
|
# - ACME wire protocol (RFC 8555 + RFC 9773 ARI)
|
||||||
|
# - SCEP wire protocol (RFC 8894)
|
||||||
|
# - EST wire protocol (RFC 7030)
|
||||||
|
# - OCSP responder
|
||||||
|
# - CRL distribution
|
||||||
|
# - Healthcheck / readiness / version endpoints
|
||||||
|
# - OIDC callback / back-channel-logout
|
||||||
|
# - SPA fallback for the embedded web UI
|
||||||
|
#
|
||||||
|
# DO NOT add entries here to silence the test on an oversight. If an
|
||||||
|
# operation should have an MCP tool and doesn't, that's the bug — add
|
||||||
|
# the tool.
|
||||||
|
|
||||||
|
exceptions: []
|
||||||
Reference in New Issue
Block a user