Files
Josh Siviour d076a434d4 Feat/e2e targeted suites (#352)
* test(e2e): add targeted suite selector + shadow-mode CI step

select-suites maps a diff to the e2e suites it must run: changed files →
services (manifest path/additional_paths) → consumer closure over
depends_on vs publishes/subscribes → suites whose covers intersect.
Special cases force-all (infra/harness/workflow, ≥3 services, submodule
bumps) or a lone suite dir; empty diffs fall back to default suites. An
AST scan of each suite's harness.Ensure* call graph derives per-test
service coverage and emits an optional -test.run subset (RUN_PATTERN).

Wired into e2e.yml as a non-gating 'Select Suites (shadow)' job that
logs the computed selection to the run summary so predicted vs actual
can be compared before the selector drives SUITES.

* ci(e2e): add dispatch-only per-suite job blocks grouped by env

Restructure the looped e2e flow into env-grouped provision -> matrix
suites -> teardown blocks so each suite surfaces as its own node in the
Actions graph instead of a single green tick:

  single_provision -> single_suite [single,cert,lb,eks] -> single_teardown
  multi_provision  -> multi_suite  [multinode]          -> multi_teardown

Provision resolves the VM/cluster and ships test binaries, emitting WAN
IP + ssh key path as job outputs; each matrix leg runs one suite over
SSH and uploads its own junit/analysis artifacts; teardown (if:always)
reuses the persisted tofu workspace on the same runner to destroy.

workflow_dispatch only -- runs alongside the gating e2e.yml for
validation and posts no commit status.

* test(e2e): add manifest drift lint (fixture-create + NATS subject)

Bead 5 of the targeted-suite-selection plan. Two ratcheted drift guards in
tests/e2e/manifest-lint, wired into make preflight and the spinifex.yml PR job:

- Fixture lint: AST-flags direct resource-create AWS calls
  (c.EC2.RunInstances, c.ELBv2.CreateLoadBalancer, ...) in suite test files,
  steering them through harness.Ensure* fixtures. ~104 pre-existing call sites
  are captured in a checked-in baseline; the lint fails only on NEW creates.
  Intentional create-path tests opt out with an inline // e2e:allow-create.

- Subject lint: extracts NATS subject literals from the natsSub{} registry and
  literal Subscribe/Publish/Request args, cross-referenced wildcard-aware
  against service-interfaces.yaml. Code subjects missing from the manifest fail
  (ratcheted); manifest subjects with no static reference warn only (many are
  produced dynamically). Fixed real drift surfaced on first run: declared the
  five spinifex.cluster.shutdown.* phase subjects under spinifex_daemon.

manifest-check is now CI-enforced alongside manifest-lint (previously
preflight/local-only).

* feat: run-scoped e2e teardown sweep

Tag every Ensure* fixture resource with e2e:run=<run-id> when
SPINIFEX_E2E_RUN_ID is set, and add a teardown.test suite that sweeps
everything carrying that tag at end of run — reclaiming resources leaked
by a crashed suite, an interrupted run, or a persistent cluster.

- harness/sweep.go: tagRunResources (EC2 CreateTags) + tagRunELB (ELBv2
  AddTags) + SweepRunResources in dependency order (LB, instances, NAT,
  snapshots, volumes, images, SGs, subnets, key pairs). Blank run id
  no-ops both tagging and sweeping, so the sweep can never touch an
  untagged or foreign resource.
- harness/fixtures.go: stamp the run tag on each created resource.
- tests/e2e/teardown: thin suite that runs the sweep; skips without a
  cluster or run id, logs the report, never fails the run.
- e2e-suites.yml: build + ship teardown.test, pass the run id into suite
  legs, sweep on the VM in both teardown jobs before tofu destroy.
2026-06-11 21:09:50 +10:00

197 lines
5.2 KiB
Go

package selectsuites
import (
"go/ast"
"go/parser"
"go/token"
"os"
"path/filepath"
"sort"
"strings"
)
// SuiteCoverage records, per suite, which services each top-level Test*
// function exercises. Coverage is derived by walking the suite's package call
// graph from each Test to the harness.Ensure* helpers it reaches, then mapping
// those fixtures to services via the manifest's fixtures block.
type SuiteCoverage struct {
Path string
Tests map[string][]string // test func name → sorted service names
}
// ScanSuites builds per-test coverage for every suite in the manifest. repoRoot
// is the spinifex repo root (suite paths are resolved beneath it). Suites whose
// directory is missing or has no test files are skipped silently — the selector
// then runs them in full.
func ScanSuites(repoRoot string, m *Manifest) (map[string]SuiteCoverage, error) {
fixSvc := map[string][]string{}
for name, f := range m.Fixtures {
fixSvc[name] = f.Services
}
out := map[string]SuiteCoverage{}
for name, s := range m.Suites {
dir := filepath.Join(repoRoot, filepath.FromSlash(s.Path))
cov, err := scanSuiteDir(dir, fixSvc)
if err != nil {
return nil, err
}
if cov == nil {
continue
}
out[name] = SuiteCoverage{Path: s.Path, Tests: cov}
}
return out, nil
}
// scanSuiteDir parses every *_test.go in dir, builds the local call graph, and
// returns testName → services. Returns nil (not an error) when the dir has no
// Go test files.
func scanSuiteDir(dir string, fixSvc map[string][]string) (map[string][]string, error) {
entries, err := os.ReadDir(dir)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
fset := token.NewFileSet()
// directEnsure[func] = set of Ensure* fixture names called directly in func.
directEnsure := map[string]map[string]bool{}
// localCalls[func] = set of package-local func names called in func.
localCalls := map[string]map[string]bool{}
tests := map[string]bool{}
found := false
for _, e := range entries {
if e.IsDir() || !strings.HasSuffix(e.Name(), "_test.go") {
continue
}
found = true
f, perr := parser.ParseFile(fset, filepath.Join(dir, e.Name()), nil, 0)
if perr != nil {
return nil, perr
}
for _, decl := range f.Decls {
fn, ok := decl.(*ast.FuncDecl)
if !ok || fn.Body == nil || fn.Recv != nil {
continue
}
name := fn.Name.Name
if isTestFunc(fn) {
tests[name] = true
}
de, lc := scanFuncBody(fn.Body)
directEnsure[name] = de
localCalls[name] = lc
}
}
if !found {
return nil, nil
}
out := map[string][]string{}
for t := range tests {
ensure := reachableEnsure(t, directEnsure, localCalls)
svc := map[string]bool{}
for fx := range ensure {
for _, s := range fixSvc[fx] {
svc[s] = true
}
}
out[t] = sortedKeys(svc)
}
return out, nil
}
// scanFuncBody collects the harness.Ensure* fixture calls and package-local
// function calls made directly inside body.
func scanFuncBody(body *ast.BlockStmt) (ensure, local map[string]bool) {
ensure = map[string]bool{}
local = map[string]bool{}
ast.Inspect(body, func(n ast.Node) bool {
call, ok := n.(*ast.CallExpr)
if !ok {
return true
}
switch fn := call.Fun.(type) {
case *ast.SelectorExpr:
if pkg, ok := fn.X.(*ast.Ident); ok && pkg.Name == "harness" &&
strings.HasPrefix(fn.Sel.Name, "Ensure") {
ensure[fn.Sel.Name] = true
}
case *ast.Ident:
local[fn.Name] = true
}
return true
})
return ensure, local
}
// reachableEnsure returns every Ensure* fixture reachable from start by
// following package-local calls (BFS over the local call graph).
func reachableEnsure(start string, direct, local map[string]map[string]bool) map[string]bool {
seen := map[string]bool{}
ensure := map[string]bool{}
queue := []string{start}
for len(queue) > 0 {
fn := queue[0]
queue = queue[1:]
if seen[fn] {
continue
}
seen[fn] = true
for e := range direct[fn] {
ensure[e] = true
}
for callee := range local[fn] {
if !seen[callee] {
queue = append(queue, callee)
}
}
}
return ensure
}
func isTestFunc(fn *ast.FuncDecl) bool {
if !strings.HasPrefix(fn.Name.Name, "Test") {
return false
}
// Exclude TestMain — it is the process entry, not a coverage unit.
return fn.Name.Name != "TestMain"
}
// runPatterns builds a suite→regex map keeping only tests that touch a covered
// service (tests with no analyzable fixture calls are always kept, so narrowing
// never drops an unanalyzable test). A pattern is emitted only when it is a
// strict, non-empty subset of the suite's tests.
func runPatterns(perTest map[string]SuiteCoverage, suites []string, covered map[string]bool) map[string]string {
out := map[string]string{}
for _, suite := range suites {
cov, ok := perTest[suite]
if !ok || len(cov.Tests) == 0 {
continue
}
var keep []string
for t, svcs := range cov.Tests {
if len(svcs) == 0 || intersects(svcs, covered) {
keep = append(keep, t)
}
}
if len(keep) == 0 || len(keep) >= len(cov.Tests) {
continue // empty or no narrowing — run the whole suite
}
sort.Strings(keep)
out[suite] = "^(" + strings.Join(keep, "|") + ")$"
}
return out
}
func intersects(svcs []string, set map[string]bool) bool {
for _, s := range svcs {
if set[s] {
return true
}
}
return false
}