mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-19 19:01:42 +00:00
d076a434d4
* test(e2e): add targeted suite selector + shadow-mode CI step
select-suites maps a diff to the e2e suites it must run: changed files →
services (manifest path/additional_paths) → consumer closure over
depends_on vs publishes/subscribes → suites whose covers intersect.
Special cases force-all (infra/harness/workflow, ≥3 services, submodule
bumps) or a lone suite dir; empty diffs fall back to default suites. An
AST scan of each suite's harness.Ensure* call graph derives per-test
service coverage and emits an optional -test.run subset (RUN_PATTERN).
Wired into e2e.yml as a non-gating 'Select Suites (shadow)' job that
logs the computed selection to the run summary so predicted vs actual
can be compared before the selector drives SUITES.
* ci(e2e): add dispatch-only per-suite job blocks grouped by env
Restructure the looped e2e flow into env-grouped provision -> matrix
suites -> teardown blocks so each suite surfaces as its own node in the
Actions graph instead of a single green tick:
single_provision -> single_suite [single,cert,lb,eks] -> single_teardown
multi_provision -> multi_suite [multinode] -> multi_teardown
Provision resolves the VM/cluster and ships test binaries, emitting WAN
IP + ssh key path as job outputs; each matrix leg runs one suite over
SSH and uploads its own junit/analysis artifacts; teardown (if:always)
reuses the persisted tofu workspace on the same runner to destroy.
workflow_dispatch only -- runs alongside the gating e2e.yml for
validation and posts no commit status.
* test(e2e): add manifest drift lint (fixture-create + NATS subject)
Bead 5 of the targeted-suite-selection plan. Two ratcheted drift guards in
tests/e2e/manifest-lint, wired into make preflight and the spinifex.yml PR job:
- Fixture lint: AST-flags direct resource-create AWS calls
(c.EC2.RunInstances, c.ELBv2.CreateLoadBalancer, ...) in suite test files,
steering them through harness.Ensure* fixtures. ~104 pre-existing call sites
are captured in a checked-in baseline; the lint fails only on NEW creates.
Intentional create-path tests opt out with an inline // e2e:allow-create.
- Subject lint: extracts NATS subject literals from the natsSub{} registry and
literal Subscribe/Publish/Request args, cross-referenced wildcard-aware
against service-interfaces.yaml. Code subjects missing from the manifest fail
(ratcheted); manifest subjects with no static reference warn only (many are
produced dynamically). Fixed real drift surfaced on first run: declared the
five spinifex.cluster.shutdown.* phase subjects under spinifex_daemon.
manifest-check is now CI-enforced alongside manifest-lint (previously
preflight/local-only).
* feat: run-scoped e2e teardown sweep
Tag every Ensure* fixture resource with e2e:run=<run-id> when
SPINIFEX_E2E_RUN_ID is set, and add a teardown.test suite that sweeps
everything carrying that tag at end of run — reclaiming resources leaked
by a crashed suite, an interrupted run, or a persistent cluster.
- harness/sweep.go: tagRunResources (EC2 CreateTags) + tagRunELB (ELBv2
AddTags) + SweepRunResources in dependency order (LB, instances, NAT,
snapshots, volumes, images, SGs, subnets, key pairs). Blank run id
no-ops both tagging and sweeping, so the sweep can never touch an
untagged or foreign resource.
- harness/fixtures.go: stamp the run tag on each created resource.
- tests/e2e/teardown: thin suite that runs the sweep; skips without a
cluster or run id, logs the report, never fails the run.
- e2e-suites.yml: build + ship teardown.test, pass the run id into suite
legs, sweep on the VM in both teardown jobs before tofu destroy.
197 lines
5.2 KiB
Go
197 lines
5.2 KiB
Go
package selectsuites
|
|
|
|
import (
|
|
"go/ast"
|
|
"go/parser"
|
|
"go/token"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
)
|
|
|
|
// SuiteCoverage records, per suite, which services each top-level Test*
|
|
// function exercises. Coverage is derived by walking the suite's package call
|
|
// graph from each Test to the harness.Ensure* helpers it reaches, then mapping
|
|
// those fixtures to services via the manifest's fixtures block.
|
|
type SuiteCoverage struct {
|
|
Path string
|
|
Tests map[string][]string // test func name → sorted service names
|
|
}
|
|
|
|
// ScanSuites builds per-test coverage for every suite in the manifest. repoRoot
|
|
// is the spinifex repo root (suite paths are resolved beneath it). Suites whose
|
|
// directory is missing or has no test files are skipped silently — the selector
|
|
// then runs them in full.
|
|
func ScanSuites(repoRoot string, m *Manifest) (map[string]SuiteCoverage, error) {
|
|
fixSvc := map[string][]string{}
|
|
for name, f := range m.Fixtures {
|
|
fixSvc[name] = f.Services
|
|
}
|
|
out := map[string]SuiteCoverage{}
|
|
for name, s := range m.Suites {
|
|
dir := filepath.Join(repoRoot, filepath.FromSlash(s.Path))
|
|
cov, err := scanSuiteDir(dir, fixSvc)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if cov == nil {
|
|
continue
|
|
}
|
|
out[name] = SuiteCoverage{Path: s.Path, Tests: cov}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// scanSuiteDir parses every *_test.go in dir, builds the local call graph, and
|
|
// returns testName → services. Returns nil (not an error) when the dir has no
|
|
// Go test files.
|
|
func scanSuiteDir(dir string, fixSvc map[string][]string) (map[string][]string, error) {
|
|
entries, err := os.ReadDir(dir)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil, nil
|
|
}
|
|
return nil, err
|
|
}
|
|
fset := token.NewFileSet()
|
|
// directEnsure[func] = set of Ensure* fixture names called directly in func.
|
|
directEnsure := map[string]map[string]bool{}
|
|
// localCalls[func] = set of package-local func names called in func.
|
|
localCalls := map[string]map[string]bool{}
|
|
tests := map[string]bool{}
|
|
found := false
|
|
|
|
for _, e := range entries {
|
|
if e.IsDir() || !strings.HasSuffix(e.Name(), "_test.go") {
|
|
continue
|
|
}
|
|
found = true
|
|
f, perr := parser.ParseFile(fset, filepath.Join(dir, e.Name()), nil, 0)
|
|
if perr != nil {
|
|
return nil, perr
|
|
}
|
|
for _, decl := range f.Decls {
|
|
fn, ok := decl.(*ast.FuncDecl)
|
|
if !ok || fn.Body == nil || fn.Recv != nil {
|
|
continue
|
|
}
|
|
name := fn.Name.Name
|
|
if isTestFunc(fn) {
|
|
tests[name] = true
|
|
}
|
|
de, lc := scanFuncBody(fn.Body)
|
|
directEnsure[name] = de
|
|
localCalls[name] = lc
|
|
}
|
|
}
|
|
if !found {
|
|
return nil, nil
|
|
}
|
|
|
|
out := map[string][]string{}
|
|
for t := range tests {
|
|
ensure := reachableEnsure(t, directEnsure, localCalls)
|
|
svc := map[string]bool{}
|
|
for fx := range ensure {
|
|
for _, s := range fixSvc[fx] {
|
|
svc[s] = true
|
|
}
|
|
}
|
|
out[t] = sortedKeys(svc)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// scanFuncBody collects the harness.Ensure* fixture calls and package-local
|
|
// function calls made directly inside body.
|
|
func scanFuncBody(body *ast.BlockStmt) (ensure, local map[string]bool) {
|
|
ensure = map[string]bool{}
|
|
local = map[string]bool{}
|
|
ast.Inspect(body, func(n ast.Node) bool {
|
|
call, ok := n.(*ast.CallExpr)
|
|
if !ok {
|
|
return true
|
|
}
|
|
switch fn := call.Fun.(type) {
|
|
case *ast.SelectorExpr:
|
|
if pkg, ok := fn.X.(*ast.Ident); ok && pkg.Name == "harness" &&
|
|
strings.HasPrefix(fn.Sel.Name, "Ensure") {
|
|
ensure[fn.Sel.Name] = true
|
|
}
|
|
case *ast.Ident:
|
|
local[fn.Name] = true
|
|
}
|
|
return true
|
|
})
|
|
return ensure, local
|
|
}
|
|
|
|
// reachableEnsure returns every Ensure* fixture reachable from start by
|
|
// following package-local calls (BFS over the local call graph).
|
|
func reachableEnsure(start string, direct, local map[string]map[string]bool) map[string]bool {
|
|
seen := map[string]bool{}
|
|
ensure := map[string]bool{}
|
|
queue := []string{start}
|
|
for len(queue) > 0 {
|
|
fn := queue[0]
|
|
queue = queue[1:]
|
|
if seen[fn] {
|
|
continue
|
|
}
|
|
seen[fn] = true
|
|
for e := range direct[fn] {
|
|
ensure[e] = true
|
|
}
|
|
for callee := range local[fn] {
|
|
if !seen[callee] {
|
|
queue = append(queue, callee)
|
|
}
|
|
}
|
|
}
|
|
return ensure
|
|
}
|
|
|
|
func isTestFunc(fn *ast.FuncDecl) bool {
|
|
if !strings.HasPrefix(fn.Name.Name, "Test") {
|
|
return false
|
|
}
|
|
// Exclude TestMain — it is the process entry, not a coverage unit.
|
|
return fn.Name.Name != "TestMain"
|
|
}
|
|
|
|
// runPatterns builds a suite→regex map keeping only tests that touch a covered
|
|
// service (tests with no analyzable fixture calls are always kept, so narrowing
|
|
// never drops an unanalyzable test). A pattern is emitted only when it is a
|
|
// strict, non-empty subset of the suite's tests.
|
|
func runPatterns(perTest map[string]SuiteCoverage, suites []string, covered map[string]bool) map[string]string {
|
|
out := map[string]string{}
|
|
for _, suite := range suites {
|
|
cov, ok := perTest[suite]
|
|
if !ok || len(cov.Tests) == 0 {
|
|
continue
|
|
}
|
|
var keep []string
|
|
for t, svcs := range cov.Tests {
|
|
if len(svcs) == 0 || intersects(svcs, covered) {
|
|
keep = append(keep, t)
|
|
}
|
|
}
|
|
if len(keep) == 0 || len(keep) >= len(cov.Tests) {
|
|
continue // empty or no narrowing — run the whole suite
|
|
}
|
|
sort.Strings(keep)
|
|
out[suite] = "^(" + strings.Join(keep, "|") + ")$"
|
|
}
|
|
return out
|
|
}
|
|
|
|
func intersects(svcs []string, set map[string]bool) bool {
|
|
for _, s := range svcs {
|
|
if set[s] {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|