mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-19 19:01:42 +00:00
Josh Siviour
d076a434d4
* test(e2e): add targeted suite selector + shadow-mode CI step
select-suites maps a diff to the e2e suites it must run: changed files →
services (manifest path/additional_paths) → consumer closure over
depends_on vs publishes/subscribes → suites whose covers intersect.
Special cases force-all (infra/harness/workflow, ≥3 services, submodule
bumps) or a lone suite dir; empty diffs fall back to default suites. An
AST scan of each suite's harness.Ensure* call graph derives per-test
service coverage and emits an optional -test.run subset (RUN_PATTERN).
Wired into e2e.yml as a non-gating 'Select Suites (shadow)' job that
logs the computed selection to the run summary so predicted vs actual
can be compared before the selector drives SUITES.
* ci(e2e): add dispatch-only per-suite job blocks grouped by env
Restructure the looped e2e flow into env-grouped provision -> matrix
suites -> teardown blocks so each suite surfaces as its own node in the
Actions graph instead of a single green tick:
single_provision -> single_suite [single,cert,lb,eks] -> single_teardown
multi_provision -> multi_suite [multinode] -> multi_teardown
Provision resolves the VM/cluster and ships test binaries, emitting WAN
IP + ssh key path as job outputs; each matrix leg runs one suite over
SSH and uploads its own junit/analysis artifacts; teardown (if:always)
reuses the persisted tofu workspace on the same runner to destroy.
workflow_dispatch only -- runs alongside the gating e2e.yml for
validation and posts no commit status.
* test(e2e): add manifest drift lint (fixture-create + NATS subject)
Bead 5 of the targeted-suite-selection plan. Two ratcheted drift guards in
tests/e2e/manifest-lint, wired into make preflight and the spinifex.yml PR job:
- Fixture lint: AST-flags direct resource-create AWS calls
(c.EC2.RunInstances, c.ELBv2.CreateLoadBalancer, ...) in suite test files,
steering them through harness.Ensure* fixtures. ~104 pre-existing call sites
are captured in a checked-in baseline; the lint fails only on NEW creates.
Intentional create-path tests opt out with an inline // e2e:allow-create.
- Subject lint: extracts NATS subject literals from the natsSub{} registry and
literal Subscribe/Publish/Request args, cross-referenced wildcard-aware
against service-interfaces.yaml. Code subjects missing from the manifest fail
(ratcheted); manifest subjects with no static reference warn only (many are
produced dynamically). Fixed real drift surfaced on first run: declared the
five spinifex.cluster.shutdown.* phase subjects under spinifex_daemon.
manifest-check is now CI-enforced alongside manifest-lint (previously
preflight/local-only).
* feat: run-scoped e2e teardown sweep
Tag every Ensure* fixture resource with e2e:run=<run-id> when
SPINIFEX_E2E_RUN_ID is set, and add a teardown.test suite that sweeps
everything carrying that tag at end of run — reclaiming resources leaked
by a crashed suite, an interrupted run, or a persistent cluster.
- harness/sweep.go: tagRunResources (EC2 CreateTags) + tagRunELB (ELBv2
AddTags) + SweepRunResources in dependency order (LB, instances, NAT,
snapshots, volumes, images, SGs, subnets, key pairs). Blank run id
no-ops both tagging and sweeping, so the sweep can never touch an
untagged or foreign resource.
- harness/fixtures.go: stamp the run tag on each created resource.
- tests/e2e/teardown: thin suite that runs the sweep; skips without a
cluster or run id, logs the report, never fails the run.
- e2e-suites.yml: build + ship teardown.test, pass the run id into suite
legs, sweep on the VM in both teardown jobs before tofu destroy.
353 lines
14 KiB
Makefile
353 lines
14 KiB
Makefile
GO_PROJECT_NAME := spx
|
|
SHELL := /bin/bash
|
|
|
|
# Detect architecture for cross-platform support
|
|
ARCH := $(shell uname -m)
|
|
ifeq ($(ARCH),x86_64)
|
|
GO_ARCH := amd64
|
|
AWS_ARCH := x86_64
|
|
else ifeq ($(ARCH),aarch64)
|
|
GO_ARCH := arm64
|
|
AWS_ARCH := aarch64
|
|
else ifeq ($(ARCH),arm64)
|
|
GO_ARCH := arm64
|
|
AWS_ARCH := aarch64
|
|
else
|
|
$(error Unsupported architecture: $(ARCH). Only x86_64 and aarch64/arm64 are supported.)
|
|
endif
|
|
|
|
# Ask Go whether workspace mode is active
|
|
IN_WORKSPACE := $(shell go env GOWORK)
|
|
|
|
# Use -mod=mod unless Go reports an active workspace path
|
|
ifeq ($(IN_WORKSPACE),)
|
|
GO_BUILD_MOD := -mod=mod
|
|
else ifeq ($(IN_WORKSPACE),off)
|
|
GO_BUILD_MOD := -mod=mod
|
|
else
|
|
GO_BUILD_MOD :=
|
|
endif
|
|
|
|
# Quiet-mode filters (active when QUIET=1, set by preflight via recursive make)
|
|
# Note: grep pipelines use PIPESTATUS[0] so the exit status of `go test`
|
|
# propagates through the filter — otherwise a test failure is swallowed by
|
|
# grep's own (success) exit code and preflight prints "passed" on red.
|
|
ifdef QUIET
|
|
_Q = @
|
|
_COVQ = 2>&1 | { grep -Ev '^\s*(ok|PASS|\?|=== RUN|--- PASS:)\s' | grep -v 'coverage: 0\.0%' || true; }; exit $${PIPESTATUS[0]}
|
|
_RACEQ = 2>&1 | { grep -Ev '^\s*(ok|PASS|\?|=== RUN|--- PASS:)\s' || true; }; exit $${PIPESTATUS[0]}
|
|
_SECQ = >
|
|
else
|
|
_Q =
|
|
_COVQ =
|
|
_RACEQ =
|
|
_SECQ = 2>&1 | tee
|
|
endif
|
|
|
|
build: go_build build-installer build-lb-agent
|
|
|
|
# Build spinifex-ui frontend (requires pnpm)
|
|
build-ui:
|
|
@echo -e "\n....Building spinifex-ui frontend...."
|
|
cd spinifex/services/spinifexui/frontend && pnpm build
|
|
|
|
# GO commands
|
|
VERSION ?= $(shell git describe --tags --always --dirty)
|
|
COMMIT ?= $(shell git rev-parse --short HEAD)
|
|
LDFLAGS := -s -w -X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Version=$(VERSION) -X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Commit=$(COMMIT)
|
|
|
|
go_build:
|
|
@echo -e "\n....Building $(GO_PROJECT_NAME)"
|
|
GOFIPS140=v1.0.0 go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go
|
|
|
|
build-installer:
|
|
@echo -e "\n....Building spinifex-installer"
|
|
GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/spinifex-installer cmd/installer/main.go
|
|
|
|
build-lb-agent:
|
|
@echo -e "\n....Building lb-agent (static)"
|
|
CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/lb-agent cmd/lb-agent/main.go
|
|
|
|
build-system-image: ## Build a system image from manifest (use IMAGE=lb or IMAGE=eks-node)
|
|
ifndef IMAGE
|
|
$(error IMAGE is required. Usage: make build-system-image IMAGE=lb)
|
|
endif
|
|
@if [ -f scripts/images/$(IMAGE).conf ]; then \
|
|
./scripts/build-system-image.sh scripts/images/$(IMAGE).conf $(if $(IMPORT),--import); \
|
|
elif [ -f scripts/images/$(IMAGE)/manifest.conf ]; then \
|
|
./scripts/build-system-image.sh scripts/images/$(IMAGE)/manifest.conf $(if $(IMPORT),--import); \
|
|
else \
|
|
echo "ERROR: no manifest at scripts/images/$(IMAGE).conf or scripts/images/$(IMAGE)/manifest.conf"; \
|
|
exit 1; \
|
|
fi
|
|
|
|
build-eks-node-image: ## Build the unified eks-node AMI (K3s server+agent; role at first boot; IMPORT=1 to register)
|
|
$(MAKE) build-system-image IMAGE=eks-node
|
|
|
|
import-eks-node-image: ## Build + register the eks-node AMI (requires a running cluster)
|
|
$(MAKE) build-system-image IMAGE=eks-node IMPORT=1
|
|
|
|
publish-eks-node-image: ## Build + publish the eks-node AMI to Cloudflare R2 (needs R2_ENDPOINT + AWS_* env)
|
|
./scripts/publish-system-image.sh scripts/images/eks-node/manifest.conf --build
|
|
|
|
MICROVM_OUT_DIR := build/microvm
|
|
MICROVM_ARTIFACTS := $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz
|
|
MICROVM_INPUTS := scripts/build-microvm-image.sh $(MICROVM_OUT_DIR)/init.sh $(MICROVM_OUT_DIR)/inittab bin/lb-agent
|
|
|
|
# Grouped target — script writes both files in one run.
|
|
$(MICROVM_ARTIFACTS) &: $(MICROVM_INPUTS)
|
|
./scripts/build-microvm-image.sh
|
|
|
|
# Only triggers when bin/lb-agent is missing; preserves the artifact's mtime so
|
|
# build-microvm-image stays correctly stale-aware.
|
|
bin/lb-agent:
|
|
$(MAKE) build-lb-agent
|
|
|
|
build-microvm-image: $(MICROVM_ARTIFACTS) ## Build microVM kernel + initramfs (incremental — skips when up to date)
|
|
|
|
install-microvm: $(MICROVM_ARTIFACTS) ## Install microVM artifacts to /usr/share/spinifex/microvm/
|
|
sudo install -d /usr/share/spinifex/microvm
|
|
sudo install -m 0644 $(MICROVM_OUT_DIR)/vmlinuz /usr/share/spinifex/microvm/vmlinuz
|
|
sudo install -m 0644 $(MICROVM_OUT_DIR)/initramfs.cpio.gz /usr/share/spinifex/microvm/initramfs.cpio.gz
|
|
|
|
# Preflight — runs the same checks as GitHub Actions (lint + vuln + tests).
|
|
# Use this before committing to catch CI failures locally.
|
|
preflight:
|
|
@$(MAKE) --no-print-directory QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-race test-harness
|
|
@echo -e "\n ✅ Preflight passed — safe to commit."
|
|
|
|
# E2E harness unit tests. Build-tagged `e2e` so they're skipped by the
|
|
# default `go test ./spinifex/...`. Runs with mocked AWS clients — no
|
|
# infrastructure required, safe to run in CI without a cluster.
|
|
test-harness:
|
|
@echo -e "\n....Running e2e harness unit tests...."
|
|
$(_Q)LOG_IGNORE=1 go test -tags=e2e -timeout 60s ./tests/e2e/harness/... $(_RACEQ)
|
|
|
|
# Validate docs/service-interfaces.yaml. Schema check + cross-reference
|
|
# of services/suites/fixtures + on-disk path existence. Subject content
|
|
# vs source is enforced separately in Bead 5 drift lint.
|
|
manifest-check:
|
|
@echo -e "\n....Checking service-interfaces.yaml...."
|
|
@go run ./tests/e2e/manifest-check/cmd/manifest-check -repo-root . -manifest docs/service-interfaces.yaml
|
|
|
|
# Drift guards (Bead 5): direct-create fixture lint + NATS subject lint,
|
|
# ratcheted against tests/e2e/manifest-lint/baseline.txt. Fails only on NEW
|
|
# drift beyond the baseline.
|
|
manifest-lint:
|
|
@echo -e "\n....Linting manifest drift (fixtures + subjects)...."
|
|
@go run ./tests/e2e/manifest-lint/cmd/manifest-lint -repo-root .
|
|
|
|
# Accept current drift into the baseline. Run after an intentional change.
|
|
manifest-lint-update:
|
|
@go run ./tests/e2e/manifest-lint/cmd/manifest-lint -repo-root . -update
|
|
|
|
# Run unit tests
|
|
test:
|
|
@echo -e "\n....Running tests for $(GO_PROJECT_NAME)...."
|
|
LOG_IGNORE=1 go test -timeout 120s ./spinifex/...
|
|
|
|
# Run unit tests with coverage profile
|
|
COVERPROFILE ?= coverage.out
|
|
test-cover:
|
|
@echo -e "\n....Running tests with coverage for $(GO_PROJECT_NAME)...."
|
|
$(_Q)LOG_IGNORE=1 go test -timeout 120s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... $(_COVQ)
|
|
@scripts/check-coverage.sh $(COVERPROFILE) $(QUIET)
|
|
|
|
# Run unit tests with race detector
|
|
test-race:
|
|
@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
|
|
$(_Q)LOG_IGNORE=1 go test -race -timeout 300s ./spinifex/... $(_RACEQ)
|
|
|
|
# Unit tests for in-repo GitHub Actions (e.g. .github/actions/e2e-analyze).
|
|
# Kept out of `test-cover` so coverage % isn't diluted by CI-only tooling.
|
|
test-actions:
|
|
@echo -e "\n....Running action tests...."
|
|
LOG_IGNORE=1 go test -timeout 60s ./.github/actions/...
|
|
|
|
# Check that new/changed code meets coverage threshold (runs tests first)
|
|
diff-coverage: test-cover
|
|
@QUIET=$(QUIET) scripts/diff-coverage.sh $(COVERPROFILE)
|
|
|
|
bench:
|
|
@echo -e "\n....Running benchmarks for $(GO_PROJECT_NAME)...."
|
|
LOG_IGNORE=1 go test -benchmem -run=. -bench=. ./...
|
|
|
|
# Fast iteration: build + install binary + restart all services.
|
|
# Microvm artifacts are reinstalled when they already exist on disk — the rule's
|
|
# input timestamps drive a rebuild only if anything actually changed. On a fresh
|
|
# checkout (no build/microvm/vmlinuz yet) the install step is skipped; run
|
|
# `make install-microvm` explicitly the first time.
|
|
deploy: build
|
|
sudo install -m 755 bin/spx /usr/local/bin/spx
|
|
@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ]; then \
|
|
$(MAKE) install-microvm; \
|
|
else \
|
|
echo "[deploy] microvm artifacts absent — run 'make install-microvm' for first-time setup"; \
|
|
fi
|
|
sudo systemctl daemon-reload
|
|
sudo systemctl restart spinifex.target
|
|
|
|
# Re-run setup.sh after changing systemd units, helper scripts, or logrotate config.
|
|
# Not needed for code-only changes — use deploy for those.
|
|
reinstall:
|
|
scripts/dev-install.sh
|
|
|
|
clean:
|
|
rm -f ./bin/$(GO_PROJECT_NAME)
|
|
rm -rf spinifex/services/spinifexui/frontend/dist
|
|
|
|
install-system:
|
|
@echo -e "\n....Installing system dependencies for $(ARCH)...."
|
|
sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
|
-o Dpkg::Options::="--force-confdef" \
|
|
-o Dpkg::Options::="--force-confold" \
|
|
nbdkit nbdkit-plugin-dev pkg-config qemu-system-x86 qemu-system-arm qemu-utils qemu-kvm \
|
|
ovmf qemu-efi-aarch64 \
|
|
libvirt-daemon-system libvirt-clients libvirt-dev make gcc jq curl \
|
|
iproute2 netcat-openbsd openssh-client wget git unzip sudo xz-utils file \
|
|
ovn-central ovn-host openvswitch-switch dhcpcd-base
|
|
|
|
install-go:
|
|
@echo -e "\n....Installing Go 1.26.4 for $(ARCH) ($(GO_ARCH))...."
|
|
@if [ ! -d "/usr/local/go" ]; then \
|
|
curl -L https://go.dev/dl/go1.26.4.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
|
|
else \
|
|
echo "Go already installed in /usr/local/go"; \
|
|
fi
|
|
@echo "Go version: $$(go version)"
|
|
|
|
install-aws:
|
|
@echo -e "\n....Installing AWS CLI v2 for $(ARCH) ($(AWS_ARCH))...."
|
|
@if ! command -v aws >/dev/null 2>&1; then \
|
|
curl "https://awscli.amazonaws.com/awscli-exe-linux-$(AWS_ARCH).zip" -o "awscliv2.zip"; \
|
|
unzip -o awscliv2.zip; \
|
|
./aws/install; \
|
|
rm -rf awscliv2.zip aws/; \
|
|
else \
|
|
echo "AWS CLI already installed"; \
|
|
fi
|
|
|
|
quickinstall: install-system install-go install-aws
|
|
@echo -e "\n✅ Quickinstall complete for $(ARCH)."
|
|
@echo " Please ensure /usr/local/go/bin is in your PATH."
|
|
|
|
lint:
|
|
@echo "Running golangci-lint..."
|
|
$(_Q)golangci-lint run ./...
|
|
@echo " golangci-lint ok"
|
|
|
|
fix:
|
|
golangci-lint run --fix ./...
|
|
|
|
govulncheck:
|
|
@echo "Running govulncheck..."
|
|
$(_Q)go tool govulncheck ./...
|
|
@echo " govulncheck ok"
|
|
|
|
# Build release tarballs — use distro-ARCH for single arch, distro for both
|
|
distro: distro-amd64 distro-arm64
|
|
@echo ""
|
|
@echo "Distribution tarballs:"
|
|
@ls -lh dist/*.tar.gz
|
|
@echo ""
|
|
@cat dist/*.sha256
|
|
|
|
distro-amd64:
|
|
@echo "Building spinifex $(VERSION) linux/amd64..."
|
|
@mkdir -p dist/
|
|
docker buildx build \
|
|
--platform linux/amd64 \
|
|
--build-arg VERSION=$(VERSION) \
|
|
-f build/Dockerfile.distro \
|
|
--output type=local,dest=dist/amd64/ \
|
|
../
|
|
@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ] && [ -f $(MICROVM_OUT_DIR)/initramfs.cpio.gz ]; then \
|
|
echo "[distro-amd64] staging microvm artifacts into tarball"; \
|
|
mkdir -p dist/amd64/microvm; \
|
|
cp $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz dist/amd64/microvm/; \
|
|
else \
|
|
echo "[distro-amd64] WARNING: microvm artifacts missing — tarball will not include them"; \
|
|
echo "[distro-amd64] run 'make build-microvm-image' before 'make distro-amd64'"; \
|
|
fi
|
|
tar -czf dist/spinifex-$(VERSION)-linux-amd64.tar.gz -C dist/amd64 .
|
|
sha256sum dist/spinifex-$(VERSION)-linux-amd64.tar.gz > dist/spinifex-$(VERSION)-linux-amd64.tar.gz.sha256
|
|
|
|
distro-arm64:
|
|
@echo "Building spinifex $(VERSION) linux/arm64..."
|
|
@mkdir -p dist/
|
|
docker buildx build \
|
|
--platform linux/arm64 \
|
|
--build-arg VERSION=$(VERSION) \
|
|
-f build/Dockerfile.distro \
|
|
--output type=local,dest=dist/arm64/ \
|
|
../
|
|
tar -czf dist/spinifex-$(VERSION)-linux-arm64.tar.gz -C dist/arm64 .
|
|
sha256sum dist/spinifex-$(VERSION)-linux-arm64.tar.gz > dist/spinifex-$(VERSION)-linux-arm64.tar.gz.sha256
|
|
|
|
distro-clean:
|
|
rm -rf dist/
|
|
|
|
# Ansible dev lifecycle (experimental, parallel to dev-*.sh / reset-dev-env.sh).
|
|
# See scripts/ansible/README.md and docs/development/improvements/ansible-dev-lifecycle.md.
|
|
#
|
|
# Variable overrides: pass EXTRA_VARS="key=val key2=val2" to forward as
|
|
# `--extra-vars` to ansible-playbook. Plain `-e` on the make command line is
|
|
# make's own flag and does NOT reach ansible.
|
|
# make ansible-dev-install EXTRA_VARS="spinifex_external_pool_start=192.168.1.90 spinifex_external_pool_end=192.168.1.99"
|
|
EXTRA_VARS ?=
|
|
_ANSIBLE_EXTRA = $(if $(strip $(EXTRA_VARS)),--extra-vars "$(EXTRA_VARS)",)
|
|
|
|
ansible-dev-preflight:
|
|
cd scripts/ansible && ansible-playbook playbooks/dev-preflight.yml $(_ANSIBLE_EXTRA)
|
|
|
|
ansible-dev-teardown:
|
|
cd scripts/ansible && ansible-playbook playbooks/dev-teardown.yml $(_ANSIBLE_EXTRA)
|
|
|
|
ansible-dev-install:
|
|
cd scripts/ansible && ansible-playbook playbooks/dev-install.yml $(_ANSIBLE_EXTRA)
|
|
|
|
ansible-dev-reset:
|
|
cd scripts/ansible && ansible-playbook playbooks/dev-reset.yml $(_ANSIBLE_EXTRA)
|
|
|
|
ansible-dev-deploy:
|
|
cd scripts/ansible && ansible-playbook playbooks/dev-deploy.yml $(_ANSIBLE_EXTRA)
|
|
|
|
ansible-dev-status:
|
|
cd scripts/ansible && ansible-playbook playbooks/dev-status.yml
|
|
|
|
ansible-dev-logs:
|
|
cd scripts/ansible && ansible-playbook playbooks/dev-logs.yml
|
|
|
|
# Snapshot / restore require an explicit -e snapshot_name=<name>. The
|
|
# wrapper passes ANSIBLE_EXTRA through so devs can run
|
|
# `make ansible-dev-snapshot ANSIBLE_EXTRA='-e snapshot_name=before-merge'`.
|
|
ansible-dev-snapshot:
|
|
cd scripts/ansible && ansible-playbook playbooks/dev-snapshot.yml $(ANSIBLE_EXTRA)
|
|
|
|
ansible-dev-restore:
|
|
cd scripts/ansible && ansible-playbook playbooks/dev-restore.yml $(ANSIBLE_EXTRA)
|
|
|
|
ansible-dev-version:
|
|
cd scripts/ansible && ansible-playbook playbooks/dev-version.yml
|
|
|
|
ansible-dev-vpc:
|
|
cd scripts/ansible && ansible-playbook playbooks/dev-vpc.yml $(ANSIBLE_EXTRA)
|
|
|
|
# Multi-node cluster bootstrap against a tofu env (env1, env2, bryn, etc).
|
|
# Inventory is generated from scripts/tofu-cluster/envs/<env>.tfvars.
|
|
# make ansible-cluster-bootstrap ENV=env1 POOL=192.168.1.150-192.168.1.159
|
|
ENV ?=
|
|
POOL ?=
|
|
ansible-cluster-bootstrap:
|
|
@test -n "$(ENV)" || { echo "ENV=<env> required (e.g. env1, env2)"; exit 64; }
|
|
cd scripts/ansible && CLUSTER_ENV=$(ENV) ansible-playbook -i inventory/tofu.py playbooks/cluster-bootstrap.yml \
|
|
-e cluster_env=$(ENV) \
|
|
$(if $(POOL),-e cluster_external_pool=$(POOL),) \
|
|
$(_ANSIBLE_EXTRA)
|
|
|
|
.PHONY: build build-ui build-installer build-lb-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-harness manifest-check manifest-lint manifest-lint-update \
|
|
deploy reinstall clean \
|
|
install-system install-go install-aws quickinstall \
|
|
lint fix govulncheck \
|
|
distro distro-amd64 distro-arm64 distro-clean
|