gsadmin/spinifex
1
0
Fork 0
mirror of https://github.com/mulgadc/spinifex.git synced 2026-10-15 09:01:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

merge dev into main

Browse Source
This commit is contained in:
Julian Sommer
2026-06-09 12:47:17 +10:00
parent 0ee9b46e38 4ef19fc488
commit 67ab37cb28
643 changed files with 47567 additions and 10011 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/CODEOWNERS
+2 -8
View File
@@ -6,16 +6,10 @@
# Set the matching branch protection rule on `dev` and `main` to
# "Require review from Code Owners" or this file is advisory only.
# Fall-through: any change needs a team review.
* @mulgadc/mulgadc-engineers
# Network layer is governed by ADR-0006 invariant tests.
# Changes here must respect those clauses — see
# spinifex/network/invariants/ and
# Changes here must respect those clauses
# see spinifex/network/invariants/
# docs/development/proposals/az-local-architecture/0006-spinifex-network-layer-contract.md
/spinifex/network/ @joshsiv-mulga
/spinifex/vpcd/ @joshsiv-mulga
# Invariants suite is the executable ADR-0006 fence — any widening or
# narrowing of the scan roots needs an explicit owner review.
/spinifex/network/invariants/ @joshsiv-mulga
.github/workflows/e2e-baremetal.yml
+48 -2
View File
@@ -2,6 +2,13 @@ name: E2E Bare-Metal
on:
workflow_dispatch:
workflow_call:
inputs:
ref:
description: "spinifex ref to checkout/test (default: triggering ref)"
required: false
type: string
default: ""
permissions:
contents: read
@@ -12,7 +19,7 @@ env:
jobs:
e2e_baremetal:
name: Bare-Metal Multi-Node E2E
name: Bare-Metal Single-Node E2E
runs-on: [ self-hosted, ci-baremetal ]
timeout-minutes: 120
concurrency:
@@ -20,11 +27,17 @@ jobs:
cancel-in-progress: false
env:
BM_ISO_CACHE: /var/cache/spinifex-iso-builder
ARTIFACT_DIR: /tmp/spinifex-e2e-baremetal-artifacts-${{ github.run_id }}
steps:
- name: Clean Workspace
run: sudo rm -rf "$GITHUB_WORKSPACE"/*
- uses: actions/checkout@v6
with:
# When called (nightly/release) the caller picks the ref (dev on
# schedule, the tag on release); standalone dispatch defaults to the
# triggering ref.
ref: ${{ inputs.ref || github.ref }}
- uses: actions/create-github-app-token@v3
id: app-token
@@ -92,6 +105,9 @@ jobs:
fi
- name: Install go-junit-report on runner
run: go install github.com/jstemmer/go-junit-report/v2@latest
- name: Run Bare-Metal E2E
env:
BM_CI_PASSWORD: ${{ secrets.BM_CI_PASSWORD }}
@@ -100,8 +116,38 @@ jobs:
BM_UBUNTU_IMAGE: ${{ env.BM_ISO_CACHE }}/ubuntu-26.04-cloudimg-amd64.img
SPX_EXTERNAL_POOL: ${{ env.SPINIFEX_CI_POOL }}
run: |
mkdir -p "${ARTIFACT_DIR}"
# Capture combined bootstrap + e2e output so failures produce a
# downloadable, analyzable artifact (the suites run go test -v on the
# init node and stream back over SSH). pipefail keeps the script's
# real exit code through the tee.
set -o pipefail
sudo -E mulga/scripts/baremetal/run-baremetal-e2e.sh \
--vmlinuz "${BM_ISO_CACHE}/vmlinuz" \
--initrd "${BM_ISO_CACHE}/initrd.img" \
--squashfs "${BM_ISO_CACHE}/spinifex.squashfs" \
--tests-tar "${RUNNER_TEMP}/spinifex-tests.tar"
--tests-tar "${RUNNER_TEMP}/spinifex-tests.tar" \
2>&1 | tee "${ARTIFACT_DIR}/test-baremetal.log"
- name: Generate JUnit report
if: always()
run: |
LOG="${ARTIFACT_DIR}/test-baremetal.log"
[ -f "$LOG" ] || exit 0
go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-baremetal.xml" || true
- name: Analyze e2e failures
if: always()
uses: ./.github/actions/e2e-analyze
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
log-dir: ${{ env.ARTIFACT_DIR }}
title: "Bare-Metal E2E failure analysis"
- uses: actions/upload-artifact@v7
if: always()
with:
name: e2e-baremetal-artifacts-${{ github.run_id }}
path: ${{ env.ARTIFACT_DIR }}
if-no-files-found: warn
retention-days: 14
.github/workflows/e2e-nightly.yml
+19
View File
@@ -216,3 +216,22 @@ jobs:
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
# Bare-metal single-node suite (PXE-reimaged physical hardware). Doesn't fit
# the tofu matrix shape (different runner/provisioning/120m timeout), so it
# runs as a sibling job via the reusable e2e-baremetal workflow. Gated like a
# matrix cell ("cell 20"): every weeknight cron, or dispatch with 20 selected
# (blank cells = all). Tests `dev` on schedule to match GIT_BRANCH above.
baremetal:
name: cell-20 (baremetal/pxe/real-hw)
if: >-
github.event_name == 'schedule' ||
inputs.cells == '' ||
contains(format(',{0},', inputs.cells), ',20,')
permissions:
contents: read
statuses: write
uses: ./.github/workflows/e2e-baremetal.yml
with:
ref: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
secrets: inherit
.github/workflows/e2e.yml
+137 -15
View File
@@ -18,7 +18,7 @@ on:
description: "PR head SHA to report status on"
required: false
suites:
description: "Space-separated Make targets to run (e.g. 'e2e-single e2e-multinode e2e-cert e2e-lb')"
description: "Space-separated Make targets to run (e.g. 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-eks')"
required: false
default: "e2e-single e2e-multinode e2e-cert e2e-lb"
test_run:
@@ -30,13 +30,6 @@ permissions:
contents: read
statuses: write
# Pushes to the same ref stack on the single `ci-single` self-hosted runner,
# and a cancelled run can leave orphan `tofu`/`go test -c` procs behind that
# wedge the runner host. Auto-cancel keeps at most one in-flight run per ref.
concurrency:
group: e2e-${{ github.ref }}
cancel-in-progress: true
env:
GIT_BRANCH: ${{ github.ref_name }}
SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb' }}
@@ -49,7 +42,7 @@ jobs:
# suites='e2e-reboot' only). Push triggers and the default SUITES list
# both include single/cert/lb so the inputs.suites null-check keeps the
# job running on plain PR pushes.
if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-lb')
if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-lb') || contains(github.event.inputs.suites, 'e2e-eks')
runs-on: [ self-hosted, ci-single ]
timeout-minutes: 45
env:
@@ -77,7 +70,15 @@ jobs:
true
- name: Clean Workspace
run: sudo rm -rf "$GITHUB_WORKSPACE"/*
run: |
# Detach any leaked mounts under the workspace before deleting. A
# killed image build can leave a `mount --bind /dev` under its build
# dir; if `rm -rf` traverses it as root it wipes the host's /dev
# (deletes /dev/null and wedges the box). umount deepest-first, then
# forbid rm from crossing any mount boundary as a hard backstop.
awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
| sort -r | xargs -r -I{} sudo umount -l -R {} || true
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
- uses: actions/create-github-app-token@v3
id: app-token
@@ -153,6 +154,85 @@ jobs:
echo "wan_ip=${WAN_IP}" >> "$GITHUB_OUTPUT"
echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
# The eks suite's CreateCluster boots a control-plane VM from the
# eks-node AMI, so that AMI must exist in the VM's image store before the
# suite runs. build-system-image.sh customizes the image with libguestfs
# (an isolated userspace appliance) — it touches no host block device, so
# it is safe on the shared runner. The build needs the spinifex source +
# network (runner only); the import needs predastore/NATS (VM only), so
# build on the runner then ship the raw and import on the VM.
- name: Build eks-node AMI (libguestfs, runner)
if: contains(env.SUITES, 'e2e-eks')
working-directory: spinifex
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
banner "build eks-node AMI (libguestfs — no host devices)"
sudo apt-get update -qq
sudo apt-get install -y --no-install-recommends libguestfs-tools qemu-utils
# Debian/Ubuntu ship /boot/vmlinuz-* mode 0600; the libguestfs
# appliance needs to read a host kernel. direct backend = no libvirt.
export LIBGUESTFS_BACKEND=direct
sudo chmod 0644 /boot/vmlinuz-* || true
group_open "make build-eks-node-image"
make build-eks-node-image
group_close
ok "raw: $(du -h /tmp/eks-node-image-build/eks-node-alpine.raw | cut -f1)"
- name: Import eks-node AMI on VM
if: contains(env.SUITES, 'e2e-eks')
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
# Pull import args straight from the manifest so they never drift.
eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
spinifex/scripts/images/eks-node/manifest.conf)"
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
VM=tf-user@${{ steps.vm.outputs.wan_ip }}
banner "import eks-node AMI to VM image store"
scp "${SSH_OPTS[@]}" \
/tmp/eks-node-image-build/eks-node-alpine.raw \
"$VM":/tmp/eks-node-alpine.raw
# Drop the import progress bar (pterm redraws to stdout — thousands of
# lines over the SSH pipe). Errors go to stderr, so they survive; the
# exit code still gates the step.
ssh "${SSH_OPTS[@]}" "$VM" \
"spx admin images import --file /tmp/eks-node-alpine.raw \
--distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
--boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null
ok "eks-node AMI imported"
# The eks data-plane subtest (TestEKS/KubectlGetNodes) drives the full
# `aws eks get-token` → kubectl → apiserver → token-webhook → gateway
# broker → STS verify path. The e2e binaries run ON the VM, so kubectl
# must be on the VM's PATH or harness.NewKubectl t.Skips the subtest.
# Pin to the k3s server version (setup.sh K3S_VERSION) — kubectl is
# ±1-minor skew tolerant; matching is simplest. Checksum-verified.
- name: Install kubectl on VM
if: contains(env.SUITES, 'e2e-eks')
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
VM=tf-user@${{ steps.vm.outputs.wan_ip }}
banner "install kubectl on VM (eks data-plane subtest)"
ssh "${SSH_OPTS[@]}" "$VM" 'bash -s' <<'EOF'
set -euo pipefail
KVER=v1.32.5
case "$(uname -m)" in
x86_64) ARCH=amd64 ;;
aarch64) ARCH=arm64 ;;
*) echo "unsupported arch $(uname -m)" >&2; exit 1 ;;
esac
cd /tmp
curl -fsSLO "https://dl.k8s.io/release/${KVER}/bin/linux/${ARCH}/kubectl"
curl -fsSLO "https://dl.k8s.io/release/${KVER}/bin/linux/${ARCH}/kubectl.sha256"
echo "$(cat kubectl.sha256) kubectl" | sha256sum -c -
sudo install -m 0755 kubectl /usr/local/bin/kubectl
kubectl version --client
EOF
ok "kubectl installed on VM"
- name: Build E2E test binaries on runner
working-directory: spinifex/tests/e2e
run: |
@@ -277,6 +357,15 @@ jobs:
ssh "${SSH_OPTS[@]}" "$VM" 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \
> "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null
ok "journal: $(wc -l < "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || echo 0) lines"
# LB/system microVM serial consoles. The lb-agent logs its nginx (NLB)
# engine activation and any `nginx -t`/reload error + active-probe
# results to the guest serial console, which the host journal never
# captures. These files persist in /run/spinifex past VM teardown, so
# collecting them here makes an NLB "0/N healthy" timeout diagnosable.
mkdir -p "${ARTIFACT_DIR}/lb-console"
ssh "${SSH_OPTS[@]}" "$VM" 'sudo sh -c "cd /run/spinifex 2>/dev/null && tar -cf - console-*.log 2>/dev/null"' \
| tar -C "${ARTIFACT_DIR}/lb-console" -xf - 2>/dev/null
ok "lb-console: $(ls "${ARTIFACT_DIR}/lb-console" 2>/dev/null | wc -l) files"
- name: Convert test logs to JUnit XML
if: always()
@@ -383,7 +472,15 @@ jobs:
true
- name: Clean Workspace
run: sudo rm -rf "$GITHUB_WORKSPACE"/*
run: |
# Detach any leaked mounts under the workspace before deleting. A
# killed image build can leave a `mount --bind /dev` under its build
# dir; if `rm -rf` traverses it as root it wipes the host's /dev
# (deletes /dev/null and wedges the box). umount deepest-first, then
# forbid rm from crossing any mount boundary as a hard backstop.
awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
| sort -r | xargs -r -I{} sudo umount -l -R {} || true
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
- uses: actions/create-github-app-token@v3
id: app-token
@@ -471,7 +568,12 @@ jobs:
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
SUITES_BUILD=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|single)$' | tr '\n' ' ')
# eks is excluded on multinode: EKS KV is a clustered (Raft) JetStream
# stream and create-cluster needs an elected leader, but multinode
# JetStream clustering isn't solid yet (mulga-omq/z2z/5yg). EKS is
# validated single-node only for now; re-enable here once clustered
# JetStream + multinode EKS are a deliberate target.
SUITES_BUILD=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|single|eks)$' | tr '\n' ' ')
if [ -z "$(echo "${SUITES_BUILD}" | tr -d ' ')" ]; then
echo "no multinode-eligible suites requested (SUITES=${SUITES}); nothing to build"
exit 0
@@ -554,7 +656,7 @@ jobs:
id: e2e
run: |
set +e
SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|single)$' | tr '\n' ' ')
SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|single|eks)$' | tr '\n' ' ')
if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then
echo "no multinode-eligible suites requested (SUITES=${SUITES}); skipping"
echo "rc=0" >> "$GITHUB_OUTPUT"
@@ -664,6 +766,18 @@ jobs:
ssh "${SSH_OPTS[@]}" tf-user@"$ip" 'sudo journalctl -u openvswitch-ipsec.service --no-pager --output=short-iso --since "60 min ago"' \
> "${ARTIFACT_DIR}/openvswitch-ipsec-node${idx}.log" 2>/dev/null
done
# LB/system microVM serial consoles, per node. The lb-agent logs its
# nginx (NLB) engine activation and any `nginx -t`/reload error +
# active-probe results to the guest serial console (never in the host
# journal). Files persist in /run/spinifex past VM teardown.
for i in "${!NODE_LIST[@]}"; do
idx=$((i + 1))
ip="${NODE_LIST[$i]}"
mkdir -p "${ARTIFACT_DIR}/lb-console/node${idx}"
ssh "${SSH_OPTS[@]}" tf-user@"$ip" 'sudo sh -c "cd /run/spinifex 2>/dev/null && tar -cf - console-*.log 2>/dev/null"' \
| tar -C "${ARTIFACT_DIR}/lb-console/node${idx}" -xf - 2>/dev/null
ok "lb-console node${idx}: $(ls "${ARTIFACT_DIR}/lb-console/node${idx}" 2>/dev/null | wc -l) files"
done
- name: Convert test logs to JUnit XML
if: always()
@@ -779,7 +893,15 @@ jobs:
true
- name: Clean Workspace
run: sudo rm -rf "$GITHUB_WORKSPACE"/*
run: |
# Detach any leaked mounts under the workspace before deleting. A
# killed image build can leave a `mount --bind /dev` under its build
# dir; if `rm -rf` traverses it as root it wipes the host's /dev
# (deletes /dev/null and wedges the box). umount deepest-first, then
# forbid rm from crossing any mount boundary as a hard backstop.
awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
| sort -r | xargs -r -I{} sudo umount -l -R {} || true
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
- uses: actions/create-github-app-token@v3
id: app-token
@@ -1004,7 +1126,7 @@ jobs:
name: Report E2E Status
if: always()
needs: [ e2e_single, e2e_real_multi, e2e_reboot ]
runs-on: [ self-hosted, ci-single ]
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Report Status to PR
.github/workflows/publish-eks-ami.yml
+92
View File
@@ -0,0 +1,92 @@
name: Publish EKS node AMI
# Builds the eks-node system AMI (libguestfs, on the runner) and publishes the
# compressed qcow2 + .sha256 sidecar to the Cloudflare R2 bucket that backs
# iso.mulgadc.com/system-ami/. Operators then install it on any cluster with:
# spx admin images import --name spinifex-eks-node
#
# Manual-only: publishing overwrites the mirror artifact, so it is a deliberate
# action, not something every push should trigger.
on:
workflow_dispatch:
inputs:
dry_run:
description: "Build + convert only; skip the R2 upload"
type: boolean
default: false
permissions:
contents: read
concurrency:
group: publish-eks-ami
cancel-in-progress: false
jobs:
publish:
name: Build + publish eks-node AMI
runs-on: ubuntu-24.04
steps:
- name: Checkout Spinifex
uses: actions/checkout@v6
with:
path: spinifex
- name: Checkout Viperblock
uses: actions/checkout@v6
with:
repository: mulgadc/viperblock
ref: dev
path: viperblock
- name: Checkout Predastore
uses: actions/checkout@v6
with:
repository: mulgadc/predastore
ref: dev
path: predastore
- name: Setup Go
uses: actions/setup-go@v6
with:
cache-dependency-path: "spinifex/go.sum"
go-version-file: "spinifex/go.mod"
- name: Setup Go Workspace
run: go work init ./spinifex ./viperblock ./predastore
- name: Install build + upload tooling
run: |
set -euo pipefail
sudo apt-get update -qq
sudo apt-get install -y --no-install-recommends libguestfs-tools qemu-utils
if ! command -v aws &>/dev/null; then
curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip
unzip -q /tmp/awscliv2.zip -d /tmp/aws-install
sudo /tmp/aws-install/aws/install
rm -rf /tmp/awscliv2.zip /tmp/aws-install
fi
- name: Build + publish to R2
working-directory: spinifex
env:
# Map the repo's R2 secrets onto the names publish-system-image.sh reads.
AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_SECRET_ACCESS_KEY }}
R2_ENDPOINT: ${{ secrets.CF_R2_ENDPOINT_URL }}
R2_BUCKET: spinifex-iso
DRY_RUN: ${{ inputs.dry_run }}
run: |
set -euo pipefail
# libguestfs needs to read a host kernel; Ubuntu ships it mode 0600.
# direct backend = no libvirt on the runner.
export LIBGUESTFS_BACKEND=direct
sudo chmod 0644 /boot/vmlinuz-* || true
FLAGS=(--build)
if [[ "${DRY_RUN}" == "true" ]]; then
FLAGS+=(--dry-run)
fi
./scripts/publish-system-image.sh \
scripts/images/eks-node/manifest.conf "${FLAGS[@]}"
.github/workflows/release.yml
+70 -28
View File
@@ -131,7 +131,16 @@ jobs:
REPO: ${{ github.repository }}
steps:
- name: Clean Workspace
run: rm -rf "$GITHUB_WORKSPACE"/*
run: |
# This runner (ci-single) is shared with the iso job, which mounts a
# kernel-global devtmpfs at <workspace>/mulga/scripts/iso-builder/.cache/rootfs/dev.
# If that job is killed its EXIT-trap unmount never runs and the bind
# leaks; a plain rm -rf then traverses it and deletes the host's /dev
# (wipes /dev/null, wedges the box). Detach any leaked mounts under the
# workspace first, then forbid rm from crossing a mount boundary.
awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
| sort -r | xargs -r -I{} sudo umount -l -R {} || true
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
- uses: actions/create-github-app-token@v3
id: app-token
@@ -169,8 +178,36 @@ jobs:
TARBALL=$(find "${GITHUB_WORKSPACE}/release-assets" -name "*-linux-amd64.tar.gz" | head -1)
./bootstrap-install.sh "${SPINIFEX_CI_ENV}" "$SETUP_SH" "$TARBALL"
- name: Teardown
if: always()
working-directory: mulga/scripts/tofu-cluster
run: |
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
# Bare-metal single-node suite on the real ISO/squashfs + PXE-boot path —
# the only release check that exercises hardware install. Runs in parallel
# with `verify` (both need: release). Promotion and ISO upload both hard-gate
# on this passing.
baremetal:
needs: release
permissions:
contents: read
statuses: write
uses: ./.github/workflows/e2e-baremetal.yml
secrets: inherit
# Promote the draft only after BOTH the tofu install (`verify`) and the
# bare-metal hardware boot (`baremetal`) pass — runs only if both succeed.
promote:
needs: [ verify, baremetal ]
runs-on: ubuntu-24.04
env:
TAG: ${{ github.ref_name }}
REPO: ${{ github.repository }}
steps:
- name: Promote Release
if: success()
env:
GH_TOKEN: ${{ github.token }}
run: |
@@ -182,29 +219,31 @@ jobs:
echo "Release ${TAG} promoted to latest"
fi
# Annotate the (still-draft) release if either gate failed.
mark-failed:
needs: [ verify, baremetal ]
if: failure()
runs-on: ubuntu-24.04
env:
TAG: ${{ github.ref_name }}
REPO: ${{ github.repository }}
steps:
- name: Mark Release Failed
if: failure()
env:
GH_TOKEN: ${{ github.token }}
run: |
EXISTING_BODY=$(gh release view "$TAG" --repo "$REPO" --json body -q .body)
NOTES_FILE=$(mktemp)
printf 'Verification failed — see workflow run for details.\n\n%s' "$EXISTING_BODY" > "$NOTES_FILE"
printf 'Verification failed (tofu install or bare-metal boot) — see workflow run for details.\n\n%s' "$EXISTING_BODY" > "$NOTES_FILE"
gh release edit "$TAG" --repo "$REPO" --notes-file "$NOTES_FILE"
rm -f "$NOTES_FILE"
- name: Teardown
if: always()
working-directory: mulga/scripts/tofu-cluster
run: |
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
iso:
needs: release
runs-on: [ self-hosted, ci-single ]
timeout-minutes: 90
# ISO build (~60-90m) overlaps the parallel baremetal run (~120m); the
# upload gate then waits for baremetal to finish, so the job must outlast it.
timeout-minutes: 180
steps:
- name: Clean Workspace
run: |
@@ -216,7 +255,7 @@ jobs:
sudo umount -l "${ROOTFS}/dev" 2>/dev/null || true
sudo umount -l "${ROOTFS}/sys" 2>/dev/null || true
sudo umount -l "${ROOTFS}/proc" 2>/dev/null || true
sudo rm -rf "$GITHUB_WORKSPACE"/*
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
- uses: actions/create-github-app-token@v3
id: app-token
@@ -289,24 +328,28 @@ jobs:
SPINIFEX_BIN="${GITHUB_WORKSPACE}/spinifex/bin/spx" \
INSTALLER_BIN="${GITHUB_WORKSPACE}/spinifex/bin/spinifex-installer"
- name: Wait for verify
- name: Wait for verify + baremetal
id: wait-verify
if: steps.build-iso.outcome == 'success'
continue-on-error: true
env:
GH_TOKEN: ${{ github.token }}
run: |
echo "Waiting for verify job to complete..."
# Gate the R2 upload on BOTH the tofu install (`verify`) and the
# real-hardware boot (`baremetal`, a reusable workflow whose job shows
# as "baremetal / ..."). Upload only if both conclude success.
echo "Waiting for verify + baremetal jobs to complete..."
while true; do
CONCLUSION=$(gh api \
"repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs" \
--jq '.jobs[] | select(.name == "verify") | .conclusion // empty')
if [ -n "$CONCLUSION" ]; then
echo "Verify conclusion: $CONCLUSION"
[ "$CONCLUSION" = "success" ] && exit 0
JOBS=$(gh api \
"repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs" --paginate)
VERIFY=$(echo "$JOBS" | jq -r '.jobs[] | select(.name == "verify") | .conclusion // empty')
BAREMETAL=$(echo "$JOBS" | jq -r '.jobs[] | select(.name | startswith("baremetal")) | .conclusion // empty')
if [ -n "$VERIFY" ] && [ -n "$BAREMETAL" ]; then
echo "verify=$VERIFY baremetal=$BAREMETAL"
{ [ "$VERIFY" = "success" ] && [ "$BAREMETAL" = "success" ]; } && exit 0
exit 1
fi
echo "Verify still running, retrying in 30s..."
echo "Still waiting (verify=${VERIFY:-pending} baremetal=${BAREMETAL:-pending}), retry 30s..."
sleep 30
done
@@ -327,14 +370,13 @@ jobs:
AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_SECRET_ACCESS_KEY }}
CF_R2_ENDPOINT_URL: ${{ secrets.CF_R2_ENDPOINT_URL }}
VERSION: ${{ github.ref_name }}
run: |
aws --endpoint-url="${CF_R2_ENDPOINT_URL}" \
s3 cp "mulga/scripts/iso-builder/dist/spinifex-${VERSION}.iso" \
"s3://spinifex-iso/spinifex-${VERSION}.iso"
s3 cp "mulga/scripts/iso-builder/dist/spinifex.iso" \
"s3://spinifex-iso/spinifex.iso"
aws --endpoint-url="${CF_R2_ENDPOINT_URL}" \
s3 cp "mulga/scripts/iso-builder/dist/spinifex-${VERSION}.iso.sha256" \
"s3://spinifex-iso/spinifex-${VERSION}.iso.sha256"
s3 cp "mulga/scripts/iso-builder/dist/spinifex.iso.sha256" \
"s3://spinifex-iso/spinifex.iso.sha256"
- name: Mark ISO failed
if: always() && (steps.build-iso.outcome == 'failure' || steps.upload.outcome == 'failure')
Makefile
+24 -10
View File
@@ -73,24 +73,27 @@ build-lb-agent:
@echo -e "\n....Building lb-agent (static)"
CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/lb-agent cmd/lb-agent/main.go
build-system-image: ## Build a system image from manifest (use IMAGE=lb or IMAGE=eks-server)
build-system-image: ## Build a system image from manifest (use IMAGE=lb or IMAGE=eks-node)
ifndef IMAGE
$(error IMAGE is required. Usage: make build-system-image IMAGE=lb)
endif
@if [ -f scripts/images/$(IMAGE).conf ]; then \
./scripts/build-system-image.sh scripts/images/$(IMAGE).conf; \
./scripts/build-system-image.sh scripts/images/$(IMAGE).conf $(if $(IMPORT),--import); \
elif [ -f scripts/images/$(IMAGE)/manifest.conf ]; then \
./scripts/build-system-image.sh scripts/images/$(IMAGE)/manifest.conf; \
./scripts/build-system-image.sh scripts/images/$(IMAGE)/manifest.conf $(if $(IMPORT),--import); \
else \
echo "ERROR: no manifest at scripts/images/$(IMAGE).conf or scripts/images/$(IMAGE)/manifest.conf"; \
exit 1; \
fi
build-eks-server-image: ## Build the eks-server AMI (K3s control-plane variant)
$(MAKE) build-system-image IMAGE=eks-server
build-eks-node-image: ## Build the unified eks-node AMI (K3s server+agent; role at first boot; IMPORT=1 to register)
$(MAKE) build-system-image IMAGE=eks-node
build-eks-agent-image: ## Build the eks-agent AMI (K3s worker variant)
$(MAKE) build-system-image IMAGE=eks-agent
import-eks-node-image: ## Build + register the eks-node AMI (requires a running cluster)
$(MAKE) build-system-image IMAGE=eks-node IMPORT=1
publish-eks-node-image: ## Build + publish the eks-node AMI to Cloudflare R2 (needs R2_ENDPOINT + AWS_* env)
./scripts/publish-system-image.sh scripts/images/eks-node/manifest.conf --build
MICROVM_OUT_DIR := build/microvm
MICROVM_ARTIFACTS := $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz
@@ -338,9 +341,20 @@ ansible-dev-version:
ansible-dev-vpc:
cd scripts/ansible && ansible-playbook playbooks/dev-vpc.yml $(ANSIBLE_EXTRA)
.PHONY: build build-ui build-installer build-lb-agent build-system-image build-eks-server-image build-eks-agent-image build-microvm-image install-microvm go_build go_run preflight test test-cover test-race diff-coverage bench run test-actions test-harness manifest-check diff-coverage bench run \
# Multi-node cluster bootstrap against a tofu env (env1, env2, bryn, etc).
# Inventory is generated from scripts/tofu-cluster/envs/<env>.tfvars.
# make ansible-cluster-bootstrap ENV=env1 POOL=192.168.1.150-192.168.1.159
ENV ?=
POOL ?=
ansible-cluster-bootstrap:
@test -n "$(ENV)" || { echo "ENV=<env> required (e.g. env1, env2)"; exit 64; }
cd scripts/ansible && CLUSTER_ENV=$(ENV) ansible-playbook -i inventory/tofu.py playbooks/cluster-bootstrap.yml \
-e cluster_env=$(ENV) \
$(if $(POOL),-e cluster_external_pool=$(POOL),) \
$(_ANSIBLE_EXTRA)
.PHONY: build build-ui build-installer build-lb-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-microvm-image install-microvm go_build go_run preflight test test-cover test-race diff-coverage bench run test-actions test-harness manifest-check diff-coverage bench run \
deploy reinstall clean \
install-system install-go install-aws quickinstall \
lint fix govulncheck \
distro distro-amd64 distro-arm64 distro-clean \
ansible-dev-preflight ansible-dev-teardown ansible-dev-install ansible-dev-reset ansible-dev-deploy ansible-dev-status ansible-dev-logs ansible-dev-snapshot ansible-dev-restore ansible-dev-version ansible-dev-vpc
distro distro-amd64 distro-arm64 distro-clean
build/microvm/init.sh
+45 -7
View File
@@ -6,20 +6,58 @@ mount -t devtmpfs dev /dev
# virtio_mmio is built-in (CONFIG_VIRTIO_MMIO=y) so virtio-net devices are
# already enumerated by the time PID 1 runs — no modprobe needed.
# qemu_fw_cfg and virtio_net are =m in stock Alpine linux-virt, so load them.
modprobe qemu_fw_cfg 2>/dev/null || modprobe fw_cfg_sysfs 2>/dev/null || true
modprobe virtio_net 2>/dev/null || true
#
# modprobe needs a usable modules.dep; when the image is built on a host whose
# depmod can't produce one, busybox modprobe silently no-ops and the module
# never loads — fw_cfg then never appears and init panics. insmod by explicit
# path needs no modules.dep, so fall back to it. Handles .ko and .ko.gz.
load_mod() {
modprobe "$1" 2>/dev/null && return 0
for ko in $(find /lib/modules -name "$1.ko" -o -name "$1.ko.gz" 2>/dev/null); do
insmod "$ko" 2>/dev/null && return 0
case "$ko" in
*.gz) gunzip -c "$ko" >/tmp/mod.ko 2>/dev/null \
&& insmod /tmp/mod.ko 2>/dev/null && return 0 ;;
esac
done
return 1
}
load_mod qemu_fw_cfg || load_mod fw_cfg_sysfs || true
# virtio_net depends on net_failover, which depends on failover. The load_mod
# insmod fallback loads a single .ko with no dependency resolution, and
# modules.dep is absent when the image is built on a kmod-less host (depmod
# skipped), so modprobe can't resolve the chain either. Load it explicitly
# bottom-up so the NIC comes up regardless of how the image was built —
# without this the agent has no network and the LB hangs in provisioning.
load_mod failover || true
load_mod net_failover || true
load_mod virtio_net || true
for i in 1 2 3 4 5; do
[ -f /sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw ] && break
sleep 0.02
NETCFG=/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw
i=0
while [ "$i" -lt 25 ]; do
[ -f "$NETCFG" ] && break
sleep 0.04
i=$((i + 1))
done
if [ ! -f /sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw ]; then
if [ ! -f "$NETCFG" ]; then
# Dump host/guest fw_cfg state to the serial console before init exits so a
# "netcfg not available" panic is diagnosable from the captured console log
# (module-not-loaded vs qemu-not-exposing-by_name vs wrong machine type).
echo "[init] FATAL: fw_cfg netcfg not available" >&2
echo "[init] dmi: vendor=$(cat /sys/class/dmi/id/sys_vendor 2>/dev/null) product=$(cat /sys/class/dmi/id/product_name 2>/dev/null) bios=$(cat /sys/class/dmi/id/bios_version 2>/dev/null)" >&2
echo "[init] qemu_fw_cfg dir present: $([ -d /sys/firmware/qemu_fw_cfg ] && echo yes || echo NO)" >&2
echo "[init] modules: $(lsmod 2>/dev/null | awk 'NR>1{print $1}' | tr '\n' ',')" >&2
echo "[init] by_name listing:" >&2
ls -la /sys/firmware/qemu_fw_cfg/by_name/ 2>&1 | sed 's/^/[init] /' >&2
ls -la /sys/firmware/qemu_fw_cfg/by_name/opt/ 2>&1 | sed 's/^/[init] /' >&2
echo "[init] dmesg fw_cfg:" >&2
dmesg 2>/dev/null | grep -i fw_cfg | sed 's/^/[init] /' >&2
exit 1
fi
. /sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw
. "$NETCFG"
for n in 0 1 2 3 4 5; do
eval mac=\$NIC${n}_MAC
build/systemd/spinifex-vpcd.service
+25 -15
View File
@@ -16,25 +16,34 @@ OOMScoreAdjust=-500
Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/vpcd/
# Filesystem access — NoNewPrivileges omitted (vpcd uses sudo for ovs-vsctl, ovn-nbctl, systemctl)
ProtectSystem=full
# vpcd MUST share the HOST mount namespace. IMDS binds one listener per VPC inside
# a per-VPC netns entered in-process via setns(CLONE_NEWNET); the /run/netns/<vpc>
# bind-mount that `ip netns add` creates must be host-visible (for diagnostics) and
# must survive a vpcd restart (so the listener can rebind). ANY filesystem-sandbox
# directive — ProtectSystem / ProtectHome / PrivateTmp / ProtectKernel* /
# ProtectControlGroups / ProtectProc / Read{Only,Write}Paths — forks a PRIVATE
# mount namespace that traps that bind-mount inside vpcd, leaving the host a stale
# handle that fails setns(2) with EINVAL (CI runs 26920490454, 26922587471). Those
# directives are therefore omitted by design, and so is the MountFlags=shared /
# spinifex-netns.service propagation workaround they required.
#
# Boundary that REMAINS: DAC (User=spinifex-vpcd) plus the seccomp / capability /
# namespace / address-family restrictions below. The durable fix that restores
# filesystem sandboxing AND drops CAP_SYS_ADMIN is the process-per-netns model
# (`ip netns exec` helper instead of in-process setns) — see
# docs/development/improvements/imds-l2-datapath.md Phase 5.
PrivateDevices=no
ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/ca.pem
ReadWritePaths=/var/lib/spinifex/vpcd /var/log/spinifex
# Hardening
ProtectHome=yes
PrivateTmp=yes
# Hardening that does NOT fork a mount namespace (kept)
RestrictSUIDSGID=no
LockPersonality=yes
RestrictRealtime=yes
RestrictNamespaces=yes
ProtectClock=yes
ProtectKernelTunables=yes
ProtectKernelModules=yes
ProtectKernelLogs=yes
ProtectControlGroups=yes
ProtectProc=invisible
# IMDS enters the per-VPC netns via setns(CLONE_NEWNET); the `ip -n <netns>` /
# `ip netns exec` plumbing also unshares a mount namespace (CLONE_NEWNS, to give
# the netns its own /sys), so `mnt` must be allowlisted alongside `net` or the
# first in-netns command fails with EPERM.
RestrictNamespaces=net mnt
SystemCallArchitectures=native
MemoryDenyWriteExecute=no
# AF_PACKET: required by the in-process DHCP client (github.com/insomniacslk/dhcp
@@ -44,9 +53,10 @@ RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK AF_PACKET
# CAP_NET_RAW: open AF_PACKET sockets.
# CAP_NET_BIND_SERVICE: covers unicast RENEW to UDP :68.
# CAP_NET_ADMIN: already used for ovs-vsctl / ip link wiring on the bridge.
# CAP_SYS_ADMIN: setns(CLONE_NEWNET) into per-VPC IMDS netns to bind the listener.
# Note, disabled for now, sudo issue with ovs
AmbientCapabilities=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID
CapabilityBoundingSet=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID
AmbientCapabilities=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID CAP_SYS_ADMIN
CapabilityBoundingSet=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID CAP_SYS_ADMIN
NoNewPrivileges=no
ProtectHostname=yes
KeyringMode=private
cmd/eks-gateway-publish/main.go
+134
View File
@@ -0,0 +1,134 @@
// eks-gateway-publish runs inside the EKS K3s control-plane VM and relays a
// single publication to the host through the AWS gateway, instead of dialing
// core NATS directly.
//
// It reads a JSON payload on stdin, wraps it as
// {accountId, channel, kind, payload}, SigV4-signs (service "eks") an HTTPS
// POST to {gateway}/clusters/{cluster}/internal-publish, and retries with
// backoff until the gateway returns 2xx or the attempt budget is exhausted —
// so a degraded link surfaces as a non-zero exit rather than a silently
// dropped message (the failure mode of fire-and-forget `nats pub`).
//
// Usage:
//
// echo '{"token":"..."}' | eks-gateway-publish -channel bootstrap -kind k3s-bootstrap-token
// kubectl ... | eks-gateway-publish -channel state
//
// Flags default to environment variables seeded by cloud-init:
// EKS_GATEWAY_URL, EKS_GATEWAY_CA, EKS_ACCESS_KEY, EKS_SECRET_KEY, EKS_REGION,
// EKS_ACCOUNT_ID, EKS_CLUSTER_NAME.
package main
import (
"bytes"
"encoding/json"
"flag"
"fmt"
"io"
"log/slog"
"os"
"time"
"github.com/mulgadc/spinifex/internal/eksgw"
_ "github.com/mulgadc/spinifex/internal/fipsboot"
)
const (
// Retry budget: the control plane reaches readiness before this runs, so a
// failing POST means a degraded link, not a cold start. Bounded so a stuck
// boot still terminates the OpenRC service.
maxAttempts = 30
retryDelay = 5 * time.Second
)
type publishBody struct {
AccountID string `json:"accountId"`
Channel string `json:"channel"`
Kind string `json:"kind"`
Payload json.RawMessage `json:"payload"`
}
func main() {
var (
gatewayURL string
gatewayCA string
accessKey string
secretKey string
region string
accountID string
clusterName string
channel string
kind string
)
flag.StringVar(&gatewayURL, "gateway", os.Getenv("EKS_GATEWAY_URL"), "Gateway URL (e.g. https://10.15.8.1:9999)")
flag.StringVar(&gatewayCA, "gateway-ca", os.Getenv("EKS_GATEWAY_CA"), "Path to gateway TLS CA PEM (optional; falls back to system trust)")
flag.StringVar(&accessKey, "access-key", os.Getenv("EKS_ACCESS_KEY"), "AWS access key ID")
flag.StringVar(&secretKey, "secret-key", os.Getenv("EKS_SECRET_KEY"), "AWS secret access key")
flag.StringVar(&region, "region", os.Getenv("EKS_REGION"), "AWS region for SigV4 signing")
flag.StringVar(&accountID, "account-id", os.Getenv("EKS_ACCOUNT_ID"), "Cluster account ID")
flag.StringVar(&clusterName, "cluster", os.Getenv("EKS_CLUSTER_NAME"), "Cluster name")
flag.StringVar(&channel, "channel", "", "Publish channel: bootstrap|state")
flag.StringVar(&kind, "kind", "", "Bootstrap subject kind (bootstrap channel only)")
flag.Parse()
switch {
case accountID == "":
fatal("--account-id is required (or set EKS_ACCOUNT_ID)")
case clusterName == "":
fatal("--cluster is required (or set EKS_CLUSTER_NAME)")
case channel != "bootstrap" && channel != "state":
fatal("--channel must be bootstrap or state")
case channel == "bootstrap" && kind == "":
fatal("--kind is required for the bootstrap channel")
}
payload, err := io.ReadAll(os.Stdin)
if err != nil {
fatal(fmt.Sprintf("read stdin payload: %v", err))
}
if len(bytes.TrimSpace(payload)) == 0 {
fatal("empty stdin payload")
}
if !json.Valid(payload) {
fatal("stdin payload is not valid JSON")
}
body, err := json.Marshal(publishBody{
AccountID: accountID,
Channel: channel,
Kind: kind,
Payload: json.RawMessage(payload),
})
if err != nil {
fatal(fmt.Sprintf("marshal request body: %v", err))
}
client, err := eksgw.New(gatewayURL, gatewayCA, accessKey, secretKey, region)
if err != nil {
fatal(fmt.Sprintf("build gateway client: %v", err))
}
path := "/clusters/" + clusterName + "/internal-publish"
var lastErr error
for attempt := 1; attempt <= maxAttempts; attempt++ {
if _, err := client.Post(path, body); err != nil {
lastErr = err
slog.Warn("eks-gateway-publish: attempt failed",
"channel", channel, "kind", kind, "attempt", attempt, "err", err)
if attempt < maxAttempts {
time.Sleep(retryDelay)
}
continue
}
slog.Info("eks-gateway-publish: published", "channel", channel, "kind", kind)
return
}
fatal(fmt.Sprintf("publish failed after %d attempts: %v", maxAttempts, lastErr))
}
func fatal(msg string) {
slog.Error("eks-gateway-publish: " + msg)
os.Exit(1)
}
cmd/eks-token-webhook/cert.go
+137
View File
@@ -0,0 +1,137 @@
package main
import (
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/base64"
"encoding/pem"
"fmt"
"math/big"
"net"
"os"
"time"
"github.com/mulgadc/spinifex/internal/tlsconfig"
)
// tlsCertificate is the webhook's loopback serving certificate.
type tlsCertificate struct {
cert tls.Certificate
}
func (c tlsCertificate) serverTLSConfig() *tls.Config {
return &tls.Config{
Certificates: []tls.Certificate{c.cert},
MinVersion: tls.VersionTLS13,
CurvePreferences: tlsconfig.Curves,
}
}
// ensureServingCert loads the persisted self-signed serving cert/key, or mints
// a fresh ECDSA-P256 pair (SAN 127.0.0.1 / localhost) and persists it. Reusing
// across restarts keeps the CA the apiserver loaded into its webhook kubeconfig
// valid even if the webhook process is restarted. Returns the parsed cert and
// the certificate PEM (for embedding as the kubeconfig CA bundle).
func ensureServingCert(certPath, keyPath string) (tlsCertificate, []byte, error) {
if certPEM, keyPEM, ok := readCertPair(certPath, keyPath); ok {
cert, err := tls.X509KeyPair(certPEM, keyPEM)
if err == nil {
return tlsCertificate{cert: cert}, certPEM, nil
}
// Fall through to regenerate on a corrupt/incompatible persisted pair.
}
certPEM, keyPEM, err := generateSelfSigned()
if err != nil {
return tlsCertificate{}, nil, err
}
if err := os.WriteFile(certPath, certPEM, 0o644); err != nil {
return tlsCertificate{}, nil, fmt.Errorf("write cert %s: %w", certPath, err)
}
if err := os.WriteFile(keyPath, keyPEM, 0o600); err != nil {
return tlsCertificate{}, nil, fmt.Errorf("write key %s: %w", keyPath, err)
}
cert, err := tls.X509KeyPair(certPEM, keyPEM)
if err != nil {
return tlsCertificate{}, nil, fmt.Errorf("parse minted keypair: %w", err)
}
return tlsCertificate{cert: cert}, certPEM, nil
}
func readCertPair(certPath, keyPath string) (certPEM, keyPEM []byte, ok bool) {
c, err := os.ReadFile(certPath)
if err != nil {
return nil, nil, false
}
k, err := os.ReadFile(keyPath)
if err != nil {
return nil, nil, false
}
return c, k, true
}
func generateSelfSigned() (certPEM, keyPEM []byte, err error) {
priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
return nil, nil, fmt.Errorf("generate key: %w", err)
}
serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
if err != nil {
return nil, nil, fmt.Errorf("serial: %w", err)
}
now := time.Now().UTC()
tmpl := x509.Certificate{
SerialNumber: serial,
Subject: pkix.Name{CommonName: "eks-token-webhook"},
NotBefore: now.Add(-time.Hour),
NotAfter: now.AddDate(10, 0, 0),
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
BasicConstraintsValid: true,
IsCA: true,
IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback},
DNSNames: []string{"localhost"},
}
der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv)
if err != nil {
return nil, nil, fmt.Errorf("create cert: %w", err)
}
keyDER, err := x509.MarshalECPrivateKey(priv)
if err != nil {
return nil, nil, fmt.Errorf("marshal key: %w", err)
}
certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
keyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
return certPEM, keyPEM, nil
}
// writeAPIServerKubeconfig writes the kubeconfig kube-apiserver loads via
// --authentication-token-webhook-config-file. It points the apiserver at the
// loopback webhook and trusts its self-signed serving cert as the CA bundle.
func writeAPIServerKubeconfig(path, addr string, certPEM []byte) error {
caB64 := base64.StdEncoding.EncodeToString(certPEM)
kubeconfig := fmt.Sprintf(`apiVersion: v1
kind: Config
clusters:
- name: mulga-eks-token-webhook
cluster:
certificate-authority-data: %s
server: https://%s/authenticate
users:
- name: kube-apiserver
contexts:
- name: webhook
context:
cluster: mulga-eks-token-webhook
user: kube-apiserver
current-context: webhook
`, caB64, addr)
if err := os.WriteFile(path, []byte(kubeconfig), 0o600); err != nil {
return fmt.Errorf("write %s: %w", path, err)
}
return nil
}
cmd/eks-token-webhook/main.go
+207 -18
View File
@@ -1,38 +1,227 @@
// eks-token-webhook is the K8s TokenReview webhook authenticator baked into
// the eks-server AMI. It decodes the SigV4-presigned GetCallerIdentity URL
// produced by `aws eks get-token`, enforces the X-K8s-Aws-Id cross-cluster
// pin, calls Mulga STS over NATS, and resolves the principal to a TokenReview
// response via the cluster's AccessEntry KV.
// the eks-server AMI. Per kubectl call the kube-apiserver POSTs the bearer
// token (produced by `aws eks get-token`); the webhook relays it through the
// AWS gateway broker (SigV4 HTTPS POST to /clusters/{name}/token-review), which
// host-side runs the STS verify (cross-cluster x-k8s-aws-id pin) + AccessEntry
// KV lookup and returns the resolved identity. The webhook never speaks core
// NATS — same gateway-broker model as ELBv2's lb-agent and eks-gateway-publish.
//
// This is a placeholder build that returns 503 Service Unavailable on every
// path. It exists so the eks-server AMI build succeeds (INSTALL_BINARIES is
// satisfied) and the K3s server VM boots cleanly. Real implementation lands
// with bead mulga-cs-eks-6c.
// It binds 127.0.0.1 only; the kube-apiserver is the sole client (loopback).
// On startup it writes the apiserver webhook kubeconfig (with its self-signed
// serving CA) so k3s can be pointed at it via
// --authentication-token-webhook-config-file.
package main
import (
"encoding/json"
"errors"
"flag"
"fmt"
"log/slog"
"net/http"
"os"
"time"
"github.com/mulgadc/spinifex/internal/eksgw"
handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
)
// config is the webhook's runtime configuration, sourced from the first-boot
// env (/etc/spinifex-eks/first-boot.env) cloud-init seeds onto the VM.
type config struct {
addr string
gatewayURL string
gatewayCA string
accessKey string
secretKey string
region string
accountID string
clusterName string
certPath string
keyPath string
kubeconfig string
}
func loadConfig() (config, error) {
c := config{
addr: flagAddr,
gatewayURL: os.Getenv("EKS_GATEWAY_URL"),
gatewayCA: os.Getenv("EKS_GATEWAY_CA"),
accessKey: os.Getenv("EKS_ACCESS_KEY"),
secretKey: os.Getenv("EKS_SECRET_KEY"),
region: os.Getenv("EKS_REGION"),
accountID: os.Getenv("EKS_ACCOUNT_ID"),
clusterName: os.Getenv("EKS_CLUSTER_NAME"),
certPath: envOr("EKS_WEBHOOK_CERT", "/etc/spinifex-eks/token-webhook.crt"),
keyPath: envOr("EKS_WEBHOOK_KEY", "/etc/spinifex-eks/token-webhook.key"),
kubeconfig: envOr("EKS_WEBHOOK_KUBECONFIG", "/etc/spinifex-eks/token-webhook.kubeconfig"),
}
switch {
case c.gatewayURL == "":
return c, errors.New("EKS_GATEWAY_URL not set")
case c.accessKey == "" || c.secretKey == "":
return c, errors.New("EKS_ACCESS_KEY / EKS_SECRET_KEY not set")
case c.accountID == "":
return c, errors.New("EKS_ACCOUNT_ID not set")
case c.clusterName == "":
return c, errors.New("EKS_CLUSTER_NAME not set")
}
return c, nil
}
func envOr(key, def string) string {
if v := os.Getenv(key); v != "" {
return v
}
return def
}
var flagAddr string
func main() {
addr := flag.String("addr", "127.0.0.1:8443", "listen address")
flag.StringVar(&flagAddr, "addr", "127.0.0.1:8443", "listen address (loopback only)")
flag.Parse()
slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo})))
mux := http.NewServeMux()
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Retry-After", "30")
w.Header().Set("Content-Type", "application/json")
http.Error(w, `{"error":"eks-token-webhook stub: real implementation not yet deployed"}`, http.StatusServiceUnavailable)
})
cfg, err := loadConfig()
if err != nil {
slog.Error("eks-token-webhook config error", "err", err)
os.Exit(1)
}
slog.Info("eks-token-webhook stub starting", "addr", *addr)
if err := http.ListenAndServe(*addr, mux); err != nil {
slog.Error("listen failed", "err", err)
if err := run(cfg); err != nil {
slog.Error("eks-token-webhook fatal", "err", err)
os.Exit(1)
}
}
func run(cfg config) error {
// Serving cert + apiserver kubeconfig. Persisted and reused across restarts
// so a webhook crash does not invalidate the CA the apiserver already loaded.
tlsCert, certPEM, err := ensureServingCert(cfg.certPath, cfg.keyPath)
if err != nil {
return fmt.Errorf("serving cert: %w", err)
}
if err := writeAPIServerKubeconfig(cfg.kubeconfig, cfg.addr, certPEM); err != nil {
return fmt.Errorf("write apiserver kubeconfig: %w", err)
}
client, err := eksgw.New(cfg.gatewayURL, cfg.gatewayCA, cfg.accessKey, cfg.secretKey, cfg.region)
if err != nil {
return fmt.Errorf("build gateway client: %w", err)
}
authr := &authenticator{
accountID: cfg.accountID,
clusterName: cfg.clusterName,
review: gatewayReviewer(client, cfg.clusterName, cfg.accountID),
}
mux := http.NewServeMux()
mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("ok"))
})
mux.HandleFunc("/authenticate", authr.handle)
server := newServer(cfg.addr, mux, tlsCert)
slog.Info("eks-token-webhook listening", "addr", cfg.addr, "cluster", cfg.clusterName)
if err := server.ListenAndServeTLS("", ""); err != nil && !errors.Is(err, http.ErrServerClosed) {
return fmt.Errorf("serve: %w", err)
}
return nil
}
// tokenReviewRequest is the body POSTed to /clusters/{name}/token-review. The
// webhook holds system SigV4 creds (system account, not the cluster account),
// so accountId names the cluster account explicitly — same as PublishInternal.
type tokenReviewRequest struct {
AccountID string `json:"accountId"`
Token string `json:"token"`
}
// gatewayReviewer returns the per-call relay: SigV4-POST {accountId,token} to
// the gateway and decode the resolved identity. A non-2xx / transport error is
// surfaced (handle() turns it into a 5xx so the apiserver retries) rather than a
// silent deny, distinguishing "webhook broker down" from "token rejected".
func gatewayReviewer(client *eksgw.Client, clusterName, accountID string) func(string) (handlers_eks.WebhookTokenReviewResult, error) {
path := "/clusters/" + clusterName + "/token-review"
return func(token string) (handlers_eks.WebhookTokenReviewResult, error) {
body, err := json.Marshal(tokenReviewRequest{AccountID: accountID, Token: token})
if err != nil {
return handlers_eks.WebhookTokenReviewResult{}, fmt.Errorf("marshal request: %w", err)
}
respBody, err := client.Post(path, body)
if err != nil {
return handlers_eks.WebhookTokenReviewResult{}, err
}
var res handlers_eks.WebhookTokenReviewResult
if err := json.Unmarshal(respBody, &res); err != nil {
return handlers_eks.WebhookTokenReviewResult{}, fmt.Errorf("decode response: %w", err)
}
return res, nil
}
}
// authenticator resolves a TokenReview by relaying to the gateway broker.
type authenticator struct {
accountID string
clusterName string
review func(token string) (handlers_eks.WebhookTokenReviewResult, error)
}
func (a *authenticator) handle(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
var review tokenReview
if err := json.NewDecoder(r.Body).Decode(&review); err != nil {
http.Error(w, "bad request", http.StatusBadRequest)
return
}
res, err := a.review(review.Spec.Token)
if err != nil {
// Broker fault (gateway down, transport error): a 5xx tells the apiserver
// to retry rather than treating a transient outage as a hard deny.
slog.Error("TokenReview relay failed", "cluster", a.clusterName, "err", err)
http.Error(w, "token review unavailable", http.StatusServiceUnavailable)
return
}
status := tokenReviewStatus{Authenticated: res.Authenticated}
if res.Authenticated {
status.User = userInfo{
Username: res.Username,
UID: res.UID,
Groups: res.Groups,
}
// Confirm the token for the audiences the apiserver asked about; an empty
// status.audiences is rejected when --api-audiences is configured.
status.Audiences = review.Spec.Audiences
}
slog.Info("TokenReview decision", "authenticated", status.Authenticated, "username", status.User.Username, "audiences", status.Audiences)
resp := tokenReview{
APIVersion: "authentication.k8s.io/v1",
Kind: "TokenReview",
Status: status,
}
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(resp); err != nil {
slog.Error("encode TokenReview response", "err", err)
}
}
// newServer builds the loopback TLS server. Split out so tests can construct it
// without binding a port.
func newServer(addr string, handler http.Handler, cert tlsCertificate) *http.Server {
return &http.Server{
Addr: addr,
Handler: handler,
ReadHeaderTimeout: 5 * time.Second,
TLSConfig: cert.serverTLSConfig(),
}
}
cmd/eks-token-webhook/main_test.go
+123
View File
@@ -0,0 +1,123 @@
package main
import (
"encoding/base64"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// postReview drives authenticator.handle with a TokenReview carrying token and
// audiences, returning the decoded response status and HTTP code.
func postReview(t *testing.T, a *authenticator, token string, audiences []string) (tokenReviewStatus, int) {
t.Helper()
body, err := json.Marshal(tokenReview{
APIVersion: "authentication.k8s.io/v1",
Kind: "TokenReview",
Spec: tokenReviewSpec{Token: token, Audiences: audiences},
})
require.NoError(t, err)
req := httptest.NewRequest(http.MethodPost, "/authenticate", strings.NewReader(string(body)))
w := httptest.NewRecorder()
a.handle(w, req)
if w.Code != http.StatusOK {
return tokenReviewStatus{}, w.Code
}
var resp tokenReview
require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp))
return resp.Status, w.Code
}
func TestHandle_GrantsAndStampsAudiences(t *testing.T) {
a := &authenticator{
accountID: "111122223333",
clusterName: "alpha",
review: func(string) (handlers_eks.WebhookTokenReviewResult, error) {
return handlers_eks.WebhookTokenReviewResult{
Authenticated: true,
Username: "arn:aws:iam::111122223333:role/admin",
UID: "AROAEXAMPLE:session",
Groups: []string{"system:masters"},
}, nil
},
}
status, code := postReview(t, a, "tok", []string{"https://kubernetes.default.svc"})
require.Equal(t, http.StatusOK, code)
require.True(t, status.Authenticated)
assert.Equal(t, "arn:aws:iam::111122223333:role/admin", status.User.Username)
assert.Equal(t, "AROAEXAMPLE:session", status.User.UID)
assert.Equal(t, []string{"system:masters"}, status.User.Groups)
assert.Equal(t, []string{"https://kubernetes.default.svc"}, status.Audiences)
}
func TestHandle_DeniesWithoutStampingAudiences(t *testing.T) {
a := &authenticator{
clusterName: "alpha",
review: func(string) (handlers_eks.WebhookTokenReviewResult, error) {
return handlers_eks.WebhookTokenReviewResult{Authenticated: false}, nil
},
}
status, code := postReview(t, a, "tok", []string{"https://kubernetes.default.svc"})
require.Equal(t, http.StatusOK, code)
assert.False(t, status.Authenticated)
assert.Empty(t, status.User.Username)
assert.Empty(t, status.Audiences, "unauthenticated review must not stamp audiences")
}
// A broker fault is a 5xx (retryable), not a silent deny — a transient gateway
// outage must not look identical to a rejected token.
func TestHandle_BrokerFaultReturns5xx(t *testing.T) {
a := &authenticator{
clusterName: "alpha",
review: func(string) (handlers_eks.WebhookTokenReviewResult, error) {
return handlers_eks.WebhookTokenReviewResult{}, errors.New("gateway returned 503")
},
}
_, code := postReview(t, a, "tok", nil)
assert.Equal(t, http.StatusServiceUnavailable, code)
}
func TestEnsureServingCert_PersistsAndReuses(t *testing.T) {
dir := t.TempDir()
certPath := filepath.Join(dir, "wh.crt")
keyPath := filepath.Join(dir, "wh.key")
c1, pem1, err := ensureServingCert(certPath, keyPath)
require.NoError(t, err)
require.NotEmpty(t, pem1)
require.NotNil(t, c1.serverTLSConfig())
// Second call reuses the persisted pair (same cert bytes).
_, pem2, err := ensureServingCert(certPath, keyPath)
require.NoError(t, err)
assert.Equal(t, pem1, pem2)
}
func TestWriteAPIServerKubeconfig(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "wh.kubeconfig")
certPEM := []byte("-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----\n")
require.NoError(t, writeAPIServerKubeconfig(path, "127.0.0.1:8443", certPEM))
data, err := os.ReadFile(path)
require.NoError(t, err)
body := string(data)
assert.Contains(t, body, "server: https://127.0.0.1:8443/authenticate")
assert.Contains(t, body, base64.StdEncoding.EncodeToString(certPEM))
assert.Contains(t, body, "current-context: webhook")
}
cmd/eks-token-webhook/tokenreview.go
+34
View File
@@ -0,0 +1,34 @@
package main
// Minimal authentication.k8s.io/v1 TokenReview wire types. Hand-rolled to avoid
// pulling k8s.io/api into the spinifex module for these few fields; the shapes
// match what kube-apiserver sends and expects on the webhook contract.
type tokenReview struct {
APIVersion string `json:"apiVersion"`
Kind string `json:"kind"`
Spec tokenReviewSpec `json:"spec,omitzero"`
Status tokenReviewStatus `json:"status"`
}
type tokenReviewSpec struct {
Token string `json:"token"`
Audiences []string `json:"audiences,omitempty"`
}
type tokenReviewStatus struct {
Authenticated bool `json:"authenticated"`
User userInfo `json:"user,omitzero"`
// Audiences echoes the request's spec.audiences on success. When the
// apiserver runs with --api-audiences, it sends spec.audiences and discards
// an authenticated=true response that does not confirm a matching audience —
// so an empty Audiences here surfaces as a 401 despite a valid principal.
Audiences []string `json:"audiences,omitempty"`
Error string `json:"error,omitempty"`
}
type userInfo struct {
Username string `json:"username,omitempty"`
UID string `json:"uid,omitempty"`
Groups []string `json:"groups,omitempty"`
}
cmd/spinifex/cmd/admin.go
+5
View File
@@ -325,6 +325,7 @@ func init() {
imagesImportCmd.Flags().String("tmp-dir", os.TempDir(), "Temporary directory for image import processing")
imagesImportCmd.Flags().String("name", "", "Import specified image by name")
imagesImportCmd.Flags().String("ami-name", "", "Override the registered AMI name (DescribeImages name). Defaults to ami-{distro}-{version}-{arch}. Use for locally-built appliances (e.g. --ami-name spinifex-eks-node).")
imagesImportCmd.Flags().String("file", "", "Import file from specified path (raw, qcow2, compressed)")
imagesImportCmd.Flags().String("distro", "", "Specified distro name (e.g debian)")
imagesImportCmd.Flags().String("version", "", "Specified distro version (e.g 12)")
@@ -380,6 +381,7 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
// and the file is used directly (no download). When only --file is set,
// metadata comes from flags and no catalog tags are applied.
imageName, _ := cmd.Flags().GetString("name")
amiNameOverride, _ := cmd.Flags().GetString("ami-name")
localFile, _ := cmd.Flags().GetString("file")
if imageName == "" && localFile == "" {
@@ -535,6 +537,9 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
// Calculate the size
manifest.AMIMetadata.Name = fmt.Sprintf("ami-%s-%s-%s", image.Distro, image.Version, image.Arch)
if amiNameOverride != "" {
manifest.AMIMetadata.Name = amiNameOverride
}
volumeId := utils.GenerateResourceID("ami")
manifest.AMIMetadata.ImageID = volumeId
cmd/spinifex/cmd/service.go
+1 -1
View File
@@ -538,7 +538,7 @@ var awsgwStartCmd = &cobra.Command{
awsgwHost := viper.GetString("host")
if awsgwHost != "" {
fmt.Println("Overwriting awsgw host to:", awsgwHost)
//nodeConfig.AWSGW.Host = awsgwHost
nodeConfig.AWSGW.Host = awsgwHost
}
awsgwTlsCert := viper.GetString("tls-cert")
docs/COMMANDS.md
+110 -159
View File
@@ -8,162 +8,102 @@ Platform management commands not exposed via the AWS gateway API. These are CLI-
Service lifecycle commands for starting, stopping, and checking status of all Spinifex cluster services. Each service subcommand supports `start`, `stop`, and `status` operations.
| Command | Flags | Prerequisites | Basic Logic | Status |
|---------|-------|---------------|-------------|--------|
| `spx service predastore start` | `--port` (default: 8443), `--host` (default: 0.0.0.0), `--base-path`, `--config-path`, `--debug`, `--tls-cert`, `--tls-key`, `--backend` (distributed/filesystem, default: distributed), `--node-id` (default: 0), `--pprof`, `--pprof-output` | TLS cert/key, base-path, config-path required | Creates predastore service instance with S3-compatible storage backend → starts service | **DONE** |
| `spx service predastore stop` | — | Predastore must be running | Stops the predastore service | **DONE** |
| `spx service predastore status` | — | None | Reports predastore service status | **DONE** |
| `spx service viperblock start` | `--s3-host` (default: 0.0.0.0:8443), `--s3-bucket` (default: predastore), `--s3-region` (default: ap-southeast-2), `--plugin-path` (auto-detected via `nbdkit --dump-config plugindir`; typically `/usr/lib/x86_64-linux-gnu/nbdkit/plugins/nbdkit-viperblock-plugin.so` on amd64, overridable via `SPINIFEX_VIPERBLOCK_PLUGIN_PATH` in `/etc/spinifex/systemd.env`) | `--config` required, plugin-path must exist on disk | Loads cluster config → connects to NATS and Predastore → starts viperblock block storage service with NBD plugin | **DONE** |
| `spx service viperblock stop` | — | Viperblock must be running | Stops the viperblock service | **DONE** |
| `spx service viperblock status` | — | None | Reports viperblock service status | **DONE** |
| `spx service nats start` | `--port` (default: 4222), `--host` (default: 0.0.0.0), `--debug`, `--data-dir`, `--jetstream` | None | Starts embedded NATS server with optional JetStream | **DONE** |
| `spx service nats stop` | — | NATS must be running | Stops the NATS service | **DONE** |
| `spx service nats status` | — | None | Reports NATS service status | **DONE** |
| `spx service spinifex start` | `--wal-dir` | `--config` required | Loads cluster config → starts spinifex daemon (VM orchestration, NATS subscriptions, health endpoint) | **DONE** |
| `spx service spinifex stop` | — | Spinifex daemon must be running | Stops the spinifex daemon service | **DONE** |
| `spx service spinifex status` | — | None | Reports spinifex daemon service status | **DONE** |
| `spx service awsgw start` | `--host` (default: 0.0.0.0:9999), `--tls-cert`, `--tls-key`, `--debug` | `--config` required | Loads cluster config → starts AWS-compatible gateway with SigV4 auth, IAM policy enforcement, TLS | **DONE** |
| `spx service awsgw stop` | — | AWS gateway must be running | Stops the AWS gateway service | **DONE** |
| `spx service awsgw status` | — | None | Reports AWS gateway service status | **DONE** |
| `spx service spinifex-ui start` | `--port` (default: 3000), `--host` (default: 0.0.0.0), `--tls-cert`, `--tls-key` | None | Starts embedded web UI server serving the React frontend. Aliases: `ui`, `spinifexui` | **DONE** |
| `spx service spinifex-ui stop` | — | spinifex-ui must be running | Stops the spinifex-ui service | **DONE** |
| `spx service spinifex-ui status` | — | None | Reports spinifex-ui service status | **DONE** |
| `spx service vpcd start` | — | `--config` required, OVN/OVS installed | Loads cluster config → starts VPC daemon (subscribes to `vpc.*` NATS events, translates to OVN logical switches/ports/routers) | **DONE** |
| `spx service vpcd stop` | — | vpcd must be running | Stops the vpcd service | **DONE** |
| `spx service vpcd status` | — | None | Reports vpcd service status | **DONE** |
| Command | Flags | Description |
|---------|-------|-------------|
| `spx service predastore start` | `--port` (default: 8443), `--host` (default: 0.0.0.0), `--base-path`, `--config-path`, `--debug`, `--tls-cert`, `--tls-key`, `--backend` (distributed/filesystem, default: distributed), `--node-id` (default: 0), `--pprof`, `--pprof-output` | Creates predastore service instance with S3-compatible storage backend → starts service |
| `spx service predastore stop` | — | Stops the predastore service |
| `spx service predastore status` | — | Reports predastore service status |
| `spx service viperblock start` | `--s3-host` (default: 0.0.0.0:8443), `--s3-bucket` (default: predastore), `--s3-region` (default: ap-southeast-2), `--plugin-path` (auto-detected via `nbdkit --dump-config plugindir`; typically `/usr/lib/x86_64-linux-gnu/nbdkit/plugins/nbdkit-viperblock-plugin.so` on amd64, overridable via `SPINIFEX_VIPERBLOCK_PLUGIN_PATH` in `/etc/spinifex/systemd.env`) | Loads cluster config → connects to NATS and Predastore → starts viperblock block storage service with NBD plugin |
| `spx service viperblock stop` | — | Stops the viperblock service |
| `spx service viperblock status` | — | Reports viperblock service status |
| `spx service nats start` | `--port` (default: 4222), `--host` (default: 0.0.0.0), `--debug`, `--data-dir`, `--jetstream` | Starts embedded NATS server with optional JetStream |
| `spx service nats stop` | — | Stops the NATS service |
| `spx service nats status` | — | Reports NATS service status |
| `spx service spinifex start` | `--wal-dir` | Loads cluster config → starts spinifex daemon (VM orchestration, NATS subscriptions, health endpoint) |
| `spx service spinifex stop` | — | Stops the spinifex daemon service |
| `spx service spinifex status` | — | Reports spinifex daemon service status |
| `spx service awsgw start` | `--host` (default: 0.0.0.0:9999), `--tls-cert`, `--tls-key`, `--debug` | Loads cluster config → starts AWS-compatible gateway with SigV4 auth, IAM policy enforcement, TLS |
| `spx service awsgw stop` | — | Stops the AWS gateway service |
| `spx service awsgw status` | — | Reports AWS gateway service status |
| `spx service spinifex-ui start` | `--port` (default: 3000), `--host` (default: 0.0.0.0), `--tls-cert`, `--tls-key` | Starts embedded web UI server serving the React frontend. Aliases: `ui`, `spinifexui` |
| `spx service spinifex-ui stop` | — | Stops the spinifex-ui service |
| `spx service spinifex-ui status` | — | Reports spinifex-ui service status |
| `spx service vpcd start` | — | Loads cluster config → starts VPC daemon (subscribes to `vpc.*` NATS events, translates to OVN logical switches/ports/routers) |
| `spx service vpcd stop` | — | Stops the vpcd service |
| `spx service vpcd status` | — | Reports vpcd service status |
### Cluster Inspection
Operational commands for inspecting cluster state. These fan out NATS requests to all nodes and aggregate responses.
| Command | Flags | Prerequisites | Basic Logic | Output Columns | Status |
|---------|-------|---------------|-------------|----------------|--------|
| `spx get nodes` | `--timeout` (default: 3s) | Cluster must be running (NATS) | Loads config → publishes to `spinifex.node.status` fan-out topic → collects responses within timeout → merges config-known nodes with NATS responders → nodes that don't respond shown as `NotReady` | NAME, STATUS, IP, REGION, AZ, UPTIME, VMs, SERVICES | **DONE** |
| `spx get vms` | `--timeout` (default: 3s) | Cluster must be running (NATS) | Publishes to `spinifex.node.vms` fan-out topic → collects VM info from all nodes → sorts by node then instance ID → prints table. Alias: `spx get instances` | INSTANCE, STATUS, TYPE, VCPU, MEM, NODE, IP, AGE | **DONE** |
| Command | Flags | Description |
|---------|-------|-------------|
| `spx get nodes` | `--timeout` (default: 3s) | Loads config → publishes to `spinifex.node.status` fan-out topic → collects responses within timeout → merges config-known nodes with NATS responders → nodes that don't respond shown as `NotReady` |
| `spx get vms` | `--timeout` (default: 3s) | Publishes to `spinifex.node.vms` fan-out topic → collects VM info from all nodes → sorts by node then instance ID → prints table. Alias: `spx get instances` |
### Resource Monitoring
| Command | Flags | Prerequisites | Basic Logic | Output | Status |
|---------|-------|---------------|-------------|--------|--------|
| `spx top nodes` | `--timeout` (default: 3s) | Cluster must be running (NATS) | Publishes to `spinifex.node.status` fan-out topic → collects CPU/memory usage per node → aggregates instance type capacity across all nodes → prints two tables: per-node resource usage and cluster-wide instance type availability | Table 1: NAME, CPU (used/total), MEM (used/total), GPU (used/total), VMs — GPU column shows `allocated/total` when passthrough is enabled, `0/count*` when the node is GPU-capable but passthrough is disabled, or `-` when no GPU is present. Table 2: INSTANCE TYPE, AVAILABLE, VCPU, MEMORY | **DONE** |
| Command | Flags | Description |
|---------|-------|-------------|
| `spx top nodes` | `--timeout` (default: 3s) | Publishes to `spinifex.node.status` fan-out topic → collects CPU/memory usage per node → aggregates instance type capacity across all nodes → prints two tables: per-node resource usage and cluster-wide instance type availability |
### Cluster Initialization
| Command | Flags | Prerequisites | Basic Logic | Test Cases | Status |
|---------|-------|---------------|-------------|------------|--------|
| `spx admin init` | `--nodes`, `--node`, `--bind`, `--port`, `--region`, `--az`, `--cluster-name`, `--cluster-bind`, `--cluster-routes`, `--predastore-nodes`, `--services`, `--formation-timeout`, `--token-ttl`, `--force` | None (first-time setup) | Generates root IAM credentials (AKIA-prefixed access key + secret) → creates master.key (AES-256, 32 bytes, 0600) → writes bootstrap.json (consumed on first start) → generates CA + server TLS certificates → generates join token (written to `join-token` file, displayed in join command) → creates NATS config with auth token → writes spinifex.toml, awsgw.toml, predastore.toml → configures AWS CLI `spx` profile → creates directory structure under `~/spinifex/` | 1. Init creates all config files<br>2. Root credentials printed once<br>3. master.key is 32 bytes, mode 0600<br>4. bootstrap.json consumed on first start<br>5. `--force` re-initializes existing config<br>6. AWS CLI profile `spx` auto-configured<br>7. Multi-node init generates join token and writes to `<config-dir>/join-token` | **DONE** |
| `spx admin join` | `--host` (required), `--node` (required), `--token` (required), `--bind`, `--port`, `--region`, `--az`, `--cluster-bind`, `--cluster-routes`, `--data-dir`, `--services` | Leader node must be running | Connects to leader node with join token (Authorization: Bearer header) → retrieves cluster configuration → configures local node to join cluster and participate in distributed operations | 1. Join existing cluster<br>2. Missing host (error)<br>3. Missing node name (error)<br>4. Missing token (error)<br>5. Invalid token (401)<br>6. Expired token (401) | **DONE** |
| Command | Flags | Description |
|---------|-------|-------------|
| `spx admin init` | `--nodes`, `--node`, `--bind`, `--port`, `--region`, `--az`, `--cluster-name`, `--cluster-bind`, `--cluster-routes`, `--predastore-nodes`, `--services`, `--formation-timeout`, `--token-ttl`, `--force` | Generates root IAM credentials (AKIA-prefixed access key + secret) → creates master.key (AES-256, 32 bytes, 0600) → writes bootstrap.json (consumed on first start) → generates CA + server TLS certificates → generates join token (written to `join-token` file, displayed in join command) → creates NATS config with auth token → writes spinifex.toml, awsgw.toml, predastore.toml → configures AWS CLI `spx` profile → creates directory structure under `~/spinifex/` |
| `spx admin join` | `--host` (required), `--node` (required), `--token` (required), `--bind`, `--port`, `--region`, `--az`, `--cluster-bind`, `--cluster-routes`, `--data-dir`, `--services` | Connects to leader node with join token (Authorization: Bearer header) → retrieves cluster configuration → configures local node to join cluster and participate in distributed operations |
### Version
| Command | Flags | Prerequisites | Basic Logic | Status |
|---------|-------|---------------|-------------|--------|
| `spx version` | — | None | Prints Spinifex version, commit hash, OS, and architecture (populated via build-time ldflags) | **DONE** |
| Command | Flags | Description |
|---------|-------|-------------|
| `spx version` | — | Prints Spinifex version, commit hash, OS, and architecture (populated via build-time ldflags) |
### Cluster Operations
| Command | Flags | Prerequisites | Basic Logic | Test Cases | Status |
|---------|-------|---------------|-------------|------------|--------|
| `spx admin cluster shutdown` | `--force` (shutdown even if nodes don't respond), `--timeout` (max wait per phase, default 120s), `--dry-run` (print phase plan without executing) | Cluster must be running | Performs coordinated, phased shutdown of entire cluster. Phases execute in order: GATE (stop API/UI) → DRAIN (stop VMs) → STORAGE (stop viperblock) → PERSIST (stop predastore) → INFRA (stop NATS/daemon). Each phase waits for all nodes to ACK before proceeding. Uses JetStream state tracking. | 1. Shutdown running cluster<br>2. All nodes stop cleanly<br>3. Force shutdown with unresponsive nodes<br>4. Dry-run prints plan | **DONE** |
| Command | Flags | Description |
|---------|-------|-------------|
| `spx admin cluster shutdown` | `--force` (shutdown even if nodes don't respond), `--timeout` (max wait per phase, default 120s), `--dry-run` (print phase plan without executing) | Performs coordinated, phased shutdown of entire cluster. Phases execute in order: GATE (stop API/UI) → DRAIN (stop VMs) → STORAGE (stop viperblock) → PERSIST (stop predastore) → INFRA (stop NATS/daemon). Each phase waits for all nodes to ACK before proceeding. Uses JetStream state tracking. |
### Certificate Management
| Command | Flags | Prerequisites | Basic Logic | Test Cases | Status |
|---------|-------|---------------|-------------|------------|--------|
| `spx admin cert renew` | `--extra-ip` (additional IPs for SANs), `--extra-dns` (additional DNS names for SANs) | Existing CA from `spx admin init` | Reads existing CA → regenerates server certificate with all current network interface IPs and machine hostname in SANs → writes new cert. Use after adding a new network interface or changing IP addresses. | 1. Renew with auto-detected IPs<br>2. Renew with extra IPs<br>3. Renew with extra DNS names | **DONE** |
| Command | Flags | Description |
|---------|-------|-------------|
| `spx admin cert renew` | `--extra-ip` (additional IPs for SANs), `--extra-dns` (additional DNS names for SANs) | Reads existing CA → regenerates server certificate with all current network interface IPs and machine hostname in SANs → writes new cert. Use after adding a new network interface or changing IP addresses. |
### Upgrade Management
| Command | Flags | Prerequisites | Basic Logic | Test Cases | Status |
|---------|-------|---------------|-------------|------------|--------|
| `spx admin upgrade` | `--yes` (apply without prompting), `--config-dir` (persistent, default: `~/spinifex/config`), `--spinifex-dir` (persistent, default: `~/spinifex`) | Existing installation (`spinifex.toml` must exist in config dir) | Reads current config versions from registry → computes pending config migrations (from→to per target) → prints version summary and pending list → prompts for confirmation unless `--yes` → runs `migrate.DefaultRegistry.RunAllConfig()` to apply migrations to config files. Intended for upgrades between Spinifex versions. Operators can skip migrations during install by setting `INSTALL_SPINIFEX_SKIP_MIGRATE=1`, then run `spx admin upgrade` manually to review and apply. After completion, services must be restarted with `sudo systemctl restart spinifex.target`. Invoked non-interactively by `scripts/setup.sh` with `--yes`. | 1. No installation present (error, suggests `spx admin init`)<br>2. No pending migrations (prints "No pending config migrations")<br>3. Pending migrations with interactive confirmation (y/yes applies, anything else aborts)<br>4. Pending migrations with `--yes` (applies without prompt)<br>5. Migration failure surfaces error and exits non-zero | **DONE** |
| Command | Flags | Description |
|---------|-------|-------------|
| `spx admin upgrade` | `--yes` (apply without prompting), `--config-dir` (persistent, default: `~/spinifex/config`), `--spinifex-dir` (persistent, default: `~/spinifex`) | Reads current config versions from registry → computes pending config migrations (from→to per target) → prints version summary and pending list → prompts for confirmation unless `--yes` → runs `migrate.DefaultRegistry.RunAllConfig()` to apply migrations to config files. Intended for upgrades between Spinifex versions. Operators can skip migrations during install by setting `INSTALL_SPINIFEX_SKIP_MIGRATE=1`, then run `spx admin upgrade` manually to review and apply. After completion, services must be restarted with `sudo systemctl restart spinifex.target`. Invoked non-interactively by `scripts/setup.sh` with `--yes`. |
### Account Management
| Command | Flags | Prerequisites | Basic Logic | Test Cases | Status |
|---------|-------|---------------|-------------|------------|--------|
| `spx admin account create` | `--name` | Cluster must be running (NATS), master key must exist | Connects to NATS → CAS loop on `spinifex-account-counter:next_id` for sequential 12-digit ID → creates Account record in `spinifex-accounts` KV → creates `admin` user in new account → creates access key for admin → creates AdministratorAccess policy (Action:*, Resource:*) → attaches policy → prints credentials | 1. Create first account (ID 000000000001)<br>2. Sequential IDs (000000000002, etc.)<br>3. Admin user has full access<br>4. Credentials printed once | **DONE** |
| `spx admin account list` | — | Cluster must be running (NATS), master key must exist | Connects to NATS → IAMService.ListAccounts() → prints table with Account ID, Name, Status, Created | 1. List all accounts<br>2. Empty list (only global account) | **DONE** |
| Command | Flags | Description |
|---------|-------|-------------|
| `spx admin account create` | `--name` | Connects to NATS → CAS loop on `spinifex-account-counter:next_id` for sequential 12-digit ID → creates Account record in `spinifex-accounts` KV → creates `admin` user in new account → creates access key for admin → creates AdministratorAccess policy (Action:*, Resource:*) → attaches policy → prints credentials |
| `spx admin account list` | — | Connects to NATS → IAMService.ListAccounts() → prints table with Account ID, Name, Status, Created |
### Image Management
| Command | Flags | Prerequisites | Basic Logic | Test Cases | Status |
|---------|-------|---------------|-------------|------------|--------|
| `spx admin images import` | `--name`, `--file`, `--distro`, `--version`, `--arch`, `--platform`, `--boot-mode` (bios/uefi/uefi-preferred), `--tag`, `--force`, `--skip-verify` | Cluster must be running; either `--name` (catalog download) or `--file` (operator-supplied media). `--file` mode additionally requires `--distro`, `--version`, `--arch`, `--boot-mode`. | Catalog imports (`--name`) download the image, fetch the catalog `Checksum` URL, verify the SHA-256/SHA-512 digest, and inherit `BootMode` from the catalog entry. `--boot-mode` overrides the catalog value when set. Mismatch fails closed; the cached file is left on disk and `--force` re-downloads. `--file` imports skip checksum verification (operator-supplied media is outside Spinifex's trust boundary, the skip is logged at INFO for audit) and require an explicit `--boot-mode` because there is no catalog metadata to inherit from. `--skip-verify` bypasses verification for catalog imports and emits a WARN slog + stderr notice; use only for debugging or when upstream mirrors are confirmed-broken. | 1. Import valid catalog image (verifies checksum)<br>2. Tampered cache hit fails with `ErrChecksumMismatch`<br>3. `--force` recovers after a mismatch<br>4. `--file` import skips verification<br>5. `--skip-verify` bypasses checksum with WARN<br>6. `--file` without `--boot-mode` rejected<br>7. `--boot-mode=invalid` rejected | **DONE** |
| `spx admin images list` | — | None | Lists available OS images that can be imported or downloaded | 1. List available images | **DONE** |
| `spx admin images remove` | `--image-id` (required), `--force`, `--yes` | Cluster must be running; predastore reachable. Targets AMIs imported via `spx admin images import` (non-account `ImageOwnerAlias`, e.g. `"system"`). | Loads `ami-<id>/config.json`, walks transitive dependents — copied snapshots whose `VolumeID == imageID`, volumes whose `SnapshotID` references the internal `snap-ami-<id>` or any derived snap, and account AMIs created via `CopyImage` whose `SnapshotID` is a derived snap — then prompts (skipped with `--yes`) before deleting `ami-<id>/config.json` (the DescribeImages barrier) followed by the rest of `ami-<id>/` and `snap-ami-<id>/`. Account-owned AMIs are refused with a hint pointing at `aws ec2 deregister-image` + `aws ec2 delete-snapshot`. `--force` bypasses the dependency, ownership and config-corrupt checks for salvage of orphaned blocks. | 1. Happy path: system AMI, no deps → deleted<br>2. Account-owned → refused with AWS-flow hint<br>3. Missing/corrupt config → `InvalidAMIID.NotFound` (salvageable with `--force`)<br>4. Dependent volume (direct or via copied snap) → refused<br>5. Dependent account AMI → refused<br>6. `--force` overrides dependents<br>7. Idempotent re-run after partial delete | **DONE** |
#### Image integrity verification (CMMC SI.L1-3.14.2)
Catalog imports (`spx admin images import --name <name>`) verify the image
against the catalog-declared SHA-256/SHA-512 digest before extraction. The sums
file is fetched from the catalog `Checksum` URL over HTTPS only (cross-scheme
redirects refused), and verification runs on both fresh downloads and cache
hits so a poisoned cache is caught on the next import.
On mismatch the import exits non-zero, the cached file is left on disk for
inspection, and the printed guidance is `spx admin images import --name <name>
--force` to re-download.
`--file` imports skip verification by design: operator-supplied media is
outside Spinifex's trust boundary and the operator is responsible for
integrity (e.g. `sha256sum` against a trusted upstream digest before import).
The skip is recorded as an INFO `slog` event with `reason=local-file-import`
so a CMMC assessor can audit the decision from journald.
`--skip-verify` bypasses the checksum step for catalog imports. The command
still downloads via the catalog URL but does not compare the image digest
against the sums file. Intended for narrow cases such as debugging upstream
mirror issues or running against a transiently-broken `latest/` path; the
skip is logged at WARN with `reason=skip-verify-flag` and printed to stderr
so operators and assessors see it. Prefer `--file` with an out-of-band
verified image over `--skip-verify` whenever possible.
**Limitation:** verification confirms the image matches the digest the mirror
served. A mirror compromise that swaps both image and sums file is not
detected; closing that gap requires GPG signature verification of the sums
file, deferred to a later phase.
#### `spx admin images remove` caveats
Admin-imported AMIs (`ImageOwnerAlias = "system"`) live
under the `ami-<id>/` S3 prefix and use a viperblock-internal snap checkpoint
at `snap-ami-<id>/` — there is no `snap-<id>/metadata.json`. The AWS handlers
(`DeregisterImage`, `DeleteSnapshot`) reject system owners with
`UnauthorizedOperation`, which is the right behaviour for tenant API callers
but leaves no AWS-flow path to reclaim space. `spx admin images remove` is
the admin-trust-boundary counterpart that performs the dependency walk and
hard-deletes the blocks directly against predastore.
`CopyImage` of a system AMI is metadata-only: it writes a fresh
`snap-<acct>/metadata.json` whose `VolumeID` points at `ami-<sys>` and a new
`ami-<acct>/config.json` referencing that snap. Volumes launched from the
copied AMI read transitively from `ami-<sys>/chunks/...`. The remove command
walks this transitive set and refuses if anything references the target.
**TOCTOU window:** between the safety scan and the `config.json` delete a
concurrent `RunInstances` against the AMI could create a new dependent
volume. The window is sub-second on a healthy cluster. The admin running
this command is expected to know the fleet's operational state; if the race
fires the result is a `vol-<id>` with deleted backing blocks, recovered by
terminating the orphaned instance.
**`--force` bypasses every safety check** (dependents, ownership, missing /
corrupt `config.json`). Use only for salvage of orphaned blocks. Running it
against a live system AMI corrupts every dependent volume on the next disk
read.
| Command | Flags | Description |
|---------|-------|-------------|
| `spx admin images import` | `--name`, `--file`, `--distro`, `--version`, `--arch`, `--platform`, `--boot-mode` (bios/uefi/uefi-preferred), `--tag`, `--force`, `--skip-verify` | Catalog imports (`--name`) download the image, fetch the catalog `Checksum` URL, verify the SHA-256/SHA-512 digest, and inherit `BootMode` from the catalog entry. `--boot-mode` overrides the catalog value when set. Mismatch fails closed; the cached file is left on disk and `--force` re-downloads. `--file` imports skip checksum verification (operator-supplied media is outside Spinifex's trust boundary, the skip is logged at INFO for audit) and require an explicit `--boot-mode` because there is no catalog metadata to inherit from. `--skip-verify` bypasses verification for catalog imports and emits a WARN slog + stderr notice; use only for debugging or when upstream mirrors are confirmed-broken. |
| `spx admin images list` | — | Lists available OS images that can be imported or downloaded |
| `spx admin images remove` | `--image-id` (required), `--force`, `--yes` | Loads `ami-<id>/config.json`, walks transitive dependents — copied snapshots whose `VolumeID == imageID`, volumes whose `SnapshotID` references the internal `snap-ami-<id>` or any derived snap, and account AMIs created via `CopyImage` whose `SnapshotID` is a derived snap — then prompts (skipped with `--yes`) before deleting `ami-<id>/config.json` (the DescribeImages barrier) followed by the rest of `ami-<id>/` and `snap-ami-<id>/`. Account-owned AMIs are refused with a hint pointing at `aws ec2 deregister-image` + `aws ec2 delete-snapshot`. `--force` bypasses the dependency, ownership and config-corrupt checks for salvage of orphaned blocks. |
### GPU Management
| Command | Flags | Prerequisites | Basic Logic | Test Cases | Status |
|---------|-------|---------------|-------------|------------|--------|
| `spx admin gpu status` | `--node` (default: local node) | Cluster must be running (NATS) | Queries `spinifex.node.status` fan-out → finds the target node response → prints Node, GPU hardware (model list or "none detected"), IOMMU state, vfio-pci state, passthrough enabled/disabled, and GPU pool allocation (`allocated/total`). Also lists GPU-capable instance types when passthrough is active. | 1. Local node with passthrough enabled (shows pool and instance types)<br>2. Local node capable but passthrough disabled (shows "run enable")<br>3. Node without GPU hardware (shows "none detected")<br>4. `--node` targets a remote node<br>5. Node not found or not responding (error) | **DONE** |
| `spx admin gpu enable` | — | Cluster must be running (NATS); node must be GPU-capable (IOMMU + vfio-pci configured) | Checks current passthrough state via NATS → errors if already enabled or prerequisites not met (directs to `setup`) → writes `gpu_passthrough = true` to `spinifex.toml` via `admin.SetGPUPassthrough` → sends SIGHUP to `spinifex-daemon` via `systemctl kill -s HUP` → polls node status for up to 30 s until daemon confirms new state → prints final `gpu status` output. Must be run directly on the target host. | 1. Enable on capable node<br>2. Already enabled (no-op)<br>3. Prerequisites not met (error, suggests `setup`)<br>4. Daemon does not confirm within 30 s (timeout error) | **DONE** |
| `spx admin gpu disable` | — | Cluster must be running (NATS); no GPU instances may be running | Checks current passthrough state via NATS → errors if already disabled or if `AllocGPUs > 0` (must terminate GPU instances first) → writes `gpu_passthrough = false` → sends SIGHUP to `spinifex-daemon` → polls for up to 30 s → prints final `gpu status` output. Must be run directly on the target host. | 1. Disable with no GPU instances running<br>2. Already disabled (no-op)<br>3. GPU instances running (error with count)<br>4. Daemon does not confirm within 30 s (timeout error) | **DONE** |
| `spx admin gpu setup` | — | Must be run as root; NVIDIA or AMD GPU must be present | Idempotent host configuration for GPU passthrough. Steps (skipped if already applied): detect GPUs via `gpu.Discover` → collect PCI IDs for all IOMMU-group siblings → check/enable IOMMU in GRUB (`intel_iommu=on iommu=pt` or `amd_iommu=on iommu=pt`) → write vfio udev rule (`/etc/udev/rules.d/99-spinifex-vfio.rules`) → blacklist nouveau (`/etc/modprobe.d/blacklist-nouveau.conf`) → blacklist amdgpu if AMD GPU present (`/etc/modprobe.d/blacklist-amdgpu.conf`) → write vfio-pci early binding config (`/etc/modprobe.d/vfio-pci.conf`) → add vfio modules to initramfs. If any change requires a reboot: runs `update-initramfs -u` and exits with reboot instructions. After reboot: verifies `vfio_pci` module is loaded → verifies each GPU is bound to `vfio-pci` (binds explicitly via `driver_override` if unbound) → calls `gpu enable` to activate passthrough. | 1. First run on unconfigured host (GRUB updated, reboot required)<br>2. Re-run after reboot (verifies bindings, enables passthrough)<br>3. Already fully configured (all steps skipped, enables passthrough)<br>4. AMD GPU present (blacklists amdgpu in addition to nouveau)<br>5. Not run as root (error)<br>6. No GPU hardware detected (error) | **DONE** |
| `spx admin gpu mig status` | — (must be run on target host) | NVIDIA GPU with MIG support present | Runs `gpu.Discover()` locally → for each GPU prints: PCI address, model, MIG capability, MIG mode (enabled/disabled/N/A); for GPUs with MIG enabled lists active MIG slices with GI ID, profile name, and mdev path. | 1. Node with MIG enabled (shows slices)<br>2. MIG capable but not enabled (shows disabled)<br>3. No MIG-capable GPU (shows N/A for MIG fields) | **DONE** |
| `spx admin gpu mig enable` | `--profile <name>` (required, e.g. `1g.10gb`), `--gpu <pci-addr>` (optional, default: all MIG-capable GPUs) | Must be run as root; MIG-capable GPU present; no GPU instances running | Checks no GPU instances running (NATS) → discovers MIG-capable GPUs via `gpu.Discover()` (filtered by `--gpu` if set) → enables MIG mode on each target (`gpu.EnableMIGMode`) → lists available profiles (`gpu.ListProfiles`) and validates requested profile name → destroys any existing instances (`gpu.DestroyAllInstances`) → creates new instances filling GPU capacity (`gpu.CreateInstances`) → writes `mig_profile` to `spinifex.toml` via `admin.SetMIGProfile` → sends SIGHUP to `spinifex-daemon`. Must be run directly on the target host. | 1. Enable MIG with valid profile<br>2. Invalid profile name (error with available profile list)<br>3. No MIG-capable GPU (error)<br>4. GPU instances running (error)<br>5. `--gpu` targets specific GPU<br>6. Not run as root (error) | **DONE** |
| `spx admin gpu mig disable` | `--gpu <pci-addr>` (optional, default: all MIG-capable GPUs) | Must be run as root; no GPU instances running | Checks no GPU instances running (NATS) → discovers MIG-capable GPUs via `gpu.Discover()` (filtered by `--gpu` if set) → destroys all GPU instances (`gpu.DestroyAllInstances`) → disables MIG mode (`gpu.DisableMIGMode`) → clears `mig_profile` in `spinifex.toml` via `admin.SetMIGProfile` → sends SIGHUP to `spinifex-daemon`. Must be run directly on the target host. | 1. Disable with no running GPU instances<br>2. GPU instances running (error)<br>3. MIG not enabled on GPU (skips that GPU)<br>4. `--gpu` targets specific GPU<br>5. Not run as root (error) | **DONE** |
| Command | Flags | Description |
|---------|-------|-------------|
| `spx admin gpu status` | `--node` (default: local node) | Queries `spinifex.node.status` fan-out → finds the target node response → prints Node, GPU hardware (model list or "none detected"), IOMMU state, vfio-pci state, passthrough enabled/disabled, and GPU pool allocation (`allocated/total`). Also lists GPU-capable instance types when passthrough is active. |
| `spx admin gpu enable` | — | Checks current passthrough state via NATS → errors if already enabled or prerequisites not met (directs to `setup`) → writes `gpu_passthrough = true` to `spinifex.toml` via `admin.SetGPUPassthrough` → sends SIGHUP to `spinifex-daemon` via `systemctl kill -s HUP` → polls node status for up to 30 s until daemon confirms new state → prints final `gpu status` output. Must be run directly on the target host. |
| `spx admin gpu disable` | — | Checks current passthrough state via NATS → errors if already disabled or if `AllocGPUs > 0` (must terminate GPU instances first) → writes `gpu_passthrough = false` → sends SIGHUP to `spinifex-daemon` → polls for up to 30 s → prints final `gpu status` output. Must be run directly on the target host. |
| `spx admin gpu setup` | — | Idempotent host configuration for GPU passthrough. Steps (skipped if already applied): detect GPUs via `gpu.Discover` → collect PCI IDs for all IOMMU-group siblings → check/enable IOMMU in GRUB (`intel_iommu=on iommu=pt` or `amd_iommu=on iommu=pt`) → write vfio udev rule (`/etc/udev/rules.d/99-spinifex-vfio.rules`) → blacklist nouveau (`/etc/modprobe.d/blacklist-nouveau.conf`) → blacklist amdgpu if AMD GPU present (`/etc/modprobe.d/blacklist-amdgpu.conf`) → write vfio-pci early binding config (`/etc/modprobe.d/vfio-pci.conf`) → add vfio modules to initramfs. If any change requires a reboot: runs `update-initramfs -u` and exits with reboot instructions. After reboot: verifies `vfio_pci` module is loaded → verifies each GPU is bound to `vfio-pci` (binds explicitly via `driver_override` if unbound) → calls `gpu enable` to activate passthrough. |
| `spx admin gpu mig status` | — (must be run on target host) | Runs `gpu.Discover()` locally → for each GPU prints: PCI address, model, MIG capability, MIG mode (enabled/disabled/N/A); for GPUs with MIG enabled lists active MIG slices with GI ID, profile name, and mdev path. |
| `spx admin gpu mig enable` | `--profile <name>` (required, e.g. `1g.10gb`), `--gpu <pci-addr>` (optional, default: all MIG-capable GPUs) | Checks no GPU instances running (NATS) → discovers MIG-capable GPUs via `gpu.Discover()` (filtered by `--gpu` if set) → enables MIG mode on each target (`gpu.EnableMIGMode`) → lists available profiles (`gpu.ListProfiles`) and validates requested profile name → destroys any existing instances (`gpu.DestroyAllInstances`) → creates new instances filling GPU capacity (`gpu.CreateInstances`) → writes `mig_profile` to `spinifex.toml` via `admin.SetMIGProfile` → sends SIGHUP to `spinifex-daemon`. Must be run directly on the target host. |
| `spx admin gpu mig disable` | `--gpu <pci-addr>` (optional, default: all MIG-capable GPUs) | Checks no GPU instances running (NATS) → discovers MIG-capable GPUs via `gpu.Discover()` (filtered by `--gpu` if set) → destroys all GPU instances (`gpu.DestroyAllInstances`) → disables MIG mode (`gpu.DisableMIGMode`) → clears `mig_profile` in `spinifex.toml` via `admin.SetMIGProfile` → sends SIGHUP to `spinifex-daemon`. Must be run directly on the target host. |
## AWS-Compatible API
@@ -295,9 +235,6 @@ by downstream services.
### EC2 — VPC Core
VPC/Subnet/ENI/SG CRUD stores metadata in NATS KV and publishes events to
vpcd for OVN translation. Single AZ for Spinifex v1.
| Command | Implemented Flags | Missing Flags | Status |
|---------|-------------------|---------------|--------|
| `create-vpc` | `--cidr-block`, `--tag-specifications` | `--instance-tenancy`, `--dry-run` | **DONE** |
@@ -393,9 +330,6 @@ EIP commands are only registered when an external IPAM pool is configured.
### EC2 — NAT Gateway
Deleted gateways move to a separate KV bucket with 1-hour TTL for Terraform
polling.
| Command | Implemented Flags | Missing Flags | Status |
|---------|-------------------|---------------|--------|
| `create-nat-gateway` | `--subnet-id`, `--allocation-id` | `--connectivity-type`, `--tag-specifications`, `--dry-run` | **DONE** |
@@ -406,9 +340,6 @@ polling.
### EC2 — Placement Groups
Strategies: `spread` (1 instance per node, strict) and `cluster` (all pinned
to single node). `partition` rejected.
| Command | Implemented Flags | Missing Flags | Status |
|---------|-------------------|---------------|--------|
| `create-placement-group` | `--group-name`, `--strategy` (spread/cluster) | `--partition-count`, `--spread-level`, `--tag-specifications`, `--dry-run` | **DONE** |
@@ -543,9 +474,6 @@ bypasses policy evaluation entirely.
### IAM — Access Keys
Max 2 keys per user. Secrets encrypted with AES-256-GCM using master key,
returned plaintext only at creation.
| Command | Implemented Flags | Missing Flags | Status |
|---------|-------------------|---------------|--------|
| `create-access-key` | `--user-name` | — | **DONE** |
@@ -555,10 +483,6 @@ returned plaintext only at creation.
### IAM — Policies
Policy documents require `Version: "2012-10-17"` and a valid Statement array.
Wildcard action matching supported (`ec2:*`, `ec2:Describe*`, `*`). Evaluation
order: explicit Deny > explicit Allow > implicit Deny.
| Command | Implemented Flags | Missing Flags | Status |
|---------|-------------------|---------------|--------|
| `create-policy` | `--policy-name`, `--policy-document`, `--path`, `--description` | `--tags` | **DONE** |
@@ -570,19 +494,6 @@ order: explicit Deny > explicit Allow > implicit Deny.
| `detach-user-policy` | `--user-name`, `--policy-arn` | — | **DONE** |
| `list-attached-user-policies` | `--user-name` | `--path-prefix`, `--max-items`, `--marker` | **DONE** |
### IAM — Policy Evaluation (internal — not an AWS API)
| Feature | Status |
|---------|--------|
| Root user bypass | **DONE** |
| Default deny | **DONE** |
| Explicit allow | **DONE** |
| Explicit deny | **DONE** |
| Wildcard matching (`*`, `ec2:*`, `ec2:Describe*`) | **DONE** |
| Account-scoped evaluation | **DONE** |
| EC2 action mapping | **DONE** |
| IAM action mapping (16 actions) | **DONE** |
### IAM — Roles
| Command | Implemented Flags | Missing Flags | Status |
@@ -599,9 +510,6 @@ order: explicit Deny > explicit Allow > implicit Deny.
### IAM — Instance Profiles
Containers for IAM roles that allow EC2 instances to assume a role. Required
for `run-instances --iam-instance-profile`. Max 1 role per profile.
| Command | Implemented Flags | Missing Flags | Status |
|---------|-------------------|---------------|--------|
| `create-instance-profile` | `--instance-profile-name`, `--path`, `--tags` | — | **DONE** |
@@ -631,11 +539,11 @@ for `run-instances --iam-instance-profile`. Max 1 role per profile.
## STS
| Command | Implemented Flags | Missing Flags (rejected if supplied) | Status |
| Command | Implemented Flags | Missing Flags | Status |
|---------|-------------------|--------------------------------------|--------|
| `get-caller-identity` | — | — | **DONE** |
| `assume-role` | `--role-arn`, `--role-session-name`, `--duration-seconds` (900–min(role MaxSessionDuration, 43200)) | `--policy`, `--policy-arns` (→ `PackedPolicyTooLarge`); `--tags`, `--transitive-tag-keys` (→ `InvalidParameterValue`); `--serial-number`, `--token-code` (→ `InvalidParameterValue`); `--external-id`, `--source-identity` (accepted and logged, **not enforced** — no Condition evaluator in v1) | **DONE** |
| `get-session-token` | — | `--duration-seconds`, `--serial-number`, `--token-code` | **NOT STARTED** |
| `get-session-token` | `--duration-seconds` (900–129600, default 43200 = 12h; clamped, not rejected) | `--serial-number`, `--token-code` (MFA → `InvalidParameterValue`) | **DONE** |
| `assume-role-with-web-identity` | — | `--role-arn`, `--role-session-name`, `--web-identity-token`, `--provider-id`, `--policy`, `--policy-arns`, `--duration-seconds` | **NOT STARTED** |
| `assume-role-with-saml` | — | `--role-arn`, `--principal-arn`, `--saml-assertion`, `--policy`, `--policy-arns`, `--duration-seconds` | **NOT STARTED** |
| `get-access-key-info` | — | `--access-key-id` | **NOT STARTED** |
@@ -649,9 +557,52 @@ Condition evaluator so accepting them would silently allow.
---
## IMDS (Instance Metadata Service)
Available at `169.254.169.254` from inside every running guest VM, matching AWS. The endpoint is reached from within a guest over plain HTTP, exactly as on EC2, with no in-VM agent to install. DHCP and fully static guests reach it identically, with no in-guest route configuration.
**IMDSv2-only.** Every read requires a session token. A tokenless (v1-style) `GET` returns `401 Unauthorized` with an empty body. Obtain a token with a `PUT /latest/api/token` carrying `X-aws-ec2-metadata-token-ttl-seconds` (1–21600), then send it back in `X-aws-ec2-metadata-token` on every read.
```bash
# Inside the guest VM:
TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" \
-H "X-aws-ec2-metadata-token-ttl-seconds: 60")
curl -s -H "X-aws-ec2-metadata-token: $TOKEN" \
http://169.254.169.254/latest/meta-data/instance-id
# Tokenless GET → 401 Unauthorized (empty body)
curl -i http://169.254.169.254/latest/meta-data/instance-id
```
### IMDS — Supported Paths
| Path | Method | Source | Status |
|------|--------|--------|--------|
| `/latest/api/token` | PUT | Issues an ENI-bound IMDSv2 token; `X-aws-ec2-metadata-token-ttl-seconds` ∈ [1, 21600] required | **DONE** |
| `/latest/meta-data/` | GET | Directory listing of supported children | **DONE** |
| `/latest/meta-data/instance-id` | GET | `vm.ID` | **DONE** |
| `/latest/meta-data/instance-type` | GET | `vm.InstanceType` | **DONE** |
| `/latest/meta-data/ami-id` | GET | launch `ImageId` | **DONE** |
| `/latest/meta-data/local-ipv4` | GET | `ENIRecord.PrivateIpAddress` (== request source IP) | **DONE** |
| `/latest/meta-data/public-ipv4` | GET | EIP, else instance public IP; empty body if none | **DONE** |
| `/latest/meta-data/mac` | GET | `ENIRecord.MacAddress` | **DONE** |
| `/latest/meta-data/security-groups` | GET | `ENIRecord.SecurityGroupIds`, newline-separated | **DONE** |
| `/latest/meta-data/hostname`, `/local-hostname` | GET | Synthesised `ip-<dashed-ip>.<region>.compute.internal` | **DONE** |
| `/latest/meta-data/placement/availability-zone` | GET | `ENIRecord.AvailabilityZone` | **DONE** |
| `/latest/meta-data/placement/region` | GET | Derived from AZ (trailing letter stripped) | **DONE** |
| `/latest/meta-data/iam/info` | GET | `{InstanceProfileArn, InstanceProfileId}`; 404 if no profile | **DONE** |
| `/latest/meta-data/iam/security-credentials/` | GET | Role name(s) under the profile, one per line; empty body if none | **DONE** |
| `/latest/meta-data/iam/security-credentials/<role>` | GET | STS `AssumeRoleForInstance` → ASIA-prefixed temporary credential JSON | **DONE** |
| `/latest/user-data` | GET | `vm.UserData`; 404 if none | **DONE** |
| `/latest/dynamic/instance-identity/document` | GET | Needs a per-cluster signing key | **NOT STARTED** (404; lands with EKS IRSA) |
| `/latest/meta-data/network/interfaces/...` | GET | Multi-ENI rendering | **NOT STARTED** (404) |
| `/latest/meta-data/tags/...` | GET | Instance-tag metadata | **NOT STARTED** (404) |
---
## ELBv2 (Application & Network Load Balancer)
The data plane uses a system-managed LB VM running HAProxy, launched automatically during `create-load-balancer`. HAProxy config is pushed via NATS on listener/target changes.
The data plane uses a system-managed LB VM, launched automatically during `create-load-balancer`. Application Load Balancers run **HAProxy** (L7: rules, fixed-response, redirect, HTTP/HTTPS). Network Load Balancers run **nginx `stream`** (L4: TCP, UDP, TLS, TCP_UDP) — HAProxy cannot load-balance UDP. The agent selects the engine from the `Engine` field on the config-delivery response.
### ELBv2 — Load Balancers
docs/administration/spinifex-admin-cli/README.md
+19
View File
@@ -82,6 +82,25 @@ spx admin images import --name debian-13-arm64
Catalog imports verify the image against the catalog-declared SHA-256/SHA-512 digest before extraction. Use `--file` to import operator-supplied media (verification skipped — operator is responsible for integrity), or `--force` to re-download after a checksum mismatch.
### EKS node image
To run EKS, import the prebuilt node image from the catalog:
```bash
spx admin images import --name spinifex-eks-node
```
This pulls the Alpine + K3s node AMI from `iso.mulgadc.com`, verifies its
checksum, and registers it tagged `spinifex:managed-by=eks`. `eks create-cluster`
and `eks create-nodegroup` resolve the boot AMI by that tag, so no further
configuration is needed.
Operators publishing a freshly built node image to the mirror run the
**Publish EKS node AMI** GitHub Actions workflow (manual dispatch; uses the
`CF_R2_*` repo secrets) or, locally, `make publish-eks-node-image` (see
`scripts/publish-system-image.sh`; requires `R2_ENDPOINT` + R2-scoped
`AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY`).
## Cluster Shutdown
docs/compute/launching-instances/README.md
+10
View File
@@ -164,6 +164,16 @@ spx admin images import --name debian-13-x86_64
cloud-init needs time to configure the instance after boot. Wait 30-60 seconds and retry.
If the connection **times out** rather than being refused, the security group is likely blocking port 22. The default security group denies all inbound traffic (matching AWS), so SSH must be explicitly allowed:
```bash
# Allow SSH from anywhere on the instance's security group
aws ec2 authorize-security-group-ingress \
--group-id $SG_ID --protocol tcp --port 22 --cidr 0.0.0.0/0
```
See [VPC Networking — Security Groups](/docs/vpc-networking#security-groups) for scoping rules to a trusted CIDR.
Verify the SSH key was specified correctly when launching:
```bash
docs/compute/vpc-networking/README.md
+78 -14
View File
@@ -48,8 +48,9 @@ NIC. Each host runs `ovn-controller` which programs OpenFlow rules on `br-int`
## Private vs Public Subnets
A subnet's behavior depends on two things: whether the VPC has an Internet
Gateway, and whether the subnet has `MapPublicIpOnLaunch` enabled.
A subnet's behavior depends on three things: whether the VPC has an Internet
Gateway, whether the subnet's route table has a default route to that IGW, and
whether the subnet has `MapPublicIpOnLaunch` enabled.
## Private Subnet (Default)
@@ -61,9 +62,10 @@ internet or be reached from the WAN.
<img src="../../../.github/assets/diagrams/vpc-private-subnet-flow.svg" alt="Private subnet — instance hits router, no default route, packet dropped" width="900">
</p>
If the VPC has an IGW attached, private subnet instances CAN reach the internet
via the VPC router's SNAT rule (outbound only — they share the gateway IP). They
still cannot be reached from the WAN because they have no public IP.
Private subnet instances reach the internet only if their route table has a
default route to the IGW (shared SNAT, outbound only — they share the gateway
IP) or to a NAT gateway. With no default route, egress is dropped. Either way
they cannot be reached from the WAN because they have no public IP.
## Public Subnet
@@ -77,8 +79,18 @@ managed by OVN — the instance OS only sees its private IP.
**Requirements for a public subnet:**
1. VPC has an Internet Gateway attached
2. Subnet has `MapPublicIpOnLaunch = true`
3. External IP pool configured in `spinifex.toml`
2. A route table associated with the subnet has a `0.0.0.0/0` route to the IGW
3. Subnet has `MapPublicIpOnLaunch = true`
4. External IP pool configured in `spinifex.toml`
Spinifex follows AWS route-table semantics: a subnet is only "public" if its
effective route table carries a default route to the IGW. A new VPC's main
route table has the local route only — Spinifex does **not** add the IGW route
for you. Without it, the subnet's egress is gated with a drop policy, so
instances cannot reach the internet (and inbound connections cannot complete
because return traffic is dropped) even with a public IP and an attached IGW.
Add the route explicitly — either to the main route table, or to a custom route
table associated with the subnet (see [Quick Start](#3-create-vpc-with-public-subnet)).
## Comparison
@@ -86,7 +98,7 @@ managed by OVN — the instance OS only sees its private IP.
| ------------------------ | --------------------------------- | ------------------------------ |
| Private IP | Yes | Yes |
| Public IP | No | Auto-assigned from pool |
| Outbound internet | Only if VPC has IGW (shared SNAT) | Yes (own public IP via SNAT) |
| Outbound internet | Only with a default route to IGW/NAT GW | Yes (own public IP via SNAT) |
| Inbound from WAN | No | Yes (via 1:1 NAT to public IP) |
| Instance sees public IP? | N/A | No — only sees private IP |
| Elastic IP support | Only if explicitly associated | Yes |
@@ -190,8 +202,7 @@ use `setup-ovn.sh --dhcp` to obtain one from the router. This is the router's
DHCP — not Spinifex's internal OVN DHCP for VMs.
**Use when:** VMs only need outbound access (apt update, pulling images). Edge
deployments behind ISP NAT. Single WAN IP available. Future: use with AWS-style
NAT Gateway for private subnet internet access.
deployments behind ISP NAT. Single WAN IP available.
```toml
[network]
@@ -935,17 +946,39 @@ spx admin init
VPC=$(aws ec2 create-vpc --cidr-block 10.200.0.0/16 \
--query Vpc.VpcId --output text)
SUBNET=$(aws ec2 create-subnet --vpc-id $VPC \
--cidr-block 10.200.1.0/24 \
--query Subnet.SubnetId --output text)
IGW=$(aws ec2 create-internet-gateway \
--query InternetGateway.InternetGatewayId --output text)
aws ec2 attach-internet-gateway \
--internet-gateway-id $IGW --vpc-id $VPC
SUBNET=$(aws ec2 create-subnet --vpc-id $VPC \
--cidr-block 10.200.1.0/24 \
--query Subnet.SubnetId --output text)
# Route table — give the subnet a default route to the IGW. Spinifex does NOT
# add this automatically; without it the subnet's egress is dropped.
RT=$(aws ec2 create-route-table --vpc-id $VPC \
--query RouteTable.RouteTableId --output text)
aws ec2 create-route --route-table-id $RT \
--destination-cidr-block 0.0.0.0/0 --gateway-id $IGW
aws ec2 associate-route-table --route-table-id $RT --subnet-id $SUBNET
# Auto-assign a public IP to instances launched into the subnet
aws ec2 modify-subnet-attribute \
--subnet-id $SUBNET --map-public-ip-on-launch
# Allow SSH + ICMP on the VPC's default security group (blocks inbound by default)
SG=$(aws ec2 describe-security-groups \
--filters Name=vpc-id,Values=$VPC \
--query 'SecurityGroups[0].GroupId' --output text)
aws ec2 authorize-security-group-ingress --group-id $SG \
--protocol tcp --port 22 --cidr 0.0.0.0/0
aws ec2 authorize-security-group-ingress --group-id $SG \
--protocol icmp --port -1 --cidr 0.0.0.0/0
```
## 4. Launch Instance
@@ -1135,6 +1168,37 @@ ip route show
sudo ovn-nbctl lr-nat-list vpc-$VPC
```
### Instance Has Public IP But No Internet
The instance shows a public IP in `describe-instances` but cannot reach the
internet, and inbound connections hang. The usual cause is a missing default
route: the subnet's effective route table has no `0.0.0.0/0` route to the IGW,
so Spinifex gates the subnet with a drop policy.
```bash
# Find the route table that applies to the subnet and check its routes
aws ec2 describe-route-tables \
--filters Name=association.subnet-id,Values=$SUBNET \
--query 'RouteTables[0].Routes'
# Expect a route with DestinationCidrBlock 0.0.0.0/0 and a GatewayId of igw-...
```
If the subnet has no explicit association it falls back to the VPC's main route
table — check that one too, then add the route to whichever table applies:
```bash
aws ec2 create-route --route-table-id $RT \
--destination-cidr-block 0.0.0.0/0 --gateway-id $IGW
```
On the host, the drop policy installed when a subnet lacks an IGW route appears
as a `Logical_Router_Policy` on the VPC router:
```bash
sudo ovn-nbctl lr-policy-list vpc-$VPC
# A drop policy matching the subnet CIDR means the subnet is gated.
```
### Public IP Not Reachable from WAN
Work through these checks in order — each eliminates a class of issues.
docs/getting-started/setting-up-your-cluster/README.md
+125 -41
View File
@@ -34,7 +34,7 @@ resources:
## Overview
This guide assumes Spinifex is already installed and running. If not, follow one of the installation guides first:
- [Single-Node Install)](/docs/install)
- [Single-Node Install](/docs/install)
- [Multi-Node Install](/docs/install-multi-node)
- [Source Install](/docs/install-source)
@@ -50,7 +50,9 @@ export AWS_PROFILE=spinifex
## 1. Import an AMI
List available images and import one matching your architecture:
### Option A: Import a bundled image
List the bundled images and import one matching your architecture:
```bash
spx admin images list
@@ -64,13 +66,11 @@ ubuntu-26.04-arm64 | ubuntu | 26.04 | arm64 | uefi
ubuntu-26.04-x86_64 | ubuntu | 26.04 | x86_64 | uefi
```
Import an image:
```bash
spx admin images import --name ubuntu-26.04-x86_64
```
Or import a local image file:
### Option B: Import a local image file
```bash
spx admin images import --file ~/images/ubuntu-26.04.img --distro ubuntu --version 26.04 --arch x86_64 --boot-mode uefi
@@ -79,7 +79,7 @@ spx admin images import --file ~/images/ubuntu-26.04.img --distro ubuntu --versi
Verify the import and note the AMI ID:
```bash
aws ec2 describe-images
AMI_ID=$(aws ec2 describe-images --query 'Images[0].ImageId' --output text)
```
## 2. Create an SSH Key
@@ -115,8 +115,6 @@ aws ec2 describe-key-pairs
```bash
VPC_ID=$(aws ec2 create-vpc --cidr-block 10.200.0.0/16 \
--query 'Vpc.VpcId' --output text)
echo "VPC: $VPC_ID"
```
### Create an Internet Gateway
@@ -130,68 +128,95 @@ IGW_ID=$(aws ec2 create-internet-gateway \
aws ec2 attach-internet-gateway \
--internet-gateway-id $IGW_ID \
--vpc-id $VPC_ID
echo "IGW: $IGW_ID (attached to $VPC_ID)"
```
### Create a Public Subnet
### Create a Subnet
A public subnet auto-assigns a routable IP to each instance, making it directly reachable from your network.
Create the subnet your instances will launch into. The routing and public-IP steps below are what make it a *public* subnet.
```bash
SUBNET_ID=$(aws ec2 create-subnet \
--vpc-id $VPC_ID \
--cidr-block 10.200.1.0/24 \
--query 'Subnet.SubnetId' --output text)
```
# Enable auto-assign public IP
### Create a Route Table
A subnet is only **public** if its route table has a default route to the Internet Gateway. Spinifex does **not** add this route automatically — a new VPC's route table only routes traffic within the VPC (matching AWS). Without a `0.0.0.0/0` route to the IGW, instances in the subnet cannot reach the internet (and inbound connections cannot complete) even with a public IP assigned.
```bash
RT_ID=$(aws ec2 create-route-table \
--vpc-id $VPC_ID \
--query 'RouteTable.RouteTableId' --output text)
# Default route to the internet via the IGW
aws ec2 create-route \
--route-table-id $RT_ID \
--destination-cidr-block 0.0.0.0/0 \
--gateway-id $IGW_ID
# Associate the route table with the subnet
aws ec2 associate-route-table \
--route-table-id $RT_ID \
--subnet-id $SUBNET_ID
```
### Enable Auto-Assign Public IP
Give every instance launched into the subnet a routable public IP, making it directly reachable from your network.
```bash
aws ec2 modify-subnet-attribute \
--subnet-id $SUBNET_ID \
--map-public-ip-on-launch
echo "Subnet: $SUBNET_ID (public)"
```
### Allow SSH and ICMP
Every VPC gets a default security group that **blocks inbound traffic from outside the group** (matching AWS). Instances launched without an explicit security group use this default, so you must authorize ingress before you can SSH or ping the instance.
```bash
SG_ID=$(aws ec2 describe-security-groups \
--filters Name=vpc-id,Values=$VPC_ID \
--query 'SecurityGroups[0].GroupId' --output text)
# Allow SSH from anywhere
aws ec2 authorize-security-group-ingress \
--group-id $SG_ID \
--protocol tcp --port 22 --cidr 0.0.0.0/0
# Allow ICMP (ping) from anywhere
aws ec2 authorize-security-group-ingress \
--group-id $SG_ID \
--protocol icmp --port -1 --cidr 0.0.0.0/0
```
**Note:** `0.0.0.0/0` opens these ports to every source. For anything beyond a quick evaluation, scope the `--cidr` to a trusted range. Rule changes apply immediately — no instance restart needed.
### Verify
```bash
aws ec2 describe-vpcs --vpc-ids $VPC_ID
aws ec2 describe-subnets --subnet-ids $SUBNET_ID
aws ec2 describe-route-tables --route-table-ids $RT_ID
aws ec2 describe-security-groups --group-ids $SG_ID
```
## 4. Launch an Instance
Query available instance types for your hardware:
Launch an instance in the public subnet on a `t3.micro` (2 vCPU / 1 GiB).
> **Note:** On an **arm64** host, use `t4g.micro` instead of `t3.micro`.
```bash
aws ec2 describe-instance-types
```
Launch an instance in the public subnet, note this will select an instance type with 2 VCPU and 1024MB of memory.
> **Note:** Replace `AMI_NAME` with the previously imported image above with the `ami-` prefix.
```bash
AMI_NAME="ami-ubuntu-26.04-x86_64"
AMI_ID=$(aws ec2 describe-images \
--filters "Name=name,Values=$AMI_NAME" \
--query 'Images | sort_by(@, &CreationDate) | [-1].ImageId' \
--output text)
INSTANCE_TYPE=$(aws ec2 describe-instance-types \
--query "sort_by(InstanceTypes[?VCpuInfo.DefaultVCpus==\`2\` && MemoryInfo.SizeInMiB>=\`1024\`], &MemoryInfo.SizeInMiB)[0].InstanceType" \
--output text)
INSTANCE_ID=$(aws ec2 run-instances \
--image-id $AMI_ID \
--instance-type $INSTANCE_TYPE \
--instance-type t3.micro \
--key-name spinifex-key \
--subnet-id $SUBNET_ID \
--count 1 \
--query 'Instances[0].InstanceId' --output text)
echo "Instance: $INSTANCE_ID"
```
Wait for the instance to reach `running` state:
@@ -328,7 +353,36 @@ PRIVATE_SUBNET=$(aws ec2 create-subnet \
--query 'Subnet.SubnetId' --output text)
```
Instances in private subnets get a private IP only. They can still reach the internet via SNAT through the VPC router, but are not directly reachable from your network.
Instances in a private subnet get a private IP only and are not reachable from your network. By default they also have **no internet access**: with no `0.0.0.0/0` route in the subnet's route table, Spinifex gates egress with a drop policy — matching AWS, where a private subnet has no route off the VPC. They can still reach other instances in the same VPC.
To give a private subnet outbound-only internet access, deploy a NAT Gateway in a public subnet and point the private subnet's default route at it (the AWS pattern):
```bash
# Allocate an Elastic IP for the NAT Gateway
NAT_EIP=$(aws ec2 allocate-address --query AllocationId --output text)
# Create the NAT Gateway in the PUBLIC subnet (the one with the IGW route)
NATGW_ID=$(aws ec2 create-nat-gateway \
--subnet-id $SUBNET_ID \
--allocation-id $NAT_EIP \
--query 'NatGateway.NatGatewayId' --output text)
# Give the private subnet its own route table with a default route to the NAT Gateway
PRIVATE_RT=$(aws ec2 create-route-table \
--vpc-id $VPC_ID \
--query 'RouteTable.RouteTableId' --output text)
aws ec2 create-route \
--route-table-id $PRIVATE_RT \
--destination-cidr-block 0.0.0.0/0 \
--nat-gateway-id $NATGW_ID
aws ec2 associate-route-table \
--route-table-id $PRIVATE_RT \
--subnet-id $PRIVATE_SUBNET
```
Instances in the private subnet now reach the internet outbound through the NAT Gateway's public IP, but remain unreachable from the WAN.
### Multiple Accounts
@@ -348,7 +402,7 @@ journalctl -u spinifex-daemon -f
aws ec2 describe-images
```
### SSH Connection Refused
### SSH Connection Refused or Times Out
cloud-init needs 30-60 seconds after boot. Check instance state:
@@ -356,6 +410,20 @@ cloud-init needs 30-60 seconds after boot. Check instance state:
aws ec2 describe-instances --instance-ids $INSTANCE_ID
```
A connection that **times out** (rather than being refused) usually means the security group is blocking port 22. Confirm the default security group allows SSH ingress:
```bash
aws ec2 describe-security-groups --group-ids $SG_ID \
--query 'SecurityGroups[0].IpPermissions'
```
If there are no rules for TCP 22, authorize it (see [Allow SSH and ICMP](#allow-ssh-and-icmp)):
```bash
aws ec2 authorize-security-group-ingress \
--group-id $SG_ID --protocol tcp --port 22 --cidr 0.0.0.0/0
```
### No Public IP Assigned
Verify the subnet has `MapPublicIpOnLaunch` enabled:
@@ -378,7 +446,23 @@ aws ec2 describe-internet-gateways
### Instance Has No Internet Access
Check the VPC router's NAT rules (from the host):
First confirm the subnet's route table has a default route to the IGW. Spinifex gates a subnet's egress with a drop policy when this route is missing, so an instance with a public IP still cannot reach the internet:
```bash
aws ec2 describe-route-tables \
--filters Name=association.subnet-id,Values=$SUBNET_ID \
--query 'RouteTables[0].Routes'
# Expect a 0.0.0.0/0 route with a GatewayId of igw-...
```
If the route is missing, add it (see [Create a Route Table](#create-a-route-table)):
```bash
aws ec2 create-route --route-table-id $RT_ID \
--destination-cidr-block 0.0.0.0/0 --gateway-id $IGW_ID
```
If the route is present, check the VPC router's NAT rules (from the host):
```bash
sudo ovn-nbctl lr-nat-list $(sudo ovn-nbctl lr-list | awk '{print $2}' | head -1)
docs/install-vm-eval/README.md
+2 -4
View File
@@ -12,8 +12,6 @@ For this tutorial download UTM.app on Mac to simplify the VM installation.
* UTM - [https://mac.getutm.app](https://mac.getutm.app)
## Download Debian ISO
Next, download the Debian ISO (ARM64, Apple Silicon)
@@ -28,7 +26,7 @@ Using the VM console, login using the `spinifex` user and your specified passwor
For a minimal Debian install additional dependencies are required prior to the Spinifex installation:
```
```bash
su root
apt update
apt install sudo curl bridge-utils
@@ -38,7 +36,7 @@ apt install sudo curl bridge-utils
Enable sudo for `spinifex` user as the root user:
```
```bash
su root
echo "spinifex ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/spinifex
```
docs/service-interfaces.yaml
+12
View File
@@ -30,10 +30,12 @@ services:
publishes:
- "ec2.*"
- "elbv2.*"
- "acm.*"
- "iam.*"
depends_on:
- "ec2.*"
- "elbv2.*"
- "acm.*"
- "iam.*"
spinifex_daemon:
@@ -50,6 +52,7 @@ services:
- spinifex/daemon
- spinifex/handlers/ec2
- spinifex/handlers/elbv2
- spinifex/handlers/acm
- spinifex/handlers/iam
subscribes:
# EC2 instance lifecycle.
@@ -95,6 +98,15 @@ services:
- "elbv2.CreateListener"
- "elbv2.DeleteListener"
- "elbv2.DescribeListeners"
# ACM-alike certificate store (handlers/acm/service_nats.go topic map).
- "acm.ImportCertificate"
- "acm.DescribeCertificate"
- "acm.ListCertificates"
- "acm.DeleteCertificate"
- "elbv2.AddListenerCertificates"
- "elbv2.RemoveListenerCertificates"
- "elbv2.DescribeListenerCertificates"
- "elbv2.DescribeSSLPolicies"
publishes:
# → vpcd
- "vpc.create"
docs/terraform-workbooks/nginx-alb/README.md
-3
View File
@@ -142,7 +142,6 @@ The ALB is internet-facing, but the DNS name Spinifex returns (`*.elb.spinifex.l
ALB_IP=$(aws elbv2 describe-load-balancers --names nginx-alb \
--query 'LoadBalancers[0].AvailabilityZones[].LoadBalancerAddresses[].IpAddress' \
--output text)
echo "ALB public IP: $ALB_IP"
```
Then hit the ALB — successive requests should alternate between Server 1 and Server 2:
@@ -163,8 +162,6 @@ TG_ARN=$(aws elbv2 describe-target-groups \
--query 'TargetGroups[0].TargetGroupArn' \
--output text)
echo "Target Group ARN: $TG_ARN"
aws elbv2 describe-target-health --target-group-arn $TG_ARN
```
go.mod
+3 -3
View File
@@ -15,7 +15,7 @@ require (
github.com/klauspost/cpuid/v2 v2.3.0
github.com/mulgadc/predastore v1.6.0
github.com/mulgadc/viperblock v1.6.0
github.com/nats-io/nats-server/v2 v2.14.1
github.com/nats-io/nats-server/v2 v2.14.2
github.com/nats-io/nats.go v1.52.0
github.com/ovn-kubernetes/libovsdb v0.8.1
github.com/pelletier/go-toml/v2 v2.3.1
@@ -101,8 +101,8 @@ require (
github.com/muesli/cancelreader v0.2.2 // indirect
github.com/muesli/termenv v0.16.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/nats-io/jwt/v2 v2.8.1 // indirect
github.com/nats-io/nkeys v0.4.15 // indirect
github.com/nats-io/jwt/v2 v2.8.2 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/pierrec/lz4/v4 v4.1.14 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
go.sum
+6 -6
View File
@@ -296,14 +296,14 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
github.com/nats-io/nats-server/v2 v2.14.1 h1:wXs/a5fw9Hzm3CvuzLxGeIwpjPulSa7gMT3eSuhGkcg=
github.com/nats-io/nats-server/v2 v2.14.1/go.mod h1:4N17zLpuS7WMbG8T9gsE2B7z9hC9PraPyulVBfpK6nU=
github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc=
github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc=
github.com/nats-io/nats-server/v2 v2.14.2 h1:Q7dRhCY03Y00rETFW3KV+KGaCIajlDfWgWUVgbMxyuk=
github.com/nats-io/nats-server/v2 v2.14.2/go.mod h1:lWpb1bSpRELZfRdlMkdz8E7lbXKKyNe8RIn0vvepIHs=
github.com/nats-io/nats.go v1.52.0 h1:n3avV4VBsCgsdwh71TppsTwtv+QdPs7ntSKM8qJLGsc=
github.com/nats-io/nats.go v1.52.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno=
github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4=
github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/ovn-kubernetes/libovsdb v0.8.1 h1:M2J8bcJt5mXCom0HqzfEtuHkT80CTSQRcYG7acT8gf4=
internal/eksgw/client.go
+103
View File
@@ -0,0 +1,103 @@
// Package eksgw is the on-VM SigV4 client EKS control-plane helpers use to
// reach the AWS gateway, instead of dialing core NATS. Both eks-gateway-publish
// (bootstrap + state relay) and eks-token-webhook (TokenReview relay) sign with
// the system (Predastore) credentials for service "eks" and POST over HTTPS;
// the gateway authorizes via IAM and relays to the host-side NATS/STS/KV.
package eksgw
import (
"bytes"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/hex"
"fmt"
"io"
"net/http"
"os"
"strings"
"time"
"github.com/mulgadc/predastore/auth"
"github.com/mulgadc/spinifex/internal/tlsconfig"
)
// Client posts SigV4-signed requests to the AWS gateway for the "eks" service.
type Client struct {
baseURL string
accessKey string
secretKey string
region string
httpClient *http.Client
}
// New builds a client. caPath optionally pins the gateway TLS CA; when empty the
// client relies on the system trust store. region defaults to us-east-1 when
// empty (SigV4 requires a non-empty region).
func New(baseURL, caPath, accessKey, secretKey, region string) (*Client, error) {
if baseURL == "" {
return nil, fmt.Errorf("eksgw: baseURL is required")
}
if accessKey == "" || secretKey == "" {
return nil, fmt.Errorf("eksgw: access key and secret key are required")
}
if region == "" {
region = "us-east-1"
}
tlsCfg := &tls.Config{
MinVersion: tls.VersionTLS13,
CurvePreferences: tlsconfig.Curves,
}
if caPath != "" {
pem, err := os.ReadFile(caPath)
if err != nil {
return nil, fmt.Errorf("eksgw: read gateway CA %q: %w", caPath, err)
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(pem) {
return nil, fmt.Errorf("eksgw: gateway CA %q has no usable certificates", caPath)
}
tlsCfg.RootCAs = pool
}
return &Client{
baseURL: strings.TrimRight(baseURL, "/"),
accessKey: accessKey,
secretKey: secretKey,
region: region,
httpClient: &http.Client{
Timeout: 10 * time.Second,
Transport: &http.Transport{TLSClientConfig: tlsCfg, MaxIdleConns: 2, IdleConnTimeout: 30 * time.Second},
},
}, nil
}
// Post SigV4-signs and sends a single POST of body to path (e.g.
// "/clusters/alpha/token-review"). It returns the response body on a 2xx, or an
// error (including the gateway status + body) otherwise. No retry — callers that
// need it wrap Post in their own loop.
func (c *Client) Post(path string, body []byte) ([]byte, error) {
req, err := http.NewRequest(http.MethodPost, c.baseURL+path, bytes.NewReader(body))
if err != nil {
return nil, fmt.Errorf("create request: %w", err)
}
req.Header.Set("Content-Type", "application/x-amz-json-1.1")
sum := sha256.Sum256(body)
if err := auth.SignReq(req, c.accessKey, c.secretKey, hex.EncodeToString(sum[:]), "eks", c.region); err != nil {
return nil, fmt.Errorf("sign request: %w", err)
}
resp, err := c.httpClient.Do(req)
if err != nil {
return nil, fmt.Errorf("send request: %w", err)
}
defer resp.Body.Close()
respBody, _ := io.ReadAll(resp.Body)
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, fmt.Errorf("gateway returned %d: %s", resp.StatusCode, strings.TrimSpace(string(respBody)))
}
return respBody, nil
}
scripts/ansible/.gitignore
-2
View File
@@ -1,2 +0,0 @@
.creds/
*.retry
scripts/ansible/README.md
-222
View File
@@ -1,222 +0,0 @@
# Ansible Dev Lifecycle (parallel to bash scripts)
Experimental Ansible-based lifecycle for a single-node Spinifex dev host.
Lives alongside the existing `scripts/dev-*.sh` and `scripts/reset-dev-env.sh`;
neither set is removed. Decide which to keep after parity testing.
Plan: `docs/development/improvements/ansible-dev-lifecycle.md` (mulga top-level).
## Prerequisites
Venv lives at sub-repo root (`spinifex/.venv/`) — one venv per sub-repo,
shared by any future python tooling in spinifex. Bootstrap from `spinifex/`:
```
sudo apt install python3.13-venv
python3 -m venv .venv
source .venv/bin/activate
pip install 'ansible>=9'
ansible-galaxy collection install -r scripts/ansible/requirements.yml
```
Then `cd scripts/ansible` to run playbooks — `ansible.cfg` resolves the
inventory and roles relative to CWD.
## Playbooks
| Playbook | Purpose | Bash parity |
|---|---|---|
| `playbooks/dev-preflight.yml` | Dependency + port check | `dev-setup.sh` §checks |
| `playbooks/dev-teardown.yml` | Wipe state only (no reinstall) | `reset-dev-env.sh` §teardown |
| `playbooks/dev-install.yml` | Build + install + init + smoketest on clean box | `dev-install.sh` |
| `playbooks/dev-reset.yml` | Capture settings → teardown → build → install → init → smoketest | `reset-dev-env.sh` (full) |
| `playbooks/dev-deploy.yml` | Rebuild + swap binaries/microvm artifacts + restart (no setup.sh, no smoketest) | `make deploy` |
| `playbooks/dev-status.yml` | Read-only health snapshot (units, ports, OVN/OVS, gateway, counts, drift) | none |
| `playbooks/dev-logs.yml` | Dump journald + `/var/log/spinifex/*` + status/networking/versions into a tarball | none |
| `playbooks/dev-snapshot.yml` | Quiesce spinifex+OVN+OVS, tar state paths to a named bundle, restart | none |
| `playbooks/dev-restore.yml` | Stop services, wipe + untar a named bundle, refresh CA, restart | none |
| `playbooks/dev-version.yml` | Build-vs-install drift report (spx, plugin, microvm) + sub-repo HEADs | none |
| `playbooks/dev-vpc.yml` | Idempotent dev VPC fixture (VPC + public subnet + IGW + default route); `-e dev_vpc_state=absent` tears down | partial overlap with `scripts/iac/examples/*` |
Upcoming (not yet implemented):
- `dev-mode-start.yml` / `dev-mode-stop.yml` — parity with `start-dev.sh` / `stop-dev.sh`
(lowest priority — deprecated bash scripts, CI-only callers).
## Invocation
```
ansible-playbook playbooks/dev-preflight.yml
ansible-playbook playbooks/dev-teardown.yml
ansible-playbook playbooks/dev-install.yml
ansible-playbook playbooks/dev-reset.yml
ansible-playbook playbooks/dev-deploy.yml
ansible-playbook playbooks/dev-status.yml
ansible-playbook playbooks/dev-logs.yml
ansible-playbook playbooks/dev-snapshot.yml -e snapshot_name=before-merge
ansible-playbook playbooks/dev-restore.yml -e snapshot_name=before-merge
ansible-playbook playbooks/dev-version.yml
ansible-playbook playbooks/dev-vpc.yml
```
Or via `make` (from `spinifex/`):
```
make ansible-dev-preflight
make ansible-dev-teardown
make ansible-dev-install
make ansible-dev-reset
make ansible-dev-deploy
make ansible-dev-status
make ansible-dev-logs
make ansible-dev-snapshot ANSIBLE_EXTRA='-e snapshot_name=before-merge'
make ansible-dev-restore ANSIBLE_EXTRA='-e snapshot_name=before-merge'
make ansible-dev-version
make ansible-dev-vpc
make ansible-dev-vpc ANSIBLE_EXTRA='-e dev_vpc_state=absent'
```
### When to use which
- First-time / clean box → `ansible-dev-install`
- Iterate on Go code, microvm initramfs, lb-agent → `ansible-dev-deploy` (fast)
- Changed systemd units, helper scripts, logrotate, setup.sh → `ansible-dev-reset` (slow, full rebuild)
- Need a clean slate without reinstall → `ansible-dev-teardown`
- "Is my dev box healthy?" → `ansible-dev-status` (read-only, never mutates)
- Filing a bug / capturing the state of a broken box → `ansible-dev-logs` (writes `/tmp/spinifex-logs-<host>-<ts>.tar.gz`; override `-e logs_since=10min` or `-e logs_include_dmesg=false`)
- Checkpoint before a risky op (branch switch, viperblock surgery, ovn restart cycle) → `ansible-dev-snapshot ANSIBLE_EXTRA='-e snapshot_name=<name>'`; roll back via `ansible-dev-restore` instead of a full `dev-reset`. Bundles live in `/var/lib/spinifex-snapshots/<name>.tar.gz` (root-owned, survives `dev-teardown`).
- "Do I need to `ansible-dev-deploy`?" → `ansible-dev-version` (drift-only check; faster than `dev-status`)
- Want a known-good VPC/subnet for manual EC2 testing without writing tofu → `ansible-dev-vpc` (smoketest will then find and use it automatically). Tear down with `ANSIBLE_EXTRA='-e dev_vpc_state=absent'`.
## Variable overrides
Defaults in `vars/defaults.yml`. Override per run with `-e`:
```
ansible-playbook playbooks/dev-teardown.yml -e spinifex_wipe_ssh_keys=true
```
Useful overrides:
| Var | Default | What it does |
|---|---|---|
| `spinifex_wipe_legacy_home` | `true` | Remove `$HOME/spinifex` (legacy dev-mode path) |
| `spinifex_wipe_ssh_keys` | `false` | Remove `~/.ssh/spinifex-key*` |
| `spinifex_wipe_aws_creds` | `false` | Remove spinifex profile from `~/.aws/credentials` |
| `teardown_process_wait_seconds` | `30` | Grace period for processes to exit |
| `spinifex_region` | `ap-southeast-2` | Region passed to `spx admin init` |
| `spinifex_az` | `{{ region }}a` | AZ passed to `spx admin init` |
| `spinifex_external_mode` | `pool` | External networking mode — `pool` or empty (NAT removed) |
| `spinifex_external_no_external` | `false` | `true` → emit `--no-external` and skip external networking |
| `spinifex_external_pool_start` | `""` | **Required** for `pool` mode — first IP of the static WAN pool |
| `spinifex_external_pool_end` | `""` | **Required** for `pool` mode — last IP of the static WAN pool |
| `spinifex_external_gateway` | `""` | WAN gateway (auto-detected when blank) |
| `spinifex_external_prefix_len` | `""` | WAN prefix length (auto-detected when blank) |
| `spinifex_external_gateway_ip` | `""` | OVN router external-IP override (`--gateway-ip`) |
`pool` mode is mandatory unless you set `spinifex_external_no_external=true`.
`spx admin init` rejects the run without a `--external-pool` range, so the
init role asserts `spinifex_external_pool_start/_end` upfront. Either persist
them in `vars/defaults.yml`, or pass them at invocation:
Direct `ansible-playbook` (use `-e` here — this is ansible's flag):
```
ansible-playbook playbooks/dev-install.yml \
-e spinifex_external_pool_start=192.168.1.150 \
-e spinifex_external_pool_end=192.168.1.250
```
Via `make` (use `EXTRA_VARS=` — make eats its own `-e`):
```
make ansible-dev-install \
EXTRA_VARS="spinifex_external_pool_start=192.168.1.150 spinifex_external_pool_end=192.168.1.250"
```
`dev-reset.yml` automatically captures `region`, `az`, `external_mode`,
pool range, gateway, prefix-len and `gateway_ip` from the existing
`/etc/spinifex/spinifex.toml` before teardown and replays them into
`init`. Legacy `external_mode = "nat"` is coerced to `"pool"` with a warn.
Use `-e` to override.
## Helper scripts
`scripts/aws-as-spinifex.sh` and `scripts/spx-as-spinifex.sh` wrap `aws`
and `spx` invocations in `sg spinifex -c '...'` so they run with the
`spinifex` supplementary group active. Both helpers preset
`AWS_PROFILE=spinifex`. All AWS-calling tasks in the roles use these
via `spinifex_aws_cli` / `spinifex_spx_cli` from `vars/defaults.yml`.
Background: `/etc/spinifex/ca.pem` lives under `drwxr-x--- root:spinifex`,
so the AWS CLI needs `spinifex` group membership to read the CA bundle.
Ansible's `become_user` is a no-op when the target user equals the
calling user (no `sudo -u` invoked → no `initgroups(3)` → supplementary
groups inherited from the operator's shell). On a fresh login the
operator's shell would have `spinifex` (added by `setup.sh`); on shells
that predate the group add it does not. The helpers sidestep this
entirely.
## Leak catalog
`roles/teardown/README.md` tracks every known state source. When teardown
misses something, add it there and in `roles/teardown/tasks/main.yml`.
## Multi-node cluster bootstrap
`playbooks/cluster-bootstrap.yml` provisions a 2+ node Spinifex cluster
across a long-running tofu env (`env1`, `env2`, `bryn`, etc). Inventory
is generated dynamically from `scripts/tofu-cluster/envs/<env>.tfvars`
via `inventory/tofu.py` — first node in `nodes = [...]` is the init host,
the rest are joins.
```bash
make ansible-cluster-bootstrap ENV=env1 POOL=192.168.1.150-192.168.1.159
```
`POOL` is optional; omit for DHCP fallback. Equivalent without make:
```bash
cd scripts/ansible
CLUSTER_ENV=env1 ansible-playbook -i inventory/tofu.py \
playbooks/cluster-bootstrap.yml \
-e cluster_env=env1 \
-e cluster_external_pool=192.168.1.150-192.168.1.159
```
Plays sequence:
1. **runner (localhost)** — `build` role produces `/tmp/spinifex-local.tar.gz`.
2. **all env hosts (parallel)** — copies tarball, stops any dev-layout
services (`~/spinifex/`), runs `preflight` + `install` roles. `setup.sh`
creates `/etc/spinifex`, the `spinifex` user/group, systemd units, and
installs `/usr/local/bin/spx`.
3. **init host only** — `cluster-init` role runs `spx admin init` in the
background, waits for the formation server, captures the join token.
CA is then copied into `/usr/local/share/ca-certificates/spinifex-ca.crt`
**before** `spinifex.target` starts (required — predastore enforces
strict TLS verify against the system trust store).
4. **join hosts (parallel)** — `cluster-join` role runs `spx admin join`
against the formation server, then installs CA, then starts
`spinifex.target`.
5. **all hosts** — polls `https://<ip>:4432/health` until `awsgw=ok`.
6. **init host only** — `cluster-post` role imports the SSH key + Ubuntu
AMI (both idempotent: skip if already present).
Re-running the playbook against an already-bootstrapped env is safe —
`setup.sh`, `spx admin init --force`, CA copy, key/AMI import are all
idempotent.
### Migration from `bootstrap.sh`
`scripts/tofu-cluster/bootstrap.sh` ran services under `~/spinifex/` as
`tf-user` via `start-dev.sh` with no systemd supervision, and installed
the cluster CA *after* services started — predastore would crash with
`x509: certificate signed by unknown authority` on every first boot.
`cluster-bootstrap.yml` replaces that flow with production-layout
(`/etc/spinifex/` + systemd) and correct CA-before-services ordering.
The first run on an env previously bootstrapped by `bootstrap.sh` will
stop the dev-layout services, install the prod layout, and re-init the
cluster from scratch via `spx admin init --force`.
`bootstrap.sh` stays in-tree for CI and pseudo-multinode envs.
scripts/ansible/ansible.cfg
-17
View File
@@ -1,17 +0,0 @@
[defaults]
inventory = inventory/localhost.yml
roles_path = roles
stdout_callback = ansible.builtin.default
host_key_checking = False
retry_files_enabled = False
interpreter_python = auto_silent
forks = 10
[inventory]
# inventory/tofu.py is an executable script inventory used by
# cluster-bootstrap.yml — opted into via `-i inventory/tofu.py` on the
# command line; not default to keep dev-* playbooks pointing at localhost.
enable_plugins = host_list, script, auto, yaml, ini, toml
[callback_default]
result_format = yaml
scripts/ansible/inventory/localhost.yml
-6
View File
@@ -1,6 +0,0 @@
---
all:
hosts:
localhost:
ansible_connection: local
ansible_python_interpreter: "{{ ansible_playbook_python }}"
scripts/ansible/inventory/tofu.py
-170
View File
@@ -1,170 +0,0 @@
#!/usr/bin/env python3
"""Dynamic ansible inventory generator from tofu-cluster tfvars.
Reads scripts/tofu-cluster/envs/<env>.tfvars (HCL subset) and emits ansible
JSON inventory. First node in the tfvars `nodes = [...]` list is the init
host (forms the cluster); the rest are join hosts.
Usage:
inventory/tofu.py --list --env env1
inventory/tofu.py --host <ip> --env env1
Selecting the env:
CLUSTER_ENV=env1 inventory/tofu.py --list
inventory/tofu.py --list --env env1
"""
from __future__ import annotations
import argparse
import json
import os
import re
import sys
from pathlib import Path
def _repo_root() -> Path:
here = Path(__file__).resolve()
for parent in here.parents:
if (parent / "go.work").exists() or (parent / ".gitmodules").exists():
return parent
raise SystemExit(f"could not locate mulga repo root from {here}")
def _tfvars_path(env: str) -> Path:
root = _repo_root()
candidate = root / "scripts" / "tofu-cluster" / "envs" / f"{env}.tfvars"
if not candidate.exists():
raise SystemExit(f"tfvars file not found: {candidate}")
return candidate
_STRING_RE = re.compile(r'^\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*=\s*"([^"]*)"\s*$')
_NUMBER_RE = re.compile(r'^\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*=\s*([0-9]+)\s*$')
_NODE_FIELD_RE = re.compile(r'(\w+)\s*=\s*"([^"]*)"')
def _parse_tfvars(path: Path) -> dict:
"""Minimal tfvars parser — handles the subset used by envs/*.tfvars.
Supports:
key = "string"
key = 12345
nodes = [
{ name = "x", wan_ip = "1.2.3.4", ... },
...
]
"""
out: dict = {"nodes": []}
text = path.read_text()
# Strip line comments
text = re.sub(r"#.*", "", text)
# Pull the nodes block out first, then process remaining lines.
nodes_match = re.search(r"nodes\s*=\s*\[(.+?)\]\s*$", text, re.DOTALL | re.MULTILINE)
if nodes_match:
nodes_body = nodes_match.group(1)
for entry in re.finditer(r"\{([^}]+)\}", nodes_body):
fields = dict(_NODE_FIELD_RE.findall(entry.group(1)))
if "name" in fields and "wan_ip" in fields:
out["nodes"].append(fields)
text = text.replace(nodes_match.group(0), "")
for line in text.splitlines():
if m := _STRING_RE.match(line):
out[m.group(1)] = m.group(2)
elif m := _NUMBER_RE.match(line):
out[m.group(1)] = int(m.group(2))
return out
def _resolve_ssh_key(value: str | None) -> str:
if not value:
return os.path.expanduser("~/.ssh/tf-user-ap-southeast-2")
return os.path.expanduser(value)
def _build_inventory(env: str) -> dict:
data = _parse_tfvars(_tfvars_path(env))
nodes = data.get("nodes") or []
if not nodes:
raise SystemExit(f"{env}.tfvars has no nodes")
ssh_user = data.get("ssh_user", "tf-user")
ssh_key = _resolve_ssh_key(data.get("ssh_public_key_path", "").replace(".pub", ""))
hostvars: dict[str, dict] = {}
all_hosts: list[str] = []
init_hosts: list[str] = []
join_hosts: list[str] = []
for idx, node in enumerate(nodes):
ip = node["wan_ip"]
role = "init" if idx == 0 else "join"
hostvars[ip] = {
"ansible_host": ip,
"ansible_user": ssh_user,
"ansible_ssh_private_key_file": ssh_key,
"ansible_ssh_common_args": "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR",
"spinifex_node_name": node["name"],
"spinifex_node_role": role,
"spinifex_node_index": idx,
"spinifex_wan_ip": ip,
}
all_hosts.append(ip)
(init_hosts if role == "init" else join_hosts).append(ip)
# Group-level vars derived from tfvars (operator can override with -e).
cluster_init_host = init_hosts[0]
group_vars = {
"cluster_env": env,
"cluster_init_host": cluster_init_host,
"cluster_init_ip": cluster_init_host,
"spinifex_node_count": len(nodes),
"spinifex_region": data.get("region", "ap-southeast-2"),
"spinifex_az": data.get("az", "ap-southeast-2a"),
}
if cidr := data.get("env_cidr"):
group_vars["cluster_env_cidr"] = cidr
inventory = {
"_meta": {"hostvars": hostvars},
env: {"hosts": all_hosts, "vars": group_vars},
"cluster": {"children": [env]},
"init": {"hosts": init_hosts},
"join": {"hosts": join_hosts},
}
return inventory
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--list", action="store_true", help="emit full inventory")
parser.add_argument("--host", help="emit hostvars for a single host (ignored — _meta carries them)")
parser.add_argument(
"--env",
default=os.environ.get("CLUSTER_ENV"),
help="tofu env name (matches scripts/tofu-cluster/envs/<env>.tfvars); also via CLUSTER_ENV env var",
)
args = parser.parse_args()
if not args.env:
sys.stderr.write("error: --env required (or CLUSTER_ENV)\n")
return 64
if args.host:
json.dump({}, sys.stdout)
sys.stdout.write("\n")
return 0
inventory = _build_inventory(args.env)
json.dump(inventory, sys.stdout, indent=2)
sys.stdout.write("\n")
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/ansible/playbooks/cluster-bootstrap.yml
-330
View File
@@ -1,330 +0,0 @@
---
# cluster-bootstrap.yml — multi-node Spinifex cluster bootstrap against a
# tofu-provisioned env. Inventory comes from inventory/tofu.py which parses
# scripts/tofu-cluster/envs/<env>.tfvars.
#
# Usage:
# cd spinifex/scripts/ansible
# ansible-playbook -i inventory/tofu.py playbooks/cluster-bootstrap.yml \
# -e cluster_env=env1 -e cluster_external_pool=192.168.1.150-192.168.1.159
#
# Or via the make wrapper:
# make ansible-cluster-bootstrap ENV=env1 POOL=192.168.1.150-192.168.1.159
#
# Behaviour:
# - all hosts: preflight + install (setup.sh) — production layout
# - init host: spx admin init (backgrounded) + CA install + spinifex.target
# - join hosts: spx admin join + CA install + spinifex.target (parallel)
# - init host: import ssh key + ubuntu AMI (idempotent)
- name: Cluster bootstrap — build artifact once on runner
hosts: localhost
gather_facts: true
vars_files:
- ../vars/defaults.yml
tasks:
- name: assert cluster_env supplied
ansible.builtin.assert:
that: cluster_env | default('') | length > 0
fail_msg: "Pass -e cluster_env=<env> (matches scripts/tofu-cluster/envs/<env>.tfvars)"
roles:
- build
- name: Cluster bootstrap — install (all nodes, parallel)
hosts: "{{ cluster_env }}"
gather_facts: true
serial: 100%
vars:
install_run_ovn: false # we drive setup-ovn ourselves with role-aware args
cluster_bootstrap_purge: true # wipe stale single-node predastore.toml + raft data
vars_files:
- ../vars/defaults.yml
pre_tasks:
- name: copy build tarball to each node
ansible.builtin.copy:
src: "{{ build_tarball_path }}"
dest: "{{ build_tarball_path }}"
mode: "0644"
- name: stop dev-layout services if running (migration from ~/spinifex/)
become: true
# Bracket trick on the first letter prevents the pattern literal in the
# shell command line from matching the running shell itself (pkill -f
# against literal patterns is otherwise self-killing — rc=-9).
# `spx [a]dmin` catches lingering formation servers from prior bootstrap
# attempts that own port 4432 and steal joins from the new init.
ansible.builtin.shell:
cmd: |
pkill -9 -f "[b]in/spx service" 2>/dev/null || true
pkill -9 -f "spx [a]dmin" 2>/dev/null || true
pkill -9 -f "spinifex-[u]i" 2>/dev/null || true
pkill -9 -f "[n]ats-server" 2>/dev/null || true
pkill -9 -f "[p]redastore" 2>/dev/null || true
pkill -9 -f "[v]iperblock" 2>/dev/null || true
sleep 2
rm -rf "{{ ansible_env.HOME }}/spinifex" 2>/dev/null || true
exit 0
changed_when: false
roles:
- install
# Wipe OVN/OVS state before setup-ovn runs. Without this, re-bootstrapping a
# previously-built cluster leaves stale chassis rows in the SB DB, stale
# external_ids on Open_vSwitch, stale br-int flows + ports, stale
# systemd-networkd unit files, and stale ovs-monitor-ipsec strongSwan certs.
# Symptom: ovn-controller flows reference dead encap endpoints; IPsec SAs
# fail AUTHENTICATION_FAILED; new joins shadow ghosts under the same
# node-name.
- name: Cluster bootstrap — OVN/OVS state reset (all nodes, parallel)
hosts: "{{ cluster_env }}"
gather_facts: false
serial: 100%
vars_files:
- ../vars/defaults.yml
tasks:
- name: reset — stop OVN/OVS services
become: true
ansible.builtin.systemd:
name: "{{ item }}"
state: stopped
loop:
- ovn-controller
- ovn-central
- openvswitch-ipsec
- openvswitch-switch
failed_when: false
- name: reset — delete OVN NB/SB databases
become: true
ansible.builtin.file:
path: "{{ item }}"
state: absent
loop:
- "{{ ovn_nb_db }}"
- "{{ ovn_sb_db }}"
- name: reset — delete OVS conf.db + system-id (force fresh ovsdb)
become: true
ansible.builtin.file:
path: "{{ item }}"
state: absent
loop:
- /etc/openvswitch/conf.db
- /etc/openvswitch/.conf.db.~lock~
- /etc/openvswitch/system-id.conf
- name: reset — drop spinifex-managed systemd-networkd unit files
become: true
ansible.builtin.shell: |
rm -f /etc/systemd/network/14-spinifex-*.netdev \
/etc/systemd/network/15-spinifex-*.netdev \
/etc/systemd/network/15-spinifex-*.network \
/etc/systemd/network/16-spinifex-*.network
networkctl reload 2>/dev/null || true
changed_when: true
- name: reset — enumerate stale spinifex veth/tap netdevs
ansible.builtin.shell: |
ip -o link show | awk -F': ' '{print $2}' | cut -d'@' -f1 | \
grep -E '^(veth-wan-br|veth-wan-ovs|spx-ext-|spx-tap-|spx-)' || true
register: reset_links
changed_when: false
- name: reset — delete stale netdevs
become: true
ansible.builtin.command: "ip link delete {{ item }}"
loop: "{{ reset_links.stdout_lines }}"
changed_when: true
failed_when: false
- name: reset — clear ovs-monitor-ipsec strongSwan state
become: true
ansible.builtin.shell: |
rm -rf /var/lib/openvswitch/pki 2>/dev/null || true
rm -f /etc/ipsec.d/certs/ovs-* /etc/ipsec.d/private/ovs-* 2>/dev/null || true
rm -f /etc/ipsec.conf.d/ovs-* 2>/dev/null || true
changed_when: true
- name: reset — start openvswitch-switch with fresh conf.db
become: true
ansible.builtin.systemd:
name: openvswitch-switch
state: started
daemon_reload: true
- name: Cluster bootstrap — OVN central (init host only)
hosts: init
gather_facts: false
vars_files:
- ../vars/defaults.yml
tasks:
- name: setup-ovn --management on init host
become: true
ansible.builtin.command:
cmd: >-
/usr/local/share/spinifex/setup-ovn.sh
--management
--encap-ip={{ spinifex_wan_ip }}
--node-name={{ hostvars[inventory_hostname]['spinifex_node_name'] }}
changed_when: true
- name: Cluster bootstrap — OVN chassis (join hosts, parallel)
hosts: join
gather_facts: false
vars_files:
- ../vars/defaults.yml
tasks:
- name: setup-ovn --ovn-remote on join hosts
become: true
ansible.builtin.command:
cmd: >-
/usr/local/share/spinifex/setup-ovn.sh
--ovn-remote=tcp:{{ cluster_init_host }}:6642
--encap-ip={{ spinifex_wan_ip }}
--node-name={{ hostvars[inventory_hostname]['spinifex_node_name'] }}
changed_when: true
- name: Cluster bootstrap — init host (start formation server)
hosts: init
gather_facts: false
vars_files:
- ../vars/defaults.yml
roles:
- cluster-init
- name: Cluster bootstrap — join hosts run spx admin join (parallel)
hosts: join
gather_facts: false
serial: 100%
vars_files:
- ../vars/defaults.yml
roles:
- cluster-join
- name: Cluster bootstrap — wait for init to finish writing multi-node configs
hosts: init
gather_facts: false
vars_files:
- ../vars/defaults.yml
tasks:
- name: wait for spx admin init async job to complete
become: true
ansible.builtin.async_status:
jid: "{{ cluster_init_async.ansible_job_id }}"
register: cluster_init_done
until: cluster_init_done.finished
retries: 60
delay: 5
- name: dump init log on failure
ansible.builtin.command: tail -100 /tmp/spx-admin-init.log
register: cluster_init_log
changed_when: false
when: cluster_init_done.rc | default(1) != 0
- name: assert init exited cleanly
ansible.builtin.assert:
that: cluster_init_done.rc == 0
fail_msg: |
spx admin init failed (rc={{ cluster_init_done.rc }})
{{ cluster_init_log.stdout | default('(log not captured)') }}
- name: confirm multi-node predastore.toml written
become: true
ansible.builtin.shell: |
if grep -q '127\.0\.0\.1' {{ spinifex_etc_dir }}/predastore/predastore.toml 2>/dev/null; then
echo "ERROR: predastore.toml still references loopback — multi-node config not written"
exit 1
fi
grep -c '^\[\[nodes\]\]' {{ spinifex_etc_dir }}/predastore/predastore.toml 2>/dev/null || echo 0
register: predastore_node_count
changed_when: false
- name: Cluster bootstrap — install CA on all nodes (before service start)
hosts: "{{ cluster_env }}"
gather_facts: false
vars_files:
- ../vars/defaults.yml
tasks:
- name: install CA into system trust
become: true
ansible.builtin.copy:
src: "{{ spinifex_ca_src }}"
dest: "{{ spinifex_ca_trust_path }}"
mode: "0644"
remote_src: true
- name: rebuild CA bundle
become: true
ansible.builtin.command: update-ca-certificates
changed_when: true
# spinifex-ui reverse-proxies localhost:9999 (awsgw) and localhost:8443
# (predastore) — hardcoded in spinifex/services/spinifexui/spinifexui.go.
# spx admin init binds those services to BindIP only, so the loopback
# connection fails (BadGateway in UI). Patch to wildcard bind; cert
# already includes 127.0.0.1 in IP SANs.
- name: patch awsgw bind to 0.0.0.0 (UI proxies via localhost)
become: true
ansible.builtin.replace:
path: "{{ spinifex_config_file }}"
regexp: '^(\s*host\s*=\s*")[^"]+(:9999")'
replace: '\g<1>0.0.0.0\g<2>'
- name: patch predastore bind to 0.0.0.0 (UI proxies via localhost)
become: true
ansible.builtin.replace:
path: "{{ spinifex_config_file }}"
regexp: '^(\s*host\s*=\s*")[^"]+(:8443")'
replace: '\g<1>0.0.0.0\g<2>'
- name: Cluster bootstrap — start services (init first, then joins)
hosts: init
gather_facts: false
vars_files:
- ../vars/defaults.yml
tasks:
- name: start spinifex.target on init
become: true
ansible.builtin.systemd:
name: spinifex.target
state: started
- name: Cluster bootstrap — start services on join hosts
hosts: join
gather_facts: false
serial: 100%
vars_files:
- ../vars/defaults.yml
tasks:
- name: start spinifex.target on joiners
become: true
ansible.builtin.systemd:
name: spinifex.target
state: started
- name: Cluster bootstrap — readiness check (all nodes)
hosts: "{{ cluster_env }}"
gather_facts: false
vars_files:
- ../vars/defaults.yml
tasks:
- name: wait for awsgw responsiveness
ansible.builtin.uri:
url: "https://{{ spinifex_wan_ip }}:{{ cluster_formation_port }}/health"
validate_certs: false
return_content: true
register: cluster_health
until:
- cluster_health.status == 200
- cluster_health.json.service_health.awsgw == "ok"
retries: 30
delay: 5
- name: Cluster bootstrap — post-bootstrap fixtures (init host only)
hosts: init
gather_facts: true
vars_files:
- ../vars/defaults.yml
roles:
- cluster-post
scripts/ansible/playbooks/dev-deploy.yml
-14
View File
@@ -1,14 +0,0 @@
---
# In-place artifact swap: rebuilds + replaces binaries/microvm artifacts on
# an already-installed node, then restarts spinifex.target. Use when iterating
# on code without altering systemd units, users, dirs, or running the full
# smoketest. For first-time or unit/script changes use dev-install / dev-reset.
- name: Dev deploy — build + swap binaries + restart (no setup.sh, no smoketest)
hosts: localhost
gather_facts: true
vars_files:
- ../vars/defaults.yml
roles:
- build
- deploy
scripts/ansible/playbooks/dev-install.yml
-67
View File
@@ -1,67 +0,0 @@
---
- name: Dev install — build, install, init, smoketest (idempotent)
hosts: localhost
gather_facts: true
vars_files:
- ../vars/defaults.yml
pre_tasks:
- name: preflight
ansible.builtin.include_role:
name: preflight
- name: health — config file present?
ansible.builtin.stat:
path: "{{ spinifex_config_file }}"
register: health_config
- name: health — spinifex.target active?
become: true
ansible.builtin.command: systemctl is-active spinifex.target
register: health_target
changed_when: false
failed_when: false
- name: health — awsgw responds?
become: true
become_user: "{{ spinifex_invoking_user }}"
environment:
AWS_PROFILE: spinifex
ansible.builtin.command: aws ec2 describe-regions
register: health_gw
changed_when: false
failed_when: false
- name: health — spx binary installed?
ansible.builtin.stat:
path: /usr/local/bin/spx
register: health_spx
- name: health — set already_installed fact
ansible.builtin.set_fact:
spinifex_already_installed: >-
{{ health_config.stat.exists
and health_spx.stat.exists
and (health_target.stdout | trim) == 'active'
and health_gw.rc == 0 }}
- name: health — summary
ansible.builtin.debug:
msg: |
Health:
spx binary: {{ 'present' if health_spx.stat.exists else 'missing' }}
config: {{ 'present' if health_config.stat.exists else 'missing' }}
target: {{ health_target.stdout | trim }}
awsgw: {{ 'ok' if health_gw.rc == 0 else 'not responding' }}
Already installed: {{ spinifex_already_installed }}
{% if spinifex_already_installed %}
→ Skipping build/install/init. Smoketest only.
{% else %}
→ Running full build → install → init → smoketest.
{% endif %}
roles:
- { role: build, when: not (spinifex_already_installed | default(false)) }
- { role: install, when: not (spinifex_already_installed | default(false)) }
- { role: init, when: not (spinifex_already_installed | default(false)) }
- smoketest
scripts/ansible/playbooks/dev-logs.yml
-17
View File
@@ -1,17 +0,0 @@
---
# Capture spinifex + ovn/ovs + qemu/nbdkit logs from journald and
# /var/log/spinifex/ into a single tarball for bug reports.
# Tail mode is intentionally not provided — use `journalctl -u 'spinifex-*' -f`
# directly; ansible is the wrong tool for follow.
- name: Dev logs — dump journald + varlog + status into a bundle
hosts: localhost
gather_facts: true
vars_files:
- ../vars/defaults.yml
vars:
logs_since: "1h"
logs_output_dir: "/tmp"
logs_include_dmesg: true
roles:
- logs
scripts/ansible/playbooks/dev-preflight.yml
-8
View File
@@ -1,8 +0,0 @@
---
- name: Dev preflight — validate host dependencies
hosts: localhost
gather_facts: true
vars_files:
- ../vars/defaults.yml
roles:
- preflight
scripts/ansible/playbooks/dev-reset.yml
-72
View File
@@ -1,72 +0,0 @@
---
- name: Dev reset — capture settings, wipe, rebuild, reinstall, re-init, smoketest
hosts: localhost
gather_facts: true
vars_files:
- ../vars/defaults.yml
pre_tasks:
- name: capture — check existing spinifex.toml
ansible.builtin.stat:
path: "{{ spinifex_config_file }}"
register: reset_config_stat
- name: capture — slurp existing config
ansible.builtin.slurp:
src: "{{ spinifex_config_file }}"
register: reset_config_raw
when: reset_config_stat.stat.exists
- name: capture — decode config
ansible.builtin.set_fact:
reset_config_txt: "{{ reset_config_raw.content | b64decode }}"
when: reset_config_stat.stat.exists
# List-concat-and-take-first pattern: missing regex match falls back to current default.
# `gateway\s*=` deliberately does not match `gateway_ip` / `gateway_lrp_*`.
- name: capture — extract scalars
ansible.builtin.set_fact:
spinifex_region: "{{ (reset_config_txt | regex_findall('region\\s*=\\s*\"([^\"]+)\"') + [spinifex_region])[0] }}"
spinifex_az: "{{ (reset_config_txt | regex_findall('\\baz\\s*=\\s*\"([^\"]+)\"') + [spinifex_az])[0] }}"
spinifex_external_mode: "{{ (reset_config_txt | regex_findall('external_mode\\s*=\\s*\"([^\"]+)\"') + [spinifex_external_mode])[0] }}"
spinifex_external_pool_start: "{{ (reset_config_txt | regex_findall('range_start\\s*=\\s*\"([^\"]+)\"') + [spinifex_external_pool_start])[0] }}"
spinifex_external_pool_end: "{{ (reset_config_txt | regex_findall('range_end\\s*=\\s*\"([^\"]+)\"') + [spinifex_external_pool_end])[0] }}"
spinifex_external_gateway: "{{ (reset_config_txt | regex_findall('\\bgateway\\s*=\\s*\"([^\"]+)\"') + [spinifex_external_gateway])[0] }}"
spinifex_external_prefix_len: "{{ (reset_config_txt | regex_findall('prefix_len\\s*=\\s*([0-9]+)') + [spinifex_external_prefix_len | string])[0] }}"
spinifex_external_gateway_ip: "{{ (reset_config_txt | regex_findall('gateway_ip\\s*=\\s*\"([^\"]+)\"') + [spinifex_external_gateway_ip])[0] }}"
when: reset_config_stat.stat.exists
# Legacy: external_mode = "nat" was removed from spx. Coerce to "pool"
# so old configs still reset cleanly; warn so the operator notices.
- name: capture — coerce legacy external_mode=nat to pool
ansible.builtin.set_fact:
spinifex_external_mode: "pool"
when: spinifex_external_mode == 'nat'
- name: capture — warn on legacy nat coercion
ansible.builtin.debug:
msg: "⚠️ Captured external_mode=nat from legacy config; coerced to 'pool' (NAT mode removed from spx admin init)."
when:
- reset_config_stat.stat.exists
- (reset_config_txt | regex_findall('external_mode\\s*=\\s*\"nat\"')) | length > 0
- name: capture — summary
ansible.builtin.debug:
msg: |
{% if reset_config_stat.stat.exists %}
Captured settings from {{ spinifex_config_file }}:
region={{ spinifex_region }}, az={{ spinifex_az }}
external_mode={{ spinifex_external_mode }}
pool={{ spinifex_external_pool_start }}-{{ spinifex_external_pool_end }}
gateway={{ spinifex_external_gateway }}, prefix={{ spinifex_external_prefix_len }}
gateway_ip={{ spinifex_external_gateway_ip }}
{% else %}
No existing config; using defaults from vars/defaults.yml (or -e overrides).
{% endif %}
roles:
- teardown
- build
- install
- init
- smoketest
scripts/ansible/playbooks/dev-restore.yml
-14
View File
@@ -1,14 +0,0 @@
---
# Restore a dev-snapshot bundle: stop services, wipe the snapshot path
# set, untar the bundle, refresh CA trust, restart.
#
# Required: -e snapshot_name=<name>
# Optional: -e snapshot_dir=/some/path
- name: Dev restore — wipe + untar snapshot + restart
hosts: localhost
gather_facts: true
vars_files:
- ../vars/defaults.yml
roles:
- restore
scripts/ansible/playbooks/dev-snapshot.yml
-17
View File
@@ -1,17 +0,0 @@
---
# Quiesce spinifex + ovn + ovs, tar /etc/spinifex + /var/lib/spinifex +
# /var/lib/ovn + /etc/openvswitch/conf.db into a named bundle, restart.
# Use to checkpoint a working state before a risky operation (branch
# switch, viperblock surgery, ovn-controller restart cycle). Roll back
# via dev-restore without a full dev-reset.
#
# Required: -e snapshot_name=<name>
# Optional: -e snapshot_dir=/some/path
- name: Dev snapshot — quiesce + tar state + restart
hosts: localhost
gather_facts: true
vars_files:
- ../vars/defaults.yml
roles:
- snapshot
scripts/ansible/playbooks/dev-status.yml
-13
View File
@@ -1,13 +0,0 @@
---
# Read-only health snapshot for a Spinifex dev node. Never mutates state.
# Run any time the operator wants to know "is my box healthy?" — pairs with
# dev-preflight (run before install) as the post-install / running-system
# diagnostic.
- name: Dev status — health snapshot (read-only)
hosts: localhost
gather_facts: true
vars_files:
- ../vars/defaults.yml
roles:
- status
scripts/ansible/playbooks/dev-teardown.yml
-8
View File
@@ -1,8 +0,0 @@
---
- name: Dev teardown — wipe Spinifex state (no reinstall)
hosts: localhost
gather_facts: true
vars_files:
- ../vars/defaults.yml
roles:
- teardown
scripts/ansible/playbooks/dev-version.yml
-11
View File
@@ -1,11 +0,0 @@
---
# Detailed build-vs-install drift report. Faster than dev-status (no AWS,
# OVN, or port probing). Answers "do I need to ansible-dev-deploy?".
- name: Dev version — build vs install drift (read-only)
hosts: localhost
gather_facts: false
vars_files:
- ../vars/defaults.yml
roles:
- version
scripts/ansible/playbooks/dev-vpc.yml
-14
View File
@@ -1,14 +0,0 @@
---
# Idempotent dev-mode VPC fixture: spinifex-dev-vpc + public subnet +
# IGW + default route. Hooks into smoketest's launch path (which finds
# the public subnet automatically). Re-runs are no-ops once converged.
#
# Default state: present. Tear down with -e dev_vpc_state=absent.
- name: Dev VPC — fixture VPC/subnet/IGW/route (idempotent)
hosts: localhost
gather_facts: false
vars_files:
- ../vars/defaults.yml
roles:
- vpc
scripts/ansible/requirements.yml
-6
View File
@@ -1,6 +0,0 @@
---
collections:
- name: community.general
version: ">=8.0.0"
- name: ansible.posix
version: ">=1.5.0"
scripts/ansible/roles/build/meta/main.yml
-7
View File
@@ -1,7 +0,0 @@
---
galaxy_info:
role_name: build
description: Build spinifex + viperblock-nbdkit, stage tarball for setup.sh consumption.
license: BSD-3-Clause
min_ansible_version: "2.14"
dependencies: []
scripts/ansible/roles/build/tasks/main.yml
-66
View File
@@ -1,66 +0,0 @@
---
- name: build — spinifex (make build)
ansible.builtin.command:
cmd: make build
chdir: "{{ spinifex_root }}"
changed_when: true
- name: build — viperblock nbdkit plugin (make go_build_nbd)
ansible.builtin.command:
cmd: make go_build_nbd
chdir: "{{ mulga_root }}/viperblock"
changed_when: true
- name: build — detect microVM build script (present only on direct-boot branches)
ansible.builtin.stat:
path: "{{ spinifex_root }}/scripts/build-microvm-image.sh"
register: build_microvm_script
- name: build — microVM kernel + initramfs (direct-boot LBs)
ansible.builtin.command:
cmd: make build-microvm-image
chdir: "{{ spinifex_root }}"
changed_when: true
when: build_microvm_script.stat.exists
- name: build — verify build artifacts present
ansible.builtin.stat:
path: "{{ item }}"
register: build_artifacts
loop:
- "{{ spinifex_root }}/bin/spx"
- "{{ mulga_root }}/viperblock/lib/nbdkit-viperblock-plugin.so"
- name: build — assert artifacts exist
ansible.builtin.assert:
that: item.stat.exists
fail_msg: "Missing build artifact: {{ item.item }}"
loop: "{{ build_artifacts.results }}"
loop_control:
label: "{{ item.item }}"
- name: build — stage tarball contents (mirrors dev-install.sh)
ansible.builtin.shell: |
set -e
STAGING=$(mktemp -d)
trap 'rm -rf "$STAGING"' EXIT
cp "{{ spinifex_root }}/bin/spx" "$STAGING/spx"
cp "{{ mulga_root }}/viperblock/lib/nbdkit-viperblock-plugin.so" "$STAGING/"
cp "{{ spinifex_root }}/scripts/setup-ovn.sh" "$STAGING/"
mkdir -p "$STAGING/systemd"
cp "{{ spinifex_root }}/build/systemd/"* "$STAGING/systemd/"
mkdir -p "$STAGING/scripts"
cp "{{ spinifex_root }}/build/scripts/"* "$STAGING/scripts/"
cp "{{ spinifex_root }}/build/logrotate/spinifex" "$STAGING/logrotate-spinifex"
# microVM artifacts ship only when the branch built them.
if [ -f "{{ spinifex_root }}/build/microvm/vmlinuz" ] && [ -f "{{ spinifex_root }}/build/microvm/initramfs.cpio.gz" ]; then
mkdir -p "$STAGING/microvm"
cp "{{ spinifex_root }}/build/microvm/vmlinuz" "$STAGING/microvm/"
cp "{{ spinifex_root }}/build/microvm/initramfs.cpio.gz" "$STAGING/microvm/"
fi
tar czf "{{ build_tarball_path }}" -C "$STAGING" .
changed_when: true
- name: build — summary
ansible.builtin.debug:
msg: "Built spx + nbdkit plugin{{ ' + microvm artifacts' if build_microvm_script.stat.exists else '' }}, tarball at {{ build_tarball_path }}"
scripts/ansible/roles/cluster-init/meta/main.yml
-9
View File
@@ -1,9 +0,0 @@
---
galaxy_info:
role_name: cluster-init
description: |
Run `spx admin init` on the cluster's init host, backgrounded so the
formation server accepts joins from the cluster-join role. Installs
cluster CA into the system trust store BEFORE starting spinifex.target
so predastore's strict TLS verify is satisfied at first start.
dependencies: []
scripts/ansible/roles/cluster-init/tasks/main.yml
-77
View File
@@ -1,77 +0,0 @@
---
- name: cluster-init — assert this is the init host
ansible.builtin.assert:
that: spinifex_node_role == "init"
fail_msg: "cluster-init role must only run against hosts in the [init] group"
# vars/defaults.yml sets spinifex_node_{name,count} for single-node dev use
# (precedence 11). Inventory hostvars (precedence 4) lose that fight, so
# we promote the inventory values via set_fact (precedence 15) before any
# task templates `{{ spinifex_node_count }}`.
- name: cluster-init — promote inventory node identity over defaults.yml
ansible.builtin.set_fact:
spinifex_node_name: "{{ hostvars[inventory_hostname]['spinifex_node_name'] }}"
spinifex_node_count: "{{ groups[cluster_env] | length }}"
- name: cluster-init — assemble external network flags
ansible.builtin.set_fact:
cluster_external_flags: >-
{% if cluster_external_pool | length > 0 %}--external-mode=pool --external-pool={{ cluster_external_pool }}{% endif %}
- name: cluster-init — run spx admin init (backgrounded, formation server waits for joins)
become: true
ansible.builtin.shell: |
set -o pipefail
spx admin init --force \
--node {{ spinifex_node_name }} \
--nodes {{ spinifex_node_count }} \
--bind {{ spinifex_wan_ip }} \
--cluster-bind {{ spinifex_wan_ip }} \
--port {{ cluster_formation_port }} \
--region {{ spinifex_region }} \
--az {{ spinifex_az }} \
{{ cluster_external_flags }} \
> /tmp/spx-admin-init.log 2>&1
async: 600
poll: 0
register: cluster_init_async
- name: cluster-init — brief pause so init can fail fast if misconfigured
ansible.builtin.pause:
seconds: 5
- name: cluster-init — check if init exited prematurely (sanity)
ansible.builtin.shell: |
if pgrep -f 'spx admin init' >/dev/null; then
echo "running"
else
tail -50 /tmp/spx-admin-init.log 2>/dev/null
exit 1
fi
register: cluster_init_alive
changed_when: false
failed_when: cluster_init_alive.rc != 0
- name: cluster-init — wait for formation server health
ansible.builtin.uri:
url: "https://{{ spinifex_wan_ip }}:{{ cluster_formation_port }}/formation/health"
validate_certs: false
status_code: 200
register: cluster_init_health
until: cluster_init_health.status == 200
retries: "{{ cluster_formation_timeout // 2 }}"
delay: 2
- name: cluster-init — fetch join token (delegated read for joiners)
become: true
ansible.builtin.slurp:
src: "{{ cluster_join_token_path }}"
register: cluster_join_token_raw
- name: cluster-init — register join token as host fact
ansible.builtin.set_fact:
cluster_join_token: "{{ cluster_join_token_raw.content | b64decode | trim }}"
- name: cluster-init — join token captured; waiting for joins handled by later play
ansible.builtin.debug:
msg: "Formation server up on {{ spinifex_wan_ip }}:{{ cluster_formation_port }}, {{ spinifex_node_count }} nodes expected"
scripts/ansible/roles/cluster-join/meta/main.yml
-8
View File
@@ -1,8 +0,0 @@
---
galaxy_info:
role_name: cluster-join
description: |
Run `spx admin join` against the cluster init host's formation server.
Installs cluster CA + starts spinifex.target after join completes
(CA-before-services ordering avoids predastore TLS-verify crashes).
dependencies: []
scripts/ansible/roles/cluster-join/tasks/main.yml
-39
View File
@@ -1,39 +0,0 @@
---
- name: cluster-join — assert this is a join host
ansible.builtin.assert:
that: spinifex_node_role == "join"
fail_msg: "cluster-join role must only run against hosts in the [join] group"
# Promote inventory hostvars over vars/defaults.yml (see cluster-init).
- name: cluster-join — promote inventory node identity over defaults.yml
ansible.builtin.set_fact:
spinifex_node_name: "{{ hostvars[inventory_hostname]['spinifex_node_name'] }}"
- name: cluster-join — pull join token from init host
ansible.builtin.set_fact:
cluster_join_token: "{{ hostvars[cluster_init_host]['cluster_join_token'] }}"
- name: cluster-join — assert token present
ansible.builtin.assert:
that: cluster_join_token | length > 0
fail_msg: "join token missing — init host did not export cluster_join_token"
- name: cluster-join — run spx admin join (retry up to 5 times)
become: true
ansible.builtin.command:
cmd: >-
spx admin join
--node {{ spinifex_node_name }}
--bind {{ spinifex_wan_ip }}
--cluster-bind {{ spinifex_wan_ip }}
--host {{ cluster_init_host }}:{{ cluster_formation_port }}
--token {{ cluster_join_token }}
--region {{ spinifex_region }}
--az {{ spinifex_az }}
register: cluster_join_result
retries: 5
delay: 3
until: cluster_join_result.rc == 0
# CA install + service start handled by cluster-bootstrap.yml plays that
# run after all joins complete and init has written multi-node configs.
scripts/ansible/roles/cluster-post/meta/main.yml
-8
View File
@@ -1,8 +0,0 @@
---
galaxy_info:
role_name: cluster-post
description: |
Post-bootstrap idempotent fixtures on the init host: import SSH key,
import Ubuntu AMI (dual-version detect 24.04/26.04). Skip when
artifacts already exist.
dependencies: []
scripts/ansible/roles/cluster-post/tasks/main.yml
-205
View File
@@ -1,205 +0,0 @@
---
- name: cluster-post — locate ssh public key for import
ansible.builtin.find:
paths: "{{ ansible_env.HOME }}/.ssh"
patterns:
- spinifex-key.pub
- tf-user-*.pub
file_type: file
register: cluster_post_ssh_pubkey
- name: cluster-post — fail when no public key found
ansible.builtin.assert:
that: cluster_post_ssh_pubkey.files | length > 0
fail_msg: "no public key in ~/.ssh — needed for spinifex-key import"
- name: cluster-post — set pubkey path fact
ansible.builtin.set_fact:
cluster_post_pubkey_path: "{{ cluster_post_ssh_pubkey.files[0].path }}"
- name: cluster-post — check whether spinifex-key already imported
ansible.builtin.command:
cmd: "{{ spinifex_aws_cli }} ec2 describe-key-pairs --key-names spinifex-key"
register: cluster_post_key_check
changed_when: false
failed_when: false
- name: cluster-post — import spinifex-key (if absent)
ansible.builtin.command:
cmd: >-
{{ spinifex_aws_cli }} ec2 import-key-pair
--key-name spinifex-key
--public-key-material fileb://{{ cluster_post_pubkey_path }}
register: cluster_post_key_import
retries: 10
delay: 6
until: cluster_post_key_import.rc == 0
when: cluster_post_key_check.rc != 0
- name: cluster-post — locate ubuntu cloud image (26.04 preferred, 24.04 fallback)
ansible.builtin.stat:
path: "{{ item }}"
register: cluster_post_ubuntu_stat
loop:
- "{{ ansible_env.HOME }}/images/ubuntu-26.04.img"
- "{{ ansible_env.HOME }}/images/ubuntu-24.04.img"
- name: cluster-post — pick first available image
ansible.builtin.set_fact:
cluster_post_ubuntu_image: "{{ (cluster_post_ubuntu_stat.results | selectattr('stat.exists') | map(attribute='item') | list | first) | default('') }}"
- name: cluster-post — fail when no ubuntu image present
ansible.builtin.assert:
that: cluster_post_ubuntu_image | length > 0
fail_msg: "no ubuntu-{24,26}.04.img in ~/images/ — gold image must stage it"
- name: cluster-post — derive ubuntu version from image filename
ansible.builtin.set_fact:
cluster_post_ubuntu_version: "{{ cluster_post_ubuntu_image | regex_search('ubuntu-([0-9]+\\.[0-9]+)\\.img', '\\1') | first }}"
- name: cluster-post — check whether ubuntu AMI already imported
ansible.builtin.command:
cmd: >-
{{ spinifex_aws_cli }} ec2 describe-images
--owners self
--filters Name=name,Values=ubuntu-{{ cluster_post_ubuntu_version }}
register: cluster_post_ami_check
changed_when: false
failed_when: false
- name: cluster-post — import ubuntu AMI (if absent)
ansible.builtin.command:
cmd: >-
{{ spinifex_spx_cli }} admin images import
--file {{ cluster_post_ubuntu_image }}
--arch x86_64
--distro ubuntu
--version {{ cluster_post_ubuntu_version }}
--boot-mode uefi
when:
- cluster_post_ami_check.rc != 0
or "ubuntu-" + cluster_post_ubuntu_version not in cluster_post_ami_check.stdout
- name: cluster-post — slurp final admin creds from ~/.aws/credentials
# Source of truth: SetupAWSCredentials writes the FINAL admin keys here.
# Init log shows obsolete keys (multi-node path silently regenerates them
# after the initial print — see cmd/spinifex/cmd/admin.go writeBootstrapFiles
# called twice in the multi-node flow).
ansible.builtin.slurp:
src: "{{ ansible_env.HOME }}/.aws/credentials"
register: cluster_post_aws_creds_raw
- name: cluster-post — extract admin access/secret keys from spinifex profile
ansible.builtin.set_fact:
cluster_post_admin_access_key: >-
{{ (cluster_post_aws_creds_raw.content | b64decode
| regex_search('\[spinifex\][^\[]*aws_access_key_id\s*=\s*(\S+)', '\1')
| first) | default('') }}
cluster_post_admin_secret_key: >-
{{ (cluster_post_aws_creds_raw.content | b64decode
| regex_search('\[spinifex\][^\[]*aws_secret_access_key\s*=\s*(\S+)', '\1')
| first) | default('') }}
- name: cluster-post — assert creds parsed
ansible.builtin.assert:
that:
- cluster_post_admin_access_key | length > 0
- cluster_post_admin_secret_key | length > 0
fail_msg: "could not parse admin keys from {{ ansible_env.HOME }}/.aws/credentials [spinifex] profile"
- name: cluster-post — ensure local .creds dir exists
delegate_to: localhost
become: false
ansible.builtin.file:
path: "{{ playbook_dir }}/../.creds"
state: directory
mode: "0700"
- name: cluster-post — write creds to operator workstation
delegate_to: localhost
become: false
ansible.builtin.copy:
dest: "{{ playbook_dir }}/../.creds/{{ cluster_env }}.txt"
mode: "0600"
content: |
# Spinifex cluster admin credentials — {{ cluster_env }}
# Generated: {{ ansible_facts['date_time']['iso8601'] }}
# Init host: {{ cluster_init_host }}
#
# UI: https://{{ cluster_init_host }}:3000
# AWS API: https://{{ cluster_init_host }}:4432
# Region: {{ spinifex_region }}
#
# Login (admin user / AWS profile spinifex):
AWS_ACCESS_KEY_ID={{ cluster_post_admin_access_key }}
AWS_SECRET_ACCESS_KEY={{ cluster_post_admin_secret_key }}
# Per-env AWS profile in operator's ~/.aws/credentials so tofu / aws CLI can
# target this cluster via AWS_PROFILE=spinifex-{{ cluster_env }}. blockinfile
# marker keeps re-runs idempotent — replaces the block in place, never
# duplicates. Does not touch the user's existing [spinifex] profile (single-
# node dev). Tofu side: provider "aws" { profile = "spinifex-env1" ... }.
- name: cluster-post — ensure local ~/.aws dir exists
delegate_to: localhost
become: false
ansible.builtin.file:
path: "{{ lookup('env', 'HOME') }}/.aws"
state: directory
mode: "0700"
- name: cluster-post — ensure local ~/.aws/credentials exists
delegate_to: localhost
become: false
ansible.builtin.copy:
dest: "{{ lookup('env', 'HOME') }}/.aws/credentials"
content: ""
force: false
mode: "0600"
- name: cluster-post — upsert spinifex-{{ cluster_env }} profile in local ~/.aws/credentials
delegate_to: localhost
become: false
ansible.builtin.blockinfile:
path: "{{ lookup('env', 'HOME') }}/.aws/credentials"
marker: "# {mark} SPINIFEX-{{ cluster_env | upper }} MANAGED BLOCK"
block: |
[spinifex-{{ cluster_env }}]
aws_access_key_id = {{ cluster_post_admin_access_key }}
aws_secret_access_key = {{ cluster_post_admin_secret_key }}
region = {{ spinifex_region }}
ca_bundle = {{ playbook_dir }}/../.creds/{{ cluster_env }}-ca.pem
endpoint_url = https://{{ cluster_init_host }}:4432
- name: cluster-post — slurp cluster CA from init host
ansible.builtin.slurp:
src: "{{ spinifex_ca_src }}"
register: cluster_post_ca_raw
become: true
- name: cluster-post — write CA pem to operator workstation
delegate_to: localhost
become: false
ansible.builtin.copy:
dest: "{{ playbook_dir }}/../.creds/{{ cluster_env }}-ca.pem"
mode: "0644"
content: "{{ cluster_post_ca_raw.content | b64decode }}"
- name: cluster-post — resolve absolute paths for summary
delegate_to: localhost
become: false
ansible.builtin.command: "readlink -f {{ playbook_dir }}/../.creds"
register: cluster_post_creds_abs
changed_when: false
- name: cluster-post — summary
ansible.builtin.debug:
msg: |
Post-bootstrap done:
spinifex-key: {{ 'imported now' if (cluster_post_key_import.changed | default(false)) else 'already present' }}
ubuntu AMI: ubuntu-{{ cluster_post_ubuntu_version }}
admin creds: {{ cluster_post_creds_abs.stdout }}/{{ cluster_env }}.txt
CA cert: {{ cluster_post_creds_abs.stdout }}/{{ cluster_env }}-ca.pem
AWS profile: spinifex-{{ cluster_env }} (in ~/.aws/credentials)
UI URL: https://{{ cluster_init_host }}:3000
Access Key: {{ cluster_post_admin_access_key }}
Tofu usage: AWS_PROFILE=spinifex-{{ cluster_env }} tofu plan
scripts/ansible/roles/deploy/meta/main.yml
-7
View File
@@ -1,7 +0,0 @@
---
galaxy_info:
role_name: deploy
description: In-place binary + microvm artifact swap for a running Spinifex node (parity with `make deploy`).
license: BSD-3-Clause
min_ansible_version: "2.14"
dependencies: []
scripts/ansible/roles/deploy/tasks/main.yml
-144
View File
@@ -1,144 +0,0 @@
---
# In-place artifact swap for an already-installed spinifex node. Mirrors
# `make deploy`: replace binaries + microvm artifacts, restart services.
# Does NOT run setup.sh (no user/dir/systemd-unit changes) or smoketest.
- name: deploy — fail fast if spinifex isn't already installed
ansible.builtin.stat:
path: "{{ spinifex_config_file }}"
register: deploy_installed_stat
failed_when: not deploy_installed_stat.stat.exists
- name: deploy — assert config present
ansible.builtin.assert:
that: deploy_installed_stat.stat.exists
fail_msg: |
{{ spinifex_config_file }} missing — this playbook only updates an
existing install. Run ansible-dev-install or ansible-dev-reset first.
- name: deploy — verify build artifacts present
ansible.builtin.stat:
path: "{{ item }}"
register: deploy_artifacts
loop:
- "{{ spinifex_root }}/bin/spx"
- "{{ mulga_root }}/viperblock/lib/nbdkit-viperblock-plugin.so"
- name: deploy — assert artifacts exist after build role
ansible.builtin.assert:
that: item.stat.exists
fail_msg: "Build artifact missing: {{ item.item }} — build role should have produced it"
loop: "{{ deploy_artifacts.results }}"
loop_control:
label: "{{ item.item }}"
- name: deploy — detect microvm artifacts (direct-boot branches only)
ansible.builtin.stat:
path: "{{ spinifex_root }}/build/microvm/{{ item }}"
register: deploy_microvm_stat
loop:
- vmlinuz
- initramfs.cpio.gz
- name: deploy — set microvm presence fact
ansible.builtin.set_fact:
deploy_microvm_present: "{{ deploy_microvm_stat.results | map(attribute='stat.exists') | min }}"
- name: deploy — detect nbdkit plugin directory
ansible.builtin.shell: |
set -e
nbdkit --dump-config | awk -F= '/^plugindir=/{print $2; exit}'
register: deploy_plugindir_cmd
changed_when: false
- name: deploy — set plugin path fact
ansible.builtin.set_fact:
deploy_plugindir: "{{ deploy_plugindir_cmd.stdout | trim }}"
- name: deploy — assert plugindir resolved
ansible.builtin.assert:
that: deploy_plugindir | length > 0
fail_msg: "Could not detect nbdkit plugin directory via `nbdkit --dump-config`"
- name: deploy — stop spinifex.target before binary swap
become: true
ansible.builtin.systemd:
name: spinifex.target
state: stopped
failed_when: false
- name: deploy — install spx binary
become: true
ansible.builtin.copy:
src: "{{ spinifex_root }}/bin/spx"
dest: /usr/local/bin/spx
mode: "0755"
owner: root
group: root
- name: deploy — install viperblock nbdkit plugin
become: true
ansible.builtin.copy:
src: "{{ mulga_root }}/viperblock/lib/nbdkit-viperblock-plugin.so"
dest: "{{ deploy_plugindir }}/nbdkit-viperblock-plugin.so"
mode: "0755"
owner: root
group: root
- name: deploy — ensure microvm dir
become: true
ansible.builtin.file:
path: /usr/share/spinifex/microvm
state: directory
mode: "0755"
owner: root
group: root
when: deploy_microvm_present
- name: deploy — install microvm kernel
become: true
ansible.builtin.copy:
src: "{{ spinifex_root }}/build/microvm/vmlinuz"
dest: /usr/share/spinifex/microvm/vmlinuz
mode: "0644"
owner: root
group: root
when: deploy_microvm_present
- name: deploy — install microvm initramfs
become: true
ansible.builtin.copy:
src: "{{ spinifex_root }}/build/microvm/initramfs.cpio.gz"
dest: /usr/share/spinifex/microvm/initramfs.cpio.gz
mode: "0644"
owner: root
group: root
when: deploy_microvm_present
- name: deploy — daemon-reload (covers any unit changes shipped via setup.sh later)
become: true
ansible.builtin.systemd:
daemon_reload: true
- name: deploy — start spinifex.target
become: true
ansible.builtin.systemd:
name: spinifex.target
state: started
- name: deploy — wait for awsgw to respond
ansible.builtin.command: "{{ spinifex_aws_cli }} ec2 describe-regions"
register: deploy_awsgw
retries: 15
delay: 2
until: deploy_awsgw.rc == 0
changed_when: false
- name: deploy — summary
ansible.builtin.debug:
msg: |
Deployed:
spx → /usr/local/bin/spx
nbdkit plugin → {{ deploy_plugindir }}/nbdkit-viperblock-plugin.so
microvm → {{ '/usr/share/spinifex/microvm/{vmlinuz,initramfs.cpio.gz}' if deploy_microvm_present else 'skipped (not built on this branch)' }}
spinifex.target restarted; awsgw responding.
scripts/ansible/roles/init/meta/main.yml
-7
View File
@@ -1,7 +0,0 @@
---
galaxy_info:
role_name: init
description: Run spx admin init, install CA trust, start services, import LB image.
license: BSD-3-Clause
min_ansible_version: "2.14"
dependencies: []
scripts/ansible/roles/init/tasks/main.yml
-90
View File
@@ -1,90 +0,0 @@
---
- name: init — base admin-init flags
ansible.builtin.set_fact:
init_base_flags: >-
--force
--region {{ spinifex_region }}
--az {{ spinifex_az }}
--node {{ spinifex_node_name }}
--nodes {{ spinifex_node_count }}
- name: init — no-external opt-out
ansible.builtin.set_fact:
init_external_flags: "--no-external"
when: spinifex_external_no_external | bool
- name: init — assert pool range supplied when mode=pool
ansible.builtin.assert:
that:
- spinifex_external_pool_start | length > 0
- spinifex_external_pool_end | length > 0
fail_msg: |
spx admin init requires --external-pool when --external-mode=pool.
Set spinifex_external_pool_start / spinifex_external_pool_end in
scripts/ansible/vars/defaults.yml, or pass:
-e spinifex_external_pool_start=192.168.1.150
-e spinifex_external_pool_end=192.168.1.250
Pass -e spinifex_external_no_external=true to skip external networking.
when:
- not (spinifex_external_no_external | bool)
- spinifex_external_mode == 'pool'
- name: init — pool-mode external flags
ansible.builtin.set_fact:
init_external_flags: >-
--external-mode=pool
--external-pool={{ spinifex_external_pool_start }}-{{ spinifex_external_pool_end }}
{% if spinifex_external_gateway | length > 0 %}--external-gateway={{ spinifex_external_gateway }}{% endif %}
{% if spinifex_external_prefix_len | string | length > 0 %}--external-prefix-len={{ spinifex_external_prefix_len }}{% endif %}
{% if spinifex_external_gateway_ip | length > 0 %}--gateway-ip={{ spinifex_external_gateway_ip }}{% endif %}
when:
- not (spinifex_external_no_external | bool)
- spinifex_external_mode == 'pool'
- name: init — assemble full flag string
ansible.builtin.set_fact:
init_all_flags: "{{ init_base_flags }} {{ init_external_flags | default('') }}"
- name: init — run spx admin init
become: true
ansible.builtin.command: "spx admin init {{ init_all_flags }}"
changed_when: true
- name: init — copy CA to system trust
become: true
ansible.builtin.copy:
src: "{{ spinifex_ca_src }}"
dest: "{{ spinifex_ca_trust_path }}"
mode: "0644"
remote_src: true
- name: init — rebuild CA bundle (not --fresh — preserve other CAs)
become: true
ansible.builtin.command: update-ca-certificates
changed_when: true
- name: init — start spinifex.target
become: true
ansible.builtin.systemd:
name: spinifex.target
state: started
# systemctl start returns as soon as units activate; awsgw still wires up
# to NATS + daemons after that. Bash start-dev.sh has internal wait loops;
# systemd path does not. Poll a benign call until gateway is fully ready.
- name: init — wait for AWS gateway readiness
ansible.builtin.command: "{{ spinifex_aws_cli }} ec2 describe-regions"
register: init_gw_ready
until: init_gw_ready.rc == 0
retries: 30
delay: 2
changed_when: false
- name: init — summary
ansible.builtin.debug:
msg: |
Init complete:
region={{ spinifex_region }}, az={{ spinifex_az }}, node={{ spinifex_node_name }}
external_mode={{ 'disabled' if (spinifex_external_no_external | bool) else spinifex_external_mode }}
spinifex.target: started
CA: {{ spinifex_ca_trust_path }}
scripts/ansible/roles/install/meta/main.yml
-7
View File
@@ -1,7 +0,0 @@
---
galaxy_info:
role_name: install
description: Invoke setup.sh and setup-ovn.sh to install Spinifex into production paths.
license: BSD-3-Clause
min_ansible_version: "2.14"
dependencies: []
scripts/ansible/roles/install/tasks/main.yml
-79
View File
@@ -1,79 +0,0 @@
---
- name: install — assert tarball exists
ansible.builtin.stat:
path: "{{ build_tarball_path }}"
register: install_tarball
- name: install — fail if tarball missing
ansible.builtin.assert:
that: install_tarball.stat.exists
fail_msg: "Tarball {{ build_tarball_path }} missing. Run the build role first."
- name: install — stop spinifex.target before reinstall
become: true
ansible.builtin.systemd:
name: spinifex.target
state: stopped
failed_when: false
- name: install — reset failed state (spinifex-* scope)
become: true
ansible.builtin.command: systemctl reset-failed 'spinifex-*'
changed_when: false
failed_when: false
- name: install — scrub stale pid/log files from prior dev-mode installs
become: true
ansible.builtin.shell: |
for dir in {{ spinifex_data_dir }} {{ spinifex_log_dir }} {{ spinifex_etc_dir }}; do
if [ -d "$dir" ]; then
find "$dir" -name '*.pid' -delete 2>/dev/null || true
find "$dir" -name '*.log' -delete 2>/dev/null || true
find "$dir" -name '*.log.*' -delete 2>/dev/null || true
fi
done
changed_when: false
- name: install — remove legacy /var/lib/spinifex/config
become: true
ansible.builtin.file:
path: "{{ spinifex_data_dir }}/config"
state: absent
- name: install — purge stale cluster state (configs + raft + NATS KV) on bootstrap
become: true
ansible.builtin.file:
path: "{{ item }}"
state: absent
loop:
- "{{ spinifex_etc_dir }}/predastore/predastore.toml"
- "{{ spinifex_etc_dir }}/spinifex.toml"
- "{{ spinifex_data_dir }}/predastore"
- "{{ spinifex_data_dir }}/nats"
- "{{ spinifex_data_dir }}/awsgw"
when: cluster_bootstrap_purge | default(false) | bool
- name: install — run setup.sh (creates users, dirs, systemd units)
become: true
ansible.builtin.command:
cmd: "bash {{ spinifex_root }}/scripts/setup.sh"
environment:
INSTALL_SPINIFEX_TARBALL: "{{ build_tarball_path }}"
INSTALL_SPINIFEX_SKIP_NEWGRP: "1"
changed_when: true
- name: install — cleanup tarball
ansible.builtin.file:
path: "{{ build_tarball_path }}"
state: absent
- name: install — run setup-ovn.sh --management
become: true
ansible.builtin.command:
cmd: "/usr/local/share/spinifex/setup-ovn.sh --management --node-name={{ spinifex_node_name }}"
changed_when: true
when: install_run_ovn | default(true) | bool
- name: install — summary
ansible.builtin.debug:
msg: "Install complete — setup.sh + setup-ovn.sh ran. Services NOT yet started (init role handles that)."
scripts/ansible/roles/logs/meta/main.yml
-7
View File
@@ -1,7 +0,0 @@
---
galaxy_info:
role_name: logs
description: Dump spinifex + ovn + system logs into a tarball for bug reports.
license: BSD-3-Clause
min_ansible_version: "2.14"
dependencies: []
scripts/ansible/roles/logs/tasks/main.yml
-153
View File
@@ -1,153 +0,0 @@
---
# Collect a self-contained log bundle for bug reports. NEVER mutates running
# state — only reads journalctl + /var/log/spinifex/* + status output, then
# tars into logs_output_dir. Errors on missing inputs are non-fatal so a
# half-broken node still produces a bundle.
- name: logs — resolve staging directory
ansible.builtin.set_fact:
logs_stage_dir: "/tmp/spinifex-logs-{{ ansible_facts['hostname'] }}-{{ ansible_facts['date_time']['iso8601_basic_short'] }}"
logs_bundle_path: "{{ logs_output_dir }}/spinifex-logs-{{ ansible_facts['hostname'] }}-{{ ansible_facts['date_time']['iso8601_basic_short'] }}.tar.gz"
- name: logs — create staging tree
ansible.builtin.file:
path: "{{ logs_stage_dir }}/{{ item }}"
state: directory
mode: "0755"
loop:
- journal
- varlog
- status
- kernel
- name: logs — capture journalctl for spinifex-* units
become: true
ansible.builtin.shell: |
journalctl --no-pager --output=short-iso \
-u 'spinifex-*' --since {{ logs_since | quote }} \
> {{ logs_stage_dir }}/journal/spinifex.log 2>&1 || true
changed_when: true
- name: logs — capture journalctl for ovn-* + openvswitch-switch
become: true
ansible.builtin.shell: |
journalctl --no-pager --output=short-iso \
-u 'ovn-*' -u openvswitch-switch \
--since {{ logs_since | quote }} \
> {{ logs_stage_dir }}/journal/ovn-ovs.log 2>&1 || true
changed_when: true
- name: logs — capture journalctl for nbdkit + qemu (best-effort match)
become: true
ansible.builtin.shell: |
journalctl --no-pager --output=short-iso \
--since {{ logs_since | quote }} \
_COMM=qemu-system-x86_64 + _COMM=qemu-system-aarch64 + _COMM=nbdkit \
> {{ logs_stage_dir }}/journal/qemu-nbdkit.log 2>&1 || true
changed_when: true
- name: logs — copy /var/log/spinifex/*
become: true
ansible.builtin.shell: |
if [ -d {{ spinifex_log_dir | quote }} ]; then
cp -a {{ spinifex_log_dir | quote }}/. {{ logs_stage_dir | quote }}/varlog/ 2>/dev/null || true
fi
changed_when: true
- name: logs — systemctl status snapshot (spinifex-*)
become: true
ansible.builtin.shell: |
systemctl status 'spinifex-*' --no-pager --all \
> {{ logs_stage_dir }}/status/spinifex-units.txt 2>&1 || true
systemctl status ovn-controller ovn-central openvswitch-switch \
--no-pager > {{ logs_stage_dir }}/status/ovn-ovs-units.txt 2>&1 || true
systemctl list-units 'spinifex-*' --all --no-legend --plain \
> {{ logs_stage_dir }}/status/spinifex-list.txt 2>&1 || true
changed_when: true
- name: logs — networking snapshot (ip, ss, ovs)
become: true
ansible.builtin.shell: |
ip -d -s link show > {{ logs_stage_dir }}/status/ip-link.txt 2>&1 || true
ip addr show > {{ logs_stage_dir }}/status/ip-addr.txt 2>&1 || true
ip route show table all > {{ logs_stage_dir }}/status/ip-route.txt 2>&1 || true
ss -tlnp > {{ logs_stage_dir }}/status/ss-tlnp.txt 2>&1 || true
if command -v ovs-vsctl >/dev/null 2>&1; then
ovs-vsctl show > {{ logs_stage_dir }}/status/ovs-show.txt 2>&1 || true
ovs-vsctl list-br > {{ logs_stage_dir }}/status/ovs-br.txt 2>&1 || true
fi
if command -v ovn-nbctl >/dev/null 2>&1; then
ovn-nbctl show > {{ logs_stage_dir }}/status/ovn-nb.txt 2>&1 || true
ovn-sbctl show > {{ logs_stage_dir }}/status/ovn-sb.txt 2>&1 || true
fi
changed_when: true
- name: logs — kernel ring buffer
become: true
ansible.builtin.shell: |
dmesg --time-format=iso > {{ logs_stage_dir }}/kernel/dmesg.txt 2>&1 || true
changed_when: true
when: logs_include_dmesg | bool
- name: logs — capture spx + plugin versions
become: true
ansible.builtin.shell: |
{
echo '## spx version'
/usr/local/bin/spx version 2>&1 | grep -vE '^[0-9]{4}/' || true
echo
echo '## nbdkit plugin sha256'
pdir=$(nbdkit --dump-config 2>/dev/null | awk -F= '/^plugindir=/{print $2; exit}')
[ -n "$pdir" ] && sha256sum "$pdir/nbdkit-viperblock-plugin.so" 2>/dev/null || true
echo
echo '## microvm artifacts'
sha256sum /usr/share/spinifex/microvm/* 2>/dev/null || true
echo
echo '## kernel + os'
uname -a
cat /etc/os-release 2>/dev/null || true
} > {{ logs_stage_dir }}/status/versions.txt 2>&1 || true
changed_when: true
- name: logs — create bundle tarball
become: true
ansible.builtin.shell: |
tar -C {{ logs_stage_dir | dirname | quote }} \
-czf {{ logs_bundle_path | quote }} \
{{ logs_stage_dir | basename | quote }}
changed_when: true
- name: logs — make bundle owned by invoking user
become: true
ansible.builtin.file:
path: "{{ logs_bundle_path }}"
owner: "{{ spinifex_invoking_user }}"
group: "{{ spinifex_invoking_user }}"
mode: "0640"
- name: logs — remove staging tree
become: true
ansible.builtin.file:
path: "{{ logs_stage_dir }}"
state: absent
- name: logs — bundle size
ansible.builtin.stat:
path: "{{ logs_bundle_path }}"
register: logs_bundle_stat
- name: logs — summary
ansible.builtin.debug:
msg: |
Log bundle written:
path : {{ logs_bundle_path }}
size : {{ (logs_bundle_stat.stat.size | int / 1024) | round(1) }} KiB
since : {{ logs_since }}
Contents:
journal/ spinifex-*, ovn-*+openvswitch-switch, qemu/nbdkit
varlog/ copy of {{ spinifex_log_dir }}
status/ systemctl status, ip/ss/ovs/ovn show, versions
kernel/ dmesg{{ '' if logs_include_dmesg else ' (skipped via -e logs_include_dmesg=false)' }}
Attach to bug report. For live tail use:
journalctl -u 'spinifex-*' -f
scripts/ansible/roles/preflight/meta/main.yml
-7
View File
@@ -1,7 +0,0 @@
---
galaxy_info:
role_name: preflight
description: Validate dev-host dependencies for Spinifex (parity with dev-setup.sh).
license: BSD-3-Clause
min_ansible_version: "2.14"
dependencies: []
scripts/ansible/roles/preflight/tasks/main.yml
-98
View File
@@ -1,98 +0,0 @@
---
- name: Check required commands
ansible.builtin.command: "which {{ item }}"
register: preflight_required
loop: "{{ spinifex_required_commands }}"
changed_when: false
failed_when: preflight_required.rc != 0
loop_control:
label: "{{ item }}"
- name: Check optional commands
ansible.builtin.command: "which {{ item }}"
register: preflight_optional
loop: "{{ spinifex_optional_commands }}"
changed_when: false
failed_when: false
loop_control:
label: "{{ item }}"
- name: Warn on missing optional commands
ansible.builtin.debug:
msg: "WARN: optional command '{{ item.item }}' not found"
loop: "{{ preflight_optional.results }}"
when: item.rc != 0
loop_control:
label: "{{ item.item }}"
- name: Check OVN/OVS commands
ansible.builtin.command: "which {{ item }}"
register: preflight_ovn
loop: "{{ spinifex_ovn_commands }}"
changed_when: false
failed_when: false
loop_control:
label: "{{ item }}"
- name: Warn on missing OVN/OVS tools
ansible.builtin.debug:
msg: "WARN: OVN/OVS tool '{{ item.item }}' not found (required for VPC networking). Install: sudo apt-get install ovn-central ovn-host openvswitch-switch"
loop: "{{ preflight_ovn.results }}"
when: item.rc != 0
loop_control:
label: "{{ item.item }}"
- name: Record OVN tool availability
ansible.builtin.set_fact:
preflight_ovs_present: "{{ (preflight_ovn.results | selectattr('item','eq','ovs-vsctl') | selectattr('rc','eq',0) | list | length) > 0 }}"
- name: Check br-int bridge
ansible.builtin.command: ovs-vsctl br-exists br-int
register: preflight_brint
changed_when: false
failed_when: false
when: preflight_ovs_present
- name: Warn if br-int missing
ansible.builtin.debug:
msg: "WARN: br-int missing. Run scripts/setup-ovn.sh --management to initialise OVN networking."
when:
- preflight_ovs_present
- preflight_brint.rc | default(0) != 0
- name: Check sibling component directories
ansible.builtin.stat:
path: "{{ playbook_dir }}/../../../../{{ item }}/go.mod"
register: preflight_components
loop: "{{ spinifex_components }}"
loop_control:
label: "{{ item }}"
- name: Warn on missing component checkouts
ansible.builtin.debug:
msg: "WARN: component '{{ item.item }}' not found at expected sibling path (no go.mod). Run scripts/clone-deps.sh."
loop: "{{ preflight_components.results }}"
when: not item.stat.exists
loop_control:
label: "{{ item.item }}"
- name: Check port availability
ansible.builtin.shell: "ss -tlnH 'sport = :{{ item.port }}' | head -1"
register: preflight_ports
loop: "{{ spinifex_ports }}"
changed_when: false
failed_when: false
loop_control:
label: "{{ item.service }} ({{ item.port }})"
- name: Warn on occupied ports
ansible.builtin.debug:
msg: "WARN: port {{ item.item.port }} occupied (needed for {{ item.item.service }})"
loop: "{{ preflight_ports.results }}"
when: item.stdout | length > 0
loop_control:
label: "{{ item.item.service }}"
- name: Preflight summary
ansible.builtin.debug:
msg: "Preflight passed. Required: {{ spinifex_required_commands | join(', ') }}. Warnings above (if any) are non-fatal."
scripts/ansible/roles/restore/meta/main.yml
-7
View File
@@ -1,7 +0,0 @@
---
galaxy_info:
role_name: restore
description: Restore a spinifex+ovn+ovs snapshot bundle, refresh CA trust, restart.
license: BSD-3-Clause
min_ansible_version: "2.14"
dependencies: []
scripts/ansible/roles/restore/tasks/main.yml
-140
View File
@@ -1,140 +0,0 @@
---
# Inverse of the snapshot role: stop services, wipe the same path set,
# untar the named bundle, refresh CA trust (because /etc/spinifex/ca.pem
# came from the snapshot but the system trust store didn't), restart.
- name: restore — require snapshot_name
ansible.builtin.assert:
that:
- snapshot_name is defined
- snapshot_name | length > 0
- snapshot_name is match('^[A-Za-z0-9._-]+$')
fail_msg: "snapshot_name required (alphanumeric/._-): -e snapshot_name=<name>"
- name: restore — resolve bundle path
ansible.builtin.set_fact:
snapshot_bundle_path: "{{ snapshot_dir }}/{{ snapshot_name }}.tar.gz"
- name: restore — assert bundle exists
become: true
ansible.builtin.stat:
path: "{{ snapshot_bundle_path }}"
register: restore_bundle_stat
- name: restore — fail if missing
ansible.builtin.assert:
that: restore_bundle_stat.stat.exists
fail_msg: |
Snapshot bundle missing: {{ snapshot_bundle_path }}.
Available bundles:
ls -lh {{ snapshot_dir }}/*.tar.gz 2>/dev/null
- name: restore — stop services in dependency order
become: true
ansible.builtin.systemd:
name: "{{ item }}"
state: stopped
loop: "{{ snapshot_services_stop_order }}"
failed_when: false
- name: restore — kill lingering QEMU VMs
become: true
ansible.builtin.command: "pkill -x {{ item }}"
loop: "{{ spinifex_qemu_processes }}"
changed_when: false
failed_when: false
- name: restore — wait for QEMU exit
become: true
ansible.builtin.shell: |
set -e
for i in $(seq 1 {{ teardown_process_wait_seconds }}); do
alive=0
for proc in {{ spinifex_qemu_processes | join(' ') }}; do
if pgrep -x "$proc" >/dev/null 2>&1; then alive=1; break; fi
done
if [ "$alive" = "0" ]; then exit 0; fi
sleep 1
done
exit 1
changed_when: false
- name: restore — wipe target paths (only those the snapshot will replace)
become: true
ansible.builtin.file:
path: "{{ item }}"
state: absent
loop: "{{ snapshot_paths }}"
- name: restore — recreate parent dirs that tar will refill
become: true
ansible.builtin.file:
path: "{{ item }}"
state: directory
owner: root
group: root
mode: "0755"
loop:
- "{{ ovn_db_dir }}"
- /etc/openvswitch
- name: restore — extract bundle (preserve permissions/ACLs)
become: true
ansible.builtin.command:
cmd: >-
tar --acls --xattrs -xzpf {{ snapshot_bundle_path | quote }}
-C /
changed_when: true
- name: restore — refresh spinifex CA in system trust
become: true
ansible.builtin.stat:
path: "{{ spinifex_ca_src }}"
register: restore_ca_stat
- name: restore — copy CA back to trust path
become: true
ansible.builtin.copy:
src: "{{ spinifex_ca_src }}"
dest: "{{ spinifex_ca_trust_path }}"
mode: "0644"
remote_src: true
when: restore_ca_stat.stat.exists
- name: restore — rebuild CA bundle
become: true
ansible.builtin.command: update-ca-certificates
changed_when: true
when: restore_ca_stat.stat.exists
- name: restore — daemon-reload (covers any unit file delta in the snapshot)
become: true
ansible.builtin.systemd:
daemon_reload: true
- name: restore — restart services in reverse order
become: true
ansible.builtin.systemd:
name: "{{ item }}"
state: started
loop: "{{ snapshot_services_stop_order | reverse | list }}"
failed_when: false
- name: restore — wait for AWS gateway readiness
ansible.builtin.command: "{{ spinifex_aws_cli }} ec2 describe-regions"
register: restore_gw_ready
retries: 30
delay: 2
until: restore_gw_ready.rc == 0
changed_when: false
failed_when: false
- name: restore — summary
ansible.builtin.debug:
msg: |
Restore complete:
snapshot : {{ snapshot_name }}
bundle : {{ snapshot_bundle_path }}
paths : {{ snapshot_paths | join(', ') }}
CA : {{ 'refreshed' if restore_ca_stat.stat.exists else 'not in bundle — system trust unchanged' }}
gateway : {{ 'ready' if restore_gw_ready.rc == 0 else 'NOT ready after restart — investigate' }}
scripts/ansible/roles/smoketest/meta/main.yml
-7
View File
@@ -1,7 +0,0 @@
---
galaxy_info:
role_name: smoketest
description: Import SSH key, import Ubuntu AMI, launch one smoke-test instance.
license: BSD-3-Clause
min_ansible_version: "2.14"
dependencies: []
scripts/ansible/roles/smoketest/tasks/main.yml
-197
View File
@@ -1,197 +0,0 @@
---
- name: smoketest — resolve invoking user's home
ansible.builtin.getent:
database: passwd
key: "{{ spinifex_invoking_user }}"
- name: smoketest — set user home fact
ansible.builtin.set_fact:
smoketest_user_home: "{{ ansible_facts['getent_passwd'][spinifex_invoking_user][4] }}"
- name: smoketest — generate SSH key if absent
become: true
become_user: "{{ spinifex_invoking_user }}"
ansible.builtin.command:
cmd: ssh-keygen -t ed25519 -f {{ smoketest_user_home }}/.ssh/spinifex-key -N ""
creates: "{{ smoketest_user_home }}/.ssh/spinifex-key"
- name: smoketest — import SSH key to spinifex
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 import-key-pair
--key-name spinifex-key
--public-key-material fileb://{{ smoketest_user_home }}/.ssh/spinifex-key.pub
register: smoketest_keypair
until: >-
smoketest_keypair.rc == 0 or
'InvalidKeyPair.Duplicate' in (smoketest_keypair.stderr | default(''))
retries: 10
delay: 3
changed_when: "'InvalidKeyPair.Duplicate' not in (smoketest_keypair.stderr | default(''))"
failed_when:
- smoketest_keypair.rc != 0
- "'InvalidKeyPair.Duplicate' not in (smoketest_keypair.stderr | default(''))"
- name: smoketest — detect architecture
ansible.builtin.set_fact:
smoketest_arch: "{{ 'arm64' if ansible_facts['architecture'] in ['aarch64','arm64'] else 'x86_64' }}"
- name: smoketest — resolve local image path
ansible.builtin.set_fact:
smoketest_local_image_path: "{{ smoketest_local_image | expanduser }}"
vars:
smoketest_local_image: "{{ spinifex_smoketest_local_image | replace('~', smoketest_user_home) }}"
- name: smoketest — check for local image
ansible.builtin.stat:
path: "{{ smoketest_local_image_path }}"
register: smoketest_local_img
- name: smoketest — import AMI from local file
ansible.builtin.command: >-
{{ spinifex_spx_cli }} admin images import
--file {{ smoketest_local_image_path }}
--distro {{ spinifex_smoketest_distro }}
--version {{ spinifex_smoketest_version }}
--arch {{ smoketest_arch }}
--boot-mode uefi
register: smoketest_import_local
until: >-
smoketest_import_local.rc == 0 or
'already exists' in (smoketest_import_local.stderr | default(''))
retries: 5
delay: 3
changed_when: smoketest_import_local.rc == 0
failed_when:
- smoketest_import_local.rc != 0
- "'already exists' not in (smoketest_import_local.stderr | default(''))"
when: smoketest_local_img.stat.exists
- name: smoketest — import AMI by name (fallback)
ansible.builtin.command: >-
{{ spinifex_spx_cli }} admin images import
--name {{ spinifex_smoketest_distro }}-{{ spinifex_smoketest_version }}-{{ smoketest_arch }}
register: smoketest_import_remote
until: >-
smoketest_import_remote.rc == 0 or
'already exists' in (smoketest_import_remote.stderr | default(''))
retries: 5
delay: 3
changed_when: smoketest_import_remote.rc == 0
failed_when:
- smoketest_import_remote.rc != 0
- "'already exists' not in (smoketest_import_remote.stderr | default(''))"
when: not smoketest_local_img.stat.exists
- name: smoketest — detect CPU vendor for instance type choice
ansible.builtin.command: grep -q AuthenticAMD /proc/cpuinfo
register: smoketest_amd
changed_when: false
failed_when: false
- name: smoketest — pick instance type
ansible.builtin.set_fact:
smoketest_instance_type: "{{ 't3a.small' if smoketest_amd.rc == 0 else 't3.small' }}"
- name: smoketest — find AMI ID
ansible.builtin.command: "{{ spinifex_aws_cli }} ec2 describe-images --query Images[0].ImageId --output text"
register: smoketest_ami
until: smoketest_ami.rc == 0
retries: 10
delay: 3
changed_when: false
- name: smoketest — find public subnet (best-effort)
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 describe-subnets
--filters Name=map-public-ip-on-launch,Values=true
--query Subnets[0].SubnetId
--output text
register: smoketest_subnet_public
changed_when: false
failed_when: false
- name: smoketest — fallback to first subnet (best-effort)
ansible.builtin.command: "{{ spinifex_aws_cli }} ec2 describe-subnets --query Subnets[0].SubnetId --output text"
register: smoketest_subnet_any
changed_when: false
failed_when: false
when:
- smoketest_subnet_public.rc != 0 or smoketest_subnet_public.stdout in ['', 'None']
- name: smoketest — resolve subnet id (empty string if none)
ansible.builtin.set_fact:
smoketest_subnet_id: >-
{%- if smoketest_subnet_public.rc == 0 and smoketest_subnet_public.stdout not in ['', 'None'] -%}
{{ smoketest_subnet_public.stdout }}
{%- elif (smoketest_subnet_any.rc | default(1)) == 0 and (smoketest_subnet_any.stdout | default('')) not in ['', 'None'] -%}
{{ smoketest_subnet_any.stdout }}
{%- else -%}{%- endif -%}
- name: smoketest — skip launch if no subnet (fresh install with no user VPC)
ansible.builtin.debug:
msg: |
No subnet found — skipping instance launch.
Services are up, CA is trusted, keypair imported, AMI imported.
Create a VPC+subnet via tofu examples (or `aws ec2 create-vpc` /
`create-subnet`) then re-run if you want a live instance.
when: smoketest_subnet_id | trim == ''
- name: smoketest — launch instance
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 run-instances
--image-id {{ smoketest_ami.stdout }}
--instance-type {{ smoketest_instance_type }}
--key-name spinifex-key
--subnet-id {{ smoketest_subnet_id | trim }}
--count 1
--query Instances[0].InstanceId
--output text
register: smoketest_instance
until: smoketest_instance.rc == 0
retries: 5
delay: 3
changed_when: true
when: smoketest_subnet_id | trim != ''
- name: smoketest — wait for instance running (proves launch path works)
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 describe-instances
--instance-ids {{ smoketest_instance.stdout }}
--query Reservations[0].Instances[0].State.Name
--output text
register: smoketest_instance_state
until: smoketest_instance_state.stdout | trim == 'running'
retries: 30
delay: 2
changed_when: false
failed_when: false
when:
- smoketest_subnet_id | trim != ''
- smoketest_instance.stdout is defined
- smoketest_instance.stdout | trim != ''
- name: smoketest — terminate instance (cleanup)
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 terminate-instances
--instance-ids {{ smoketest_instance.stdout }}
register: smoketest_terminate
until: smoketest_terminate.rc == 0
retries: 5
delay: 3
changed_when: true
failed_when: false
when:
- smoketest_subnet_id | trim != ''
- smoketest_instance.stdout is defined
- smoketest_instance.stdout | trim != ''
- name: smoketest — summary
ansible.builtin.debug:
msg: |
Smoketest complete:
AMI: {{ smoketest_ami.stdout }}
Subnet: {{ smoketest_subnet_id | trim | default('(none — launch skipped)', true) }}
Instance type: {{ smoketest_instance_type }}
Instance ID: {{ smoketest_instance.stdout | default('(skipped)', true) }}
Final state: {{ smoketest_instance_state.stdout | default('(skipped)', true) | trim }}
Terminated: {{ 'yes' if smoketest_terminate.rc | default(1) == 0 else 'no (launch skipped or terminate failed)' }}
scripts/ansible/roles/snapshot/meta/main.yml
-7
View File
@@ -1,7 +0,0 @@
---
galaxy_info:
role_name: snapshot
description: Quiesce spinifex + ovn + ovs, tar state paths, restart.
license: BSD-3-Clause
min_ansible_version: "2.14"
dependencies: []
scripts/ansible/roles/snapshot/tasks/main.yml
-154
View File
@@ -1,154 +0,0 @@
---
# Snapshot a quiesced spinifex node: stop spinifex.target → ovn → ovs in order,
# tar /etc/spinifex + /var/lib/spinifex + /var/lib/ovn + /etc/openvswitch/conf.db,
# restart in reverse. Lets devs checkpoint a working state before risky ops
# (branch switch, viperblock surgery, ovn-controller restart cycle) and roll
# back via dev-restore without a full ~10-min dev-reset.
#
# Caveat: /var/lib/spinifex contains viperblock volumes — snapshot size scales
# with VM disks. Default gzip; switch to a faster compressor by hand if needed.
- name: snapshot — require snapshot_name
ansible.builtin.assert:
that:
- snapshot_name is defined
- snapshot_name | length > 0
- snapshot_name is match('^[A-Za-z0-9._-]+$')
fail_msg: "snapshot_name required (alphanumeric/._-): -e snapshot_name=<name>"
- name: snapshot — resolve bundle path
ansible.builtin.set_fact:
snapshot_bundle_path: "{{ snapshot_dir }}/{{ snapshot_name }}.tar.gz"
- name: snapshot — refuse to overwrite existing bundle
ansible.builtin.stat:
path: "{{ snapshot_bundle_path }}"
register: snapshot_existing
- name: snapshot — assert not already present
ansible.builtin.assert:
that: not snapshot_existing.stat.exists
fail_msg: |
Snapshot {{ snapshot_bundle_path }} already exists. Pick another
name, or `sudo rm` the existing bundle first.
- name: snapshot — ensure snapshot dir exists
become: true
ansible.builtin.file:
path: "{{ snapshot_dir }}"
state: directory
owner: root
group: root
mode: "0750"
- name: snapshot — stop services in dependency order
become: true
ansible.builtin.systemd:
name: "{{ item }}"
state: stopped
loop: "{{ snapshot_services_stop_order }}"
failed_when: false
- name: snapshot — kill lingering QEMU VMs
become: true
ansible.builtin.command: "pkill -x {{ item }}"
loop: "{{ spinifex_qemu_processes }}"
changed_when: false
failed_when: false
- name: snapshot — wait for QEMU exit (up to {{ teardown_process_wait_seconds }}s)
become: true
ansible.builtin.shell: |
set -e
for i in $(seq 1 {{ teardown_process_wait_seconds }}); do
alive=0
for proc in {{ spinifex_qemu_processes | join(' ') }}; do
if pgrep -x "$proc" >/dev/null 2>&1; then alive=1; break; fi
done
if [ "$alive" = "0" ]; then exit 0; fi
sleep 1
done
echo "QEMU still running after {{ teardown_process_wait_seconds }}s" >&2
exit 1
changed_when: false
- name: snapshot — verify which paths actually exist (skip missing silently)
ansible.builtin.stat:
path: "{{ item }}"
register: snapshot_path_stats
loop: "{{ snapshot_paths }}"
loop_control:
label: "{{ item }}"
- name: snapshot — assemble list of present paths
ansible.builtin.set_fact:
snapshot_present_paths: >-
{{ snapshot_path_stats.results
| selectattr('stat.exists') | map(attribute='item') | list }}
- name: snapshot — assert at least one path present
ansible.builtin.assert:
that: snapshot_present_paths | length > 0
fail_msg: |
None of the snapshot paths exist:
{{ snapshot_paths | join(', ') }}
Has spinifex ever been installed on this node?
- name: snapshot — create bundle (tar czf with absolute paths)
become: true
ansible.builtin.command:
cmd: >-
tar --acls --xattrs -czpf {{ snapshot_bundle_path | quote }}
{{ snapshot_present_paths | map('quote') | join(' ') }}
changed_when: true
- name: snapshot — record manifest inside the bundle dir for traceability
become: true
ansible.builtin.copy:
dest: "{{ snapshot_dir }}/{{ snapshot_name }}.manifest"
mode: "0640"
content: |
snapshot_name: {{ snapshot_name }}
created_at: {{ ansible_facts['date_time']['iso8601'] }}
hostname: {{ ansible_facts['hostname'] }}
paths:
{% for p in snapshot_present_paths %} - {{ p }}
{% endfor %}
services_quiesced:
{% for s in snapshot_services_stop_order %} - {{ s }}
{% endfor %}
- name: snapshot — restart services in reverse order
become: true
ansible.builtin.systemd:
name: "{{ item }}"
state: started
loop: "{{ snapshot_services_stop_order | reverse | list }}"
failed_when: false
- name: snapshot — wait for AWS gateway readiness
ansible.builtin.command: "{{ spinifex_aws_cli }} ec2 describe-regions"
register: snapshot_gw_ready
retries: 30
delay: 2
until: snapshot_gw_ready.rc == 0
changed_when: false
failed_when: false
- name: snapshot — bundle size
become: true
ansible.builtin.stat:
path: "{{ snapshot_bundle_path }}"
register: snapshot_bundle_stat
- name: snapshot — summary
ansible.builtin.debug:
msg: |
Snapshot complete:
name : {{ snapshot_name }}
path : {{ snapshot_bundle_path }}
size : {{ (snapshot_bundle_stat.stat.size | int / 1024 / 1024) | round(1) }} MiB
paths : {{ snapshot_present_paths | join(', ') }}
gateway : {{ 'ready' if snapshot_gw_ready.rc == 0 else 'NOT ready after restart — investigate' }}
Restore with: ansible-playbook playbooks/dev-restore.yml -e snapshot_name={{ snapshot_name }}
scripts/ansible/roles/status/meta/main.yml
-7
View File
@@ -1,7 +0,0 @@
---
galaxy_info:
role_name: status
description: Read-only health snapshot for a Spinifex dev node (doctor).
license: BSD-3-Clause
min_ansible_version: "2.14"
dependencies: []
scripts/ansible/roles/status/tasks/main.yml
-259
View File
@@ -1,259 +0,0 @@
---
# Read-only health snapshot. NEVER mutates state. Every probe must use
# `changed_when: false` and `failed_when: false` so the role always reaches
# the summary task even when the node is half-broken — surfacing brokenness
# is the whole point.
- name: status — spinifex.target activation
ansible.builtin.command: systemctl is-active spinifex.target
register: status_target
changed_when: false
failed_when: false
- name: status — list spinifex-* units (active + failed)
ansible.builtin.shell: |
systemctl list-units 'spinifex-*' --all --no-legend --plain \
--output=json 2>/dev/null || \
systemctl list-units 'spinifex-*' --all --no-legend --plain
register: status_units
changed_when: false
failed_when: false
- name: status — failed spinifex-* units (one line per)
ansible.builtin.shell: |
systemctl list-units 'spinifex-*' --state=failed --no-legend --plain \
| awk '{print $1}'
register: status_failed_units
changed_when: false
failed_when: false
- name: status — port listeners (which process holds each spinifex port)
ansible.builtin.shell: |
ss -tlnpH 'sport = :{{ item.port }}' | head -1
register: status_ports
loop: "{{ spinifex_ports }}"
changed_when: false
failed_when: false
loop_control:
label: "{{ item.service }} ({{ item.port }})"
- name: status — OVS tooling present?
ansible.builtin.command: which ovs-vsctl
register: status_ovs_which
changed_when: false
failed_when: false
- name: status — OVS bridges
become: true
ansible.builtin.command: ovs-vsctl list-br
register: status_ovs_brs
changed_when: false
failed_when: false
when: status_ovs_which.rc == 0
- name: status — br-int exists?
become: true
ansible.builtin.command: ovs-vsctl br-exists br-int
register: status_brint
changed_when: false
failed_when: false
when: status_ovs_which.rc == 0
- name: status — ovn-controller active?
ansible.builtin.command: systemctl is-active ovn-controller
register: status_ovn_ctrl
changed_when: false
failed_when: false
- name: status — ovn-central active?
ansible.builtin.command: systemctl is-active ovn-central
register: status_ovn_central
changed_when: false
failed_when: false
- name: status — OVN NB/SB DB files present?
become: true
ansible.builtin.stat:
path: "{{ item }}"
register: status_ovn_dbs
loop:
- "{{ ovn_nb_db }}"
- "{{ ovn_sb_db }}"
loop_control:
label: "{{ item }}"
- name: status — spinifex.toml present?
become: true
ansible.builtin.stat:
path: "{{ spinifex_config_file }}"
register: status_config
- name: status — CA trust present?
become: true
ansible.builtin.stat:
path: "{{ spinifex_ca_trust_path }}"
register: status_ca
- name: status — AWS gateway round-trip (describe-regions)
ansible.builtin.command: "{{ spinifex_aws_cli }} ec2 describe-regions --output text"
register: status_awsgw
changed_when: false
failed_when: false
- name: status — installed spx binary
become: true
ansible.builtin.stat:
path: /usr/local/bin/spx
checksum_algorithm: sha256
register: status_spx_installed
- name: status — built spx binary
ansible.builtin.stat:
path: "{{ spinifex_root }}/bin/spx"
checksum_algorithm: sha256
register: status_spx_built
- name: status — spx version (installed)
ansible.builtin.shell: |
/usr/local/bin/spx version 2>/dev/null | grep -vE '^[0-9]{4}/' | head -1
register: status_spx_version
changed_when: false
failed_when: false
when: status_spx_installed.stat.exists
- name: status — installed nbdkit plugin
become: true
ansible.builtin.shell: |
set -e
pdir=$(nbdkit --dump-config | awk -F= '/^plugindir=/{print $2; exit}')
[ -f "$pdir/nbdkit-viperblock-plugin.so" ] && \
sha256sum "$pdir/nbdkit-viperblock-plugin.so" | awk '{print $1}'
register: status_plugin_installed_sha
changed_when: false
failed_when: false
- name: status — built nbdkit plugin
ansible.builtin.stat:
path: "{{ mulga_root }}/viperblock/lib/nbdkit-viperblock-plugin.so"
checksum_algorithm: sha256
register: status_plugin_built
- name: status — microvm artifacts (built)
ansible.builtin.stat:
path: "{{ spinifex_root }}/build/microvm/{{ item }}"
checksum_algorithm: sha256
register: status_microvm_built
loop:
- vmlinuz
- initramfs.cpio.gz
- name: status — microvm artifacts (installed)
become: true
ansible.builtin.stat:
path: /usr/share/spinifex/microvm/{{ item }}
checksum_algorithm: sha256
register: status_microvm_installed
loop:
- vmlinuz
- initramfs.cpio.gz
- name: status — running QEMU VMs
ansible.builtin.shell: pgrep -fa 'qemu-system-' | wc -l
register: status_qemu_count
changed_when: false
failed_when: false
- name: status — count AMIs
ansible.builtin.command: "{{ spinifex_aws_cli }} ec2 describe-images --query length(Images) --output text"
register: status_ami_count
changed_when: false
failed_when: false
- name: status — count keypairs
ansible.builtin.command: "{{ spinifex_aws_cli }} ec2 describe-key-pairs --query length(KeyPairs) --output text"
register: status_key_count
changed_when: false
failed_when: false
- name: status — count volumes
ansible.builtin.command: "{{ spinifex_aws_cli }} ec2 describe-volumes --query length(Volumes) --output text"
register: status_vol_count
changed_when: false
failed_when: false
- name: status — count running instances
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 describe-instances
--filters Name=instance-state-name,Values=running
--query length(Reservations[].Instances[]) --output text
register: status_instance_count
changed_when: false
failed_when: false
- name: status — compute drift facts
ansible.builtin.set_fact:
status_spx_drift: >-
{{ status_spx_installed.stat.exists and status_spx_built.stat.exists and
(status_spx_installed.stat.checksum | default('')) !=
(status_spx_built.stat.checksum | default('')) }}
status_plugin_drift: >-
{{ status_plugin_built.stat.exists and
(status_plugin_installed_sha.stdout | default('') | trim) !=
(status_plugin_built.stat.checksum | default('')) }}
status_microvm_built_present: "{{ status_microvm_built.results
| map(attribute='stat.exists') | min }}"
- name: status — render port lines
ansible.builtin.set_fact:
status_port_lines: >-
{{ status_port_lines | default([]) +
[('%-24s' | format(item.item.service ~ ' (' ~ item.item.port ~ ')'))
~ ': '
~ ('listening' if (item.stdout | length) > 0 else 'free')] }}
loop: "{{ status_ports.results }}"
loop_control:
label: "{{ item.item.service }}"
- name: status — assemble summary
ansible.builtin.set_fact:
status_summary_lines: >-
{{
[
'── Spinifex dev-node status ──────────────────────────────────────────────',
'target : ' ~ (status_target.stdout | trim),
'failed units : ' ~ (status_failed_units.stdout_lines | length)
~ ((' → ' ~ (status_failed_units.stdout_lines | join(', '))) if (status_failed_units.stdout_lines | length) > 0 else ''),
'── Ports ──'
]
+ status_port_lines
+ [
'── Networking ──',
'ovs-vsctl : ' ~ ('present' if status_ovs_which.rc == 0 else 'MISSING'),
'ovs bridges : ' ~ ((status_ovs_brs.stdout_lines | default([]) | join(', ')) or '(none)'),
'br-int : ' ~ ('present' if (status_brint.rc | default(1)) == 0 else 'MISSING'),
'ovn-controller : ' ~ (status_ovn_ctrl.stdout | trim),
'ovn-central : ' ~ (status_ovn_central.stdout | trim),
'ovn NB DB : ' ~ ('present' if status_ovn_dbs.results[0].stat.exists else 'absent'),
'ovn SB DB : ' ~ ('present' if status_ovn_dbs.results[1].stat.exists else 'absent'),
'── Config / trust ──',
'spinifex.toml : ' ~ ('present' if status_config.stat.exists else 'MISSING') ~ ' (' ~ spinifex_config_file ~ ')',
'CA trust : ' ~ ('present' if status_ca.stat.exists else 'MISSING') ~ ' (' ~ spinifex_ca_trust_path ~ ')',
'── AWS gateway ──',
'describe-regions : ' ~ ('OK' if status_awsgw.rc == 0 else 'FAIL (rc=' ~ status_awsgw.rc ~ ')'),
'AMIs : ' ~ (status_ami_count.stdout | default('?') | trim),
'keypairs : ' ~ (status_key_count.stdout | default('?') | trim),
'volumes : ' ~ (status_vol_count.stdout | default('?') | trim),
'running instances: ' ~ (status_instance_count.stdout | default('?') | trim),
'running QEMU : ' ~ (status_qemu_count.stdout | default('?') | trim),
'── Versions / drift ──',
'spx (installed) : ' ~ ((status_spx_version.stdout | default('') | trim) or '(not installed)'),
'spx drift : ' ~ ('YES (built ≠ installed) — run ansible-dev-deploy' if status_spx_drift else 'no'),
'plugin drift : ' ~ ('YES (built ≠ installed) — run ansible-dev-deploy' if status_plugin_drift else 'no'),
'microvm built : ' ~ ('present' if status_microvm_built_present else 'not built on this branch'),
'──────────────────────────────────────────────────────────────────────────'
]
}}
- name: status — summary
ansible.builtin.debug:
msg: "{{ status_summary_lines }}"
scripts/ansible/roles/teardown/README.md
-54
View File
@@ -1,54 +0,0 @@
# teardown — Leak Catalog
Every task in this role corresponds to a known state source that survived a
previous reset. When the operator reports "reset didn't clean X", add a new
task and a new row here citing the symptom.
## Initial catalog (from reset-dev-env.sh + reset-dev-env.md target)
| State source | Cleaned by | Symptom if leaked |
|---|---|---|
| `spinifex.target` active | `systemctl stop` | Services keep running, file locks held |
| `spinifex-*` units in failed state | `systemctl reset-failed 'spinifex-*'` | `systemctl status` shows red units, next start refuses |
| Spinifex processes (spx, nats, predastore, etc.) | `pkill` pattern + wait loop | Stale listeners on 4222/8443/9999, start fails with "address in use" |
| QEMU VMs | `pkill qemu-system-*` + wait loop | Zombie VMs hold tap interfaces and viperblock state |
| `/etc/spinifex/` | `file: state=absent` | Stale TOML, stale CA, service users can't read fresh config |
| `/var/lib/spinifex/` | `file: state=absent` | Stale WAL, stale volumes, viperblock mounts fail |
| `/var/log/spinifex/` | `file: state=absent` | Logrotate confusion, permission drift after user changes |
| `/run/spinifex/` | `file: state=absent` | Stale sockets, pidfiles from prior systemd run |
| `$HOME/spinifex/` (legacy) | `file: state=absent` | Old dev-mode state shadowing new install |
| OVS bridges (all) | `ovs-vsctl del-br` loop | `br-int` populated with stale ports → `ovn-controller` commit loop |
| OVS `external_ids` | `ovs-vsctl clear Open_vSwitch .` | Stale chassis entries in SBDB |
| `/var/lib/ovn/ovnnb_db.db`, `ovnsb_db.db` | `file: state=absent` | "OVNSB commit failed, force recompute" loop after reset |
| `veth-wan-br` / `veth-wan-ovs` | `ip link delete` | Pair left dangling after setup-ovn.sh re-run |
| `spx-ext-*` macvlan interfaces | `ip link delete` loop | Stale macvlan on WAN NIC, packet duplication |
| Spinifex CA in trust store | remove + `update-ca-certificates` | TLS clients trust old CA, new cert fails verification |
## Not cleaned by default (opt-in)
| State source | Flag | Why opt-in |
|---|---|---|
| `~/.ssh/spinifex-key*` | `-e spinifex_wipe_ssh_keys=true` | Operator may want key to persist across resets |
| `~/.aws/credentials` spinifex profile | `-e spinifex_wipe_aws_creds=true` | `spx admin init --force` rewrites this block anyway; wipe only on demand |
## Known traps
- **`pgrep -f` self-match.** A literal pattern that matches the pattern
string itself causes `pgrep -f` to match our own shell. Mitigation:
bracket trick (`[b]in/spx` matches `bin/spx` but not `[b]in/spx`).
- **Log followers / ansible wrapper false positives.** `journalctl -f
_SYSTEMD_UNIT=spinifex-*` and ansible's own module shell matches the
service-name patterns in their cmdlines. Mitigation:
`spinifex_process_exclude` filter drops `journalctl|ansible|/bin/sh|pgrep|pkill|tail -[fF]`.
- **SIGTERM survivors.** `spx service spinifex start` (and potentially
other service-supervisor processes) ignore or delay SIGTERM. Mitigation:
kill task escalates SIGTERM → SIGKILL after 5s.
## Adding a new entry
1. Add a task to `tasks/main.yml` in the right section (services / state-dirs /
ovn / ovs / netlink / trust / user-owned).
2. Prefix the task `name:` with the state class (e.g. `netlink:`, `ovn:`).
3. Add a row to the catalog above — symptom is what the operator described.
4. If the leak class is general (e.g. "any interface matching spx-*"),
generalise the task; do not pattern-match on a single instance name.
scripts/ansible/roles/teardown/meta/main.yml
-7
View File
@@ -1,7 +0,0 @@
---
galaxy_info:
role_name: teardown
description: Wipe Spinifex dev state. Central leak-enumeration surface — every state location cleaned here is a known leak source. See README.md for catalog.
license: BSD-3-Clause
min_ansible_version: "2.14"
dependencies: []
scripts/ansible/roles/teardown/tasks/main.yml
-242
View File
@@ -1,242 +0,0 @@
---
- name: guard — detect node count in spinifex.toml
ansible.builtin.shell: |
set -e
if [ -f {{ spinifex_config_file | quote }} ]; then
grep -cE '^\[nodes\.[^.]+\]' {{ spinifex_config_file | quote }} || echo 0
else
echo 0
fi
register: teardown_node_count
changed_when: false
- name: guard — refuse to run on multi-node cluster
ansible.builtin.assert:
that: teardown_node_count.stdout | int <= 1
fail_msg: >-
Multi-node cluster detected ({{ teardown_node_count.stdout }} nodes in
{{ spinifex_config_file }}). Teardown supports single-node only;
use 'spx admin cluster shutdown' per node.
- name: services — stop spinifex.target
become: true
ansible.builtin.systemd:
name: spinifex.target
state: stopped
failed_when: false
- name: services — reset failed state (spinifex-* scope only)
become: true
ansible.builtin.command: systemctl reset-failed 'spinifex-*'
changed_when: false
failed_when: false
- name: processes — kill spinifex processes (SIGTERM, escalate to SIGKILL after 5s)
become: true
ansible.builtin.shell: |
pids=$(pgrep -af "{{ spinifex_process_pattern }}" \
| grep -vE "{{ spinifex_process_exclude }}" \
| awk '{print $1}')
if [ -n "$pids" ]; then
kill $pids 2>/dev/null || true
sleep 5
pids=$(pgrep -af "{{ spinifex_process_pattern }}" \
| grep -vE "{{ spinifex_process_exclude }}" \
| awk '{print $1}')
if [ -n "$pids" ]; then
kill -9 $pids 2>/dev/null || true
fi
fi
changed_when: false
failed_when: false
- name: processes — wait for exit (up to {{ teardown_process_wait_seconds }}s)
become: true
ansible.builtin.shell: |
for i in $(seq 1 {{ teardown_process_wait_seconds }}); do
pids=$(pgrep -af "{{ spinifex_process_pattern }}" \
| grep -vE "{{ spinifex_process_exclude }}" \
| awk '{print $1}')
if [ -z "$pids" ]; then
exit 0
fi
sleep 1
done
echo "still running after {{ teardown_process_wait_seconds }}s:" >&2
pgrep -af "{{ spinifex_process_pattern }}" \
| grep -vE "{{ spinifex_process_exclude }}" >&2 || true
exit 1
changed_when: false
- name: qemu — kill lingering VMs
become: true
ansible.builtin.command: "pkill -x {{ item }}"
loop: "{{ spinifex_qemu_processes }}"
changed_when: false
failed_when: false
- name: qemu — wait for exit (up to {{ teardown_process_wait_seconds }}s)
become: true
ansible.builtin.shell: |
set -e
for i in $(seq 1 {{ teardown_process_wait_seconds }}); do
alive=0
for proc in {{ spinifex_qemu_processes | join(' ') }}; do
if pgrep -x "$proc" >/dev/null 2>&1; then alive=1; break; fi
done
if [ "$alive" = "0" ]; then exit 0; fi
sleep 1
done
echo "QEMU still running after {{ teardown_process_wait_seconds }}s" >&2
exit 1
changed_when: false
- name: state-dirs — remove production dirs
become: true
ansible.builtin.file:
path: "{{ item }}"
state: absent
loop:
- "{{ spinifex_etc_dir }}"
- "{{ spinifex_data_dir }}"
- "{{ spinifex_log_dir }}"
- "{{ spinifex_run_dir }}"
- name: state-dirs — remove legacy $HOME/spinifex (opt-in)
become: true
become_user: "{{ spinifex_invoking_user }}"
ansible.builtin.file:
path: "{{ spinifex_legacy_home_dir }}"
state: absent
when: spinifex_wipe_legacy_home | bool
- name: ovs — detect tooling
ansible.builtin.command: which ovs-vsctl
register: teardown_ovs_which
changed_when: false
failed_when: false
- name: ovs — ensure openvswitch-switch is running for cleanup
become: true
ansible.builtin.systemd:
name: openvswitch-switch
state: started
when: teardown_ovs_which.rc == 0
failed_when: false
- name: ovs — list bridges
become: true
ansible.builtin.command: ovs-vsctl list-br
register: teardown_ovs_brs
changed_when: false
failed_when: false
when: teardown_ovs_which.rc == 0
- name: ovs — delete all bridges
become: true
ansible.builtin.command: "ovs-vsctl --if-exists del-br {{ item }}"
loop: "{{ teardown_ovs_brs.stdout_lines | default([]) }}"
changed_when: true
when: teardown_ovs_which.rc == 0
- name: ovs — clear external_ids
become: true
ansible.builtin.command: ovs-vsctl --if-exists clear Open_vSwitch . external_ids
changed_when: true
failed_when: false
when: teardown_ovs_which.rc == 0
- name: ovs — stop openvswitch-switch after cleanup
become: true
ansible.builtin.systemd:
name: openvswitch-switch
state: stopped
when: teardown_ovs_which.rc == 0
failed_when: false
- name: ovn — stop ovn-central
become: true
ansible.builtin.systemd:
name: ovn-central
state: stopped
failed_when: false
- name: ovn — stop ovn-controller
become: true
ansible.builtin.systemd:
name: ovn-controller
state: stopped
failed_when: false
- name: ovn — delete NB/SB DB files
become: true
ansible.builtin.file:
path: "{{ item }}"
state: absent
loop:
- "{{ ovn_nb_db }}"
- "{{ ovn_sb_db }}"
- name: netlink — enumerate spinifex-managed veth/tap
ansible.builtin.shell: |
ip -o link show | awk -F': ' '{print $2}' | cut -d'@' -f1 | \
grep -E '^(veth-wan-br|veth-wan-ovs|spx-ext-|spx-tap-|spx-)' || true
register: teardown_links
changed_when: false
- name: netlink — delete enumerated interfaces
become: true
ansible.builtin.command: "ip link delete {{ item }}"
loop: "{{ teardown_links.stdout_lines }}"
changed_when: true
failed_when: false
- name: trust — remove spinifex CA from system store
become: true
ansible.builtin.file:
path: "{{ spinifex_ca_trust_path }}"
state: absent
register: teardown_ca_removed
- name: trust — rebuild CA bundle (without --fresh, preserve other CAs)
become: true
ansible.builtin.command: update-ca-certificates
changed_when: true
when: teardown_ca_removed.changed
- name: user-owned — remove spinifex profile from AWS credentials (opt-in)
become: true
become_user: "{{ spinifex_invoking_user }}"
ansible.builtin.blockinfile:
path: "~/.aws/credentials"
marker: "# {mark} SPINIFEX MANAGED BLOCK"
block: ""
state: absent
when: spinifex_wipe_aws_creds | bool
failed_when: false
- name: user-owned — remove spinifex SSH key (opt-in)
become: true
become_user: "{{ spinifex_invoking_user }}"
ansible.builtin.file:
path: "~/.ssh/{{ item }}"
state: absent
loop:
- spinifex-key
- spinifex-key.pub
when: spinifex_wipe_ssh_keys | bool
- name: summary
ansible.builtin.debug:
msg: |
Teardown complete. Catalog: roles/teardown/README.md
systemd : spinifex.target stopped, spinifex-* failed cleared
procs : {{ spinifex_process_pattern }}
qemu : {{ spinifex_qemu_processes | join(', ') }}
dirs : {{ spinifex_etc_dir }}, {{ spinifex_data_dir }}, {{ spinifex_log_dir }}, {{ spinifex_run_dir }}{{ ', ' ~ spinifex_legacy_home_dir if spinifex_wipe_legacy_home else '' }}
ovs : all bridges deleted, external_ids cleared
ovn : NB/SB DB files removed
netlink : {{ teardown_links.stdout_lines | length }} interface(s) removed
trust : {{ spinifex_ca_trust_path }} removed{% if spinifex_wipe_aws_creds %}
aws : credentials block removed{% endif %}{% if spinifex_wipe_ssh_keys %}
ssh : spinifex-key pair removed{% endif %}
scripts/ansible/roles/version/meta/main.yml
-7
View File
@@ -1,7 +0,0 @@
---
galaxy_info:
role_name: version
description: Detailed build-vs-install drift report for spinifex artifacts.
license: BSD-3-Clause
min_ansible_version: "2.14"
dependencies: []
scripts/ansible/roles/version/tasks/main.yml
-161
View File
@@ -1,161 +0,0 @@
---
# Detailed build-vs-install drift report for the four artifacts that
# `ansible-dev-deploy` swaps: spx, the nbdkit viperblock plugin, and the
# two microvm artifacts (kernel + initramfs). Pure read-only; faster
# than dev-status (no AWS / OVN / port probing).
- name: version — built spx
ansible.builtin.stat:
path: "{{ spinifex_root }}/bin/spx"
checksum_algorithm: sha256
register: ver_spx_built
- name: version — installed spx
become: true
ansible.builtin.stat:
path: /usr/local/bin/spx
checksum_algorithm: sha256
register: ver_spx_installed
- name: version — installed spx version string
ansible.builtin.shell: |
/usr/local/bin/spx version 2>/dev/null | grep -vE '^[0-9]{4}/' | head -1
register: ver_spx_version
changed_when: false
failed_when: false
when: ver_spx_installed.stat.exists
- name: version — built plugin
ansible.builtin.stat:
path: "{{ mulga_root }}/viperblock/lib/nbdkit-viperblock-plugin.so"
checksum_algorithm: sha256
register: ver_plugin_built
- name: version — nbdkit plugin dir
ansible.builtin.shell: |
nbdkit --dump-config 2>/dev/null | awk -F= '/^plugindir=/{print $2; exit}'
register: ver_plugindir
changed_when: false
failed_when: false
- name: version — installed plugin
become: true
ansible.builtin.stat:
path: "{{ ver_plugindir.stdout | trim }}/nbdkit-viperblock-plugin.so"
checksum_algorithm: sha256
register: ver_plugin_installed
when: ver_plugindir.stdout | trim | length > 0
- name: version — built microvm artifacts
ansible.builtin.stat:
path: "{{ spinifex_root }}/build/microvm/{{ item }}"
checksum_algorithm: sha256
register: ver_microvm_built
loop:
- vmlinuz
- initramfs.cpio.gz
- name: version — installed microvm artifacts
become: true
ansible.builtin.stat:
path: /usr/share/spinifex/microvm/{{ item }}
checksum_algorithm: sha256
register: ver_microvm_installed
loop:
- vmlinuz
- initramfs.cpio.gz
- name: version — sub-repo HEAD short-rev (spinifex)
ansible.builtin.command: git -C {{ spinifex_root | quote }} rev-parse --short HEAD
register: ver_spinifex_head
changed_when: false
failed_when: false
- name: version — sub-repo HEAD short-rev (viperblock)
ansible.builtin.command: git -C {{ mulga_root }}/viperblock rev-parse --short HEAD
register: ver_viperblock_head
changed_when: false
failed_when: false
- name: version — sub-repo HEAD short-rev (predastore)
ansible.builtin.command: git -C {{ mulga_root }}/predastore rev-parse --short HEAD
register: ver_predastore_head
changed_when: false
failed_when: false
- name: version — assemble per-artifact rows
ansible.builtin.set_fact:
ver_artifact_rows: >-
{{
[
('%-18s built=%s installed=%s drift=%s' | format(
'spx',
(ver_spx_built.stat.checksum | default('-') | truncate(13, true, ''))
if ver_spx_built.stat.exists else 'MISSING',
(ver_spx_installed.stat.checksum | default('-') | truncate(13, true, ''))
if ver_spx_installed.stat.exists else 'MISSING',
('YES' if ver_spx_built.stat.exists and ver_spx_installed.stat.exists
and ver_spx_built.stat.checksum != ver_spx_installed.stat.checksum
else 'no')
)),
('%-18s built=%s installed=%s drift=%s' | format(
'plugin (nbdkit)',
(ver_plugin_built.stat.checksum | default('-') | truncate(13, true, ''))
if ver_plugin_built.stat.exists else 'MISSING',
(ver_plugin_installed.stat.checksum | default('-') | truncate(13, true, ''))
if (ver_plugin_installed.stat | default({})).get('exists', false) else 'MISSING',
('YES' if ver_plugin_built.stat.exists
and (ver_plugin_installed.stat | default({})).get('exists', false)
and ver_plugin_built.stat.checksum != ver_plugin_installed.stat.checksum
else 'no')
)),
('%-18s built=%s installed=%s drift=%s' | format(
'microvm vmlinuz',
(ver_microvm_built.results[0].stat.checksum | default('-') | truncate(13, true, ''))
if ver_microvm_built.results[0].stat.exists else 'MISSING',
(ver_microvm_installed.results[0].stat.checksum | default('-') | truncate(13, true, ''))
if ver_microvm_installed.results[0].stat.exists else 'MISSING',
('YES' if ver_microvm_built.results[0].stat.exists
and ver_microvm_installed.results[0].stat.exists
and ver_microvm_built.results[0].stat.checksum
!= ver_microvm_installed.results[0].stat.checksum
else 'no')
)),
('%-18s built=%s installed=%s drift=%s' | format(
'microvm initramfs',
(ver_microvm_built.results[1].stat.checksum | default('-') | truncate(13, true, ''))
if ver_microvm_built.results[1].stat.exists else 'MISSING',
(ver_microvm_installed.results[1].stat.checksum | default('-') | truncate(13, true, ''))
if ver_microvm_installed.results[1].stat.exists else 'MISSING',
('YES' if ver_microvm_built.results[1].stat.exists
and ver_microvm_installed.results[1].stat.exists
and ver_microvm_built.results[1].stat.checksum
!= ver_microvm_installed.results[1].stat.checksum
else 'no')
))
]
}}
- name: version — compute any-drift flag
ansible.builtin.set_fact:
ver_any_drift: "{{ 'YES' in (ver_artifact_rows | join(' ')) }}"
- name: version — summary
ansible.builtin.debug:
msg: >-
{{
[
'── Spinifex build vs install ─────────────────────────────────────────────',
'spx (installed): ' ~ ((ver_spx_version.stdout | default('') | trim) or '(not installed)')
]
+ ver_artifact_rows
+ [
'── Sub-repo HEADs ──',
'spinifex : ' ~ ((ver_spinifex_head.stdout | default('') | trim) or '(not a git repo)'),
'viperblock : ' ~ ((ver_viperblock_head.stdout | default('') | trim) or '(missing)'),
'predastore : ' ~ ((ver_predastore_head.stdout | default('') | trim) or '(missing)'),
'──────────────────────────────────────────────────────────────────────────',
('DRIFT detected — run: make ansible-dev-deploy' if ver_any_drift
else 'No drift; built = installed for all artifacts.')
]
}}
scripts/ansible/roles/vpc/meta/main.yml
-7
View File
@@ -1,7 +0,0 @@
---
galaxy_info:
role_name: vpc
description: Idempotent dev-mode VPC + public subnet + IGW + default route fixture.
license: BSD-3-Clause
min_ansible_version: "2.14"
dependencies: []
scripts/ansible/roles/vpc/tasks/absent.yml
-128
View File
@@ -1,128 +0,0 @@
---
# Tear down the dev VPC fixture. Deletes in reverse dependency order:
# route-table associations, route table, subnet, IGW (detach + delete),
# VPC. Each step is best-effort so a partially-converged fixture still
# converges to "gone" on re-run.
- name: vpc — find VPC by Name tag
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 describe-vpcs
--filters Name=tag:Name,Values={{ dev_vpc_name }}
--query Vpcs[0].VpcId --output text
register: absent_vpc
changed_when: false
- name: vpc — nothing to do
ansible.builtin.debug:
msg: "VPC tagged Name={{ dev_vpc_name }} not found; nothing to delete."
when: absent_vpc.stdout | trim in ['', 'None']
- name: vpc — short-circuit when no VPC
ansible.builtin.meta: end_play
when: absent_vpc.stdout | trim in ['', 'None']
- name: vpc — set VPC id fact
ansible.builtin.set_fact:
absent_vpc_id: "{{ absent_vpc.stdout | trim }}"
- name: vpc — find subnet
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 describe-subnets
--filters Name=vpc-id,Values={{ absent_vpc_id }}
Name=tag:Name,Values={{ dev_vpc_name }}-public
--query Subnets[0].SubnetId --output text
register: absent_subnet
changed_when: false
# Enumerate ALL non-main route tables in the VPC (main RT can't be
# deleted; it goes when the VPC does). Catches orphans from earlier
# runs that landed before the create-tags workaround.
- name: vpc — find all non-main route tables in VPC
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 describe-route-tables
--filters Name=vpc-id,Values={{ absent_vpc_id }}
--query RouteTables[?!(Associations[?Main==`true`])].RouteTableId
--output text
register: absent_rt_list
changed_when: false
- name: vpc — split RT id list
ansible.builtin.set_fact:
absent_rt_ids: "{{ absent_rt_list.stdout.split() | reject('equalto', '') | list }}"
- name: vpc — find all subnet/RT associations to disassociate
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 describe-route-tables
--route-table-ids {{ item }}
--query RouteTables[0].Associations[?SubnetId!=null].RouteTableAssociationId
--output text
register: absent_assocs
loop: "{{ absent_rt_ids }}"
changed_when: false
failed_when: false
- name: vpc — disassociate subnets from route tables
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 disassociate-route-table
--association-id {{ item }}
loop: "{{ absent_assocs.results
| map(attribute='stdout')
| map('default', '')
| map('split')
| flatten
| reject('equalto', '')
| list }}"
changed_when: true
failed_when: false
- name: vpc — delete all non-main route tables
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 delete-route-table
--route-table-id {{ item }}
loop: "{{ absent_rt_ids }}"
changed_when: true
failed_when: false
- name: vpc — delete subnet
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 delete-subnet
--subnet-id {{ absent_subnet.stdout | trim }}
when: absent_subnet.stdout | trim not in ['', 'None']
changed_when: true
failed_when: false
- name: vpc — find IGW
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 describe-internet-gateways
--filters Name=attachment.vpc-id,Values={{ absent_vpc_id }}
--query InternetGateways[0].InternetGatewayId --output text
register: absent_igw
changed_when: false
- name: vpc — detach IGW
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 detach-internet-gateway
--internet-gateway-id {{ absent_igw.stdout | trim }}
--vpc-id {{ absent_vpc_id }}
when: absent_igw.stdout | trim not in ['', 'None']
changed_when: true
failed_when: false
- name: vpc — delete IGW
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 delete-internet-gateway
--internet-gateway-id {{ absent_igw.stdout | trim }}
when: absent_igw.stdout | trim not in ['', 'None']
changed_when: true
failed_when: false
- name: vpc — delete VPC
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 delete-vpc
--vpc-id {{ absent_vpc_id }}
changed_when: true
failed_when: false
- name: vpc — summary (absent)
ansible.builtin.debug:
msg: "Dev VPC {{ dev_vpc_name }} (was {{ absent_vpc_id }}) torn down."
scripts/ansible/roles/vpc/tasks/main.yml
-3
View File
@@ -1,3 +0,0 @@
---
- name: vpc — dispatch by state
ansible.builtin.include_tasks: "{{ dev_vpc_state }}.yml"
scripts/ansible/roles/vpc/tasks/present.yml
-209
View File
@@ -1,209 +0,0 @@
---
# Idempotent create. Every step describes-then-creates: re-running on an
# already-converged VPC produces zero changes. Tag everything with
# Name={{ dev_vpc_name }} so a single tag query finds the whole graph.
- name: vpc — find existing VPC by Name tag
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 describe-vpcs
--filters Name=tag:Name,Values={{ dev_vpc_name }}
--query Vpcs[0].VpcId --output text
register: vpc_existing
changed_when: false
- name: vpc — create VPC if missing
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 create-vpc
--cidr-block {{ dev_vpc_cidr }}
--tag-specifications
ResourceType=vpc,Tags=[{Key=Name,Value={{ dev_vpc_name }}}]
--query Vpc.VpcId --output text
register: vpc_created
when: vpc_existing.stdout | trim in ['', 'None']
changed_when: true
- name: vpc — set VPC id fact
ansible.builtin.set_fact:
dev_vpc_id: >-
{{ vpc_created.stdout | default('') | trim
if (vpc_existing.stdout | trim in ['', 'None'])
else vpc_existing.stdout | trim }}
- name: vpc — assert VPC id resolved
ansible.builtin.assert:
that: dev_vpc_id | length > 0
fail_msg: "VPC id could not be resolved after describe/create"
- name: vpc — find existing IGW attached to this VPC
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 describe-internet-gateways
--filters Name=attachment.vpc-id,Values={{ dev_vpc_id }}
--query InternetGateways[0].InternetGatewayId --output text
register: vpc_igw_existing
changed_when: false
- name: vpc — create IGW if missing
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 create-internet-gateway
--tag-specifications
ResourceType=internet-gateway,Tags=[{Key=Name,Value={{ dev_vpc_name }}-igw}]
--query InternetGateway.InternetGatewayId --output text
register: vpc_igw_created
when: vpc_igw_existing.stdout | trim in ['', 'None']
changed_when: true
- name: vpc — set IGW id fact
ansible.builtin.set_fact:
dev_vpc_igw_id: >-
{{ vpc_igw_created.stdout | default('') | trim
if (vpc_igw_existing.stdout | trim in ['', 'None'])
else vpc_igw_existing.stdout | trim }}
- name: vpc — attach IGW to VPC if needed
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 attach-internet-gateway
--internet-gateway-id {{ dev_vpc_igw_id }}
--vpc-id {{ dev_vpc_id }}
register: vpc_igw_attach
when: vpc_igw_existing.stdout | trim in ['', 'None']
changed_when: true
failed_when:
- vpc_igw_attach.rc | default(0) != 0
- "'Resource.AlreadyAssociated' not in (vpc_igw_attach.stderr | default(''))"
- name: vpc — find existing subnet by Name tag
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 describe-subnets
--filters Name=vpc-id,Values={{ dev_vpc_id }}
Name=tag:Name,Values={{ dev_vpc_name }}-public
--query Subnets[0].SubnetId --output text
register: vpc_subnet_existing
changed_when: false
- name: vpc — create subnet if missing
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 create-subnet
--vpc-id {{ dev_vpc_id }}
--cidr-block {{ dev_vpc_subnet_cidr }}
--availability-zone {{ dev_vpc_subnet_az }}
--tag-specifications
ResourceType=subnet,Tags=[{Key=Name,Value={{ dev_vpc_name }}-public}]
--query Subnet.SubnetId --output text
register: vpc_subnet_created
when: vpc_subnet_existing.stdout | trim in ['', 'None']
changed_when: true
- name: vpc — set subnet id fact
ansible.builtin.set_fact:
dev_vpc_subnet_id: >-
{{ vpc_subnet_created.stdout | default('') | trim
if (vpc_subnet_existing.stdout | trim in ['', 'None'])
else vpc_subnet_existing.stdout | trim }}
- name: vpc — ensure subnet has map-public-ip-on-launch
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 modify-subnet-attribute
--subnet-id {{ dev_vpc_subnet_id }}
--map-public-ip-on-launch
register: vpc_subnet_mapip
changed_when: vpc_subnet_existing.stdout | trim in ['', 'None']
failed_when: false
# Try tag-based lookup first. If the gateway didn't tag a previously
# created RT (spinifex gateway silently drops --tag-specifications on
# create-route-table AND no-ops create-tags for route tables — see
# mulga-siv-94; works correctly for subnets), fall back to "RT
# associated with our subnet" which doesn't need tags.
- name: vpc — find existing route table by Name tag
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 describe-route-tables
--filters Name=vpc-id,Values={{ dev_vpc_id }}
Name=tag:Name,Values={{ dev_vpc_name }}-public
--query RouteTables[0].RouteTableId --output text
register: vpc_rt_existing
changed_when: false
- name: vpc — fallback lookup by subnet association
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 describe-route-tables
--filters Name=association.subnet-id,Values={{ dev_vpc_subnet_id }}
--query RouteTables[0].RouteTableId --output text
register: vpc_rt_byassoc
changed_when: false
when: vpc_rt_existing.stdout | trim in ['', 'None']
- name: vpc — create route table if neither lookup found one
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 create-route-table
--vpc-id {{ dev_vpc_id }}
--query RouteTable.RouteTableId --output text
register: vpc_rt_created
when:
- vpc_rt_existing.stdout | trim in ['', 'None']
- (vpc_rt_byassoc.stdout | default('') | trim) in ['', 'None']
changed_when: true
- name: vpc — set route-table id fact
ansible.builtin.set_fact:
dev_vpc_rt_id: >-
{%- if (vpc_rt_existing.stdout | trim) not in ['', 'None'] -%}
{{ vpc_rt_existing.stdout | trim }}
{%- elif (vpc_rt_byassoc.stdout | default('') | trim) not in ['', 'None'] -%}
{{ vpc_rt_byassoc.stdout | default('') | trim }}
{%- else -%}
{{ vpc_rt_created.stdout | default('') | trim }}
{%- endif -%}
- name: vpc — assert route-table id resolved
ansible.builtin.assert:
that: dev_vpc_rt_id | length > 0
fail_msg: "Route table id could not be resolved after describe/create"
# Spinifex gateway dropped --tag-specifications on the create call;
# apply via create-tags so re-runs can find the RT via tag filter.
- name: vpc — tag route table (workaround for create-tag-specifications gap)
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 create-tags
--resources {{ dev_vpc_rt_id }}
--tags Key=Name,Value={{ dev_vpc_name }}-public
changed_when: false
failed_when: false
- name: vpc — add default route via IGW (ignore RouteAlreadyExists)
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 create-route
--route-table-id {{ dev_vpc_rt_id }}
--destination-cidr-block 0.0.0.0/0
--gateway-id {{ dev_vpc_igw_id }}
register: vpc_route_add
changed_when:
- vpc_route_add.rc == 0
- "'RouteAlreadyExists' not in (vpc_route_add.stderr | default(''))"
failed_when:
- vpc_route_add.rc != 0
- "'RouteAlreadyExists' not in (vpc_route_add.stderr | default(''))"
- name: vpc — associate subnet with route table (ignore already-associated)
ansible.builtin.command: >-
{{ spinifex_aws_cli }} ec2 associate-route-table
--route-table-id {{ dev_vpc_rt_id }}
--subnet-id {{ dev_vpc_subnet_id }}
register: vpc_assoc
changed_when:
- vpc_assoc.rc == 0
- "'Resource.AlreadyAssociated' not in (vpc_assoc.stderr | default(''))"
failed_when:
- vpc_assoc.rc != 0
- "'Resource.AlreadyAssociated' not in (vpc_assoc.stderr | default(''))"
- name: vpc — summary (present)
ansible.builtin.debug:
msg: |
Dev VPC ready:
VPC : {{ dev_vpc_id }} ({{ dev_vpc_cidr }})
IGW : {{ dev_vpc_igw_id }}
Subnet : {{ dev_vpc_subnet_id }} ({{ dev_vpc_subnet_cidr }} @ {{ dev_vpc_subnet_az }}, public)
Route table : {{ dev_vpc_rt_id }}
Smoketest now finds this subnet via its existing
Name=map-public-ip-on-launch,Values=true filter.
scripts/ansible/scripts/aws-as-spinifex.sh
-27
View File
@@ -1,27 +0,0 @@
#!/usr/bin/env bash
# Run `aws` CLI inside a shell that has the `spinifex` supplementary group
# active, with AWS_PROFILE=spinifex preset. Works around mulga-siv-92:
# ansible's `become` is a no-op when the target user equals the calling
# user, so `sudo -u` is never invoked and supplementary groups are not
# re-initialised. Without the `spinifex` group, /etc/spinifex/ca.pem (the
# AWS endpoint CA bundle) is unreadable and aws fails with SSL Errno 13.
#
# Usage from ansible tasks:
# ansible.builtin.command:
# cmd: "{{ playbook_dir }}/../scripts/aws-as-spinifex.sh ec2 describe-regions"
#
# All args are forwarded to `aws`. Quoting is preserved via printf %q so
# things like `--query 'length(Images)'` reach aws intact.
set -euo pipefail
if [[ $# -eq 0 ]]; then
echo "usage: $0 <aws-cli-args>" >&2
exit 64
fi
cmd="AWS_PROFILE=spinifex aws"
for arg in "$@"; do
cmd+=" $(printf '%q' "$arg")"
done
exec sg spinifex -c "$cmd"
scripts/ansible/scripts/spx-as-spinifex.sh
-18
View File
@@ -1,18 +0,0 @@
#!/usr/bin/env bash
# Run `spx` inside a shell that has the `spinifex` supplementary group
# active. Works around mulga-siv-92 (see aws-as-spinifex.sh for details).
# All args are forwarded to /usr/local/bin/spx via printf %q so flag
# values with spaces or shell metachars survive intact.
set -euo pipefail
if [[ $# -eq 0 ]]; then
echo "usage: $0 <spx-args>" >&2
exit 64
fi
cmd="AWS_PROFILE=spinifex /usr/local/bin/spx"
for arg in "$@"; do
cmd+=" $(printf '%q' "$arg")"
done
exec sg spinifex -c "$cmd"
scripts/ansible/vars/defaults.yml
-137
View File
@@ -1,137 +0,0 @@
---
# Production layout (per spinifex/scripts/setup.sh).
spinifex_etc_dir: /etc/spinifex
spinifex_data_dir: /var/lib/spinifex
spinifex_log_dir: /var/log/spinifex
spinifex_run_dir: /run/spinifex
spinifex_config_file: "{{ spinifex_etc_dir }}/spinifex.toml"
spinifex_ca_trust_path: /usr/local/share/ca-certificates/spinifex-ca.crt
spinifex_ca_src: "{{ spinifex_etc_dir }}/ca.pem"
# Legacy $HOME layout (still present on older dev boxes).
spinifex_legacy_home_dir: "{{ lookup('env','HOME') }}/spinifex"
# OVN state.
ovn_db_dir: /var/lib/ovn
ovn_nb_db: "{{ ovn_db_dir }}/ovnnb_db.db"
ovn_sb_db: "{{ ovn_db_dir }}/ovnsb_db.db"
# Region / init.
spinifex_region: ap-southeast-2
spinifex_az: "{{ spinifex_region }}a"
spinifex_node_name: node1
spinifex_node_count: 1
# External network.
# spx admin init contract (as of network-layer-redesign):
# external_mode : "pool" or "" (NAT path removed)
# external_pool : MANDATORY when mode=pool — set start/end below or pass -e.
# external_gw : auto-detected from default route when blank.
# external_prefix: auto-detected from WAN when blank.
# external_gw_ip : OVN router external-IP override (was nat_gateway_ip).
# no_external : true → emit --no-external (skip external networking).
spinifex_external_mode: pool
spinifex_external_no_external: false
spinifex_external_pool_start: ""
spinifex_external_pool_end: ""
spinifex_external_gateway: ""
spinifex_external_prefix_len: ""
spinifex_external_gateway_ip: ""
# Preflight checks.
spinifex_ports:
- { port: 4222, service: NATS }
- { port: 8443, service: Predastore }
- { port: 9999, service: "Spinifex Gateway" }
spinifex_required_commands:
- go
- make
- nbdkit
spinifex_optional_commands:
- air
spinifex_ovn_commands:
- ovs-vsctl
- ovn-controller
- ovn-nbctl
spinifex_components:
- viperblock
- predastore
# Process patterns for teardown.
# Bracket trick ([b]in/spx etc.) prevents the pattern literal from matching
# our own pgrep/shell cmdline. Exclude filter drops benign log followers and
# ansible's own wrappers that also mention these service names.
spinifex_process_pattern: "([b]in/spx|spinifex-[u]i|nats-[s]erver|[p]redastore|[v]iperblock|[v]pcd)"
spinifex_process_exclude: "(journalctl|ansible|/bin/sh|pgrep|pkill|tail -[fF])"
spinifex_qemu_processes:
- qemu-system-x86_64
- qemu-system-aarch64
# Invoking user (when run under sudo).
spinifex_invoking_user: "{{ lookup('env','SUDO_USER') | default(ansible_facts['user_id'], true) }}"
# Opt-in wipes.
spinifex_wipe_ssh_keys: false
spinifex_wipe_aws_creds: false
spinifex_wipe_legacy_home: true
# Timeouts.
teardown_process_wait_seconds: 30
# Path resolution — playbook_dir is scripts/ansible/playbooks/ at runtime.
spinifex_root: "{{ lookup('pipe', 'readlink -f ' ~ playbook_dir ~ '/../../..') }}"
mulga_root: "{{ lookup('pipe', 'readlink -f ' ~ playbook_dir ~ '/../../../..') }}"
# Wrappers around `aws` and `spx` that re-enter a shell with the
# `spinifex` supplementary group active. Needed because ansible's
# `become_user` is a no-op when target user equals current user, so
# supplementary groups are never re-initialised and /etc/spinifex/ca.pem
# (drwxr-x--- root:spinifex) is unreadable.
spinifex_aws_cli: "{{ playbook_dir }}/../scripts/aws-as-spinifex.sh"
spinifex_spx_cli: "{{ playbook_dir }}/../scripts/spx-as-spinifex.sh"
# dev-vpc fixture. Deterministic VPC + public subnet + IGW + default
# route. Idempotent: re-running detects existing resources by Name tag.
# Used by smoketest's launch path (which looks for any
# map-public-ip-on-launch=true subnet).
dev_vpc_name: spinifex-dev-vpc
dev_vpc_cidr: 10.99.0.0/16
dev_vpc_subnet_cidr: 10.99.1.0/24
dev_vpc_subnet_az: "{{ spinifex_az }}"
dev_vpc_state: present # present | absent
# Build output.
build_tarball_path: /tmp/spinifex-local.tar.gz
# Snapshot / restore. Snapshots quiesce the full spinifex+ovn+ovs stack,
# tar these paths, then restart services. Restore wipes + untars + restarts.
# Default snapshot dir is root-owned so it survives `dev-teardown`
# (teardown only removes the four spinifex_*_dir paths above).
snapshot_dir: /var/lib/spinifex-snapshots
snapshot_paths:
- "{{ spinifex_etc_dir }}"
- "{{ spinifex_data_dir }}"
- "{{ ovn_db_dir }}"
- /etc/openvswitch/conf.db
# Order matters: stop spinifex first (depends on OVN/OVS), then OVN, then OVS.
# Restart in reverse.
snapshot_services_stop_order:
- spinifex.target
- ovn-controller
- ovn-central
- openvswitch-switch
# Smoketest image — parameterised so tofu examples can share the same base.
spinifex_smoketest_distro: debian
spinifex_smoketest_version: "13"
# Local image file path (if present, imported directly; otherwise fall back to --name).
spinifex_smoketest_local_image: "~/images/{{ spinifex_smoketest_distro }}-{{ spinifex_smoketest_version }}.img"
# Multi-node cluster bootstrap (cluster-bootstrap.yml).
# cluster_env, cluster_init_host, spinifex_node_count come from inventory/tofu.py.
cluster_external_pool: "" # "<start>-<end>" or empty for DHCP fallback
cluster_join_token_path: /etc/spinifex/join-token
cluster_formation_port: 4432
cluster_formation_timeout: 60 # seconds (uri retries = value // 2)
scripts/build-microvm-image.sh
+74 -16
View File
@@ -19,14 +19,21 @@ INITTAB="$BUILD_DIR/inittab"
ALPINE_VERSION="${ALPINE_VERSION:-3.20}"
ALPINE_MIRROR="${ALPINE_MIRROR:-https://dl-cdn.alpinelinux.org/alpine}"
ALPINE_PACKAGES="busybox linux-virt haproxy iproute2 ca-certificates"
# haproxy: ALB (L7) data plane. nginx + nginx-mod-stream: NLB (L4) data plane —
# the `stream` module (a separate Alpine package) load-balances TCP/UDP/TLS,
# which haproxy cannot do for UDP.
ALPINE_PACKAGES="busybox linux-virt haproxy nginx nginx-mod-stream iproute2 ca-certificates"
echo "[build-microvm-image] repo root: $REPO_ROOT"
echo "[build-microvm-image] build dir: $BUILD_DIR"
mkdir -p "$BUILD_DIR"
# --- Verify non-negotiable tools ---
for tool in cpio gzip find; do
# fakeroot is mandatory: device nodes are written into the cpio archive inside a
# faked-root environment so no real device node is ever created on the host
# filesystem (which previously required sudo mknod and risked clobbering host
# /dev/null on cleanup).
for tool in cpio gzip find fakeroot; do
if ! command -v "$tool" >/dev/null 2>&1; then
echo "ERROR: required tool not found: $tool" >&2
exit 1
@@ -35,6 +42,13 @@ done
# --- Create chroot ---
CHROOT_DIR=$(mktemp -d)
# Guard: every destructive op below targets "$CHROOT_DIR/...". An empty or
# unexpected value would aim rm -rf at the host (e.g. /dev). Refuse anything that
# is not a freshly-created temp dir.
case "${CHROOT_DIR:?mktemp -d returned empty}" in
/tmp/*|/var/tmp/*|"${TMPDIR:-/nonexistent}"/*) ;;
*) echo "ERROR: refusing to use unexpected CHROOT_DIR: $CHROOT_DIR" >&2; exit 1 ;;
esac
CONTAINER_TOOL=""
CONTAINER_CID=""
cleanup() {
@@ -93,16 +107,14 @@ else
$CONTAINER_TOOL export "$CONTAINER_CID" | tar -x -C "$CHROOT_DIR"
fi
# --- Fix /dev device nodes ---
# --- /dev: empty mountpoint only ---
# tar extraction without root silently creates 0-byte regular files for device
# nodes (mknod requires CAP_MKNOD). The kernel opens /dev/console to wire
# init's stdio to the serial console; a regular file there causes all init
# output to be silently discarded. Recreate the two nodes needed before
# devtmpfs mounts (which provides the rest at runtime).
rm -rf "$CHROOT_DIR/dev"
# nodes. Drop them and leave an empty /dev mountpoint. The two static nodes the
# kernel needs (/dev/console to wire init's stdio, /dev/null) are written into
# the cpio archive under fakeroot at pack time — NOT created on the host. init
# mounts devtmpfs over /dev as its first action for everything else at runtime.
rm -rf "${CHROOT_DIR:?}/dev"
mkdir "$CHROOT_DIR/dev"
sudo mknod -m 600 "$CHROOT_DIR/dev/console" c 5 1
sudo mknod -m 666 "$CHROOT_DIR/dev/null" c 1 3
# --- Place init script ---
echo "[build-microvm-image] installing init.sh as /init..."
@@ -161,6 +173,21 @@ else
echo "[build-microvm-image] fw_cfg: module found in lib/modules"
fi
# --- Assert nginx + stream module (NLB L4 data plane) ---
# The lb-agent runs `nginx -c` with a config that `load_module`s the stream
# module; if either the binary or the .so is missing the agent never enters
# nginx mode, never probes, and NLB targets stay at 0/N healthy. Fail the build
# loudly here rather than discover it as an opaque e2e health timeout.
if [ ! -x "$CHROOT_DIR/usr/sbin/nginx" ] && [ ! -x "$CHROOT_DIR/usr/bin/nginx" ]; then
echo "ERROR: nginx binary missing from image — NLB data plane will not start" >&2
exit 1
fi
if ! find "$CHROOT_DIR/usr/lib/nginx/modules" -name "ngx_stream_module.so" 2>/dev/null | grep -q .; then
echo "ERROR: ngx_stream_module.so missing from image — NLB stream config fails to load" >&2
exit 1
fi
echo "[build-microvm-image] nginx: binary + stream module present"
# --- Strip kernel modules (Phase 2 keep-list approach) ---
# Only modules needed by a virtio-mmio microvm guest are kept:
# virtio*.ko* — virtio bus, net, blk, rng, console, and helpers
@@ -197,8 +224,34 @@ for kver_dir in "$CHROOT_DIR/lib/modules"/*/; do
done)
rm -rf "$tmp_keep"
# Regenerate module dependency map from the surviving modules.
depmod -b "$CHROOT_DIR" "$kver" 2>/dev/null || true
# Decompress kept modules to plain .ko. Alpine ships .ko.gz, but a host
# kmod that lacks gzip support (e.g. Debian trixie's kmod 34) silently
# produces an EMPTY modules.dep from compressed modules — the guest's
# modprobe then no-ops and qemu_fw_cfg never loads. Plain .ko sidesteps the
# host-kmod dependency entirely and the guest needs no decompressor either.
find "$kernel_dir" -name "*.ko.gz" -exec gunzip -f {} +
# Regenerate the module dependency map when a host depmod is available. A
# populated modules.dep lets the guest init's load_mod() resolve modules via
# modprobe; without it, load_mod() falls back to insmod-by-path (see
# build/microvm/init.sh), so modules.dep is an optimisation, not a boot
# requirement. depmod (kmod) is absent on some self-hosted CI runners and
# cannot be apt-installed there — skip rather than hard-fail. When depmod
# IS present, still assert a non-empty result so the trixie-kmod-can't-read
# -compressed-modules bug (empty modules.dep from .ko.gz) fails loudly.
if command -v depmod >/dev/null 2>&1; then
if ! depmod -b "$CHROOT_DIR" "$kver"; then
echo "ERROR: depmod failed for $kver" >&2
exit 1
fi
if [ ! -s "$kver_dir/modules.dep" ]; then
echo "ERROR: ${kver_dir}modules.dep missing or empty after depmod" >&2
exit 1
fi
else
echo "[build-microvm-image] depmod not found; skipping modules.dep" \
"(guest init load_mod() will insmod modules by path)" >&2
fi
done
# --- Strip package-manager artifacts (not needed at runtime) ---
@@ -238,10 +291,15 @@ rm -rf "$CHROOT_DIR/boot"
# --- Build initramfs ---
INITRAMFS_OUT="$BUILD_DIR/initramfs.cpio.gz"
echo "[build-microvm-image] building initramfs: $INITRAMFS_OUT"
(
cd "$CHROOT_DIR"
find . | cpio --quiet -o -H newc | gzip -9 > "$INITRAMFS_OUT"
)
# Create the static device nodes and pack the archive inside a single fakeroot
# session: mknod/cpio see faked device entries and write real char-device
# records into the cpio, while the host filesystem never gets an actual node.
fakeroot sh -c '
cd "$1" || exit 1
mknod -m 600 dev/console c 5 1
mknod -m 666 dev/null c 1 3
find . | cpio --quiet -o -H newc | gzip -9 > "$2"
' _ "$CHROOT_DIR" "$INITRAMFS_OUT"
# --- Log artifact sizes ---
echo ""
scripts/build-system-image.sh
+102 -209
View File
@@ -3,17 +3,28 @@ set -euo pipefail
# build-system-image.sh — Build a minimal system AMI from a manifest
#
# Supports Alpine Linux and Ubuntu 24.04 as base distros.
# Creates a pre-baked image with custom packages, binaries, and setup scripts
# installed, ready for import as a Spinifex AMI.
# Supports Alpine Linux and Ubuntu 24.04+ as base distros. Creates a pre-baked
# image with custom packages, binaries, and setup scripts installed, ready for
# import as a Spinifex AMI.
#
# Requirements: qemu-nbd, qemu-img, sudo (for mount/chroot), curl
# Customization runs entirely inside the libguestfs appliance (its own kernel +
# userspace), so the build touches NO host block device, NO host mount, and
# needs NO sudo. This deliberately replaces the previous qemu-nbd + `sudo mount`
# + chroot + /dev bind-mount flow, whose leaked binds could be traversed by a
# later `rm -rf` and wipe the host's /dev (it wedged CI runners). libguestfs
# isolates all of that in a throwaway VM.
#
# Requirements: virt-customize, guestfish, virt-resize (libguestfs-tools),
# qemu-img, curl
# Usage: ./scripts/build-system-image.sh <manifest.conf> [--import]
# --import Also import the image as an AMI via spx admin images import
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
# No libvirt on CI runners — drive qemu directly. Harmless when libvirt exists.
export LIBGUESTFS_BACKEND="${LIBGUESTFS_BACKEND:-direct}"
usage() {
echo "Usage: $0 <manifest.conf> [--import]"
echo ""
@@ -60,8 +71,6 @@ done
# Derived paths
BUILD_DIR="/tmp/${IMAGE_NAME}-image-build"
NBD_DEV="/dev/nbd0"
MOUNT_DIR="${BUILD_DIR}/mnt"
case "$DISTRO" in
alpine)
@@ -74,7 +83,6 @@ case "$DISTRO" in
OUTPUT_IMAGE="${BUILD_DIR}/${IMAGE_NAME}-alpine.qcow2"
OUTPUT_RAW="${BUILD_DIR}/${IMAGE_NAME}-alpine.raw"
DISTRO_VERSION="${ALPINE_VERSION}"
ROOT_PART="${NBD_DEV}"
BOOT_MODE="bios"
;;
ubuntu)
@@ -87,7 +95,6 @@ case "$DISTRO" in
OUTPUT_IMAGE="${BUILD_DIR}/${IMAGE_NAME}-ubuntu.qcow2"
OUTPUT_RAW="${BUILD_DIR}/${IMAGE_NAME}-ubuntu.raw"
DISTRO_VERSION="${UBUNTU_VERSION}"
ROOT_PART="${NBD_DEV}p1"
BOOT_MODE="uefi"
;;
*)
@@ -96,19 +103,20 @@ case "$DISTRO" in
;;
esac
cleanup() {
echo "Cleaning up..."
# Ubuntu chroot has bind mounts that must be released first
sudo umount "${MOUNT_DIR}/dev/pts" 2>/dev/null || true
sudo umount "${MOUNT_DIR}/dev" 2>/dev/null || true
sudo umount "${MOUNT_DIR}/sys" 2>/dev/null || true
sudo umount "${MOUNT_DIR}/proc" 2>/dev/null || true
sudo umount "${MOUNT_DIR}" 2>/dev/null || true
sudo qemu-nbd --disconnect "${NBD_DEV}" 2>/dev/null || true
exec 9>&- 2>/dev/null || true # release nbd lock if held
echo "Done."
# import_ami registers the built raw image as a Spinifex AMI. Shared by the
# fresh-build and cached-build paths.
import_ami() {
echo "Importing as AMI..."
rm -f "$OUTPUT_IMAGE"
local args=(--file "$OUTPUT_RAW" --distro "${DISTRO}" --version "${DISTRO_VERSION}" --arch x86_64 --boot-mode "${BOOT_MODE}")
if [[ -n "${AMI_NAME:-}" ]]; then
args+=(--ami-name "$AMI_NAME")
fi
if [[ -n "${SYSTEM_TAG:-}" ]]; then
args+=(--tag "$SYSTEM_TAG")
fi
spx admin images import "${args[@]}"
}
trap cleanup EXIT
# In quiet mode, redirect build output to /dev/null (import output still shown)
if [[ "$QUIET" == true ]]; then
@@ -116,7 +124,7 @@ if [[ "$QUIET" == true ]]; then
exec 1>/dev/null # suppress build output
fi
echo "=== System Image Builder ==="
echo "=== System Image Builder (libguestfs) ==="
echo "Image: ${IMAGE_NAME} — ${IMAGE_DESCRIPTION:-}"
echo "Distro: ${DISTRO} ${DISTRO_VERSION}"
echo "Size: ${IMAGE_SIZE}"
@@ -124,18 +132,18 @@ echo "Build: ${BUILD_DIR}"
echo ""
# Step 0: Check prerequisites
if ! command -v qemu-nbd &>/dev/null; then
echo "ERROR: qemu-nbd not found. Install qemu-utils."
for tool in virt-customize qemu-img; do
if ! command -v "$tool" &>/dev/null; then
echo "ERROR: $tool not found. Install libguestfs-tools + qemu-utils."
exit 1
fi
done
if [[ "$DISTRO" == "alpine" ]] && ! command -v guestfish &>/dev/null; then
echo "ERROR: guestfish not found. Install libguestfs-tools."
exit 1
fi
if ! command -v qemu-img &>/dev/null; then
echo "ERROR: qemu-img not found. Install qemu-utils."
exit 1
fi
if [[ "$DISTRO" == "ubuntu" ]] && ! command -v parted &>/dev/null && ! [[ -x /usr/sbin/parted ]]; then
echo "ERROR: parted not found. Install parted."
if [[ "$DISTRO" == "ubuntu" ]] && ! command -v virt-resize &>/dev/null; then
echo "ERROR: virt-resize not found. Install libguestfs-tools."
exit 1
fi
@@ -166,11 +174,11 @@ if [[ -n "${INSTALL_BINARIES:-}" ]]; then
done
fi
# Serialize the entire image build with flock — concurrent builds on the same
# host (e.g. CI single-node + multi-node jobs) share /dev/nbd0 and BUILD_DIR.
NBD_LOCK="/tmp/build-system-image.lock"
# Serialize the build with flock — concurrent builds on the same host (e.g. CI
# single-node + multi-node jobs) share BUILD_DIR + the cached raw image.
BUILD_LOCK="/tmp/build-system-image.lock"
echo "Acquiring build lock..."
exec 9>"$NBD_LOCK"
exec 9>"$BUILD_LOCK"
flock 9
echo "Lock acquired"
@@ -189,18 +197,12 @@ if [[ -f "$OUTPUT_RAW" ]] && [[ $(( $(date +%s) - $(stat -c %Y "$OUTPUT_RAW") ))
echo " raw: $OUTPUT_RAW ($(du -h "$OUTPUT_RAW" | cut -f1))"
if [[ "$DO_IMPORT" == true ]]; then
echo "Importing as AMI..."
rm -f "$OUTPUT_IMAGE"
IMPORT_ARGS=(--file "$OUTPUT_RAW" --distro "${DISTRO}" --version "${DISTRO_VERSION}" --arch x86_64 --boot-mode "${BOOT_MODE}")
if [[ -n "${SYSTEM_TAG:-}" ]]; then
IMPORT_ARGS+=(--tag "$SYSTEM_TAG")
fi
spx admin images import "${IMPORT_ARGS[@]}"
import_ami
fi
exit 0
fi
mkdir -p "$BUILD_DIR" "$MOUNT_DIR"
mkdir -p "$BUILD_DIR"
# Step 1: Download base cloud image
if [[ -f "${BUILD_DIR}/${SOURCE_IMAGE}" ]]; then
@@ -225,126 +227,66 @@ rm -f "$OUTPUT_IMAGE"
echo "Copying image for customization..."
cp "${BUILD_DIR}/${SOURCE_IMAGE}" "$OUTPUT_IMAGE"
# Resize the image to provide room for packages
qemu-img resize "$OUTPUT_IMAGE" "$IMAGE_SIZE"
# Step 3: Connect via qemu-nbd
echo "Connecting image via qemu-nbd..."
sudo modprobe nbd max_part=4 2>/dev/null || true
if [[ ! -e "${NBD_DEV}" ]]; then
echo "ERROR: ${NBD_DEV} does not exist. Is the nbd kernel module loaded? Try: sudo modprobe nbd"
exit 1
# Step 3: Resize to provide room for packages. Both paths grow the root
# filesystem entirely inside the libguestfs appliance — no host loop/nbd device.
echo "Resizing image to ${IMAGE_SIZE}..."
if [[ "$DISTRO" == "alpine" ]]; then
# Alpine cloud images have ext4 directly on the whole disk (no partition
# table); grow the disk file then expand the fs from inside the appliance.
qemu-img resize "$OUTPUT_IMAGE" "$IMAGE_SIZE"
guestfish --rw -a "$OUTPUT_IMAGE" run \
: e2fsck /dev/sda forceall:true \
: resize2fs /dev/sda
else
# Ubuntu images use a GPT partition table with the root fs on /dev/sda1;
# virt-resize expands the partition + fs into a fresh IMAGE_SIZE disk.
RESIZED="${OUTPUT_IMAGE}.resized"
rm -f "$RESIZED"
qemu-img create -f qcow2 "$RESIZED" "$IMAGE_SIZE"
virt-resize --expand /dev/sda1 "$OUTPUT_IMAGE" "$RESIZED"
mv -f "$RESIZED" "$OUTPUT_IMAGE"
fi
sudo qemu-nbd --disconnect "${NBD_DEV}" 2>/dev/null || true
sudo qemu-nbd --connect="${NBD_DEV}" "$OUTPUT_IMAGE"
# Wait for the nbd device to be ready
for i in $(seq 1 10); do
if sudo blockdev --getsize64 "${NBD_DEV}" &>/dev/null; then
break
fi
sleep 1
done
sleep 1
# Ubuntu images have a partition table; wait for partition devices then resize
# the partition to fill the expanded image before resizing the filesystem.
# Step 4: Assemble the virt-customize operation list. virt-customize runs the
# operations in the order they appear on the command line, so this mirrors the
# old chroot sequence: packages → binaries → files → setup → services → clean.
CUST=(virt-customize -a "$OUTPUT_IMAGE" --network)
if [[ "$DISTRO" == "ubuntu" ]]; then
for i in $(seq 1 10); do
if [[ -e "${ROOT_PART}" ]]; then break; fi
sleep 1
done
echo "Resizing partition..."
# After qemu-img resize the GPT backup header sits at the old end of disk.
# Relocate it to the new end (sfdisk --relocate is util-linux built-in,
# no extra packages needed), then re-read the partition table before parted
# runs — without this, parted sees stale GPT geometry and fails with
# "Unable to satisfy all constraints on the partition."
sudo sgdisk --move-second-header "${NBD_DEV}"
sudo partprobe "${NBD_DEV}" 2>/dev/null || true
sleep 1
sudo parted --script "${NBD_DEV}" resizepart 1 100%
# DKMS driver bakes (GPU images) need real resources in the appliance.
CUST+=(--memsize 6144 --smp 4)
else
CUST+=(--memsize 2048)
fi
# Alpine cloud images have ext4 directly on the block device (no partition table).
echo "Checking filesystem..."
sudo e2fsck -f -y "${ROOT_PART}" || {
ec=$?
if [[ $ec -gt 1 ]]; then
echo "ERROR: e2fsck failed with exit code $ec on ${ROOT_PART}"
exit 1
fi
}
echo "Resizing filesystem..."
if ! sudo resize2fs "${ROOT_PART}"; then
echo "ERROR: resize2fs failed on ${ROOT_PART}"
exit 1
fi
# Step 4: Mount and customize
echo "Mounting root filesystem..."
sudo mount "${ROOT_PART}" "$MOUNT_DIR"
# Set up resolv.conf for DNS inside chroot.
# Ubuntu cloud images symlink /etc/resolv.conf → /run/systemd/resolve/stub-resolv.conf
# which doesn't exist yet. Remove the symlink and write a real file.
sudo rm -f "${MOUNT_DIR}/etc/resolv.conf"
sudo cp /etc/resolv.conf "${MOUNT_DIR}/etc/resolv.conf"
# Ubuntu chroot requires /proc /sys /dev bind mounts for systemd and DKMS
if [[ "$DISTRO" == "ubuntu" ]]; then
sudo mount --bind /proc "${MOUNT_DIR}/proc"
sudo mount --bind /sys "${MOUNT_DIR}/sys"
sudo mount --bind /dev "${MOUNT_DIR}/dev"
sudo mount --bind /dev/pts "${MOUNT_DIR}/dev/pts"
fi
# Install packages
# Packages
if [[ "$DISTRO" == "alpine" ]] && [[ -n "${APK_PACKAGES:-}" ]]; then
echo "Installing packages: ${APK_PACKAGES}..."
sudo chroot "$MOUNT_DIR" /bin/sh -c "
set -e
# Enable community repo
sed -i 's|^#\(.*community\)|\1|' /etc/apk/repositories 2>/dev/null || true
# Ensure community repo is present
if ! grep -q community /etc/apk/repositories; then
MIRROR=\$(grep main /etc/apk/repositories | head -1 | sed 's|/main|/community|')
echo \"\$MIRROR\" >> /etc/apk/repositories
fi
apk update
apk add ${APK_PACKAGES}
"
echo "Will install packages: ${APK_PACKAGES}"
CUST+=(--run-command 'sed -i "s|^#\(.*community\)|\1|" /etc/apk/repositories 2>/dev/null || true')
CUST+=(--run-command 'grep -q community /etc/apk/repositories || grep main /etc/apk/repositories | head -1 | sed "s|/main|/community|" >> /etc/apk/repositories')
CUST+=(--run-command 'apk update')
CUST+=(--run-command "apk add ${APK_PACKAGES}")
elif [[ "$DISTRO" == "ubuntu" ]] && [[ -n "${APT_PACKAGES:-}" ]]; then
echo "Installing packages: ${APT_PACKAGES}..."
sudo chroot "$MOUNT_DIR" /bin/bash -c "
export DEBIAN_FRONTEND=noninteractive
apt-get update
apt-get install -y --no-install-recommends ${APT_PACKAGES}
"
echo "Will install packages: ${APT_PACKAGES}"
CUST+=(--run-command 'export DEBIAN_FRONTEND=noninteractive; apt-get update')
CUST+=(--run-command "export DEBIAN_FRONTEND=noninteractive; apt-get install -y --no-install-recommends ${APT_PACKAGES}")
fi
# Step 5: Copy binaries into the image (before setup script, which may reference them)
# Binaries (mode 0755). --mkdir is idempotent (mkdir -p semantics).
if [[ -n "${INSTALL_BINARIES:-}" ]]; then
echo "Installing binaries..."
IFS=' ' read -ra BINARY_PAIRS <<< "$INSTALL_BINARIES"
for pair in "${BINARY_PAIRS[@]}"; do
src="${pair%%:*}"
dst="${pair#*:}"
src_path="${PROJECT_DIR}/${src}"
echo " ${src} -> ${dst}"
sudo cp "$src_path" "${MOUNT_DIR}${dst}"
sudo chmod 755 "${MOUNT_DIR}${dst}"
CUST+=(--mkdir "$(dirname "$dst")")
CUST+=(--upload "${src_path}:${dst}")
CUST+=(--chmod "0755:${dst}")
done
fi
# Copy auxiliary files (systemd units, OpenRC initd scripts, cron entries, configs).
# Same src:dst pair grammar as INSTALL_BINARIES; mode 0644, parent dirs auto-created.
# Optional — manifests without INSTALL_FILES (e.g. existing GPU images) skip this step.
# Auxiliary files (systemd units, OpenRC initd scripts, cron entries, configs).
# Mode 0644; setup scripts may chmod role-specific files to 0755 afterwards.
if [[ -n "${INSTALL_FILES:-}" ]]; then
echo "Installing files..."
IFS=' ' read -ra FILE_PAIRS <<< "$INSTALL_FILES"
for pair in "${FILE_PAIRS[@]}"; do
src="${pair%%:*}"
@@ -354,85 +296,45 @@ if [[ -n "${INSTALL_FILES:-}" ]]; then
echo "ERROR: INSTALL_FILES source not found: $src_path"
exit 1
fi
echo " ${src} -> ${dst}"
sudo mkdir -p "${MOUNT_DIR}$(dirname "$dst")"
sudo cp "$src_path" "${MOUNT_DIR}${dst}"
sudo chmod 0644 "${MOUNT_DIR}${dst}"
CUST+=(--mkdir "$(dirname "$dst")")
CUST+=(--upload "${src_path}:${dst}")
CUST+=(--chmod "0644:${dst}")
done
fi
# Run custom setup script inside chroot (after binaries are installed)
# Custom setup script (uploaded + executed + removed by --run, inside the guest).
if [[ -n "${SETUP_SCRIPT:-}" ]]; then
setup_path="${PROJECT_DIR}/${SETUP_SCRIPT}"
if [[ ! -f "$setup_path" ]]; then
echo "ERROR: Setup script not found: $setup_path"
exit 1
fi
echo "Running setup script: ${SETUP_SCRIPT}..."
sudo cp "$setup_path" "${MOUNT_DIR}/tmp/setup.sh"
sudo chmod 755 "${MOUNT_DIR}/tmp/setup.sh"
if [[ "$DISTRO" == "ubuntu" ]]; then
if [[ ! -x "${MOUNT_DIR}/bin/bash" ]]; then
echo "ERROR: /bin/bash not found or not executable in chroot"
exit 1
fi
sudo chroot "$MOUNT_DIR" /bin/bash -x /tmp/setup.sh
else
sudo chroot "$MOUNT_DIR" /tmp/setup.sh
fi
sudo rm -f "${MOUNT_DIR}/tmp/setup.sh"
CUST+=(--run "$setup_path")
fi
# Enable services (runs after INSTALL_FILES + SETUP_SCRIPT so init scripts the
# manifest references are already on disk).
# Enable services (after files + setup so referenced init scripts exist).
if [[ -n "${ENABLE_SERVICES:-}" ]]; then
echo "Enabling services: ${ENABLE_SERVICES}..."
IFS=' ' read -ra SERVICES <<< "$ENABLE_SERVICES"
for svc in "${SERVICES[@]}"; do
if [[ "$DISTRO" == "alpine" ]]; then
if ! sudo chroot "$MOUNT_DIR" /bin/sh -c "rc-update add ${svc} default"; then
echo "ERROR: Failed to enable service '${svc}' — does it exist in the image?"
exit 1
fi
CUST+=(--run-command "rc-update add ${svc} default")
else
if ! sudo chroot "$MOUNT_DIR" /bin/bash -c "systemctl enable ${svc}"; then
echo "ERROR: Failed to enable service '${svc}'"
exit 1
fi
CUST+=(--run-command "systemctl enable ${svc}")
fi
done
fi
# Step 6: Clean up and unmount
echo "Cleaning up image..."
# Final cleanup of package caches + tmp.
if [[ "$DISTRO" == "alpine" ]]; then
sudo chroot "$MOUNT_DIR" /bin/sh -c '
apk cache clean 2>/dev/null || true
rm -rf /var/cache/apk/* /tmp/*
'
CUST+=(--run-command 'apk cache clean 2>/dev/null || true; rm -rf /var/cache/apk/* /tmp/* 2>/dev/null || true')
else
sudo chroot "$MOUNT_DIR" /bin/bash -c '
export DEBIAN_FRONTEND=noninteractive
apt-get clean
rm -rf /var/lib/apt/lists/* /tmp/*
'
CUST+=(--run-command 'export DEBIAN_FRONTEND=noninteractive; apt-get clean; rm -rf /var/lib/apt/lists/* /tmp/* 2>/dev/null || true')
fi
# Restore the systemd-resolved symlink (Ubuntu default); cloud-init sets DNS on boot.
sudo rm -f "${MOUNT_DIR}/etc/resolv.conf"
sudo ln -sf /run/systemd/resolve/stub-resolv.conf "${MOUNT_DIR}/etc/resolv.conf"
echo "Customizing image (libguestfs appliance)..."
"${CUST[@]}"
echo "Unmounting..."
if [[ "$DISTRO" == "ubuntu" ]]; then
sudo umount "${MOUNT_DIR}/dev/pts"
sudo umount "${MOUNT_DIR}/dev"
sudo umount "${MOUNT_DIR}/sys"
sudo umount "${MOUNT_DIR}/proc"
fi
sudo umount "$MOUNT_DIR"
sudo qemu-nbd --disconnect "${NBD_DEV}"
# Step 7: Convert to raw for import
# Step 5: Convert to raw for import
echo "Converting to raw format..."
qemu-img convert -f qcow2 -O raw "$OUTPUT_IMAGE" "$OUTPUT_RAW"
@@ -449,23 +351,14 @@ echo " raw: $OUTPUT_RAW ($(du -h "$OUTPUT_RAW" | cut -f1))"
echo ""
if [[ "$DO_IMPORT" == true ]]; then
echo "Importing as AMI..."
rm -f "$OUTPUT_IMAGE"
IMPORT_ARGS=(--file "$OUTPUT_RAW" --distro "${DISTRO}" --version "${DISTRO_VERSION}" --arch x86_64 --boot-mode "${BOOT_MODE}")
if [[ -n "${AMI_NAME:-}" ]]; then
IMPORT_ARGS+=(--name "$AMI_NAME")
fi
if [[ -n "${SYSTEM_TAG:-}" ]]; then
IMPORT_ARGS+=(--tag "$SYSTEM_TAG")
fi
spx admin images import "${IMPORT_ARGS[@]}"
import_ami
else
echo "To import as AMI, run:"
echo " spx admin images import \\"
echo " --file $OUTPUT_RAW \\"
NAME_HINT=""
if [[ -n "${AMI_NAME:-}" ]]; then
NAME_HINT=" \\\n --name ${AMI_NAME}"
NAME_HINT=" \\\n --ami-name ${AMI_NAME}"
fi
if [[ -n "${SYSTEM_TAG:-}" ]]; then
echo -e " --distro ${DISTRO} --version ${DISTRO_VERSION} --arch x86_64 --boot-mode ${BOOT_MODE}${NAME_HINT} \\"
scripts/dev-install.sh
+11
View File
@@ -90,6 +90,17 @@ fi
echo "=== Building and installing microVM artifacts ==="
cd "$PROJECT_ROOT" && make build-lb-agent install-microvm
# Unified EKS node AMI (server + agent; role chosen at first boot). Heavy (full
# Alpine + K3s build, needs sudo + network), so it is opt-in: set
# DEV_INSTALL_EKS=1 to build + register it. Without it, `aws eks create-cluster`
# fails AMI resolution until the operator runs `make import-eks-node-image`.
if [ "${DEV_INSTALL_EKS:-0}" = "1" ]; then
echo "=== Building + importing eks-node AMI ==="
cd "$PROJECT_ROOT" && make import-eks-node-image
else
echo "=== Skipping eks-node AMI (set DEV_INSTALL_EKS=1 to enable, or run 'make import-eks-node-image') ==="
fi
echo "=== Done ==="
echo "Services: sudo systemctl status spinifex.target"
echo "Logs: journalctl -u 'spinifex-*' -f"
scripts/images/eks-agent/manifest.conf
-25
View File
@@ -1,25 +0,0 @@
IMAGE_NAME="eks-agent"
IMAGE_DESCRIPTION="EKS K3s agent VM — Alpine + K3s v1.32.5+k3s1 (worker node)"
DISTRO="alpine"
ALPINE_VERSION="3.21.7"
# 2G for the worker — kubelet + containerd + node images. Customer workload
# storage attaches via separate EBS volumes (managed by viperblock); this is
# the OS root only.
IMAGE_SIZE="2G"
APK_PACKAGES="ca-certificates curl iptables ip6tables conntrack-tools cgroup-tools openrc"
# K3s binary downloaded by setup.sh (same pinned version + SHA path as
# eks-server) to keep server / agent in lock-step.
INSTALL_FILES="\
scripts/images/eks-agent/k3s-agent.initd:/etc/init.d/k3s-agent \
scripts/images/eks-agent/mulga-eks-state-report.sh:/etc/periodic/15min/mulga-eks-state-report"
ENABLE_SERVICES="crond k3s-agent"
SETUP_SCRIPT="scripts/images/eks-agent/setup.sh"
AMI_NAME="spinifex-eks-agent"
SYSTEM_TAG="spinifex:managed-by=eks"
scripts/images/eks-agent/mulga-eks-state-report.sh
-38
View File
@@ -1,38 +0,0 @@
#!/bin/sh
set -eu
# mulga-eks-state-report (agent variant) — periodic NATS publish of agent
# liveness signal. Subject: eks.state.{accountID}.{clusterName}.agent.{nodeName}
# Payload: JSON { kubelet: "ok"|"fail", ts: <unix> }
#
# The agent has no API server to query for its own status; we shell-check the
# local kubelet socket via crictl, and fall back to a process check.
ENVFILE=/etc/spinifex-eks/state-report.env
[ -f "${ENVFILE}" ] || { logger -t mulga-eks-state-report "${ENVFILE} missing"; exit 0; }
# shellcheck disable=SC1090
. "${ENVFILE}"
: "${SPINIFEX_NATS_URL:?}"
: "${EKS_ACCOUNT_ID:?}"
: "${EKS_CLUSTER_NAME:?}"
: "${EKS_NODE_NAME:?}"
if pgrep -f 'k3s agent' >/dev/null 2>&1; then
KUBELET=ok
else
KUBELET=fail
fi
TS=$(date +%s)
PAYLOAD=$(printf '{"kubelet":"%s","ts":%s}' "${KUBELET}" "${TS}")
SUBJ="eks.state.${EKS_ACCOUNT_ID}.${EKS_CLUSTER_NAME}.agent.${EKS_NODE_NAME}"
NATS_ARGS="-s ${SPINIFEX_NATS_URL}"
if [ -n "${SPINIFEX_NATS_CREDS_FILE:-}" ] && [ -f "${SPINIFEX_NATS_CREDS_FILE}" ]; then
NATS_ARGS="${NATS_ARGS} --creds ${SPINIFEX_NATS_CREDS_FILE}"
fi
# shellcheck disable=SC2086
printf '%s' "${PAYLOAD}" | nats ${NATS_ARGS} pub "${SUBJ}" --stdin 2>&1 \
| logger -t mulga-eks-state-report
scripts/images/eks-agent/setup.sh
-47
View File
@@ -1,47 +0,0 @@
#!/bin/sh
set -eu
# setup.sh — chroot customisation for the eks-agent AMI.
# Pulls the same K3s release as eks-server (lock-step versioning per Q12) and
# wires the k3s-agent OpenRC service.
K3S_VERSION="v1.32.5+k3s1"
K3S_URL_BASE="https://github.com/k3s-io/k3s/releases/download/${K3S_VERSION}"
echo "[eks-agent-setup] fetching k3s checksums ${K3S_VERSION}"
curl -fsSL -o /tmp/k3s-checksums.txt "${K3S_URL_BASE}/sha256sum-amd64.txt"
K3S_SHA256=$(awk '/[ \t]k3s$/{print $1; exit}' /tmp/k3s-checksums.txt)
if [ -z "${K3S_SHA256}" ]; then
echo "[eks-agent-setup] could not parse k3s sha256 from upstream checksums"
cat /tmp/k3s-checksums.txt
exit 1
fi
echo "[eks-agent-setup] downloading k3s ${K3S_VERSION} (sha256=${K3S_SHA256})"
curl -fsSL -o /usr/local/bin/k3s "${K3S_URL_BASE}/k3s"
echo "${K3S_SHA256} /usr/local/bin/k3s" > /tmp/k3s.sha256
if ! sha256sum -c /tmp/k3s.sha256; then
echo "[eks-agent-setup] k3s SHA256 mismatch — refusing to bake AMI"
exit 1
fi
chmod 0755 /usr/local/bin/k3s
ln -sf /usr/local/bin/k3s /usr/local/bin/kubectl
ln -sf /usr/local/bin/k3s /usr/local/bin/crictl
ln -sf /usr/local/bin/k3s /usr/local/bin/ctr
# nats-cli — see eks-server setup.sh for rationale.
NATS_CLI_VERSION="0.4.0"
NATS_CLI_URL="https://github.com/nats-io/natscli/releases/download/v${NATS_CLI_VERSION}/nats-${NATS_CLI_VERSION}-linux-amd64.zip"
echo "[eks-agent-setup] downloading nats-cli ${NATS_CLI_VERSION}"
curl -fsSL -o /tmp/nats-cli.zip "${NATS_CLI_URL}"
apk add --no-cache unzip
unzip -q -d /tmp/nats-cli /tmp/nats-cli.zip
install -m 0755 /tmp/nats-cli/nats-${NATS_CLI_VERSION}-linux-amd64/nats /usr/local/bin/nats
rm -rf /tmp/nats-cli /tmp/nats-cli.zip
apk del --no-cache unzip
chmod 0755 /etc/init.d/k3s-agent
chmod 0755 /etc/periodic/15min/mulga-eks-state-report
mkdir -p /etc/spinifex-eks
echo "[eks-agent-setup] done"
scripts/images/eks-node/eks-node-role.initd
Executable
+27
View File
@@ -0,0 +1,27 @@
#!/sbin/openrc-run
# eks-node-role — one-shot first-boot service that selects the node's role
# (server vs agent) from the cloud-init env and enables + starts the matching
# K3s services. The unified eks-node AMI bakes every role's services but leaves
# them out of the default runlevel; this selector adds the correct ones and then
# removes itself so it runs exactly once.
description="EKS node role selector (server vs agent)"
# Backgrounded: the selector calls `rc-service k3s start`, which blocks on k3s'
# start-stop-daemon --wait window; backgrounding keeps it off the runlevel's
# critical path. command_background is also required for output_log/error_log.
command="/usr/local/sbin/eks-node-role"
command_background="yes"
pidfile="/run/eks-node-role.pid"
# Route to the serial console so the role decision + any failure is visible in
# the orchestrator's host-side console capture — the VM has no operator login.
output_log="/dev/console"
error_log="/dev/console"
depend() {
# Needs the same substrate the role services do, since it starts them.
need net cgroups
# cloud-init write_files delivers /etc/spinifex-eks/first-boot.env (the role
# source) in the cloud-config/cloud-final stages. Order after them so the
# selector never races ahead and reads a missing env file on a fast boot.
after firewall cloud-config cloud-final
}
scripts/images/eks-node/eks-node-role.sh
Executable
+71
View File
@@ -0,0 +1,71 @@
#!/bin/sh
set -eu
# eks-node-role — first-boot role selector for the unified eks-node AMI.
#
# The same image boots either as a K3s control-plane server or a worker agent;
# the role is chosen per-instance at first boot from SPINIFEX_K3S_ROLE in the
# cloud-init env file. This script enables + starts the matching OpenRC services
# (none are in the default runlevel at bake time), records the resolved role to
# ROLE_FILE so later boots and the etcd-snapshot cron can branch without
# re-parsing, then removes itself from the runlevel so it runs exactly once.
#
# server → eks-token-webhook, k3s (server), k3s-first-boot (bootstrap publish)
# agent → k3s-agent
#
# Paths are overridable via env so the selection logic is unit-testable (bats)
# without root or a real /etc.
ROLE_FILE="${EKS_NODE_ROLE_FILE:-/etc/spinifex-eks/role}"
ENVFILE="${EKS_NODE_ENVFILE:-/etc/spinifex-eks/first-boot.env}"
AGENT_ENVFILE="${EKS_NODE_AGENT_ENVFILE:-/etc/spinifex-eks/agent.env}"
LOGTAG="eks-node-role"
log() { echo "[${LOGTAG}] $*"; }
die() { log "ERROR: $*"; exit 1; }
# Already resolved on a prior boot — the role services are in the runlevel and
# this service should have been pulled, but guard anyway so a stray re-run is a
# no-op rather than a double-enable.
if [ -f "${ROLE_FILE}" ]; then
log "role already resolved ($(cat "${ROLE_FILE}")); nothing to do"
exit 0
fi
# Resolve the role. Server seeds SPINIFEX_K3S_ROLE in first-boot.env; a worker
# seed that predates the explicit var is inferred from the presence of agent.env
# (K3S_URL/K3S_TOKEN live there).
ROLE=""
if [ -f "${ENVFILE}" ]; then
ROLE=$(. "${ENVFILE}"; printf '%s' "${SPINIFEX_K3S_ROLE:-}")
fi
if [ -z "${ROLE}" ] && [ -f "${AGENT_ENVFILE}" ]; then
ROLE="agent"
fi
case "${ROLE}" in
server)
log "configuring server role"
rc-update add eks-token-webhook default
rc-update add k3s default
rc-update add k3s-first-boot default
rc-update add mulga-eks-state-report default
rc-service eks-token-webhook start
rc-service k3s start
rc-service k3s-first-boot start
rc-service mulga-eks-state-report start
;;
agent)
log "configuring agent role"
rc-update add k3s-agent default
rc-service k3s-agent start
;;
*)
die "SPINIFEX_K3S_ROLE missing or invalid: '${ROLE}'"
;;
esac
mkdir -p "$(dirname "${ROLE_FILE}")"
printf '%s\n' "${ROLE}" > "${ROLE_FILE}"
rc-update del eks-node-role default 2>/dev/null || true
log "role '${ROLE}' configured"
scripts/images/eks-node/eks-node-role_test.sh
Executable
+103
View File
@@ -0,0 +1,103 @@
#!/bin/sh
# Self-contained POSIX test for eks-node-role.sh role selection. No bats/root
# needed: rc-update / rc-service are stubbed on PATH and record their calls; the
# script's file paths are redirected into a temp dir via its EKS_NODE_* env knobs.
#
# Run: sh scripts/images/eks-node/eks-node-role_test.sh
set -eu
SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
SELECTOR="${SCRIPT_DIR}/eks-node-role.sh"
WORK=$(mktemp -d)
trap 'rm -rf "${WORK}"' EXIT
# Stub rc-update / rc-service: append "cmd args" to ${CALLS} when invoked.
STUBBIN="${WORK}/bin"
mkdir -p "${STUBBIN}"
for tool in rc-update rc-service; do
cat > "${STUBBIN}/${tool}" <<EOF
#!/bin/sh
echo "${tool} \$*" >> "\${CALLS}"
exit 0
EOF
chmod +x "${STUBBIN}/${tool}"
done
PASS=0
FAIL=0
check() { # check <desc> <expected> <actual>
if [ "$2" = "$3" ]; then
PASS=$((PASS + 1))
else
FAIL=$((FAIL + 1))
echo "FAIL: $1"
echo " expected: [$2]"
echo " actual: [$3]"
fi
}
# run_case <name>: resets a per-case dir, exports the knobs; caller seeds env
# files then invokes the selector. Sets CASE (dir), CALLS (log), ROLE_FILE.
run_case() {
CASE="${WORK}/$1"
mkdir -p "${CASE}"
CALLS="${CASE}/calls"
: > "${CALLS}"
ROLE_FILE="${CASE}/role"
export CALLS
export EKS_NODE_ROLE_FILE="${ROLE_FILE}"
export EKS_NODE_ENVFILE="${CASE}/first-boot.env"
export EKS_NODE_AGENT_ENVFILE="${CASE}/agent.env"
}
# Run the selector, capturing its exit code in ${rc} without tripping set -e.
invoke() { rc=0; PATH="${STUBBIN}:${PATH}" sh "${SELECTOR}" >/dev/null 2>&1 || rc=$?; }
# --- Case 1: explicit server role ---
run_case server
printf 'SPINIFEX_K3S_ROLE=server\nEKS_CLUSTER_NAME=alpha\n' > "${EKS_NODE_ENVFILE}"
invoke
check "server: exit 0" 0 "${rc}"
check "server: role file" "server" "$(cat "${ROLE_FILE}")"
check "server: enables webhook" "yes" "$(grep -q 'rc-update add eks-token-webhook default' "${CALLS}" && echo yes || echo no)"
check "server: enables k3s" "yes" "$(grep -q 'rc-update add k3s default' "${CALLS}" && echo yes || echo no)"
check "server: enables k3s-first-boot" "yes" "$(grep -q 'rc-update add k3s-first-boot default' "${CALLS}" && echo yes || echo no)"
check "server: starts k3s" "yes" "$(grep -q 'rc-service k3s start' "${CALLS}" && echo yes || echo no)"
check "server: no k3s-agent" "no" "$(grep -q 'k3s-agent' "${CALLS}" && echo yes || echo no)"
check "server: self-disables" "yes" "$(grep -q 'rc-update del eks-node-role default' "${CALLS}" && echo yes || echo no)"
# --- Case 2: explicit agent role ---
run_case agent_explicit
printf 'SPINIFEX_K3S_ROLE=agent\n' > "${EKS_NODE_ENVFILE}"
invoke
check "agent: exit 0" 0 "${rc}"
check "agent: role file" "agent" "$(cat "${ROLE_FILE}")"
check "agent: enables k3s-agent" "yes" "$(grep -q 'rc-update add k3s-agent default' "${CALLS}" && echo yes || echo no)"
check "agent: no server services" "no" "$(grep -qE 'eks-token-webhook|k3s-first-boot|add k3s default' "${CALLS}" && echo yes || echo no)"
# --- Case 3: agent inferred from agent.env (no explicit role) ---
run_case agent_inferred
printf 'K3S_URL=https://nlb:443\nK3S_TOKEN=abc\n' > "${EKS_NODE_AGENT_ENVFILE}"
invoke
check "infer: exit 0" 0 "${rc}"
check "infer: role file" "agent" "$(cat "${ROLE_FILE}")"
check "infer: enables k3s-agent" "yes" "$(grep -q 'rc-update add k3s-agent default' "${CALLS}" && echo yes || echo no)"
# --- Case 4: missing role + no env files → fail, no role file, no services ---
run_case missing
invoke
check "missing: nonzero exit" "1" "${rc}"
check "missing: no role file" "absent" "$([ -f "${ROLE_FILE}" ] && echo present || echo absent)"
check "missing: no rc calls" "" "$(cat "${CALLS}")"
# --- Case 5: already resolved (role file present) → no-op, exit 0 ---
run_case resolved
printf 'SPINIFEX_K3S_ROLE=server\n' > "${EKS_NODE_ENVFILE}"
printf 'server\n' > "${ROLE_FILE}"
invoke
check "resolved: exit 0" 0 "${rc}"
check "resolved: no rc calls" "" "$(cat "${CALLS}")"
echo "---"
echo "PASS=${PASS} FAIL=${FAIL}"
[ "${FAIL}" -eq 0 ]
scripts/images/eks-node/eks-token-webhook.initd
Executable
+41
View File
@@ -0,0 +1,41 @@
#!/sbin/openrc-run
# eks-token-webhook — K8s TokenReview authenticator for IAM-backed kubectl.
# Binds 127.0.0.1:8443 only; kube-apiserver is the sole client (loopback path).
# Relays `aws eks get-token` bearer tokens through the AWS gateway broker (SigV4
# HTTPS), which host-side verifies via Mulga STS + resolves the principal to a
# TokenReview via the cluster AccessEntry KV — the VM never speaks core NATS.
# Ordered `before k3s` so its apiserver kubeconfig exists before k3s reads
# --authentication-token-webhook-config-file.
description="EKS Token Review webhook (Mulga)"
command="/usr/local/bin/eks-token-webhook"
command_args="--addr=127.0.0.1:8443"
command_background="yes"
pidfile="/run/eks-token-webhook.pid"
# Headless appliance: no SSH. Route the webhook's stdout/stderr to the serial
# console so its startup + per-request decisions land in the host-captured
# console log (the only off-VM window into webhook health).
output_log="/dev/console"
error_log="/dev/console"
# Gateway endpoint + SigV4 creds + cluster identity come from the cloud-init-
# seeded first-boot env. Sourced here and exported so start-stop-daemon hands
# them to the binary, which reads EKS_GATEWAY_* / EKS_ACCESS_KEY / EKS_SECRET_KEY
# / EKS_REGION / EKS_ACCOUNT_ID / EKS_CLUSTER_NAME from its environment.
if [ -f /etc/spinifex-eks/first-boot.env ]; then
# shellcheck disable=SC1091
. /etc/spinifex-eks/first-boot.env
export EKS_GATEWAY_URL EKS_GATEWAY_CA EKS_ACCESS_KEY EKS_SECRET_KEY EKS_REGION EKS_ACCOUNT_ID EKS_CLUSTER_NAME
fi
depend() {
need net
before k3s
}
start_pre() {
if [ ! -f /etc/spinifex-eks/first-boot.env ]; then
eerror "/etc/spinifex-eks/first-boot.env not present — cloud-init did not seed gateway creds"
return 1
fi
}
scripts/images/eks-agent/k3s-agent.initd → scripts/images/eks-node/k3s-agent.initd
Regular → Executable
View File
scripts/images/eks-node/k3s-first-boot.initd
Executable
+30
View File
@@ -0,0 +1,30 @@
#!/sbin/openrc-run
# k3s-first-boot — one-shot OpenRC service that runs the first-boot publisher
# after K3s is up. Self-disables by removing itself from the default runlevel
# once the sentinel file is gone.
description="K3s first-boot publisher (node-token + admin kubeconfig)"
# Backgrounded: the publisher waits up to 5min for K3s /healthz before it can
# read the node-token + kubeconfig, so it must not block the default runlevel.
# command_background is also required for output_log/error_log — start-stop-daemon
# rejects --stdout/--stderr without --background ("Failed to start k3s-first-boot").
command="/usr/local/sbin/k3s-first-boot"
command_background="yes"
pidfile="/run/k3s-first-boot.pid"
# Route to the serial console so the bootstrap publisher's progress (healthz
# wait, token/kubeconfig publish, errors) is visible in the orchestrator's
# host-side console capture — the VM has no operator login. Low volume.
output_log="/dev/console"
error_log="/dev/console"
depend() {
need k3s
after k3s
}
start_pre() {
if [ ! -f /var/lib/spinifex-eks/first-boot.pending ]; then
einfo "first-boot already complete; nothing to do"
return 0
fi
}
scripts/images/eks-node/k3s-first-boot.sh
Executable
+139
View File
@@ -0,0 +1,139 @@
#!/bin/sh
set -eu
# k3s-first-boot — runs once after the K3s server reaches a healthy state.
# Reads the bootstrap node-token and admin kubeconfig that K3s writes at
# server startup, rewrites the kubeconfig server address to the cluster's
# NLB endpoint (so workers and external kubectl can use it), and publishes
# the bootstrap artifacts to the host via the AWS gateway (SigV4-signed HTTPS
# POST — the eks-gateway-publish helper) for the spinifex cluster reconciler
# to consume into KV. The gateway relays onto the eks.bus.* NATS subjects; the
# VM never speaks NATS directly.
#
# Required env (from cloud-init user-data /etc/spinifex-eks/first-boot.env):
# EKS_GATEWAY_URL https://{mgmt-host}:9999 (AWS gateway)
# EKS_GATEWAY_CA /etc/spinifex-eks/gateway-ca.pem (TLS CA bundle)
# EKS_ACCESS_KEY system SigV4 access key id
# EKS_SECRET_KEY system SigV4 secret access key
# EKS_REGION SigV4 signing region
# EKS_ACCOUNT_ID
# EKS_CLUSTER_NAME
# EKS_NLB_ENDPOINT https://{cluster}.{accountID}.eks.{region}.{suffix}
#
# Idempotent: a sentinel file at /var/lib/spinifex-eks/first-boot.pending gates
# execution. On success the sentinel is removed and the OpenRC service is
# pulled from the default runlevel so it does not retry on subsequent boots.
SENTINEL=/var/lib/spinifex-eks/first-boot.pending
ENVFILE=/etc/spinifex-eks/first-boot.env
LOGTAG="k3s-first-boot"
log() { echo "[${LOGTAG}] $*"; }
die() { log "ERROR: $*"; exit 1; }
if [ ! -f "${SENTINEL}" ]; then
log "sentinel missing — first boot already complete"
exit 0
fi
if [ ! -f "${ENVFILE}" ]; then
die "${ENVFILE} not found — cloud-init did not seed first-boot env"
fi
# set -a so the sourced KEY=value lines are exported: the eks-gateway-publish
# child reads its config from the environment (EKS_ACCOUNT_ID etc.). A bare
# source leaves them unexported and the helper exits "--account-id is required".
set -a
# shellcheck disable=SC1090
. "${ENVFILE}"
set +a
for v in EKS_GATEWAY_URL EKS_ACCESS_KEY EKS_SECRET_KEY EKS_ACCOUNT_ID EKS_CLUSTER_NAME EKS_NLB_ENDPOINT; do
eval "val=\${$v:-}"
[ -n "${val}" ] || die "env ${v} not set"
done
# Paths K3s writes during server startup: the bootstrap node-token and the
# admin kubeconfig (used both to gate readiness and to publish downstream).
TOKEN_FILE=/var/lib/rancher/k3s/server/node-token
KUBECONFIG_FILE=/etc/rancher/k3s/k3s.yaml
# 1. Wait for the apiserver to be ready to serve, gating on /readyz. K3s runs
# the apiserver with anonymous-auth=false, so an anonymous probe returns 401,
# never "ok" — the probe must be authenticated. Use the node's admin
# kubeconfig (kubectl get --raw), which k3s writes early in startup. kubectl
# exits 0 only when /readyz returns 200; on a failing sub-check it exits
# non-zero, so the body "ok" check plus exit status gates correctly. Every
# 30s the failing /readyz checks are logged so a stuck boot is diagnosable
# from the captured serial console.
log "waiting for K3s apiserver readiness (/readyz) on 127.0.0.1:6443"
i=0
ready=0
while [ "${i}" -lt 300 ]; do
if [ -r "${KUBECONFIG_FILE}" ] && \
[ "$(kubectl --kubeconfig "${KUBECONFIG_FILE}" get --raw='/readyz' 2>/dev/null)" = "ok" ]; then
log "K3s apiserver ready after ${i}s"
ready=1
break
fi
if [ $((i % 30)) -eq 0 ]; then
log "apiserver not ready after ${i}s:"
if [ -r "${KUBECONFIG_FILE}" ]; then
kubectl --kubeconfig "${KUBECONFIG_FILE}" get --raw='/readyz?verbose' 2>&1 \
| grep -E '^\[-\]|failed$' | head -20 | while IFS= read -r l; do log " ${l}"; done
else
log " (admin kubeconfig ${KUBECONFIG_FILE} not written yet)"
fi
fi
i=$((i + 5))
sleep 5
done
if [ "${ready}" -ne 1 ]; then
log "K3s apiserver not ready within 5 minutes; last /readyz body:"
kubectl --kubeconfig "${KUBECONFIG_FILE}" get --raw='/readyz?verbose' 2>&1 \
| head -40 | while IFS= read -r l; do log " ${l}"; done
log "last 40 lines of /var/log/k3s.log follow:"
tail -n 40 /var/log/k3s.log 2>/dev/null || log "(no /var/log/k3s.log)"
die "K3s apiserver not ready within 5 minutes"
fi
# 2. Read the four bootstrap artifacts K3s wrote at server startup.
[ -r "${TOKEN_FILE}" ] || die "${TOKEN_FILE} unreadable"
[ -r "${KUBECONFIG_FILE}" ] || die "${KUBECONFIG_FILE} unreadable"
NODE_TOKEN=$(cat "${TOKEN_FILE}")
# K3s ships kubeconfig with server: https://127.0.0.1:6443 — rewrite to the
# NLB endpoint so it works from outside the control plane VM.
KUBECONFIG_REWRITTEN=$(sed "s|server: https://127\.0\.0\.1:6443|server: ${EKS_NLB_ENDPOINT}|" "${KUBECONFIG_FILE}")
# The cluster CA the daemon records (base64 PEM) is exactly the
# certificate-authority-data the admin kubeconfig already embeds.
CA_B64=$(awk '/certificate-authority-data:/ {print $2; exit}' "${KUBECONFIG_FILE}")
[ -n "${CA_B64}" ] || die "no certificate-authority-data in ${KUBECONFIG_FILE}"
# The OIDC JWKS the apiserver serves from the signing key cloud-init seeded. The
# daemon cross-checks its kid/kty against the controller-generated keypair.
JWKS=$(kubectl --kubeconfig "${KUBECONFIG_FILE}" get --raw='/openid/v1/jwks' 2>/dev/null)
[ -n "${JWKS}" ] || die "apiserver returned empty /openid/v1/jwks"
# 3. Publish the four bootstrap messages through the AWS gateway. Each is a
# BootstrapEnvelope JSON document (handlers/eks/nats_bootstrap.go); jq encodes
# the values so embedded newlines/quotes in the kubeconfig and JWKS stay valid
# JSON. eks-gateway-publish SigV4-signs the POST and retries until the gateway
# acks, then the gateway relays onto eks.bus.{account}.{cluster}.{kind}. It
# reads EKS_GATEWAY_URL/EKS_GATEWAY_CA/EKS_ACCESS_KEY/EKS_SECRET_KEY/EKS_REGION/
# EKS_ACCOUNT_ID/EKS_CLUSTER_NAME from the env file already sourced above.
# publish_envelope <kind-suffix>: reads the envelope JSON from stdin.
publish_envelope() {
log "publishing $1 via gateway"
eks-gateway-publish -channel bootstrap -kind "$1"
}
jq -n --arg t "${NODE_TOKEN}" '{token: $t}' | publish_envelope k3s-bootstrap-token
jq -n --arg k "${KUBECONFIG_REWRITTEN}" '{adminKubeconfig: $k}' | publish_envelope k3s-admin-kubeconfig
jq -n --arg j "${JWKS}" '{jwks: $j}' | publish_envelope k3s-oidc-jwks
jq -n --arg c "${CA_B64}" '{certificateAuthority: $c}' | publish_envelope k3s-ca
# 4. Self-disable. Remove sentinel, pull from runlevel.
rm -f "${SENTINEL}"
rc-update del k3s-first-boot default 2>/dev/null || true
log "first boot complete"
scripts/images/eks-node/k3s.initd
Executable
+65
View File
@@ -0,0 +1,65 @@
#!/sbin/openrc-run
# K3s server control plane. The per-cluster config.yaml sets cluster-init, so
# k3s uses the embedded etcd datastore (required for multi-server HA). Data
# lives at /var/lib/rancher/k3s; subsequent starts reuse it.
description="K3s Kubernetes server (Mulga EKS control plane)"
command="/usr/local/bin/k3s"
command_args="server --config /etc/rancher/k3s/config.yaml"
pidfile="/run/k3s.pid"
# Supervise the apiserver. A transient k3s crash must self-heal: without a
# supervisor a single exit leaves the control plane permanently dead, the NLB
# target never recovers, and the cluster wedges ACTIVE-but-unreachable.
supervisor="supervise-daemon"
respawn_delay=5
respawn_max=0
output_log="/var/log/k3s.log"
error_log="/var/log/k3s.log"
depend() {
need net cgroups
after firewall
}
start_pre() {
# config.yaml is written by k3s-first-boot from the skeleton + cloud-init
# data. Refuse to start if neither exists.
if [ ! -f /etc/rancher/k3s/config.yaml ]; then
if [ -f /etc/rancher/k3s/config.yaml.skel ]; then
cp /etc/rancher/k3s/config.yaml.skel /etc/rancher/k3s/config.yaml
else
eerror "no /etc/rancher/k3s/config.yaml and no .skel to derive from"
return 1
fi
fi
# Deferred built-in ingress: when this cluster wants traefik
# (EKS_DEFER_TRAEFIK=1) but the control plane has not yet stabilised (no
# traefik-enabled sentinel), stage a deploy-controller .skip marker so k3s
# writes traefik.yaml/traefik-crd.yaml but does NOT apply them. Staged here,
# pre-start, so the marker exists before the deploy controller first scans —
# racing it post-start could apply traefik (and crash etcd) anyway.
# mulga-eks-state-report removes the marker once the apiserver is healthy.
if [ -f /etc/spinifex-eks/first-boot.env ]; then
defer=$(. /etc/spinifex-eks/first-boot.env 2>/dev/null; echo "${EKS_DEFER_TRAEFIK:-0}")
if [ "${defer}" = "1" ] && [ ! -f /var/lib/spinifex-eks/traefik-enabled ]; then
mkdir -p /var/lib/rancher/k3s/server/manifests
: > /var/lib/rancher/k3s/server/manifests/traefik.yaml.skip
: > /var/lib/rancher/k3s/server/manifests/traefik-crd.yaml.skip
fi
fi
# The apiserver is configured with --authentication-token-webhook-config-file
# pointing at the kubeconfig eks-token-webhook writes at startup. `before k3s`
# orders the webhook's start but it backgrounds, so wait (bounded) for the
# kubeconfig to land before letting the apiserver load a missing file.
i=0
while [ ! -f /etc/spinifex-eks/token-webhook.kubeconfig ] && [ "${i}" -lt 30 ]; do
sleep 1
i=$((i + 1))
done
if [ ! -f /etc/spinifex-eks/token-webhook.kubeconfig ]; then
ewarn "eks-token-webhook kubeconfig absent after ${i}s; IAM token auth will be unavailable until the webhook starts"
fi
}
scripts/images/eks-node/manifest.conf
+47
View File
@@ -0,0 +1,47 @@
IMAGE_NAME="eks-node"
IMAGE_DESCRIPTION="Unified EKS K3s node VM — Alpine + K3s v1.32.5+k3s1 + eks-token-webhook; role (server|agent) chosen at first boot"
DISTRO="alpine"
ALPINE_VERSION="3.21.7"
# 4G — server superset (K3s server binary + embedded-etcd datastore working set under
# /var/lib/rancher/k3s on the root disk). Agents use the same image; the ~10MB
# of unused server binary + webhook is dwarfed by the headroom for node images.
IMAGE_SIZE="4G"
# Server superset of both package sets (etcd-ctl is ~5MB, not worth branching).
# K3s pulls cgroups + iptables; jq builds the bootstrap-message JSON envelopes;
# etcd-client used by the snapshot script (server role only). The control plane
# reaches the host via the AWS gateway (eks-gateway-publish), not a NATS CLI.
APK_PACKAGES="ca-certificates curl jq iptables ip6tables conntrack-tools cgroup-tools etcd-ctl openrc"
# eks-token-webhook binary always installed (agents never start it).
# eks-gateway-publish relays the bootstrap envelopes + state reports to the AWS
# gateway over SigV4 HTTPS (server role only). The k3s binary is downloaded +
# SHA256-verified by setup.sh (network in chroot).
BUILD_COMMANDS="CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-token-webhook ./cmd/eks-token-webhook && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-gateway-publish ./cmd/eks-gateway-publish"
INSTALL_BINARIES="bin/eks-token-webhook:/usr/local/bin/eks-token-webhook bin/eks-gateway-publish:/usr/local/bin/eks-gateway-publish"
# All role services ship; none but the selector are enabled at bake time.
# eks-node-role picks server vs agent at first boot and enables the rest.
INSTALL_FILES="\
scripts/images/eks-node/eks-node-role.initd:/etc/init.d/eks-node-role \
scripts/images/eks-node/eks-node-role.sh:/usr/local/sbin/eks-node-role \
scripts/images/eks-node/k3s.initd:/etc/init.d/k3s \
scripts/images/eks-node/k3s-agent.initd:/etc/init.d/k3s-agent \
scripts/images/eks-node/eks-token-webhook.initd:/etc/init.d/eks-token-webhook \
scripts/images/eks-node/k3s-first-boot.initd:/etc/init.d/k3s-first-boot \
scripts/images/eks-node/k3s-first-boot.sh:/usr/local/sbin/k3s-first-boot \
scripts/images/eks-node/mulga-eks-state-report.initd:/etc/init.d/mulga-eks-state-report \
scripts/images/eks-node/mulga-eks-state-report.sh:/usr/local/sbin/mulga-eks-state-report \
scripts/images/eks-node/mulga-eks-etcd-snapshot.sh:/etc/periodic/daily/mulga-eks-etcd-snapshot"
# Only crond (etcd snapshot) + the role selector at bake. The selector adds the
# role-specific services (k3s/eks-token-webhook/k3s-first-boot/state-report or
# k3s-agent) to the default runlevel on first boot.
ENABLE_SERVICES="crond eks-node-role"
SETUP_SCRIPT="scripts/images/eks-node/setup.sh"
AMI_NAME="spinifex-eks-node"
SYSTEM_TAG="spinifex:managed-by=eks"
scripts/images/eks-server/mulga-eks-etcd-snapshot.sh → scripts/images/eks-node/mulga-eks-etcd-snapshot.sh
Regular → Executable
+6
View File
@@ -12,6 +12,12 @@ set -eu
# Retention: managed via S3 lifecycle policy on the bucket; this script does
# not prune.
# Worker nodes share the unified eks-node AMI but have no etcd/SQLite datastore
# to snapshot. The role file is written once by eks-node-role at first boot.
if [ "$(cat /etc/spinifex-eks/role 2>/dev/null)" = "agent" ]; then
exit 0
fi
ENVFILE=/etc/spinifex-eks/etcd-snapshot.env
[ -f "${ENVFILE}" ] || { logger -t mulga-eks-etcd-snapshot "${ENVFILE} missing"; exit 0; }
# shellcheck disable=SC1090
scripts/images/eks-node/mulga-eks-state-report.initd
+29
View File
@@ -0,0 +1,29 @@
#!/sbin/openrc-run
# mulga-eks-state-report — server-role service that publishes K3s health +
# node_count to the management NATS bus every STATE_REPORT_INTERVAL seconds. The
# spinifex cluster reconciler consumes this to gate CREATING → ACTIVE and to
# track nodegroup join/scale, since the apiserver (bound to the VPC node-ip) is
# unreachable from the host. Enabled on the server role by eks-node-role.sh.
description="Mulga EKS control-plane state reporter"
command="/usr/local/sbin/mulga-eks-state-report"
command_background="yes"
pidfile="/run/mulga-eks-state-report.pid"
output_log="/var/log/mulga-eks-state-report.log"
error_log="/var/log/mulga-eks-state-report.log"
# Cadence the report loop sleeps between publishes. 30s is well under the
# reconciler's 90s stale window, so a missed tick does not flap health.
export STATE_REPORT_INTERVAL="${STATE_REPORT_INTERVAL:-30}"
depend() {
need net
after k3s
}
start_pre() {
if [ ! -f /etc/spinifex-eks/first-boot.env ]; then
eerror "/etc/spinifex-eks/first-boot.env not present — cloud-init did not seed NATS creds"
return 1
fi
}

Some files were not shown because too many files have changed in this diff Show More