Compare commits

..

1 Commits

Author SHA1 Message Date
joshsiv-mulga fa8f35b764 test(e2e): report retained local WAL before the churn test detaches
The storage-growth suite measures backend bytes. Local WAL retention sits
upstream of that boundary and nothing under tests/e2e ever looked at it, so the
condition this suite produces on every run has never once appeared in a result.

TestContinuousChurn is the only workload shaped to show it, because it holds a
single volume open for minutes while writing concurrently. The measurement has
to be taken before the detach: Close calls RemoveLocalFiles and deletes the
per-volume directory, so afterwards the number always reads zero.

It reports rather than asserts. Retention is unconditional today, so an
assertion would be red on every branch and would teach readers to scroll past
it. The readout is a banner plus a GitHub workflow warning annotation, so it
lands on the run summary and not only in the log body. It goes quiet on its own
once reclaim lands, at which point it can become a hard assertion.

Every probe step runs under sudo, the directory test included. The viperblock
base dir is 0700 spinifex-viperblock, so an unprivileged [ -d ] reads false even
when the volume is sitting right there, which would have silently downgraded
this to "no local state found" on every run.

Verified against a real retained WAL on a live node: 14853396868 bytes across
2028 files, rendered as 13.8 GiB at 99% of bytes written.
2026-09-22 22:58:50 +10:00
6947 changed files with 399397 additions and 711434 deletions
-3
View File
@@ -1,3 +0,0 @@
# The committed bundle carries the AWS SDK's template strings verbatim,
# including trailing spaces that are part of their values.
spinifex/runtime/roles/spinifexui/frontend/dist/** -whitespace
+15
View File
@@ -0,0 +1,15 @@
# CODEOWNERS — GitHub PR review enforcement
#
# Format: <path pattern> <owner> [<owner> …]
# Order matters — last matching pattern wins.
#
# Set the matching branch protection rule on `dev` and `main` to
# "Require review from Code Owners" or this file is advisory only.
# Network layer is governed by ADR-0006 invariant tests.
# Changes here must respect those clauses
# see spinifex/network/invariants/
# docs/development/proposals/az-local-architecture/0006-spinifex-network-layer-contract.md
/spinifex/network/ @joshsiv-mulga
/spinifex/vpcd/ @joshsiv-mulga
/spinifex/network/invariants/ @joshsiv-mulga
+1 -1
View File
@@ -8,7 +8,7 @@ Actions job summary and emits one `::error` annotation per failed test case.
```yaml
- name: Publish test summary
if: always()
uses: $/.github/actions/junit-report
uses: ./spinifex/.github/actions/junit-report # or your own checkout path
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
title: "Go E2E results"
+1 -1
View File
@@ -27,7 +27,7 @@
<line x1="510.0" y1="212.0" x2="510.0" y2="220.0" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
<rect x="28.0" y="224.0" width="964.0" height="94.0" rx="13" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
<text x="48.0" y="251.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#EE4B12" font-weight="700" text-anchor="start" letter-spacing="1.5">AWS GATEWAY</text>
<text x="48.0" y="272.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">spinifex/runtime/roles/awsgw · TLS termination · SigV4 verification · IAM policy eval · NATS routing</text>
<text x="48.0" y="272.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">spinifex/services/awsgw · TLS termination · SigV4 verification · IAM policy eval · NATS routing</text>
<rect x="46.0" y="287.0" width="68.2" height="22.0" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="80.1" y="301.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">TLS 1.3</text>
<rect x="123.2" y="287.0" width="88.0" height="22.0" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>

Before

Width:  |  Height:  |  Size: 20 KiB

After

Width:  |  Height:  |  Size: 20 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 167 KiB

+1 -1
View File
@@ -51,7 +51,7 @@
<!-- ═══ AWS GATEWAY ════════════════════════════════════ -->
<rect x="16" y="154" width="868" height="104" rx="8" fill="url(#gw-grad)" stroke="#DC2626" stroke-width="1.3" stroke-opacity="0.75"/>
<text x="32" y="176" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#F87171" font-weight="700" letter-spacing="1">AWS GATEWAY · port 9999</text>
<text x="876" y="176" font-family="'SF Mono', ui-monospace, monospace" font-size="9" fill="#71717a" text-anchor="end">spinifex/runtime/roles/awsgw/awsgw.go</text>
<text x="876" y="176" font-family="'SF Mono', ui-monospace, monospace" font-size="9" fill="#71717a" text-anchor="end">spinifex/services/awsgw/awsgw.go</text>
<!-- Two-column feature bullets -->
<g font-family="'Segoe UI', sans-serif" font-size="10.5" fill="#f5f5f5">

Before

Width:  |  Height:  |  Size: 12 KiB

After

Width:  |  Height:  |  Size: 12 KiB

@@ -1,79 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" width="1020" height="836" viewBox="0 0 1020 836" role="img" aria-label="Spinifex on a single Exoscale instance: the instance sits on the zone&#x27;s public network with one interface, eth0, holding its own address and the default route. Manual Elastic IPs are delivered to eth0 and routed by the host into OVN through the spx-nat transit veth, and an Exoscale block storage volume backs /var/lib/spinifex.">
<rect x="0" y="0" width="1020" height="836" rx="20" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="28" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2.5">SPINIFEX ON EXOSCALE — SINGLE NODE</text>
<text x="992" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="end">one instance · routed NAT · Elastic IPs</text>
<text x="28" y="72" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="23" fill="#F5F3EE" font-weight="800" text-anchor="start">One Exoscale instance, single Spinifex node</text>
<rect x="28" y="92" width="964" height="76" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="48" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">INTERNET</text>
<text x="972" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">AWS SDKs, SSH and guest egress all arrive here</text>
<rect x="46" y="126" width="260.0" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="176.0" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">aws-cli / Terraform / boto3 → :9999</text>
<rect x="315.0" y="126" width="171.6" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="400.8" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">ssh → guest Elastic IP</text>
<rect x="495.6" y="126" width="205.6" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="598.4" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">guest egress ← same address</text>
<line x1="510" y1="170" x2="510" y2="186" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
<rect x="28" y="188" width="964" height="582" rx="16" fill="#EE4B12" fill-opacity="0.022" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.45" stroke-dasharray="6 5"/>
<text x="50" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2">EXOSCALE ZONE DE-FRA-1 · PUBLIC NETWORK 91.92.116.0/23</text>
<text x="970" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">gateway 91.92.116.1 · security group open to all</text>
<rect x="48" y="232" width="924" height="520" rx="14" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
<text x="68" y="266" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="16" fill="#F5F3EE" font-weight="800" text-anchor="start">standard.huge</text>
<text x="212" y="266" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="400" text-anchor="start">spinifex-node-01 · 8 vCPU / 32 GiB · Ubuntu 26.04</text>
<text x="952" y="266" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">one host = one Spinifex region</text>
<line x1="68" y1="282" x2="952" y2="282" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
<rect x="68" y="298" width="434" height="108" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="88" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">eth0 — PUBLIC INTERFACE</text>
<text x="482" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">the only NIC</text>
<text x="88" y="356" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">91.92.116.250/23 · holds the default route</text>
<text x="88" y="378" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">advertise IP · API :9999 · SSH · host masquerade</text>
<rect x="518" y="298" width="434" height="108" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="538" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">ELASTIC IPs — MANUAL</text>
<text x="932" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">attached to the instance</text>
<text x="538" y="356" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">89.145.163.1/32 → delivered to eth0</text>
<text x="538" y="378" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">the host routes each /32 into OVN, never owns it</text>
<line x1="735" y1="406" x2="735" y2="422" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
<rect x="68" y="424" width="884" height="40" rx="11" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
<text x="88" y="449" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#EE4B12" font-weight="600" text-anchor="start">spx-nat-host ⇄ spx-nat-ovs</text>
<text x="296" y="449" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#9AA1A8" font-weight="400" text-anchor="start">transit 100.127.0.0/24 · EIP/32 routed in · masquerade out eth0 for the rest</text>
<line x1="510" y1="464" x2="510" y2="480" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
<rect x="68" y="482" width="884" height="100" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="88" y="512" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">SPINIFEX CONTROL PLANE</text>
<text x="932" y="512" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">every service is one binary: spx service &lt;name&gt;</text>
<rect x="86" y="530" width="93.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="132.5" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">awsgw :9999</text>
<rect x="188.0" y="530" width="58.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="217.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">daemon</text>
<rect x="255.0" y="530" width="44.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="277.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">vpcd</text>
<rect x="308.0" y="530" width="44.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="330.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">NATS</text>
<rect x="361.0" y="530" width="86.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="404.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">predastore</text>
<rect x="456.0" y="530" width="86.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="499.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">viperblock</text>
<rect x="551.0" y="530" width="79.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="590.5" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">northstar</text>
<rect x="639.0" y="530" width="72.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="675.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">ui :3000</text>
<text x="86" y="568" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">SigV4 in, NATS in the middle, QEMU/KVM (nested) · OVN/OVS · Predastore · Viperblock underneath</text>
<line x1="248" y1="582" x2="248" y2="598" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
<line x1="772" y1="582" x2="772" y2="598" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
<rect x="68" y="600" width="532" height="130" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="88" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">TENANT VPCs — OVN br-int</text>
<text x="580" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">geneve-local</text>
<rect x="86" y="644" width="496" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="100" y="666" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">vpc-a 172.31.0.0/16</text>
<text x="300" y="666" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-… 172.31.0.4 → 89.145.163.1</text>
<rect x="86" y="686" width="496" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="100" y="708" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">vpc-b 10.0.0.0/16</text>
<text x="300" y="708" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-… 10.0.1.9 · private, egress only</text>
<rect x="616" y="600" width="336" height="130" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="636" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">BLOCK STORAGE VOLUME</text>
<text x="932" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">256 GiB+</text>
<text x="636" y="658" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">virtio-blk → /dev/vdb</text>
<text x="636" y="680" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">/var/lib/spinifex</text>
<text x="636" y="700" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11.5" fill="#5B636C" font-weight="400" text-anchor="start">found by-id, defaults,nofail</text>
<text x="636" y="718" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11.5" fill="#5B636C" font-weight="400" text-anchor="start">viperblock · predastore · JetStream</text>
<text x="28" y="796" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">A guest&#x27;s public address is a manual Exoscale Elastic IP, delivered to eth0 addressed to itself — no private/public pair to translate.</text>
<text x="28" y="814" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">The host routes it into OVN, so inbound and outbound are the same address and the guest sees its private IP only.</text>
</svg>

Before

Width:  |  Height:  |  Size: 14 KiB

@@ -1,123 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" width="1020" height="796" viewBox="0 0 1020 796" role="img" aria-label="Spinifex across three Exoscale instances in one zone, spread by an anti-affinity group. Each has a public eth0 carrying its own Elastic IPs, a private-network eth1 for the cluster mesh, its own block storage volume and its share of the guests. Geneve, NATS JetStream, predastore erasure coding and the OVN raft join them east-west over the private network. This topology is not yet tested.">
<rect x="0" y="0" width="1020" height="796" rx="20" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="28" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2.5">SPINIFEX ON EXOSCALE — THREE NODES</text>
<text x="992" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="end">anti-affinity group · one zone · not yet tested</text>
<text x="28" y="72" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="23" fill="#F5F3EE" font-weight="800" text-anchor="start">Three Exoscale instances, redundant multi-node install</text>
<rect x="28" y="92" width="964" height="76" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="48" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">INTERNET</text>
<text x="972" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">one endpoint, whichever node answers</text>
<rect x="46" y="126" width="260.0" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="176.0" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">aws-cli / Terraform / boto3 → :9999</text>
<rect x="315.0" y="126" width="171.6" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="400.8" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">ssh → guest Elastic IP</text>
<rect x="495.6" y="126" width="205.6" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="598.4" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">guest egress ← same address</text>
<line x1="510" y1="170" x2="510" y2="186" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
<rect x="28" y="188" width="964" height="440" rx="16" fill="#EE4B12" fill-opacity="0.022" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.45" stroke-dasharray="6 5"/>
<text x="50" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2">EXOSCALE ZONE DE-FRA-1 · PUBLIC NETWORK + PRIVATE NETWORK</text>
<text x="970" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">one security group · one anti-affinity group</text>
<rect x="48" y="232" width="300" height="380" rx="14" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
<text x="64" y="260" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-01</text>
<text x="332" y="260" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">standard.huge</text>
<line x1="64" y1="272" x2="332" y2="272" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
<rect x="64" y="284" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="78" y="305" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth0 91.92.116.250</text>
<text x="78" y="323" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">public · default route · API</text>
<rect x="64" y="342" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="78" y="363" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth1 10.10.0.11</text>
<text x="78" y="381" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">private network · mesh · advertise · encap</text>
<rect x="64" y="400" width="268" height="50" rx="10" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
<text x="78" y="421" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start">EIP 89.145.163.1</text>
<text x="78" y="439" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">→ eth0 · /32 via spx-nat-host</text>
<rect x="64" y="462" width="51.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="89.5" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">awsgw</text>
<rect x="124.0" y="462" width="57.6" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="152.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">daemon</text>
<rect x="190.6" y="462" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="212.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">vpcd</text>
<rect x="64" y="492" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="86.2" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">nats</text>
<rect x="117.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="159.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">predastore</text>
<rect x="210.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="252.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">viperblock</text>
<rect x="64" y="526" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="76" y="548" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-23d2… 172.31.0.4 → 89.145.163.1</text>
<rect x="64" y="568" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="76" y="590" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">volume /dev/vdb → /var/lib/spinifex</text>
<rect x="360" y="232" width="300" height="380" rx="14" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
<text x="376" y="260" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-02</text>
<text x="644" y="260" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">standard.huge</text>
<line x1="376" y1="272" x2="644" y2="272" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
<rect x="376" y="284" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="390" y="305" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth0 91.92.117.14</text>
<text x="390" y="323" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">public · default route · API</text>
<rect x="376" y="342" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="390" y="363" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth1 10.10.0.12</text>
<text x="390" y="381" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">private network · mesh · advertise · encap</text>
<rect x="376" y="400" width="268" height="50" rx="10" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
<text x="390" y="421" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start">EIP 89.145.160.107</text>
<text x="390" y="439" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">→ eth0 · /32 via spx-nat-host</text>
<rect x="376" y="462" width="51.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="401.5" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">awsgw</text>
<rect x="436.0" y="462" width="57.6" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="464.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">daemon</text>
<rect x="502.6" y="462" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="524.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">vpcd</text>
<rect x="376" y="492" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="398.2" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">nats</text>
<rect x="429.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="471.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">predastore</text>
<rect x="522.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="564.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">viperblock</text>
<rect x="376" y="526" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="388" y="548" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-6b91… 172.31.0.9 → 89.145.160.107</text>
<rect x="376" y="568" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="388" y="590" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">volume /dev/vdb → /var/lib/spinifex</text>
<rect x="672" y="232" width="300" height="380" rx="14" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
<text x="688" y="260" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-03</text>
<text x="956" y="260" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">standard.huge</text>
<line x1="688" y1="272" x2="956" y2="272" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
<rect x="688" y="284" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="702" y="305" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth0 91.92.116.88</text>
<text x="702" y="323" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">public · default route · API</text>
<rect x="688" y="342" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="702" y="363" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth1 10.10.0.13</text>
<text x="702" y="381" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">private network · mesh · advertise · encap</text>
<rect x="688" y="400" width="268" height="50" rx="10" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
<text x="702" y="421" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start">EIP 89.145.162.131</text>
<text x="702" y="439" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">→ eth0 · /32 via spx-nat-host</text>
<rect x="688" y="462" width="51.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="713.5" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">awsgw</text>
<rect x="748.0" y="462" width="57.6" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="776.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">daemon</text>
<rect x="814.6" y="462" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="836.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">vpcd</text>
<rect x="688" y="492" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="710.2" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">nats</text>
<rect x="741.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="783.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">predastore</text>
<rect x="834.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="876.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">viperblock</text>
<rect x="688" y="526" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="700" y="548" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-0ae4… 172.31.1.6 → 89.145.162.131</text>
<rect x="688" y="568" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="700" y="590" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">volume /dev/vdb → /var/lib/spinifex</text>
<line x1="510" y1="628" x2="510" y2="644" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
<rect x="28" y="646" width="964" height="70" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="48" y="676" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">EAST–WEST, OVER THE EXOSCALE PRIVATE NETWORK</text>
<text x="972" y="676" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">no Elastic IP is involved · MTU 1500</text>
<rect x="46" y="684" width="124.0" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="108.0" y="698.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">Geneve UDP 6081</text>
<rect x="179.0" y="684" width="137.6" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="247.8" y="698.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">NATS JetStream R3</text>
<rect x="325.6" y="684" width="144.4" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="397.8" y="698.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">predastore RS(2,1)</text>
<rect x="479.0" y="684" width="151.2" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="554.6" y="698.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">OVN Northbound raft</text>
<rect x="639.2" y="684" width="56.0" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="667.2" y="698.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">OVSDB</text>
<text x="28" y="744" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Guest-to-guest traffic rides Geneve between the nodes&#x27; private-network interfaces — it never touches eth0 or an Elastic IP.</text>
<text x="28" y="762" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Each node carries the Elastic IPs of its own guests. Moving one between nodes is an Exoscale API call, not yet automated.</text>
<text x="28" y="780" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Three nodes is the smallest cluster that tolerates losing one; the anti-affinity group keeps them on different hypervisors.</text>
</svg>

Before

Width:  |  Height:  |  Size: 22 KiB

@@ -1,80 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" width="1020" height="836" viewBox="0 0 1020 836" role="img" aria-label="Spinifex on a single OCI instance: the instance sits inside the VCN public subnet 10.200.0.0/23; its primary VNIC carries the default route and the cluster mesh, its secondary VNIC is the br-wan bridge holding registered secondary private IPs paired with reserved public IPs; guests reach them through the spx-nat transit veth, and an iSCSI block volume backs /var/lib/spinifex.">
<defs><linearGradient id="nats" x1="0" y1="0" x2="1" y2="0"><stop offset="0" stop-color="#F5652E"/><stop offset="1" stop-color="#C63C0E"/></linearGradient></defs>
<rect x="0" y="0" width="1020" height="836" rx="20" fill="#14181D" stroke="#2A2F36" stroke-width="1"/>
<text x="28" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2.5">SPINIFEX ON OCI — SINGLE NODE</text>
<text x="992" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="end">one instance · routed NAT · real public addresses</text>
<text x="28" y="72" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="23" fill="#F5F3EE" font-weight="800" text-anchor="start">One OCI instance, single Spinifex node</text>
<rect x="28" y="92" width="964" height="76" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="48" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">INTERNET</text>
<text x="972" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">AWS SDKs, SSH and guest egress all arrive here</text>
<rect x="46" y="126" width="272.7" height="22" rx="7" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="182.3" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">aws-cli / Terraform / boto3 → :9999</text>
<rect x="327.7" y="126" width="177.2" height="22" rx="7" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="416.3" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">ssh → guest public IP</text>
<rect x="513.92" y="126" width="218.1" height="22" rx="7" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="623.0" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">guest egress ← same address</text>
<line x1="510.0" y1="170" x2="510.0" y2="186" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
<rect x="28" y="188" width="964" height="582" rx="16" fill="#EE4B12" fill-opacity="0.022" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.45" stroke-dasharray="6 5"/>
<text x="50" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2">OCI VCN 10.200.0.0/16 · PUBLIC SUBNET 10.200.0.0/23</text>
<text x="970" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">virtual router 10.200.0.1 · internet gateway</text>
<rect x="48" y="232" width="924" height="520" rx="14" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
<text x="68" y="266" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="16" fill="#F5F3EE" font-weight="800" text-anchor="start">BM.Standard.E2.64</text>
<text x="260" y="266" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="400" text-anchor="start">spinifex-node-01 · FAULT-DOMAIN-1</text>
<text x="952" y="266" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">one host = one Spinifex region</text>
<line x1="68" y1="282" x2="952" y2="282" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
<rect x="68" y="298" width="434.0" height="108" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="88" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">VNIC 0 — PRIMARY</text>
<text x="482.0" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">enp0s9</text>
<text x="88" y="356" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">10.200.0.252/23 · holds the default route</text>
<text x="88" y="378" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">advertise IP · ovn-encap-ip · NATS · API :9999</text>
<rect x="518.0" y="298" width="434.0" height="108" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="538.0" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">VNIC 1 — SECONDARY</text>
<text x="932.0" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">br-wan (Linux bridge)</text>
<text x="538.0" y="356" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">10.200.1.233/32 + ip rule → table 200</text>
<text x="538.0" y="378" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">secondary private IPs ⇄ reserved public IPs</text>
<line x1="735.0" y1="406" x2="735.0" y2="422" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
<rect x="68" y="424" width="884" height="40" rx="11" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
<text x="88" y="449" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#EE4B12" font-weight="600" text-anchor="start">spx-nat-host ⇄ spx-nat-ovs</text>
<text x="296" y="449" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#9AA1A8" font-weight="400" text-anchor="start">transit 100.127.0.0/24 · nft chain spinifex-nat-egress masquerades out br-wan</text>
<line x1="510.0" y1="464" x2="510.0" y2="480" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
<rect x="68" y="482" width="884" height="100" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="88" y="512" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">SPINIFEX CONTROL PLANE</text>
<text x="932" y="512" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">every service is one binary: spx service &lt;name&gt;</text>
<rect x="86" y="530" width="109.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="140.5" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">awsgw :9999</text>
<rect x="204.01999999999998" y="530" width="74.9" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="241.5" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">daemon</text>
<rect x="287.94" y="530" width="61.3" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="318.6" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">vpcd</text>
<rect x="358.22" y="530" width="61.3" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="388.9" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">NATS</text>
<rect x="428.5" y="530" width="102.2" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="479.6" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">predastore</text>
<rect x="539.7" y="530" width="102.2" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="590.8" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">viperblock</text>
<rect x="650.9000000000001" y="530" width="95.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="698.6" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">northstar</text>
<rect x="755.2800000000001" y="530" width="88.6" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="799.6" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">ui :3000</text>
<text x="86" y="568" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">SigV4 in, NATS in the middle, QEMU/KVM · OVN/OVS · Predastore · Viperblock underneath</text>
<line x1="248" y1="582" x2="248" y2="598" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
<line x1="772" y1="582" x2="772" y2="598" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
<rect x="68" y="600" width="532" height="130" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="88" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">TENANT VPCs — OVN br-int</text>
<text x="580" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">geneve-local</text>
<rect x="86" y="644" width="496" height="34" rx="9" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="100" y="666" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">vpc-a 10.0.0.0/16</text>
<text x="310" y="666" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-… 10.0.1.4 → 150.230.13.131</text>
<rect x="86" y="686" width="496" height="34" rx="9" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="100" y="708" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">vpc-b 172.31.0.0/16</text>
<text x="310" y="708" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-… 172.31.4.9 · private, egress only</text>
<rect x="616" y="600" width="336" height="130" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="636" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">BLOCK VOLUME — iSCSI</text>
<text x="932" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">256 GB+</text>
<text x="636" y="658" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">169.254.2.2:3260 → /dev/sdb</text>
<text x="636" y="680" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">/var/lib/spinifex</text>
<text x="636" y="700" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11.5" fill="#5B636C" font-weight="400" text-anchor="start">mounted by UUID, defaults,_netdev,nofail</text>
<text x="636" y="718" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11.5" fill="#5B636C" font-weight="400" text-anchor="start">viperblock · predastore · JetStream</text>
<text x="28" y="796" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">A guest's public address is a real OCI reserved public IP, attached to a secondary private IP on br-wan.</text>
<text x="28" y="814" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Inbound and outbound are therefore the same address, and OCI's per-VNIC source check passes.</text>
</svg>

Before

Width:  |  Height:  |  Size: 15 KiB

-116
View File
@@ -1,116 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" width="1020" height="796" viewBox="0 0 1020 796" role="img" aria-label="Spinifex across three OCI instances: all three sit inside the VCN public subnet 10.200.0.0/23, one per fault domain. Each carries a primary VNIC for the cluster mesh and a br-wan secondary VNIC holding its reserved public addresses, its own iSCSI block volume, and its share of the guests. Geneve, NATS JetStream, predastore erasure coding and the OVN raft join them east-west.">
<defs><linearGradient id="nats" x1="0" y1="0" x2="1" y2="0"><stop offset="0" stop-color="#F5652E"/><stop offset="1" stop-color="#C63C0E"/></linearGradient></defs>
<rect x="0" y="0" width="1020" height="796" rx="20" fill="#14181D" stroke="#2A2F36" stroke-width="1"/>
<text x="28" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2.5">SPINIFEX ON OCI — THREE NODES</text>
<text x="992" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="end">three fault domains · one region · one API endpoint</text>
<text x="28" y="72" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="23" fill="#F5F3EE" font-weight="800" text-anchor="start">Three OCI instances, redundant multi-node install</text>
<rect x="28" y="92" width="964" height="76" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="48" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">INTERNET</text>
<text x="972" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">one endpoint, whichever node answers</text>
<rect x="46" y="126" width="272.7" height="22" rx="7" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="182.3" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">aws-cli / Terraform / boto3 → :9999</text>
<rect x="327.7" y="126" width="177.2" height="22" rx="7" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="416.3" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">ssh → guest public IP</text>
<rect x="513.92" y="126" width="218.1" height="22" rx="7" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="623.0" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">guest egress ← same address</text>
<line x1="510.0" y1="170" x2="510.0" y2="186" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
<rect x="28" y="188" width="964" height="464" rx="16" fill="#EE4B12" fill-opacity="0.022" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.45" stroke-dasharray="6 5"/>
<text x="50" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2">OCI VCN 10.200.0.0/16 · PUBLIC SUBNET 10.200.0.0/23</text>
<text x="970" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">virtual router 10.200.0.1 · internet gateway</text>
<rect x="48.0" y="232" width="297.3" height="272" rx="13" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
<text x="66.0" y="262" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-01</text>
<text x="327.3333333333333" y="262" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#EE4B12" font-weight="400" text-anchor="end" fill-opacity="0.85">FAULT-DOMAIN-1</text>
<text x="66.0" y="280" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">BM.Standard.E2.64</text>
<line x1="66.0" y1="292" x2="327.3333333333333" y2="292" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
<text x="66.0" y="314" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">VNIC 0 10.200.0.252</text>
<text x="66.0" y="330" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11" fill="#5B636C" font-weight="400" text-anchor="start">mesh · advertise · encap</text>
<text x="66.0" y="352" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">VNIC 1 br-wan 10.200.1.233/32</text>
<text x="66.0" y="368" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11" fill="#5B636C" font-weight="400" text-anchor="start">public IP pairs · table 200</text>
<rect x="66.0" y="382" width="51.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="91.5" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">awsgw</text>
<rect x="123.0" y="382" width="57.2" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="151.6" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">daemon</text>
<rect x="186.2" y="382" width="44.8" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="208.6" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">vpcd</text>
<rect x="66.0" y="408" width="44.8" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="88.4" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">nats</text>
<rect x="116.8" y="408" width="82.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="157.8" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">predastore</text>
<rect x="204.8" y="408" width="82.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="245.8" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">viperblock</text>
<rect x="66.0" y="438" width="261.3" height="24" rx="8" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="76.0" y="454" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-23d2… 10.0.1.4 → 150.230.13.131</text>
<rect x="66.0" y="466" width="261.3" height="24" rx="8" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="76.0" y="482" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">iSCSI /dev/sdb → /var/lib/spinifex</text>
<line x1="345.3333333333333" y1="364" x2="361.3333333333333" y2="364" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.5" stroke-dasharray="2 4"/>
<rect x="361.3333333333333" y="232" width="297.3" height="272" rx="13" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
<text x="379.3333333333333" y="262" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-02</text>
<text x="640.6666666666666" y="262" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#EE4B12" font-weight="400" text-anchor="end" fill-opacity="0.85">FAULT-DOMAIN-2</text>
<text x="379.3333333333333" y="280" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">BM.Standard.E2.64</text>
<line x1="379.3333333333333" y1="292" x2="640.6666666666666" y2="292" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
<text x="379.3333333333333" y="314" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">VNIC 0 10.200.0.118</text>
<text x="379.3333333333333" y="330" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11" fill="#5B636C" font-weight="400" text-anchor="start">mesh · advertise · encap</text>
<text x="379.3333333333333" y="352" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">VNIC 1 br-wan 10.200.1.96/32</text>
<text x="379.3333333333333" y="368" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11" fill="#5B636C" font-weight="400" text-anchor="start">public IP pairs · table 200</text>
<rect x="379.3333333333333" y="382" width="51.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="404.8" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">awsgw</text>
<rect x="436.3333333333333" y="382" width="57.2" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="464.9" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">daemon</text>
<rect x="499.5333333333333" y="382" width="44.8" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="521.9" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">vpcd</text>
<rect x="379.3333333333333" y="408" width="44.8" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="401.7" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">nats</text>
<rect x="430.1333333333333" y="408" width="82.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="471.1" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">predastore</text>
<rect x="518.1333333333333" y="408" width="82.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="559.1" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">viperblock</text>
<rect x="379.3333333333333" y="438" width="261.3" height="24" rx="8" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="389.3333333333333" y="454" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-6b91… 10.0.1.9 → 137.23.9.125</text>
<rect x="379.3333333333333" y="466" width="261.3" height="24" rx="8" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="389.3333333333333" y="482" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">iSCSI /dev/sdb → /var/lib/spinifex</text>
<line x1="658.6666666666666" y1="364" x2="674.6666666666666" y2="364" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.5" stroke-dasharray="2 4"/>
<rect x="674.6666666666666" y="232" width="297.3" height="272" rx="13" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
<text x="692.6666666666666" y="262" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-03</text>
<text x="954.0" y="262" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#EE4B12" font-weight="400" text-anchor="end" fill-opacity="0.85">FAULT-DOMAIN-3</text>
<text x="692.6666666666666" y="280" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">BM.Standard.E2.64</text>
<line x1="692.6666666666666" y1="292" x2="954.0" y2="292" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
<text x="692.6666666666666" y="314" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">VNIC 0 10.200.0.41</text>
<text x="692.6666666666666" y="330" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11" fill="#5B636C" font-weight="400" text-anchor="start">mesh · advertise · encap</text>
<text x="692.6666666666666" y="352" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">VNIC 1 br-wan 10.200.1.12/32</text>
<text x="692.6666666666666" y="368" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11" fill="#5B636C" font-weight="400" text-anchor="start">public IP pairs · table 200</text>
<rect x="692.6666666666666" y="382" width="51.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="718.2" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">awsgw</text>
<rect x="749.6666666666666" y="382" width="57.2" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="778.3" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">daemon</text>
<rect x="812.8666666666667" y="382" width="44.8" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="835.3" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">vpcd</text>
<rect x="692.6666666666666" y="408" width="44.8" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="715.1" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">nats</text>
<rect x="743.4666666666666" y="408" width="82.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="784.5" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">predastore</text>
<rect x="831.4666666666666" y="408" width="82.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="872.5" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">viperblock</text>
<rect x="692.6666666666666" y="438" width="261.3" height="24" rx="8" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="702.6666666666666" y="454" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-0ae4… 10.0.2.6 → 150.230.4.88</text>
<rect x="692.6666666666666" y="466" width="261.3" height="24" rx="8" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="702.6666666666666" y="482" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">iSCSI /dev/sdb → /var/lib/spinifex</text>
<rect x="48" y="524" width="924" height="104" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
<text x="68" y="554" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">EAST–WEST, INSIDE THE VCN</text>
<text x="952" y="554" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">no public address is involved</text>
<rect x="66" y="574" width="136.3" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="134.2" y="588.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">Geneve UDP 6081</text>
<rect x="211.3" y="574" width="149.9" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="286.3" y="588.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">NATS JetStream R3</text>
<rect x="370.24" y="574" width="156.8" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="448.6" y="588.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">predastore RS(2,1)</text>
<rect x="536.0" y="574" width="163.6" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="617.8" y="588.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">OVN Northbound raft</text>
<rect x="708.58" y="574" width="68.1" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
<text x="742.6" y="588.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">OVSDB</text>
<text x="66" y="612" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Guest-to-guest traffic rides Geneve between the nodes' primary VNICs — it never leaves the subnet and never touches br-wan.</text>
<rect x="28" y="668" width="964" height="46" rx="12" fill="#EE4B12" fill-opacity="0.07" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.55"/>
<text x="48" y="688" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="13" fill="#F5F3EE" font-weight="700" text-anchor="start">A guest that moves takes its address with it.</text>
<text x="48" y="705" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#9AA1A8" font-weight="400" text-anchor="start">The reserved public IP follows the secondary private IP to whichever node's br-wan the guest lands on — the tenant sees no change.</text>
<text x="28" y="756" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Three nodes is the smallest cluster that tolerates losing one: NATS JetStream keeps quorum, predastore RS(2,1)</text>
<text x="28" y="774" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">keeps the data readable, and the OVN Northbound raft keeps a leader. One node per fault domain is what makes that real.</text>
</svg>

Before

Width:  |  Height:  |  Size: 21 KiB

+46 -33
View File
@@ -19,8 +19,8 @@
</defs>
<rect width="900" height="480" fill="url(#bg)" rx="12"/>
<text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">NGINX + ALB · PUBLIC SUBNETS · SPREAD ACROSS NODES</text>
<text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">10.20.0.0/16 · server_count = 3</text>
<text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">NGINX + ALB · VPC WITH PUBLIC AND PRIVATE SUBNETS</text>
<text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">10.20.0.0/16 · multi-AZ</text>
<!-- Internet -->
<rect x="350" y="54" width="200" height="40" rx="8" fill="url(#inet-grad)" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.55"/>
@@ -41,45 +41,58 @@
<line x1="230" y1="184" x2="230" y2="202" stroke="#DC2626" stroke-width="1" stroke-dasharray="3,3" opacity="0.6"/>
<line x1="670" y1="184" x2="670" y2="202" stroke="#DC2626" stroke-width="1" stroke-dasharray="3,3" opacity="0.6"/>
<!-- public_a -->
<g transform="translate(108, 202)">
<rect width="244" height="150" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
<rect width="244" height="104" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
<text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">public_a</text>
<text x="224" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">10.20.1.0/24</text>
<text x="224" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">az-a</text>
<line x1="16" y1="32" x2="228" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
<rect x="22" y="42" width="200" height="22" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
<text x="122" y="57" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle" font-weight="600">ALB ENI</text>
<rect x="22" y="70" width="200" height="36" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
<text x="122" y="86" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle" font-weight="600">nginx-alb-1</text>
<text x="122" y="100" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">public IP · Ubuntu · cloud-init</text>
<rect x="22" y="110" width="200" height="36" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
<text x="122" y="126" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle" font-weight="600">nginx-alb-3</text>
<text x="122" y="140" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">public IP · Ubuntu · cloud-init</text>
<rect x="20" y="42" width="200" height="22" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
<text x="120" y="57" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle" font-weight="600">ALB ENI</text>
<rect x="20" y="70" width="200" height="22" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
<text x="120" y="85" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle" font-weight="600">NAT GW</text>
</g>
<!-- public_b -->
<g transform="translate(548, 202)">
<rect width="244" height="150" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
<rect width="244" height="104" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
<text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">public_b</text>
<text x="224" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">10.20.2.0/24</text>
<text x="224" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">az-b</text>
<line x1="16" y1="32" x2="228" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
<rect x="22" y="42" width="200" height="22" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
<text x="122" y="57" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle" font-weight="600">ALB ENI</text>
<rect x="22" y="70" width="200" height="36" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
<text x="122" y="86" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle" font-weight="600">nginx-alb-2</text>
<text x="122" y="100" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">public IP · Ubuntu · cloud-init</text>
<rect x="20" y="56" width="200" height="22" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
<text x="120" y="71" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle" font-weight="600">ALB ENI</text>
</g>
<!-- spread placement group -->
<rect x="108" y="372" width="684" height="88" rx="8" fill="url(#priv-grad)" stroke="#a855f7" stroke-width="1.2" stroke-opacity="0.55"/>
<text x="128" y="394" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#c084fc" font-weight="700">nginx-alb-spread</text>
<text x="772" y="394" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#d8b4fe" text-anchor="end">placement group · strategy spread · one server per node</text>
<line x1="124" y1="402" x2="776" y2="402" stroke="#a855f7" stroke-opacity="0.3" stroke-width="0.5"/>
<rect x="128" y="412" width="204" height="36" rx="5" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
<text x="230" y="428" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#d8b4fe" text-anchor="middle" font-weight="700">node1</text>
<text x="230" y="442" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">one web server</text>
<rect x="348" y="412" width="204" height="36" rx="5" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
<text x="450" y="428" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#d8b4fe" text-anchor="middle" font-weight="700">node2</text>
<text x="450" y="442" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">one web server</text>
<rect x="568" y="412" width="204" height="36" rx="5" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
<text x="670" y="428" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#d8b4fe" text-anchor="middle" font-weight="700">node3</text>
<text x="670" y="442" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">one web server</text>
<!-- SNAT arrow from public_a NAT → private row -->
<line x1="230" y1="306" x2="230" y2="340" stroke="#F87171" stroke-width="1.2" stroke-dasharray="3,3" opacity="0.65"/>
<polygon points="226,336 234,336 230,344" fill="#F87171" opacity="0.8"/>
<rect x="256" y="314" width="240" height="20" rx="10" fill="#0a0a0a" stroke="#F87171" stroke-width="0.6" stroke-opacity="0.55"/>
<text x="376" y="328" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#FCA5A5" text-anchor="middle">SNAT for private subnets (egress)</text>
<!-- private_a -->
<g transform="translate(108, 350)">
<rect width="244" height="104" rx="8" fill="url(#priv-grad)" stroke="#a855f7" stroke-width="1.2" stroke-opacity="0.55"/>
<text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#c084fc" font-weight="700">private_a</text>
<text x="224" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#d8b4fe" text-anchor="end">az-a</text>
<line x1="16" y1="32" x2="228" y2="32" stroke="#a855f7" stroke-opacity="0.3" stroke-width="0.5"/>
<rect x="20" y="44" width="200" height="48" rx="5" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
<text x="120" y="64" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#d8b4fe" text-anchor="middle" font-weight="700">nginx_1</text>
<text x="120" y="80" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">Debian 12 · cloud-init</text>
</g>
<!-- private_b -->
<g transform="translate(548, 350)">
<rect width="244" height="104" rx="8" fill="url(#priv-grad)" stroke="#a855f7" stroke-width="1.2" stroke-opacity="0.55"/>
<text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#c084fc" font-weight="700">private_b</text>
<text x="224" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#d8b4fe" text-anchor="end">az-b</text>
<line x1="16" y1="32" x2="228" y2="32" stroke="#a855f7" stroke-opacity="0.3" stroke-width="0.5"/>
<rect x="20" y="44" width="200" height="48" rx="5" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
<text x="120" y="64" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#d8b4fe" text-anchor="middle" font-weight="700">nginx_2</text>
<text x="120" y="80" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">Debian 12 · cloud-init</text>
</g>
</svg>

Before

Width:  |  Height:  |  Size: 7.5 KiB

After

Width:  |  Height:  |  Size: 7.1 KiB

@@ -1,110 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" width="900" height="580" viewBox="0 0 900 580">
<defs>
<linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
<stop offset="0%" style="stop-color:#0a0a0a"/>
<stop offset="100%" style="stop-color:#0d1117"/>
</linearGradient>
<linearGradient id="vpc-grad" x1="0%" y1="0%" x2="0%" y2="100%">
<stop offset="0%" style="stop-color:#58a6ff14"/>
<stop offset="100%" style="stop-color:#58a6ff05"/>
</linearGradient>
<linearGradient id="node-grad" x1="0%" y1="0%" x2="0%" y2="100%">
<stop offset="0%" style="stop-color:#3fb9501f"/>
<stop offset="100%" style="stop-color:#3fb95008"/>
</linearGradient>
<linearGradient id="svc-grad" x1="0%" y1="0%" x2="0%" y2="100%">
<stop offset="0%" style="stop-color:#a855f71f"/>
<stop offset="100%" style="stop-color:#a855f708"/>
</linearGradient>
<marker id="arrow" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse">
<path d="M0,0 L10,5 L0,10 z" fill="#71717a"/>
</marker>
</defs>
<rect width="900" height="580" fill="url(#bg)" rx="12"/>
<text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">SPINIFEX ON VULTR · ONE NODE OR THREE</text>
<text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">--nodes 1 or 3 · vx1-g-16c-64g or bare metal</text>
<rect x="30" y="52" width="250" height="56" rx="8" fill="#e4e4e70d" stroke="#e4e4e7" stroke-width="1.1" stroke-opacity="0.6"/>
<text x="155.0" y="76" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#e4e4e7" text-anchor="middle" font-weight="700">Your machine</text>
<text x="155.0" y="94" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">deploy-vultr.sh · OpenTofu or Terraform</text>
<rect x="325" y="52" width="250" height="56" rx="8" fill="#3fb9500d" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.6"/>
<text x="450.0" y="76" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#3fb950" text-anchor="middle" font-weight="700">Internet</text>
<text x="450.0" y="94" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#7ee787" text-anchor="middle">customers reach your EC2 and load balancers</text>
<rect x="620" y="52" width="250" height="56" rx="8" fill="#58a6ff12" stroke="#58a6ff" stroke-width="1.1" stroke-opacity="0.6"/>
<text x="745.0" y="76" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#79c0ff" text-anchor="middle" font-weight="700">Vultr API</text>
<text x="745.0" y="94" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#a5d6ff" text-anchor="middle">VPC · servers · reserved IPs</text>
<rect x="30" y="160" width="840" height="400" rx="10" fill="url(#vpc-grad)" stroke="#58a6ff" stroke-width="1.2" stroke-opacity="0.55" stroke-dasharray="6,4"/>
<text x="50" y="184" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#79c0ff" font-weight="700">Vultr VPC</text>
<text x="850" y="184" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#58a6ff" text-anchor="end">private network 10.10.8.0/24</text>
<line x1="155" y1="108" x2="155" y2="158" stroke="#71717a" stroke-width="1.2" opacity="0.8" marker-end="url(#arrow)"/>
<text x="165" y="138" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#a1a1aa" text-anchor="start">Terraform builds · SSH installs</text>
<line x1="450" y1="108" x2="450" y2="158" stroke="#71717a" stroke-width="1.2" opacity="0.8" marker-end="url(#arrow)"/>
<text x="460" y="138" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#a1a1aa" text-anchor="start">reserved IPs</text>
<line x1="745" y1="158" x2="745" y2="110" stroke="#71717a" stroke-width="1.2" opacity="0.8" marker-end="url(#arrow)"/>
<text x="735" y="138" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#a1a1aa" text-anchor="end">Spinifex calls the API</text>
<rect x="60" y="205" width="250" height="118" rx="8" fill="url(#node-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.6"/>
<text x="78" y="229" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#3fb950" font-weight="700">node1</text>
<text x="292" y="229" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">Ubuntu 24.04 · Spinifex</text>
<line x1="74" y1="237" x2="296" y2="237" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.6"/>
<rect x="76" y="250" width="74" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
<text x="113.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">AWS API</text>
<rect x="160" y="250" width="60" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
<text x="190.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">EBS</text>
<rect x="230" y="250" width="60" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
<text x="260.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">S3</text>
<text x="185" y="300" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">VPC networking · your instances</text>
<rect x="325" y="205" width="250" height="118" rx="8" fill="url(#node-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.6" stroke-dasharray="5,3"/>
<text x="343" y="229" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#3fb950" font-weight="700">node2</text>
<text x="557" y="229" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">with --nodes 3</text>
<line x1="339" y1="237" x2="561" y2="237" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.6"/>
<rect x="341" y="250" width="74" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
<text x="378.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">AWS API</text>
<rect x="425" y="250" width="60" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
<text x="455.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">EBS</text>
<rect x="495" y="250" width="60" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
<text x="525.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">S3</text>
<text x="450" y="300" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">VPC networking · your instances</text>
<rect x="590" y="205" width="250" height="118" rx="8" fill="url(#node-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.6" stroke-dasharray="5,3"/>
<text x="608" y="229" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#3fb950" font-weight="700">node3</text>
<text x="822" y="229" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">with --nodes 3</text>
<line x1="604" y1="237" x2="826" y2="237" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.6"/>
<rect x="606" y="250" width="74" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
<text x="643.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">AWS API</text>
<rect x="690" y="250" width="60" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
<text x="720.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">EBS</text>
<rect x="760" y="250" width="60" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
<text x="790.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">S3</text>
<text x="715" y="300" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">VPC networking · your instances</text>
<line x1="310" y1="264" x2="325" y2="264" stroke="#58a6ff" stroke-width="1.6" stroke-dasharray="3,3" opacity="0.9"/>
<line x1="575" y1="264" x2="590" y2="264" stroke="#58a6ff" stroke-width="1.6" stroke-dasharray="3,3" opacity="0.9"/>
<text x="450" y="348" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#79c0ff" text-anchor="middle">nodes cluster over the private network: control plane, EBS replicas and S3 shards</text>
<rect x="60" y="370" width="780" height="166" rx="8" fill="url(#svc-grad)" stroke="#a855f7" stroke-width="1.2" stroke-opacity="0.55"/>
<text x="80" y="394" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#c084fc" font-weight="700">AWS services across the fleet</text>
<text x="820" y="394" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#d8b4fe" text-anchor="end">spread across the nodes; with three, moved to a survivor if one fails</text>
<line x1="76" y1="402" x2="824" y2="402" stroke="#a855f7" stroke-opacity="0.3" stroke-width="0.6"/>
<rect x="76" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
<text x="119.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">EC2</text>
<text x="119.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">instances</text>
<rect x="169" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
<text x="212.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">ALB / NLB</text>
<text x="212.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">load balancers</text>
<rect x="262" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
<text x="305.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">EKS</text>
<text x="305.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">Kubernetes</text>
<rect x="355" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
<text x="398.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">ECS</text>
<text x="398.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">containers</text>
<rect x="448" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
<text x="491.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">RDS</text>
<text x="491.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">databases</text>
<rect x="541" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
<text x="584.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">S3</text>
<text x="584.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">object storage</text>
<rect x="634" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
<text x="677.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">EBS</text>
<text x="677.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">volumes</text>
<rect x="727" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
<text x="770.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">IAM</text>
<text x="770.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">access</text>
<text x="450" y="500" font-family="'Segoe UI', system-ui, sans-serif" font-size="10.5" fill="#a1a1aa" text-anchor="middle">Use the AWS CLI, SDKs and Terraform against your own endpoint, unmodified.</text>
<text x="450" y="520" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#71717a" text-anchor="middle">Public addresses are Vultr reserved IPs, attached to the node each resource runs on.</text>
</svg>

Before

Width:  |  Height:  |  Size: 13 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 725 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 743 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 151 KiB

+6 -26
View File
@@ -23,6 +23,11 @@ updates:
timezone: Australia/Sydney
cooldown:
default-days: 3
ignore:
- dependency-name: "*"
update-types:
- "version-update:semver-minor"
- "version-update:semver-patch"
groups:
github-actions:
patterns:
@@ -30,7 +35,7 @@ updates:
- package-ecosystem: npm
target-branch: dev
directory: /spinifex/runtime/roles/spinifexui/frontend
directory: /spinifex/services/spinifexui/frontend
schedule:
interval: weekly
day: sunday
@@ -45,28 +50,3 @@ updates:
npm-dependencies:
patterns:
- "*"
# Base images are pinned by digest, so this is what moves them. The golang
# version is deliberate and changes with go.mod; only its digest moves here.
- package-ecosystem: docker
target-branch: dev
directories:
- /build
- /scripts
- /docs/terraform-workbooks/demo-app
schedule:
interval: weekly
day: sunday
timezone: Australia/Sydney
cooldown:
default-days: 3
ignore:
- dependency-name: golang
update-types:
- "version-update:semver-major"
- "version-update:semver-minor"
- "version-update:semver-patch"
groups:
docker-images:
patterns:
- "*"
+3 -109
View File
@@ -8,8 +8,7 @@
"arch": "x86_64",
"runner": "ci-single",
"extra": "",
"suite": "",
"timeout_minutes": 60
"suite": ""
},
{
"cell": 2,
@@ -20,8 +19,7 @@
"arch": "x86_64",
"runner": "ci-single",
"extra": "",
"suite": "",
"timeout_minutes": 60
"suite": ""
},
{
"cell": 3,
@@ -32,8 +30,7 @@
"arch": "x86_64",
"runner": "ci-single",
"extra": "",
"suite": "",
"timeout_minutes": 60
"suite": ""
},
{
"cell": 8,
@@ -146,108 +143,5 @@
"extra": "",
"suite": "storagefault",
"timeout_minutes": 100
},
{
"cell": 30,
"install": "binary",
"topo": "real-multi",
"net": "nat",
"host": "debian-13",
"arch": "x86_64",
"runner": "ci-multi",
"extra": "nat-multi",
"suite": ""
},
{
"cell": 31,
"install": "binary",
"topo": "real-multi",
"net": "static",
"host": "debian-13",
"arch": "x86_64",
"runner": "ci-multi",
"extra": "",
"suite": "instancerecovery",
"timeout_minutes": 125
},
{
"cell": 32,
"install": "binary",
"topo": "real-multi",
"net": "static",
"host": "debian-13",
"arch": "x86_64",
"runner": "ci-multi",
"extra": "",
"suite": "lbrecovery",
"timeout_minutes": 90
},
{
"cell": 33,
"install": "binary",
"topo": "real-multi",
"net": "static",
"host": "debian-13",
"arch": "x86_64",
"runner": "ci-multi",
"extra": "",
"suite": "kvquorum",
"timeout_minutes": 55
},
{
"cell": 34,
"install": "binary",
"topo": "single",
"net": "dhcp",
"host": "oracle-10",
"arch": "x86_64",
"runner": "ci-single",
"extra": "",
"suite": "",
"timeout_minutes": 60
},
{
"cell": 35,
"install": "binary",
"topo": "single",
"net": "static",
"host": "oracle-10",
"arch": "x86_64",
"runner": "ci-single",
"extra": "reboot-single",
"suite": ""
},
{
"cell": 36,
"install": "binary",
"topo": "single",
"net": "nat",
"host": "oracle-10",
"arch": "x86_64",
"runner": "ci-single",
"extra": "nat-single",
"suite": ""
},
{
"cell": 37,
"install": "binary",
"topo": "real-multi",
"net": "dhcp",
"host": "oracle-10",
"arch": "x86_64",
"runner": "ci-multi",
"extra": "",
"suite": ""
},
{
"cell": 38,
"install": "binary",
"topo": "real-multi",
"net": "nat",
"host": "oracle-10",
"arch": "x86_64",
"runner": "ci-multi",
"extra": "nat-multi",
"suite": ""
}
]
-4
View File
@@ -5,7 +5,3 @@
## Changes
<!-- The notable changes, as bullets. -->
## Community Contribution License
- [ ] All community contributions in this pull request are licensed to Mulga under the terms of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). By creating this pull request I represent that I have the right to license these contributions to Mulga under the Apache License 2.0.
-148
View File
@@ -1,148 +0,0 @@
#!/usr/bin/env bash
# e2e-select.sh push <before> <sha> | suites "<e2e-x ...>"
# Prints the e2e gate's single_legs, multi_suites and multi_timeout outputs.
# Exits non-zero on anything it cannot read, which fails the e2e status.
set -euo pipefail
SCRIPTS_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
# shellcheck source=suite-sets.sh
. "${SCRIPTS_DIR}/suite-sets.sh"
# Provision, install and teardown around the suites; the floor is what the
# jobs have always had, so a budget can only be raised from it.
SETUP_ALLOWANCE_MIN=15
FLOOR_MIN=60
BASE_RUNNER=ci-single
SERVICE_RUNNER=ci-pseudo
log() { printf '[e2e-select] %s\n' "$*" >&2; }
die() { log "error: $*"; exit 1; }
# in_list <word> <list>
in_list() {
local w
for w in $2; do [ "$w" = "$1" ] && return 0; done
return 1
}
# keep <list> <eligible>: the words of <list> that are in <eligible>, deduplicated.
keep() {
local out="" s
for s in $1; do
in_list "$s" "$2" || continue
in_list "$s" "$out" || out="${out:+$out }$s"
done
printf '%s' "$out"
}
# budget <list>: max over the suites' own timeouts plus the setup allowance.
# A sum would demand hours for a set that shares one job and never takes them.
budget() {
local want="${FLOOR_MIN}" s t
for s in $1; do
t="$(e2e_suite_timeout "$s")"
t=$((${t%m} + SETUP_ALLOWANCE_MIN))
[ "$t" -gt "$want" ] && want="$t"
done
printf '%s' "$want"
}
prefixed() {
local out="" s
for s in $1; do out="${out:+$out }e2e-$s"; done
printf '%s' "$out"
}
# leg <name> <runner> <suites>: one matrix entry, or nothing for no suites.
leg() {
[ -n "$3" ] || return 0
jq -nc --arg leg "$1" --arg runner "$2" --arg suites "$(prefixed "$3")" \
--argjson timeout "$(budget "$3")" \
'{leg: $leg, runner: $runner, suites: $suites, timeout: $timeout}'
}
# emit <base-single> <service-single> <multi>
emit() {
local legs
legs=$( { leg base "${BASE_RUNNER}" "$1"; leg service "${SERVICE_RUNNER}" "$2"; } | jq -sc .)
log "base leg: ${1:-none}"
log "service leg: ${2:-none}"
log "multi: ${3:-none}"
printf 'single_legs=%s\n' "$legs"
printf 'multi_suites=%s\n' "$(prefixed "$3")"
printf 'multi_timeout=%s\n' "$(budget "$3")"
}
# select_for_paths <newline-separated paths> [all]: base set plus matches.
# With "all", every mapped suite is selected, for a push with no baseline.
select_for_paths() {
local matched="" p s
if [ "${2:-}" = all ]; then
matched=$(e2e_path_suites_all | tr '\n' ' ')
else
while IFS= read -r p; do
[ -n "$p" ] || continue
for s in $(e2e_suites_for_path "$p"); do
in_list "$s" "$matched" || { matched="${matched:+$matched }$s"; log "${p} -> ${s}"; }
done
done <<<"$1"
fi
# A matched suite runs where it is eligible, preferring one node, so route53
# goes to the service leg and lb to the multi job.
local extra_single="" extra_multi=""
for s in $matched; do
if in_list "$s" "${E2E_SUITES_SINGLE}"; then
extra_single="${extra_single:+$extra_single }$s"
elif in_list "$s" "${E2E_SUITES_MULTI}"; then
extra_multi="${extra_multi:+$extra_multi }$s"
else
die "E2E_PATH_SUITES names '${s}', which no topology can run"
fi
done
local base_single base_multi
base_single=$(keep "${E2E_SUITES_PUSH_BASE}" "${E2E_SUITES_SINGLE}")
base_multi=$(keep "${E2E_SUITES_PUSH_BASE}" "${E2E_SUITES_MULTI}")
# A matched suite already in the base leg runs there, not twice.
extra_single=$(for s in $extra_single; do in_list "$s" "$base_single" || printf '%s ' "$s"; done)
emit "$base_single" "$(keep "$extra_single" "${E2E_SUITES_SINGLE}")" \
"$(keep "$base_multi $extra_multi" "${E2E_SUITES_MULTI}")"
}
mode="${1:-}"
case "$mode" in
push)
[ $# -eq 3 ] || die "usage: $0 push <before> <sha>"
before="$2" sha="$3"
git cat-file -e "${sha}^{commit}" 2>/dev/null || die "commit ${sha} is not in this checkout"
if [ -z "${before//0/}" ]; then
log "no previous commit for this push; selecting every mapped suite"
select_for_paths "" all
exit 0
fi
if ! git cat-file -e "${before}^{commit}" 2>/dev/null; then
git fetch -q --filter=blob:none origin "${before}" 2>/dev/null || true
fi
if ! git cat-file -e "${before}^{commit}" 2>/dev/null; then
log "previous commit ${before} is gone; selecting every mapped suite"
select_for_paths "" all
exit 0
fi
# A rename counts as both paths, so moving code out of a service selects it.
changed=$(git diff --no-renames --name-only "${before}" "${sha}") || die "git diff ${before} ${sha} failed"
log "$(printf '%s\n' "$changed" | grep -c . || true) paths changed ${before:0:9}..${sha:0:9}"
select_for_paths "$changed"
;;
suites)
[ $# -eq 2 ] || die "usage: $0 suites \"<e2e-x ...>\""
if [ -z "${2// /}" ]; then
select_for_paths ""
exit 0
fi
requested=$(printf '%s' "$2" | sed 's/e2e-//g')
emit "$(keep "$requested" "${E2E_SUITES_SINGLE}")" "" "$(keep "$requested" "${E2E_SUITES_MULTI}")"
;;
*)
die "usage: $0 push <before> <sha> | suites \"<list>\""
;;
esac
-45
View File
@@ -1,45 +0,0 @@
#!/usr/bin/env bash
# Trims GOCACHE to the entries one CI run used. Go refreshes an entry's mtime
# on use only when it is over an hour old, so "mark" backdates every restored
# entry first and "trim" drops whatever this run neither read nor wrote.
set -euo pipefail
cache="$(go env GOCACHE)"
mark="${RUNNER_TEMP:?}/go-build-cache.mark"
# stale prints each entry this run did not touch and no touched action entry
# names. Go skips rewriting an output it already holds, so a fresh action
# entry can point at a backdated output.
stale() {
local keep="${RUNNER_TEMP}/go-build-cache.keep"
find "$cache" -mindepth 2 -maxdepth 2 -name '*-a' -newer "$mark" -exec cat {} + |
awk '{ print "/" $3 "-d" }' >"$keep"
find "$cache" -mindepth 2 -maxdepth 2 ! -newer "$mark" | { grep -vFf "$keep" || true; }
}
case "${1:-}" in
mark)
mkdir -p "$cache"
find "$cache" -mindepth 2 -maxdepth 2 -exec touch -m -d '2 hours ago' {} +
touch "$mark"
;;
stale | trim)
if [[ ! -f "$mark" ]]; then
echo "go-build-cache: no mark, leaving the cache untrimmed" >&2
exit 0
fi
if [[ "$1" == stale ]]; then
stale
exit 0
fi
before=$(du -sm "$cache" | cut -f1)
# An entry is a file, or a directory holding one cached executable whose
# mtime Go refreshes in place of the binary's, so it goes as a unit.
stale | xargs -r rm -rf --
echo "trimmed ${before}MB -> $(du -sm "$cache" | cut -f1)MB"
;;
*)
echo "usage: $0 mark|stale|trim" >&2
exit 2
;;
esac
@@ -1,66 +0,0 @@
#!/usr/bin/env bash
# Usage: hypervisor-disk-preflight.sh <env.tfvars>
#
# Fails fast when a hypervisor named in the tofu env is short of disk, before
# `tofu apply` tries to define a domain there. libvirt writes each domain's XML
# under /etc/libvirt/qemu on the root filesystem and its disks into the pool, and
# a full one fails the apply minutes in as "Domain Creation Failed".
#
# Hosts are reached the way the libvirt provider reaches them (ssh <user>@<name>,
# resolved through ~/.ssh/config). A host that cannot be reached is reported and
# skipped: tofu's own error for it is clearer than anything said here.
#
# Env: HV_SSH_USER (tf-user), HV_MIN_ROOT_FREE_MB (1024), HV_MIN_POOL_FREE_GB (20).
set -uo pipefail
tfvars="${1:?usage: $0 <env.tfvars>}"
user="${HV_SSH_USER:-tf-user}"
min_root_kb=$(( ${HV_MIN_ROOT_FREE_MB:-1024} * 1024 ))
min_pool_kb=$(( ${HV_MIN_POOL_FREE_GB:-20} * 1024 * 1024 ))
if [ ! -r "$tfvars" ]; then
echo "::error::hypervisor disk preflight: cannot read $tfvars" >&2
exit 1
fi
# One node per line in every env file: { name = "...", ..., storage_pool = "..." }.
pairs=$(grep -v '^[[:space:]]*#' "$tfvars" |
sed -nE 's/.*name[[:space:]]*=[[:space:]]*"([^"]+)".*storage_pool[[:space:]]*=[[:space:]]*"([^"]+)".*/\1 \2/p')
if [ -z "$pairs" ]; then
echo "::warning::hypervisor disk preflight: no nodes with a storage_pool in $tfvars; nothing checked"
exit 0
fi
# Prints "<root-avail-kb> <pool-avail-kb> <pool-path>" for the pool named in $1.
# shellcheck disable=SC2016 # expanded on the hypervisor, not here
remote='set -u
root=$(df -Pk /etc/libvirt/qemu | awk "NR==2{print \$4}")
path=$(virsh -c qemu:///system pool-dumpxml "$1" 2>/dev/null | sed -n "s:.*<path>\(.*\)</path>.*:\1:p" | head -n1)
pool=-
[ -n "$path" ] && pool=$(df -Pk "$path" | awk "NR==2{print \$4}")
echo "$root $pool ${path:--}"'
fail=0
while read -r host pool; do
[ -n "$host" ] || continue
if ! out=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "${user}@${host}" bash -s -- "$pool" <<<"$remote" 2>&1); then
echo "::warning::hypervisor disk preflight: cannot reach $host ($out); its disk is unchecked"
continue
fi
read -r root_kb pool_kb pool_path <<<"$(tail -n1 <<<"$out")"
if ! [[ "$root_kb" =~ ^[0-9]+$ ]]; then
echo "::warning::hypervisor disk preflight: unexpected answer from $host: $out"
continue
fi
echo "$host: root fs $((root_kb / 1024)) MiB free; pool $pool ($pool_path) $([[ "$pool_kb" =~ ^[0-9]+$ ]] && echo "$((pool_kb / 1024 / 1024)) GiB free" || echo "not found")"
if [ "$root_kb" -lt "$min_root_kb" ]; then
echo "::error::hypervisor $host has $((root_kb / 1024)) MiB free on the filesystem holding /etc/libvirt/qemu (need ${HV_MIN_ROOT_FREE_MB:-1024} MiB): libvirt cannot write a domain definition there. Free space on $host before re-running." >&2
fail=1
fi
if [[ "$pool_kb" =~ ^[0-9]+$ ]] && [ "$pool_kb" -lt "$min_pool_kb" ]; then
echo "::error::hypervisor $host has $((pool_kb / 1024 / 1024)) GiB free in storage pool $pool at $pool_path (need ${HV_MIN_POOL_FREE_GB:-20} GiB): the node's disk cannot grow. Free space on $host before re-running." >&2
fail=1
fi
done <<<"$pairs"
exit "$fail"
@@ -1,29 +0,0 @@
#!/usr/bin/env bash
# Resolve the tag of each sibling repo (viperblock, predastore, northstar,
# bluebottle) a release build checks out. A release ships what go.mod requires,
# so each sibling is built at the version go.mod names, never a branch tip. A
# pseudo-version has no tag to check out, so it fails the release rather than
# silently building something else.
#
# Takes the path to spinifex's go.mod and writes viperblock=<tag>,
# predastore=<tag>, northstar=<tag> and bluebottle=<tag> to $GITHUB_OUTPUT for
# the checkout steps to consume.
set -euo pipefail
gomod="${1:?usage: $0 <path to spinifex go.mod>}"
: "${GITHUB_OUTPUT:?GITHUB_OUTPUT must be set}"
for repo in viperblock predastore northstar bluebottle; do
module="github.com/mulgadc/${repo}"
version=$(awk -v m="$module" '$1 == m { print $2; exit }' "$gomod")
if [[ -z "$version" ]]; then
echo "::error::${gomod} does not require ${module}"
exit 1
fi
if [[ ! "$version" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ || "$version" =~ [0-9]{14}-[0-9a-f]{12}$ ]]; then
echo "::error::${module} is required at ${version}, which is not a release tag; tag ${repo} and bump go.mod before releasing"
exit 1
fi
echo "${repo}: building against ${version}"
echo "${repo}=${version}" >>"$GITHUB_OUTPUT"
done
+2 -109
View File
@@ -28,9 +28,6 @@
# that runs identically on either, and because a quota left switched on would
# cap every suite sharing the environment — so it must be proved to restore
# itself wherever it runs, not only on one node.
# route53 belongs to both topologies because every node serves every hosted
# zone from one shared bucket: on one node it proves the zone loads at all, on
# three that each node's sync reaches the same set.
# storagefault runs last in both lists and belongs to both topologies, for
# different reasons. On one node it is RS(1,0), so freezing the only predastore
# is a total outage and the guest-survival assertion is unambiguous. On three it
@@ -40,93 +37,15 @@
# received. It is last because it SIGSTOPs a cluster-wide
# service: anything sharing the environment would see the same outage and report
# a failure that says nothing about itself.
E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota route53 storagefault"
E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota storagefault"
# Suites runnable against a multi-node environment.
#
# instancerecovery is multi-only and needs three: recovery is inert below that,
# because there is no survivor to move a guest to that is not the node that
# failed. Like storagefault it takes a node away and freezes a cluster-wide
# service, so it runs last and owns its environment.
#
# lbrecovery is multi-only and needs three for two reasons rather than one: a
# survivor to move the load balancer to, and a node for its backends that is not
# the load balancer's, so taking that node away does not take the backends with
# it. It runs beside instancerecovery at the end because it also takes a node
# away and owns its environment while it does.
#
# kvquorum is multi-only and needs three, for the same reason as those two: a
# surviving majority. It is what proves the control plane's own state is
# replicated rather than sitting on one node, so it runs before the pair that
# take a node away — a cluster whose buckets are already wrong should report
# that, not report a recovery failing.
E2E_SUITES_MULTI="multinode lb cert quota route53 kvquorum lbrecovery instancerecovery storagefault"
E2E_SUITES_MULTI="multinode lb cert quota storagefault"
# grep -xE alternation form, for narrowing a requested set to the eligible one.
E2E_SUITES_SINGLE_RE="$(printf '%s' "${E2E_SUITES_SINGLE}" | tr ' ' '|')"
E2E_SUITES_MULTI_RE="$(printf '%s' "${E2E_SUITES_MULTI}" | tr ' ' '|')"
# The push gate always runs this set. A push adds to it the suites whose code
# it changed, by E2E_PATH_SUITES below; a dispatch names its own set instead.
E2E_SUITES_PUSH_BASE="single multinode cert iam partialblock"
# "suite prefix" pairs: a push changing a path under prefix adds suite. The
# recovery and fault suites stay nightly, and shared code adds nothing because
# the base set covers it. Image directories carry each guest's own scripts.
E2E_PATH_SUITES="
rds cmd/rds-agent/
rds internal/rdsgw/
rds spinifex/agents/rds/
rds spinifex/domains/rds/
rds tests/e2e/rds/
rds scripts/images/rds-mariadb/
rds scripts/images/rds-postgres/
ecs cmd/ecs-agent/
ecs contracts/ecs/
ecs internal/ecsgw/
ecs spinifex/agents/ecs/
ecs spinifex/domains/ecs/
ecs tests/e2e/ecs/
ecs scripts/images/ecs-agent/
eks cmd/eks-
eks contracts/eks/
eks internal/eksgw/
eks spinifex/agents/eks/
eks spinifex/domains/eks/
eks tests/e2e/eks/
eks scripts/images/eks-node/
rds scripts/images/common/
ecs scripts/images/common/
eks scripts/images/common/
rds scripts/build-system-image.sh
ecs scripts/build-system-image.sh
eks scripts/build-system-image.sh
route53 spinifex/domains/route53/
route53 tests/e2e/route53/
lb cmd/lb-agent/
lb spinifex/agents/elbv2/
lb spinifex/domains/elbv2/
lb tests/e2e/lb/
"
# Every suite E2E_PATH_SUITES can name, once each, in table order.
# e2e_path_suites_all
e2e_path_suites_all() {
printf '%s\n' "${E2E_PATH_SUITES}" | awk 'NF == 2 && !seen[$1]++ { print $1 }'
}
# The suites a change to <path> triggers, one per line.
# e2e_suites_for_path <path>
e2e_suites_for_path() {
local path="$1" suite prefix
while read -r suite prefix; do
[ -n "${prefix}" ] || continue
case "${path}" in
"${prefix}"*) printf '%s\n' "${suite}" ;;
esac
done <<<"${E2E_PATH_SUITES}"
}
# What every e2e-nightly permutation cell must prove, deliberately narrower
# than the two sets above: those cells have a 35-minute budget and exist to show
# that every install / network / host-OS combination boots and serves. The
@@ -145,10 +64,6 @@ e2e_suites_for_path() {
# after a failed seal do not exist on one node, and they are the assertions
# worth a nightly.
E2E_SUITES_NIGHTLY_SINGLE="single cert iam"
#
# The multi-node nightly subset carries kvquorum's read-only half through
# `multinode`, which sweeps every KV bucket rather than the three the daemon
# owns. The node-loss half is too slow for a 35-minute cell and has its own.
E2E_SUITES_NIGHTLY_MULTI="multinode cert lb"
# How long a suite gets, for the same reason the lists above live here: the
@@ -173,28 +88,6 @@ e2e_suite_timeout() {
# SIGSTOPped for seven minutes to prove a guest survives it. A full pass on
# dev-prod is ~63 minutes, so the 30m default truncated it mid-suite.
storagefault) echo "90m" ;;
# instancerecovery is almost entirely waiting, and none of it is avoidable:
# a dead owner has to be seen stale twice, a guest has to boot, and the
# storage case waits for a pause measured at 4m47s and then holds four
# minutes to prove nothing moved. Three tests, each taking a node away and
# bringing it back, and the partition case adds a lease validity, a lease
# TTL and a boot on top of everything the other two wait for. The data
# test lets its guest age past the pending watchdog twice, about 20 more.
instancerecovery) echo "110m" ;;
# lbrecovery pays for a load balancer twice: once to stand one up with two
# backends placed around it, and once for the recovery itself — a stale
# owner, a lease, a boot, HAProxy's own start, and its health checks against
# the backends before it forwards anything. Then the node comes back and has
# to be shown not to answer for an address it no longer holds.
lbrecovery) echo "75m" ;;
# kvquorum audits every bucket in seconds and then spends the rest of its
# budget on a node stopping and starting: seven units down, a raft election,
# and the whole service stack back up with every stream caught up.
kvquorum) echo "40m" ;;
# single is the broadest suite and finishes, it does not hang: passing runs
# took 24-29m, and an Oracle host ran out at 30m inside the last test's
# cleanup. The default left it no headroom at all.
single) echo "40m" ;;
*) echo "30m" ;;
esac
}
-34
View File
@@ -1,34 +0,0 @@
#!/usr/bin/env bash
# Usage: wait-for-memory.sh <MiB>
#
# Waits until the runner's MemAvailable reaches <MiB>, then returns. Runners on
# one host share its RAM, and a neighbour's build holds several GiB for a few
# minutes, so one sample taken during it is not a verdict on the host.
# Gives up after MEM_WAIT_TIMEOUT_S and names the biggest processes.
#
# Env: MEM_WAIT_TIMEOUT_S (900), MEM_WAIT_POLL_S (30), MEMINFO (/proc/meminfo).
set -uo pipefail
want_mb="${1:?usage: $0 <MiB>}"
timeout_s="${MEM_WAIT_TIMEOUT_S:-900}"
poll_s="${MEM_WAIT_POLL_S:-30}"
meminfo="${MEMINFO:-/proc/meminfo}"
avail_mb() { awk '/^MemAvailable:/ {print int($2 / 1024)}' "$meminfo"; }
start=$SECONDS
while :; do
have=$(avail_mb)
if [ -n "$have" ] && [ "$have" -ge "$want_mb" ]; then
echo "memory: ${have} MiB available, need ${want_mb} MiB"
exit 0
fi
waited=$(( SECONDS - start ))
if [ "$waited" -ge "$timeout_s" ]; then
echo "::error::need ${want_mb} MiB available, have ${have:-unknown} MiB after waiting ${waited}s"
ps -eo rss,pid,args --sort=-rss | head -11 | cut -c1-200
exit 1
fi
echo "memory: ${have:-unknown} MiB available, need ${want_mb} MiB; waiting (${waited}s of ${timeout_s}s)"
sleep "$poll_s"
done
-143
View File
@@ -1,143 +0,0 @@
# This file is machine-generated by `gh actions-lock`.
# Do not edit by hand; run `gh actions-lock` to update.
# Docs: https://gh.io/actions-lockfile
version: 'v0.0.3'
workflows:
'.github/workflows/apt-packages.yml':
- 'actions/checkout@v7.0.1'
'.github/workflows/dev-release.yml':
- 'actions/checkout@v7.0.1'
- 'actions/setup-go@v7.0.0'
- 'docker/setup-buildx-action@v4.4.1'
'.github/workflows/e2e-baremetal.yml':
- 'actions/checkout@v7.0.1'
- 'actions/create-github-app-token@v3.3.0'
- 'actions/upload-artifact@v7.0.2'
'.github/workflows/e2e-cloudvendor-nightly.yml':
- 'actions/checkout@v7.0.1'
- 'actions/create-github-app-token@v3.3.0'
- 'actions/upload-artifact@v7.0.2'
- 'hashicorp/setup-terraform@v4.0.1'
'.github/workflows/e2e-cloudvendor-vultr.yml':
- 'actions/checkout@v7.0.1'
- 'actions/create-github-app-token@v3.3.0'
- 'actions/upload-artifact@v7.0.2'
- 'hashicorp/setup-terraform@v4.0.1'
'.github/workflows/e2e-ddil.yml':
- 'actions/checkout@v7.0.1'
- 'actions/create-github-app-token@v3.3.0'
- 'actions/upload-artifact@v7.0.2'
'.github/workflows/e2e-nightly.yml':
- 'actions/checkout@v7.0.1'
- 'actions/create-github-app-token@v3.3.0'
- 'actions/download-artifact@v8.0.2'
- 'actions/upload-artifact@v7.0.2'
'.github/workflows/e2e-suites.yml':
- 'actions/checkout@v7.0.1'
- 'actions/create-github-app-token@v3.3.0'
- 'actions/setup-go@v7.0.0'
- 'actions/upload-artifact@v7.0.2'
'.github/workflows/e2e.yml':
- 'actions/checkout@v7.0.1'
- 'actions/create-github-app-token@v3.3.0'
- 'actions/download-artifact@v8.0.2'
- 'actions/setup-go@v7.0.0'
- 'actions/upload-artifact@v7.0.2'
'.github/workflows/gpu-e2e.yml':
- 'actions/checkout@v7.0.1'
- 'actions/setup-go@v7.0.0'
- 'actions/upload-artifact@v7.0.2'
'.github/workflows/image-scripts.yml':
- 'actions/checkout@v7.0.1'
- 'actions/setup-go@v7.0.0'
'.github/workflows/perf-3node.yml':
- 'actions/checkout@v7.0.1'
- 'actions/create-github-app-token@v3.3.0'
- 'actions/setup-go@v7.0.0'
- 'actions/upload-artifact@v7.0.2'
'.github/workflows/publish-system-ami.yml':
- 'actions/checkout@v7.0.1'
- 'actions/setup-go@v7.0.0'
'.github/workflows/release.yml':
- 'actions/checkout@v7.0.1'
- 'actions/create-github-app-token@v3.3.0'
- 'actions/download-artifact@v8.0.2'
- 'actions/setup-go@v7.0.0'
- 'actions/upload-artifact@v7.0.2'
- 'docker/setup-buildx-action@v4.4.1'
- 'softprops/action-gh-release@v3.0.3'
'.github/workflows/segscan-oracle.yml':
- 'actions/checkout@v7.0.1'
- 'actions/create-github-app-token@v3.3.0'
- 'actions/setup-go@v7.0.0'
'.github/workflows/spinifex.yml':
- 'actions/cache@v6.1.0'
- 'actions/checkout@v7.0.1'
- 'actions/setup-go@v7.0.0'
- 'golangci/golangci-lint-action@v9.3.0'
'.github/workflows/ui.yml':
- 'actions/checkout@v7.0.1'
- 'actions/setup-node@v7.1.0'
- 'actions/upload-artifact@v7.0.2'
- 'pnpm/action-setup@v6.1.0'
dependencies:
'actions/cache@v6.1.0':
ref: 'v6.1.0'
commit: 'sha1-55cc8345863c7cc4c66a329aec7e433d2d1c52a9'
owner_id: 44036562
repo_id: 215566462
'actions/checkout@v7.0.1':
ref: 'v7.0.1'
commit: 'sha1-3d3c42e5aac5ba805825da76410c181273ba90b1'
owner_id: 44036562
repo_id: 197814629
'actions/create-github-app-token@v3.3.0':
ref: 'v3.3.0'
commit: 'sha1-64ab729a6294000f5aa4a4bc0b73d1f32bc45510'
owner_id: 44036562
repo_id: 642580244
'actions/download-artifact@v8.0.2':
ref: 'v8.0.2'
commit: 'sha1-9000827ccba6bdab643e8b6fd33ac0654aef8333'
owner_id: 44036562
repo_id: 192626254
'actions/setup-go@v7.0.0':
ref: 'v7.0.0'
commit: 'sha1-b7ad1dad31e06c5925ef5d2fc7ad053ef454303e'
owner_id: 44036562
repo_id: 192624594
'actions/setup-node@v7.1.0':
ref: 'v7.1.0'
commit: 'sha1-949feb2413d6458794dcd2491c4babbbce0c15c1'
owner_id: 44036562
repo_id: 189476904
'actions/upload-artifact@v7.0.2':
ref: 'v7.0.2'
commit: 'sha1-cf430e030ddbb5b0abf93d22962f4752f3646cd9'
owner_id: 44036562
repo_id: 192625955
'docker/setup-buildx-action@v4.4.1':
ref: 'v4.4.1'
commit: 'sha1-f87e5991a6d7451dcb8d9637bfbc97413f497069'
owner_id: 5429470
repo_id: 288485773
'golangci/golangci-lint-action@v9.3.0':
ref: 'v9.3.0'
commit: 'sha1-ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a'
owner_id: 35628013
repo_id: 232539879
'hashicorp/setup-terraform@v4.0.1':
ref: 'v4.0.1'
commit: 'sha1-dfe3c3f87815947d99a8997f908cb6525fc44e9e'
owner_id: 761456
repo_id: 258271226
'pnpm/action-setup@v6.1.0':
ref: 'v6.1.0'
commit: 'sha1-ea17c68df8912ef543352723c149a84f56e3d413'
owner_id: 21320719
repo_id: 262289747
'softprops/action-gh-release@v3.0.3':
ref: 'v3.0.3'
commit: 'sha1-efb35369e0ad2afab669f228072c1b0d510eae64'
owner_id: 2242
repo_id: 204253808
+2 -16
View File
@@ -2,8 +2,7 @@ name: Apt Packages
# Path-filtered like image-scripts.yml: this resolves setup.sh's runtime package
# list against each supported suite's real indices, so a renamed, dropped, or
# ambiguously virtual package fails here rather than on a user's machine. The
# dnf job does the same for Oracle Linux, through setup.sh's own repo setup.
# ambiguously virtual package fails here rather than on a user's machine.
on:
push:
branches:
@@ -12,7 +11,6 @@ on:
paths:
- scripts/setup.sh
- scripts/lint-apt-packages.sh
- scripts/lint-dnf-packages.sh
- .github/workflows/apt-packages.yml
pull_request:
branches:
@@ -21,7 +19,6 @@ on:
paths:
- scripts/setup.sh
- scripts/lint-apt-packages.sh
- scripts/lint-dnf-packages.sh
- .github/workflows/apt-packages.yml
workflow_dispatch:
@@ -43,18 +40,7 @@ jobs:
steps:
- name: Checkout Code
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
- name: Resolve runtime package list
run: ./scripts/lint-apt-packages.sh ${{ matrix.image }}
dnf_packages:
name: Resolve dnf list (oraclelinux:10)
runs-on: ubuntu-26.04
steps:
- name: Checkout Code
uses: actions/checkout@v7.0.1
- name: Resolve runtime package list
run: ./scripts/lint-dnf-packages.sh oraclelinux:10
-21
View File
@@ -1,21 +0,0 @@
# Codex review gate: calls bluebottle's reusable workflows from main, so a PR cannot change its own review.
name: codex-review
on:
pull_request_target:
branches: [dev]
types: [opened, synchronize, reopened, ready_for_review, edited]
issue_comment:
types: [created]
permissions: {}
jobs:
review:
if: github.event_name == 'pull_request_target'
uses: mulgadc/bluebottle/.github/workflows/codex-pr-review.yml@main
secrets: inherit
waive:
if: github.event_name == 'issue_comment' && github.event.issue.pull_request
uses: mulgadc/bluebottle/.github/workflows/codex-waive.yml@main
secrets: inherit
-190
View File
@@ -1,190 +0,0 @@
name: Dev Prerelease
# Publishes the dev channel that install.mulgadc.com already serves. setup.sh has
# understood INSTALL_SPINIFEX_CHANNEL=dev since it was written and the install
# Worker resolves that channel to the newest GitHub prerelease, but nothing has
# ever produced one -- so /download/dev/amd64 returned 404. This is the producer.
#
# A separate workflow rather than conditionals in release.yml, because a dev build
# is a different artifact and not a release with flags off: amd64 only, no ISO, no
# AMI refresh, no bare-metal gate, and nothing that touches a self-hosted runner.
on:
push:
branches: [dev]
# Nothing here changes the binary, and a prerelease per docs commit would
# churn the channel our contacts are installing from.
paths-ignore:
- "docs/**"
- "**/*.md"
workflow_dispatch:
permissions:
contents: write
# Supersede rather than queue: on a run of merges to dev only the newest commit is
# worth publishing, and the older build would land after it and become the channel.
concurrency:
group: dev-release
cancel-in-progress: true
jobs:
prerelease:
runs-on: ubuntu-26.04
env:
# Dev prereleases, newest published first: GitHub lists releases by tag name
# as text, so list order is not build order. The install Worker uses this rule.
DEV_RELEASES_JQ: '[.[] | select(.draft == false and .prerelease and .published_at != null and (.tag_name | test("-dev(\\.|$)")))] | sort_by(.published_at) | reverse'
steps:
# Full history and tags, because the tag name is git describe and a shallow
# clone has no tag to describe against.
- name: Checkout Spinifex
uses: actions/checkout@v7.0.1
with:
path: spinifex
fetch-depth: 0
# dev, not each repository's default branch, which is what release.yml takes.
# A dev build of spinifex against main of the others is a pairing nobody runs.
- name: Checkout Predastore
uses: actions/checkout@v7.0.1
with:
repository: mulgadc/predastore
ref: dev
path: predastore
- name: Checkout Viperblock
uses: actions/checkout@v7.0.1
with:
repository: mulgadc/viperblock
ref: dev
path: viperblock
- name: Checkout Northstar
uses: actions/checkout@v7.0.1
with:
repository: mulgadc/northstar
ref: dev
path: northstar
- name: Checkout Bluebottle
uses: actions/checkout@v7.0.1
with:
repository: mulgadc/bluebottle
ref: dev
path: bluebottle
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4.4.1
- name: Set up Go
uses: actions/setup-go@v7.0.0
with:
go-version-file: spinifex/go.mod
cache: false
# The tag carries the describe string so the commit is recoverable from the
# version alone, and the -dev suffix is what release.yml's own convention
# already treats as a prerelease. VERSION must equal the tag: the install
# Worker derives the asset name from the tag it resolves.
- name: Resolve the version
id: version
working-directory: spinifex
run: |
set -euo pipefail
# Describing against a dev tag would nest it, growing the name every release.
TAG="$(git describe --tags --always --exclude '*-dev')-dev"
case "$TAG" in
v*) ;;
*) echo "::error::git describe gave '$TAG', which is not a v-prefixed version"; exit 1 ;;
esac
if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
echo "::notice::$TAG already published; nothing to do"
echo "exists=true" >> "$GITHUB_OUTPUT"
fi
echo "tag=$TAG" >> "$GITHUB_OUTPUT"
echo "Publishing $TAG"
env:
GH_TOKEN: ${{ github.token }}
- name: Build microVM artifacts (lb-agent + kernel + initramfs + qboot + SQS store and worker images)
if: steps.version.outputs.exists != 'true'
run: cd spinifex && make build-microvm-image build-sqs-worker-microvm-image
- name: Build amd64 Tarball
if: steps.version.outputs.exists != 'true'
env:
VERSION: ${{ steps.version.outputs.tag }}
run: cd spinifex && make distro-amd64 VERSION="$VERSION"
# --prerelease on create, never a later promote step. release.yml sets it in
# a job that runs after its verification gates, which leaves a window where
# the release is not flagged; GitHub's releases/latest excludes prereleases
# only, so in that window a dev build is what every customer curl installs.
- name: Publish the prerelease
if: steps.version.outputs.exists != 'true'
working-directory: spinifex
env:
GH_TOKEN: ${{ github.token }}
TAG: ${{ steps.version.outputs.tag }}
run: |
set -euo pipefail
# A file, not an inline string: the notes are Markdown, and leading
# whitespace from indenting a shell literal turns them into code blocks.
cat > "$RUNNER_TEMP/notes.md" <<EOF
Automated dev-channel build of \`$GITHUB_SHA\`.
**Not a release.** amd64 only, no ISO and no system images, and it has not been through the release verification gates. Install it with:
\`\`\`
curl -sfL https://install.mulgadc.com | sudo bash -s -- --channel dev
\`\`\`
EOF
sed -i 's/^ //' "$RUNNER_TEMP/notes.md"
gh release create "$TAG" \
--repo "$GITHUB_REPOSITORY" \
--target "$GITHUB_SHA" \
--prerelease \
--title "$TAG" \
--notes-file "$RUNNER_TEMP/notes.md" \
"dist/spinifex-$TAG-linux-amd64.tar.gz" \
"dist/spinifex-$TAG-linux-amd64.tar.gz.sha256" \
dist/qboot-*.tar.gz \
"scripts/setup.sh"
# The assertion that matters. A prerelease must never become the latest
# release, because latest is what an unqualified curl | bash installs -- so
# this checks the published state rather than trusting the flag we passed.
- name: Assert the stable channel is untouched
if: steps.version.outputs.exists != 'true'
env:
GH_TOKEN: ${{ github.token }}
TAG: ${{ steps.version.outputs.tag }}
run: |
set -euo pipefail
latest="$(gh api "repos/$GITHUB_REPOSITORY/releases/latest" --jq .tag_name)"
if [ "$latest" = "$TAG" ]; then
echo "::error::$TAG became the latest release: every customer curl | bash would install it"
gh release edit "$TAG" --repo "$GITHUB_REPOSITORY" --prerelease
exit 1
fi
echo "latest is still $latest"
dev="$(gh api "repos/$GITHUB_REPOSITORY/releases?per_page=100" --jq "$DEV_RELEASES_JQ | .[0].tag_name")"
[ "$dev" = "$TAG" ] || { echo "::error::the dev channel resolves to $dev, not $TAG"; exit 1; }
# Kept rather than unbounded: the Worker reads one page of releases to resolve
# the channel, so enough prereleases would eventually push the newest off it.
# Five leaves a few recent builds to roll back to.
- name: Prune old dev prereleases
if: steps.version.outputs.exists != 'true'
env:
GH_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
gh api "repos/$GITHUB_REPOSITORY/releases?per_page=100" \
--jq "$DEV_RELEASES_JQ | .[5:] | .[].tag_name" \
> stale.txt
while read -r tag; do
[ -n "$tag" ] || continue
echo "deleting $tag"
gh release delete "$tag" --repo "$GITHUB_REPOSITORY" --yes --cleanup-tag
done < stale.txt
+10 -32
View File
@@ -71,31 +71,24 @@ jobs:
ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ inputs.cell || '20' }}-${{ github.run_id }}
SPINIFEX_OTEL_ATTRS: ${{ inputs.otel-attrs || format('ci.run_id={0},ci.workflow={1},ci.cell={2},ci.matrix=baremetal/pxe/real-hw', github.run_id, github.workflow, inputs.cell || '20') }}
steps:
# A cancelled job's root steps outlive it, since the runner cannot reach
# them once it kills sudo, and would keep writing here and to the cache.
# A rootfs killed before bin/bash exists is one the build refuses to reuse.
- name: Clean Workspace
run: |
sudo systemctl stop spinifex-bm-pxe.scope spinifex-bm-e2e.scope 2>/dev/null || true
sudo umount -R -l "${BM_ISO_CACHE}/rootfs" 2>/dev/null || true
if [ -d "${BM_ISO_CACHE}/rootfs" ] && [ ! -x "${BM_ISO_CACHE}/rootfs/bin/bash" ]; then
sudo rm -rf --one-file-system "${BM_ISO_CACHE}/rootfs"
fi
if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
ref: ${{ inputs.ref || github.ref }}
- uses: actions/create-github-app-token@v3.3.0
- uses: actions/create-github-app-token@v3
id: app-token
with:
client-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
repositories: mulga
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
repository: mulgadc/mulga
token: ${{ steps.app-token.outputs.token }}
@@ -120,26 +113,17 @@ jobs:
"$GITHUB_WORKSPACE" \
"${RUNNER_TEMP}/spinifex-tests.tar" \
"${RUNNER_TEMP}/bm-bins"
# Built here, as the runner user, so the PXE build finds it current.
make -C "$GITHUB_WORKSPACE" build-microvm-image
- name: Build PXE Artifacts
run: |
# -o build-spinifex: the binaries come from Build Artifacts, and a
# rebuild here would write root-owned files into the workspace.
sudo systemd-run --scope --collect --quiet --unit=spinifex-bm-pxe -- \
make -C mulga/scripts/iso-builder -o build-spinifex \
sudo make -C mulga/scripts/iso-builder \
build-rootfs build-squashfs build-initramfs \
CACHE="${BM_ISO_CACHE}" \
SPINIFEX_BIN="${RUNNER_TEMP}/bm-bins/spx" \
INSTALLER_BIN="${RUNNER_TEMP}/bm-bins/spinifex-installer" \
SPINIFEX_SRC="${GITHUB_WORKSPACE}" \
VIPERBLOCK_SRC="${RUNNER_TEMP}/bm-bins/viperblock" \
VERSION="$(cd "$GITHUB_WORKSPACE" && git describe --tags --always --exclude '*-dev' 2>/dev/null || echo dev)"
if root_owned="$(find "$GITHUB_WORKSPACE" -user root -print -quit)" && [ -n "$root_owned" ]; then
echo "::error::the PXE build wrote root-owned files into the workspace, e.g. $root_owned"
exit 1
fi
VERSION="$(cd "$GITHUB_WORKSPACE" && git describe --tags --always 2>/dev/null || echo dev)"
- name: Write ePDU credentials
run: |
@@ -167,8 +151,7 @@ jobs:
fi
- name: Install go-junit-report on runner
# Version from go.mod's tool directive, checked against go.sum.
run: GOWORK=off go install github.com/jstemmer/go-junit-report/v2
run: go install github.com/jstemmer/go-junit-report/v2@latest
# The eks suite's CreateCluster boots a control-plane VM from the eks-node
# AMI. Built here because libguestfs needs source and network, which the
@@ -211,18 +194,13 @@ jobs:
done
export BM_TEST_TIMEOUT
echo "suites=${BM_SUITES} test-timeout=${BM_TEST_TIMEOUT}"
sudo -E systemd-run --scope --collect --quiet --unit=spinifex-bm-e2e -- \
mulga/scripts/baremetal/run-baremetal-e2e.sh \
sudo -E mulga/scripts/baremetal/run-baremetal-e2e.sh \
--vmlinuz "${BM_ISO_CACHE}/vmlinuz" \
--initrd "${BM_ISO_CACHE}/initrd.img" \
--squashfs "${BM_ISO_CACHE}/spinifex.squashfs" \
--tests-tar "${RUNNER_TEMP}/spinifex-tests.tar" \
2>&1 | tee "${ARTIFACT_DIR}/test-baremetal.log"
- name: Stop this job's root steps
if: always()
run: sudo systemctl stop spinifex-bm-pxe.scope spinifex-bm-e2e.scope 2>/dev/null || true
- name: Generate JUnit report
if: always()
run: |
@@ -243,20 +221,20 @@ jobs:
- name: Ship test results to Elasticsearch
if: always()
continue-on-error: true
uses: $/.github/actions/junit-to-es
uses: ./.github/actions/junit-to-es
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
cell: ${{ env.BM_CELL }}
- name: Analyze e2e failures
if: always()
uses: $/.github/actions/e2e-analyze
uses: ./.github/actions/e2e-analyze
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
log-dir: ${{ env.ARTIFACT_DIR }}
title: "E2E failure analysis (cell ${{ env.BM_CELL }})"
- uses: actions/upload-artifact@v7.0.2
- uses: actions/upload-artifact@v7
if: always()
with:
name: e2e-artifacts-cell-${{ env.BM_CELL }}-${{ github.run_id }}
@@ -1,287 +0,0 @@
name: E2E Cloud Vendor Nightly
# Dispatched by the same Cloudflare Worker that drives e2e-nightly.yml, once per
# ref, rather than by a `schedule:` here. GitHub registers a schedule only from
# the default branch, so a cron in this file could never run `dev` and a feature
# branch as separate nightlies, and it would always execute the default branch's
# copy of this file -- leaving a fix to the workflow itself untestable until it
# had already shipped. A dispatch takes both the file and the source from its ref.
on:
workflow_dispatch:
inputs:
topologies:
description: "Space-separated topologies (blank = vm-single vm-multi bm vm-single-principal). One name runs one topology."
required: false
workbooks:
description: "Space-separated workbooks (blank = the driver's default list)"
required: false
source:
description: "tree (build this ref) or release (the published installer)"
required: false
default: tree
channel:
description: "With source=release: latest, or dev for the newest prerelease"
required: false
default: latest
keep_on_fail:
description: "Leave a failed topology up to inspect. It keeps billing."
required: false
type: boolean
default: false
auth:
description: "How nodes reach the OCI API: instance_principal (needs the tenancy dynamic group) or api_key (needs the credential hook). The vm-single-principal topology always uses instance_principal."
required: false
default: instance_principal
type: choice
options: [instance_principal, api_key]
no_external_pool:
description: "Form without the OCI allocator. Skips every workbook needing a public address. Not applied to vm-single-principal."
required: false
type: boolean
default: false
runner:
description: "Runner label (blank = the OCI_E2E_RUNNER variable, else ubuntu-26.04)"
required: false
permissions:
contents: read
# Every topology is built in one OCI compartment under one set of resource names,
# so two runs at once would each see the other's instances and the sweep in
# e2e-oci.sh could destroy a cluster the other run was still testing. Queue the
# second rather than cancelling it: a dispatch is usually a verification run.
concurrency:
group: e2e-oci
cancel-in-progress: false
jobs:
oci:
# Hosted by default: this job only drives Terraform and SSH, and the work
# happens on OCI, so nothing here needs a hypervisor. Overridable because the
# argument for a per-provider self-hosted runner is control, not capacity --
# a dedicated label keeps a two-hour cloud wait off the hypervisor labels and
# lifts the six-hour cap a hosted job carries.
runs-on: ${{ inputs.runner || vars.OCI_E2E_RUNNER || 'ubuntu-26.04' }}
# Three topologies including a bare-metal build, each with its own install,
# formation and workbook pass. The sweep still runs if this fires.
timeout-minutes: 300
steps:
# Set here rather than in a job-level env block: the runner context is not
# available at job level, only from a step onwards, and a workflow that
# references it there is rejected as invalid before any job starts.
#
# Outside github.workspace on purpose. The checkout wipes untracked files, so
# on a persistent runner it would delete the Terraform state of a killed run
# before the next sweep could read it, leaving instances nothing can find.
- name: Put the Terraform state outside the checkout
run: echo "OCI_STATE_ROOT=$RUNNER_WORKSPACE/oci-state" >> "$GITHUB_ENV"
- uses: actions/checkout@v7.0.1
with:
path: spinifex
- uses: actions/create-github-app-token@v3.3.0
id: app-token
with:
client-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
# mulga alone. It is the only private repository of the six, and the
# five that build-install-artifacts.sh clones are public, so the builder
# needs no credential and this token is not widened to reach them.
repositories: mulga
# build-install-artifacts.sh lives in mulga. The workbooks and their driver
# are in spinifex, so only the one directory is needed here.
- uses: actions/checkout@v7.0.1
with:
repository: mulgadc/mulga
path: mulga
token: ${{ steps.app-token.outputs.token }}
sparse-checkout: scripts/tofu-cluster
- uses: hashicorp/setup-terraform@v4.0.1
with:
terraform_wrapper: false
# Written to the runner's own filesystem and nowhere else. The OCI API
# credential is not here at all: it reaches Terraform as OCI_PRIVATE_KEY in
# the environment, so no key file is ever created.
- name: Install the throwaway SSH key
env:
OCI_E2E_SSH_PRIVATE_KEY: ${{ secrets.OCI_E2E_SSH_PRIVATE_KEY }}
OCI_E2E_SSH_PUBLIC_KEY: ${{ secrets.OCI_E2E_SSH_PUBLIC_KEY }}
run: |
set -euo pipefail
mkdir -p ~/.ssh
printf '%s\n' "$OCI_E2E_SSH_PRIVATE_KEY" > ~/.ssh/oci-spx
printf '%s\n' "$OCI_E2E_SSH_PUBLIC_KEY" > ~/.ssh/oci-spx.pub
chmod 600 ~/.ssh/oci-spx
ssh-keygen -y -f ~/.ssh/oci-spx > /dev/null
- name: Run the OCI topologies
working-directory: spinifex/scripts/terraform/oci-spx
env:
OCI_TENANCY_OCID: ${{ secrets.OCI_TENANCY_OCID }}
OCI_USER_OCID: ${{ secrets.OCI_USER_OCID }}
OCI_FINGERPRINT: ${{ secrets.OCI_FINGERPRINT }}
OCI_PRIVATE_KEY: ${{ secrets.OCI_PRIVATE_KEY }}
OCI_REGION: ${{ vars.OCI_REGION || 'ap-sydney-1' }}
TF_VAR_compartment_ocid: ${{ secrets.OCI_COMPARTMENT_OCID }}
MULGA_ROOT: ${{ github.workspace }}/mulga
SPX_REF: ${{ github.ref_name }}
OCI_TOPOLOGIES: ${{ inputs.topologies }}
OCI_SOURCE: ${{ inputs.source }}
OCI_CHANNEL: ${{ inputs.channel }}
# Independent of the pool, deliberately. These were one switch, which made
# the combination we actually need -- an API key with the allocator running
# -- impossible to ask for, so public-IP allocation went untested.
OCI_INSTANCE_PRINCIPAL: ${{ inputs.auth == 'api_key' && '0' || '1' }}
# Skipping the pool records the allocator and every public-address workbook
# as SKIPPED, so a green run with this set is not a claim about guest
# networking. It exists for a formation-only check, not as a way past a
# missing credential.
OCI_NO_EXTERNAL_POOL: ${{ inputs.no_external_pool && '1' || '0' }}
# Only an API-key run needs one: an instance principal is the node's own
# certificate, so there is nothing to hand over. Unset is not an error, and
# the allocator gate then fails rather than passing a node that cannot
# authenticate.
OCI_CREDENTIAL_HOOK: ${{ inputs.auth == 'api_key' && vars.OCI_CREDENTIAL_HOOK || '' }}
# vm-single-principal builds in its own tenancy. The dynamic group it needs
# is a tenancy-root resource, and the shared test tenancy grants us only
# compartment rights, so that topology can never pass there however correct
# the code is. Unset skips it rather than spending a provision to fail.
OCI_PRINCIPAL_TENANCY_OCID: ${{ secrets.OCI_PRINCIPAL_TENANCY_OCID }}
OCI_PRINCIPAL_USER_OCID: ${{ secrets.OCI_PRINCIPAL_USER_OCID }}
OCI_PRINCIPAL_FINGERPRINT: ${{ secrets.OCI_PRINCIPAL_FINGERPRINT }}
OCI_PRINCIPAL_PRIVATE_KEY: ${{ secrets.OCI_PRINCIPAL_PRIVATE_KEY }}
OCI_PRINCIPAL_COMPARTMENT_OCID: ${{ secrets.OCI_PRINCIPAL_COMPARTMENT_OCID }}
OCI_PRINCIPAL_REGION: ${{ vars.OCI_PRINCIPAL_REGION }}
# Service limits are per tenancy, so the shape the other topologies use
# can have zero quota in this one. Ours has no E6 cores at all, and the
# apply then fails before anything about the principal is exercised.
OCI_PRINCIPAL_SHAPE: ${{ vars.OCI_PRINCIPAL_SHAPE }}
OCI_ARTIFACT_DIR: ${{ runner.temp }}/e2e-oci
OCI_KEEP_ON_FAIL: ${{ inputs.keep_on_fail && '1' || '0' }}
# Through the environment, never interpolated into the script below: a
# dispatch input is attacker-controlled text as far as the shell is
# concerned, and a quote in it would otherwise end the string.
WORKBOOKS_INPUT: ${{ inputs.workbooks }}
run: |
set -uo pipefail
# WORKBOOKS is exported only when the dispatch named some: the harness
# distinguishes unset (the driver's default list) from empty (run none).
if [ -n "$WORKBOOKS_INPUT" ]; then export WORKBOOKS="$WORKBOOKS_INPUT"; fi
./e2e-oci.sh 2>&1 | tee "$RUNNER_TEMP/e2e-oci.log"
exit "${PIPESTATUS[0]}"
# Before the sweep, which destroys the state the tables are read from, and
# always(): a run whose verdict is a failure is exactly the one somebody
# wants a table for. The successful path writes the same tables itself, so
# this step is for the run that did not get there.
#
# The markdown lives in the script rather than here. A heredoc in a YAML
# string is the one place in this repository nothing can lint or run.
- name: Publish the result tables
if: always()
working-directory: spinifex/scripts/terraform/oci-spx
env:
OCI_REGION: ${{ vars.OCI_REGION || 'ap-sydney-1' }}
OCI_ARTIFACT_DIR: ${{ runner.temp }}/e2e-oci
OCI_TOPOLOGIES: ${{ inputs.topologies }}
OCI_SOURCE: ${{ inputs.source }}
OCI_CHANNEL: ${{ inputs.channel }}
SPX_REF: ${{ github.ref_name }}
# The same two the run step passes. The header states what was and was not
# tested, so a summary rendered without them would claim more than the run.
OCI_INSTANCE_PRINCIPAL: ${{ inputs.auth == 'api_key' && '0' || '1' }}
OCI_NO_EXTERNAL_POOL: ${{ inputs.no_external_pool && '1' || '0' }}
run: ./e2e-oci.sh --summary-only
# The sweep below deletes the per-topology state directories, so anything
# still only in one of them at that point is unrecoverable -- and a failing
# workbook writes its assertion output to exactly there. Copying into the
# artifact directory first is what makes a failure diagnosable by whoever
# reads the run page, rather than only on the runner that produced it.
- name: Collect the per-topology logs
if: always()
run: |
set -uo pipefail
dest="$RUNNER_TEMP/e2e-oci"
mkdir -p "$dest"
# The topologies this run drove, taken from the per-topology log the
# harness writes rather than from the dispatch input: a run that names
# none gets the harness's own default, and reading that back here would
# mean a second copy of the list to disagree with the first. A state
# directory left by an earlier run is then not collected, so the artifact
# cannot show a topology this run never attempted.
topologies=""
for log in "$dest"/*.log; do
[ -f "$log" ] || continue
name="${log##*/}"; name="${name%.log}"
[ -d "$OCI_STATE_ROOT/.validate-$name" ] && topologies="$topologies $name"
done
echo "collecting:${topologies:- nothing}"
for topology in $topologies; do
for file in "$OCI_STATE_ROOT/.validate-$topology"/*.log \
"$OCI_STATE_ROOT/.validate-$topology"/*.txt \
"$OCI_STATE_ROOT/.validate-$topology"/*.tsv; do
[ -f "$file" ] || continue
# The hook handles a private key and the one thing known about an
# operator's executable is that it is not ours to publish.
case "${file##*/}" in credential-hook.log) continue ;; esac
cp -- "$file" "$dest/$topology-${file##*/}"
done
done
ls -l "$dest"
# always(), and before the artifact upload: an OCI bare-metal host left
# running by a cancelled or timed-out job is the most expensive way this
# workflow can fail. The sweep works from the Terraform state files, so it
# can only destroy what this run created.
- name: Sweep anything left behind
if: always() && !inputs.keep_on_fail
working-directory: spinifex/scripts/terraform/oci-spx
env:
OCI_TENANCY_OCID: ${{ secrets.OCI_TENANCY_OCID }}
OCI_USER_OCID: ${{ secrets.OCI_USER_OCID }}
OCI_FINGERPRINT: ${{ secrets.OCI_FINGERPRINT }}
OCI_PRIVATE_KEY: ${{ secrets.OCI_PRIVATE_KEY }}
OCI_REGION: ${{ vars.OCI_REGION || 'ap-sydney-1' }}
TF_VAR_compartment_ocid: ${{ secrets.OCI_COMPARTMENT_OCID }}
# The sweep destroys by state file, so it needs whichever tenancy each
# topology built in. Without these a leftover principal deployment is
# swept against a tenancy that never held it and reported clean.
OCI_PRINCIPAL_TENANCY_OCID: ${{ secrets.OCI_PRINCIPAL_TENANCY_OCID }}
OCI_PRINCIPAL_USER_OCID: ${{ secrets.OCI_PRINCIPAL_USER_OCID }}
OCI_PRINCIPAL_FINGERPRINT: ${{ secrets.OCI_PRINCIPAL_FINGERPRINT }}
OCI_PRINCIPAL_PRIVATE_KEY: ${{ secrets.OCI_PRINCIPAL_PRIVATE_KEY }}
OCI_PRINCIPAL_COMPARTMENT_OCID: ${{ secrets.OCI_PRINCIPAL_COMPARTMENT_OCID }}
OCI_PRINCIPAL_REGION: ${{ vars.OCI_PRINCIPAL_REGION }}
OCI_ARTIFACT_DIR: ${{ runner.temp }}/e2e-oci-sweep
run: ./e2e-oci.sh --sweep-only
# Explicit, not left to the runner's own cleanup: a hosted runner is
# discarded after the job, but a self-hosted one keeps $HOME, so the key
# would outlive the run that needed it and sit there between nightlies.
- name: Remove the throwaway SSH key
if: always()
run: rm -f ~/.ssh/oci-spx ~/.ssh/oci-spx.pub
# Named extensions rather than a directory glob: the artifact directory also
# holds the distro tarball and the test archive, which are hundreds of
# megabytes and reproducible from the ref. Everything here is a file the
# collect step above chose, so terraform.tfstate cannot reach an artifact.
- name: Upload the run
if: always()
uses: actions/upload-artifact@v7.0.2
with:
name: e2e-oci-${{ github.run_id }}
path: |
${{ runner.temp }}/e2e-oci.log
${{ runner.temp }}/e2e-oci/*.log
${{ runner.temp }}/e2e-oci/verdict.txt
${{ runner.temp }}/e2e-oci/summary.md
${{ runner.temp }}/e2e-oci/*.txt
${{ runner.temp }}/e2e-oci/*.tsv
${{ runner.temp }}/e2e-oci-sweep/*.log
retention-days: 14
-142
View File
@@ -1,142 +0,0 @@
name: E2E Cloud Vendor Vultr
# On demand only. Deploys a ref to Vultr with deploy-vultr.sh, runs the same
# published Terraform workbooks as the hypervisor nightly and OCI, and destroys
# the deployment, failing if anything is left billing.
on:
workflow_dispatch:
inputs:
nodes:
description: "1 or 3 nodes"
required: false
default: "1"
type: choice
options: ["1", "3"]
plan:
description: "Vultr plan: vx1-g-16c-64g (VM) or vbm-8c-64gb-amd (bare metal)"
required: false
default: vx1-g-16c-64g
region:
description: "Vultr region"
required: false
default: ord
workbooks:
description: "Space-separated workbooks (blank = the driver's full list)"
required: false
source:
description: "tree (build this ref) or release (the published installer)"
required: false
default: tree
type: choice
options: [tree, release]
channel:
description: "With source=release: latest, or dev for the newest prerelease"
required: false
default: latest
keep_on_fail:
description: "Leave a failed deployment up to inspect. It keeps billing."
required: false
type: boolean
default: false
runner:
description: "Runner label (blank = the VULTR_E2E_RUNNER variable, else ubuntu-26.04)"
required: false
permissions:
contents: read
# Deployments are named per run, but the account's reserved-IP quota is shared,
# so a second dispatch queues rather than competing with the first.
concurrency:
group: e2e-vultr
cancel-in-progress: false
jobs:
vultr:
# Hosted by default: the job only drives Terraform and SSH. The API key's
# access control must admit this runner, so a hosted runner needs a key
# whose list holds 0.0.0.0/0; a self-hosted runner can use a fixed address.
runs-on: ${{ inputs.runner || vars.VULTR_E2E_RUNNER || 'ubuntu-26.04' }}
timeout-minutes: 240
env:
VULTR_NAME: e2e-${{ github.run_id }}
VULTR_NODES: ${{ inputs.nodes }}
VULTR_PLAN: ${{ inputs.plan }}
VULTR_REGION: ${{ inputs.region }}
VULTR_SOURCE: ${{ inputs.source }}
VULTR_CHANNEL: ${{ inputs.channel }}
VULTR_ARTIFACT_DIR: ${{ github.workspace }}/e2e-vultr
SPX_REF: ${{ github.ref_name }}
steps:
- uses: actions/checkout@v7.0.1
with:
path: spinifex
fetch-depth: 0
- uses: actions/create-github-app-token@v3.3.0
id: app-token
with:
client-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
repositories: mulga
# build-install-artifacts.sh lives in mulga; the workbooks and their driver
# are in spinifex, so only the one directory is needed here.
- uses: actions/checkout@v7.0.1
with:
repository: mulgadc/mulga
path: mulga
token: ${{ steps.app-token.outputs.token }}
sparse-checkout: scripts/tofu-cluster
- uses: hashicorp/setup-terraform@v4.0.1
with:
terraform_wrapper: false
- name: Run Vultr
working-directory: spinifex/scripts/terraform/vultr-spx
env:
VULTR_API_KEY: ${{ secrets.VULTR_API_KEY }}
MULGA_ROOT: ${{ github.workspace }}/mulga
# Through the environment, never interpolated into the script: a
# dispatch input is untrusted text to the shell.
WORKBOOKS_INPUT: ${{ inputs.workbooks }}
VULTR_KEEP_ON_FAIL: ${{ inputs.keep_on_fail && '1' || '0' }}
run: |
set -uo pipefail
if [ -n "$WORKBOOKS_INPUT" ]; then export WORKBOOKS="$WORKBOOKS_INPUT"; fi
./e2e-vultr.sh 2>&1 | tee "$RUNNER_TEMP/e2e-vultr.log"
exit "${PIPESTATUS[0]}"
# always(): a cancelled or timed-out job is the one that would leave
# servers and reserved IPs billing. destroy works from this run's state.
- name: Sweep anything left behind
if: always() && !inputs.keep_on_fail
working-directory: spinifex/scripts/terraform/vultr-spx
env:
VULTR_API_KEY: ${{ secrets.VULTR_API_KEY }}
run: ./e2e-vultr.sh --sweep-only
- name: Publish the verdict
if: always()
run: |
{
echo "### Vultr: ${VULTR_NODES} x ${VULTR_PLAN} in ${VULTR_REGION}, ${VULTR_SOURCE} build of ${SPX_REF}"
echo '```'
cat "$VULTR_ARTIFACT_DIR/verdict.txt" 2>/dev/null || echo "no verdict written"
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
# Logs and the verdict only. The throwaway SSH key and the Terraform state
# under ~/.spinifex never reach an artifact.
- name: Upload the run
if: always()
uses: actions/upload-artifact@v7.0.2
with:
name: e2e-vultr-${{ github.run_id }}
path: |
${{ runner.temp }}/e2e-vultr.log
${{ github.workspace }}/e2e-vultr/*.log
${{ github.workspace }}/e2e-vultr/verdict.txt
${{ github.workspace }}/e2e-vultr/node*/*.log
retention-days: 14
+8 -8
View File
@@ -38,19 +38,19 @@ jobs:
- name: Clean Workspace
run: sudo rm -rf "$GITHUB_WORKSPACE"/*
- uses: actions/create-github-app-token@v3.3.0
- uses: actions/create-github-app-token@v3
id: app-token
with:
client-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
repositories: mulga
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
name: Checkout Spinifex
with:
path: spinifex
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
name: Checkout Mulga (tofu-cluster)
with:
repository: mulgadc/mulga
@@ -81,7 +81,7 @@ jobs:
# Adds ci.job_id alongside ci.cell, before Bootstrap bakes the attrs into
# the VMs, so a telemetry doc points back at this job's log directly.
- name: Resolve CI job id
uses: $/.github/actions/otel-job-id
uses: ./spinifex/.github/actions/otel-job-id
with:
job-name: DDIL Real Multi-Node
@@ -175,7 +175,7 @@ jobs:
- name: Publish test summary
if: always()
uses: $/.github/actions/junit-report
uses: ./spinifex/.github/actions/junit-report
with:
junit-glob: ${{ runner.temp }}/ddil-results/junit-*.xml
title: "DDIL scenario results"
@@ -185,7 +185,7 @@ jobs:
- name: Ship test results to Elasticsearch
if: always()
continue-on-error: true
uses: $/.github/actions/junit-to-es
uses: ./spinifex/.github/actions/junit-to-es
with:
junit-glob: ${{ runner.temp }}/ddil-results/junit-*.xml
cell: ddil
@@ -214,7 +214,7 @@ jobs:
- name: Upload DDIL Results
if: always()
uses: actions/upload-artifact@v7.0.2
uses: actions/upload-artifact@v7
with:
name: ddil-results-${{ github.run_id }}
path: ${{ runner.temp }}/ddil-results
@@ -234,7 +234,7 @@ jobs:
- name: Upload tofu log
if: always()
uses: actions/upload-artifact@v7.0.2
uses: actions/upload-artifact@v7
with:
name: tofu-log-ddil-${{ github.run_id }}
path: ${{ env.TOFU_LOG }}
+54 -177
View File
@@ -6,32 +6,11 @@ name: E2E Nightly
# `main` and `dev` as separate nightlies, and it would always execute `main`'s
# copy of this file — leaving a fix to the workflow itself untestable until it
# had already shipped. A dispatch takes both the file and the source from its ref.
#
# Cells (matrix cells are defined in .github/e2e-cells.json):
#
# Single node Three nodes
# 1 Debian 13, DHCP 8 Debian 13, DHCP
# 2 Ubuntu 26.04, static 9 Ubuntu 26.04, static
# 3 Ubuntu 26.04, DHCP 10 Ubuntu 26.04, DHCP
# 17 tofu examples 29 storage fault injection
# 18 reboot 30 routed NAT
# 19 routed NAT 31 instance recovery
# 25 eks 32 load balancer recovery
# 26 ecs 33 KV bucket quorum under node loss
# 27 rds 37 Oracle Linux 10, DHCP
# 34 Oracle Linux 10, DHCP 38 Oracle Linux 10, routed NAT
# 35 Oracle Linux 10, reboot
# 36 Oracle Linux 10, routed NAT
#
# Standalone jobs
# 20 bare metal (PXE) 23 predastore throughput
# 21 ISO, single node 24 predastore fault injection
# 22 ISO, three nodes 28 guest disk performance (bare metal)
on:
workflow_dispatch:
inputs:
cells:
description: "Comma-separated cell numbers, no spaces, e.g. 1,25,27 (blank = all). The cell list is at the top of .github/workflows/e2e-nightly.yml."
description: "Comma-separated cell numbers to run, no spaces (blank = all). 1-3, 8-10, 17-19 tofu permutation matrix, 20 baremetal, 21-22 ISO, 23 predastore throughput, 24 predastore fault injection, 25-27 service suites (eks, ecs, rds), 28 guest disk performance, 29 storage fault injection"
required: false
permissions:
@@ -54,7 +33,6 @@ concurrency:
env:
GIT_BRANCH: ${{ github.ref_name }}
UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-26.04.qcow2
ORACLE_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-oracle-10.qcow2
SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
@@ -65,7 +43,7 @@ jobs:
cells: ${{ steps.pick.outputs.cells }}
any: ${{ steps.pick.outputs.any }}
steps:
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
sparse-checkout: |
.github/e2e-cells.json
@@ -150,21 +128,21 @@ jobs:
if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
- uses: actions/create-github-app-token@v3.3.0
- uses: actions/create-github-app-token@v3
id: app-token
with:
client-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
repositories: mulga
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
repository: mulgadc/mulga
token: ${{ steps.app-token.outputs.token }}
sparse-checkout: scripts/tofu-cluster
path: mulga
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
path: spinifex
@@ -173,7 +151,7 @@ jobs:
# links each cell's telemetry to the job log that produced it. Runs
# before Bootstrap bakes SPINIFEX_OTEL_ATTRS into the VM's telemetry.env.
- name: Resolve CI job id
uses: $/.github/actions/otel-job-id
uses: ./spinifex/.github/actions/otel-job-id
with:
job-name: "cell-${{ matrix.cell }} (${{ matrix.topo }}/${{ matrix.net }}/${{ matrix.host }}${{ matrix.extra && format('/{0}', matrix.extra) || '' }})"
@@ -205,14 +183,6 @@ jobs:
echo "running qemu: $(pgrep -c qemu-system-x86 || echo 0)"
} >> "${ARTIFACT_DIR}/hypervisor-load.txt"
# A hypervisor out of disk fails the apply minutes in, as "Domain Creation
# Failed" from libvirt writing the domain XML. Say so before starting.
- name: Preflight hypervisor disk
working-directory: mulga/scripts/tofu-cluster
run: |
"${GITHUB_WORKSPACE}/spinifex/.github/scripts/hypervisor-disk-preflight.sh" \
"envs/${SPINIFEX_CI_ENV}.tfvars"
- name: Provision VMs
working-directory: mulga/scripts/tofu-cluster
env:
@@ -225,8 +195,7 @@ jobs:
# back onto the debian gold.
TOFU_VAR_ARGS=""
case "$MATRIX_HOST" in
ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE} -var=host_os=ubuntu" ;;
oracle-*) TOFU_VAR_ARGS="-var=host_image_override=${ORACLE_GOLD_IMAGE} -var=host_os=oracle" ;;
ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}" ;;
esac
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
@@ -270,11 +239,6 @@ jobs:
. "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
e2e_suites_assert_subset "e2e-nightly single" "${E2E_SUITES_NIGHTLY_SINGLE}" "${E2E_SUITES_SINGLE}"
# The select job sized this cell's budget from these same timeouts, so
# resolve them here and carry the pairs into the remote loop.
SUITE_TMOS=""
for s in ${E2E_SUITES_NIGHTLY_SINGLE}; do SUITE_TMOS="${SUITE_TMOS}${s}:$(e2e_suite_timeout "${s}") "; done
INVENTORY=$(tofu output -json inventory)
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
@@ -284,10 +248,9 @@ jobs:
for SUITE in ${E2E_SUITES_NIGHTLY_SINGLE}; do \
echo \"::group::suite \$SUITE\"; \
date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
TMO=30m; for kv in ${SUITE_TMOS}; do case \$kv in \$SUITE:*) TMO=\${kv#*:} ;; esac; done; \
GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
ARTIFACT_DIR='${ARTIFACT_DIR}' \
./\$SUITE.test -test.v -test.timeout=\$TMO \
./\$SUITE.test -test.v -test.timeout=30m \
2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log'; \
rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \
if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
@@ -315,31 +278,8 @@ jobs:
rc=\${PIPESTATUS[0]}; echo '::endgroup::'; \
if [ \$rc -ne 0 ]; then echo 'natuplink: FAILED' >> '${ARTIFACT_DIR}/suite-status.txt'; fi; exit \$rc"
# natuplink proves a guest with a public IP reaches the internet. The two
# surfaces it never touches are a private subnet's isolation and its NAT
# gateway, and TestVPCEgressPaths is the authority on both — it skipped in
# nat mode until the fixture learned that routed NAT carries EIPs too.
- name: Run VPC Egress Paths in NAT mode
if: matrix.extra == 'nat-single'
working-directory: mulga/scripts/tofu-cluster
run: |
INVENTORY=$(tofu output -json inventory)
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
SSH_KEY="${SSH_KEY/#\~/$HOME}"
ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
"set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; \
echo '::group::suite single/TestVPCEgressPaths'; \
date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-vpcegress-nat.start'; \
GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
ARTIFACT_DIR='${ARTIFACT_DIR}' \
./single.test -test.v -test.timeout=30m -test.run '^TestVPCEgressPaths\$' \
2>&1 | tee '${ARTIFACT_DIR}/test-vpcegress-nat.log'; \
rc=\${PIPESTATUS[0]}; echo '::endgroup::'; \
if [ \$rc -ne 0 ]; then echo 'vpcegress-nat: FAILED' >> '${ARTIFACT_DIR}/suite-status.txt'; fi; exit \$rc"
- name: Run Multi Node E2E
if: matrix.topo == 'real-multi' && matrix.suite == '' && matrix.extra == ''
if: matrix.topo == 'real-multi' && matrix.suite == ''
working-directory: mulga/scripts/tofu-cluster
run: |
set -euo pipefail
@@ -359,59 +299,21 @@ jobs:
NODES_CSV="${NODES_CSV:+$NODES_CSV,}$ip"
done
SUITE_TMOS=""
for s in ${E2E_SUITES_NIGHTLY_MULTI}; do SUITE_TMOS="${SUITE_TMOS}${s}:$(e2e_suite_timeout "${s}") "; done
ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
"set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; FAIL=0; \
for SUITE in ${E2E_SUITES_NIGHTLY_MULTI}; do \
echo \"::group::suite \$SUITE\"; \
date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
TMO=30m; for kv in ${SUITE_TMOS}; do case \$kv in \$SUITE:*) TMO=\${kv#*:} ;; esac; done; \
GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \
SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
ARTIFACT_DIR='${ARTIFACT_DIR}' \
./\$SUITE.test -test.v -test.timeout=\$TMO \
./\$SUITE.test -test.v -test.timeout=30m \
2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log'; \
rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \
if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
done; exit \$FAIL"
# The same suite as the nat-single cell, run against three nodes: there it
# proves routed NAT works at all, here that each node delivers its own
# guests' public IPs and no others — which one node cannot show.
- name: Run NAT Uplink Multi E2E
if: matrix.extra == 'nat-multi'
working-directory: mulga/scripts/tofu-cluster
run: |
set -euo pipefail
INVENTORY=$(tofu output -json inventory)
NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
SSH_KEY="${SSH_KEY/#\~/$HOME}"
NODES_CSV=""
NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
for i in $(seq 0 $((NODE_COUNT - 1))); do
ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
NODES_CSV="${NODES_CSV:+$NODES_CSV,}$ip"
done
ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
"set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; \
if [ ! -f natuplink.test ]; then echo 'natuplink.test not in tests tar (branch predates suite) — skipping'; exit 0; fi; \
echo '::group::suite natuplink'; \
date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-natuplink.start'; \
GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \
SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
ARTIFACT_DIR='${ARTIFACT_DIR}' \
./natuplink.test -test.v -test.timeout=30m \
2>&1 | tee '${ARTIFACT_DIR}/test-natuplink.log'; \
rc=\${PIPESTATUS[0]}; echo '::endgroup::'; \
if [ \$rc -ne 0 ]; then echo 'natuplink: FAILED' >> '${ARTIFACT_DIR}/suite-status.txt'; fi; exit \$rc"
# The dedicated suite cells only. Resolved once here rather than in each
# step below, which would repeat the same three lines five times.
- name: Resolve VM SSH details
@@ -578,8 +480,6 @@ jobs:
echo \"::group::suite ${SUITE}\"; \
date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-${SUITE}.start'; \
GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
SPINIFEX_NODES='${{ steps.vm.outputs.wan_ip }}' SPINIFEX_SSH_USER=tf-user \
SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
ARTIFACT_DIR='${ARTIFACT_DIR}' \
./${SUITE}.test -test.v -test.timeout=${TMO} \
2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log'; \
@@ -730,7 +630,7 @@ jobs:
'echo "host: $(hostname)"; echo "loadavg: $(cat /proc/loadavg)";
grep -E "^(MemTotal|MemAvailable|SwapFree):" /proc/meminfo;
echo "cpu(steal is field 8): $(grep "^cpu " /proc/stat)";
echo "running qemu: $(pgrep -c "qemu-system-x86|qemu-kvm" || echo 0)"' 2>&1
echo "running qemu: $(pgrep -c qemu-system-x86 || echo 0)"' 2>&1
} >> "${ARTIFACT_DIR}/node-load.txt"
exit 0
@@ -838,8 +738,7 @@ jobs:
- name: Install go-junit-report
if: always()
run: |
# Version from go.mod's tool directive, checked against go.sum.
GOWORK=off go -C spinifex install github.com/jstemmer/go-junit-report/v2
go install github.com/jstemmer/go-junit-report/v2@latest
echo "$HOME/go/bin" >> "$GITHUB_PATH"
- name: Convert test logs to JUnit XML
@@ -856,20 +755,20 @@ jobs:
- name: Ship test results to Elasticsearch
if: always()
continue-on-error: true
uses: $/.github/actions/junit-to-es
uses: ./spinifex/.github/actions/junit-to-es
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
cell: ${{ matrix.cell }}
- name: Analyze e2e failures
if: always()
uses: $/.github/actions/e2e-analyze
uses: ./spinifex/.github/actions/e2e-analyze
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
log-dir: ${{ env.ARTIFACT_DIR }}
title: "E2E failure analysis (cell ${{ matrix.cell }})"
- uses: actions/upload-artifact@v7.0.2
- uses: actions/upload-artifact@v7
if: always()
with:
name: e2e-artifacts-cell-${{ matrix.cell }}-${{ github.run_id }}
@@ -881,7 +780,7 @@ jobs:
# pulling nineteen journals it has no use for.
- name: Upload junit only
if: always()
uses: actions/upload-artifact@v7.0.2
uses: actions/upload-artifact@v7
with:
name: junit-cell-${{ matrix.cell }}-${{ github.run_id }}
path: ${{ env.ARTIFACT_DIR }}/junit-*.xml
@@ -901,8 +800,7 @@ jobs:
# different image path than apply did.
TOFU_VAR_ARGS=""
case "$MATRIX_HOST" in
ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE} -var=host_os=ubuntu" ;;
oracle-*) TOFU_VAR_ARGS="-var=host_image_override=${ORACLE_GOLD_IMAGE} -var=host_os=oracle" ;;
ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}" ;;
esac
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
@@ -913,7 +811,7 @@ jobs:
- name: Upload tofu log
if: always()
uses: actions/upload-artifact@v7.0.2
uses: actions/upload-artifact@v7
with:
name: tofu-log-cell-${{ matrix.cell }}-${{ github.run_id }}
path: ${{ env.TOFU_LOG }}
@@ -1018,20 +916,20 @@ jobs:
{ sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
- uses: actions/create-github-app-token@v3.3.0
- uses: actions/create-github-app-token@v3
id: app-token
with:
client-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
repositories: mulga
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
repository: mulgadc/mulga
token: ${{ steps.app-token.outputs.token }}
path: mulga
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
ref: ${{ env.GIT_BRANCH }}
path: mulga/spinifex
@@ -1040,28 +938,28 @@ jobs:
id: siblings
env:
BRANCH: ${{ env.GIT_BRANCH }}
run: bash .github/scripts/resolve-sibling-refs.sh
run: bash .github/resolve-sibling-refs.sh
working-directory: mulga/spinifex
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
repository: mulgadc/viperblock
ref: ${{ steps.siblings.outputs.viperblock }}
path: mulga/viperblock
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
repository: mulgadc/predastore
ref: ${{ steps.siblings.outputs.predastore }}
path: mulga/predastore
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
repository: mulgadc/northstar
ref: ${{ steps.siblings.outputs.northstar }}
path: mulga/northstar
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
repository: mulgadc/bluebottle
ref: ${{ steps.siblings.outputs.bluebottle }}
@@ -1080,6 +978,9 @@ jobs:
{ echo "::error::br-wan not permitted in /etc/qemu/bridge.conf"; exit 1; }
command -v sshpass >/dev/null ||
{ echo "::error::sshpass not installed"; exit 1; }
free_mb=$(free -m | awk '/^Mem:/ {print $7}')
[ "$free_mb" -ge 8192 ] ||
{ echo "::error::need 8 GiB available, have ${free_mb} MiB"; exit 1; }
# A 40G image on tmpfs competes with the 8 GiB the VM itself needs.
fstype=$(df --output=fstype "$TMPDIR" | tail -1 | tr -d ' ')
[ "$fstype" != tmpfs ] ||
@@ -1110,10 +1011,6 @@ jobs:
ssh-keygen -t ed25519 -N "" -f "$HOME/.ssh/$k" -C "$k-ci"
done
# The node's 8 GiB, claimed when the VM boots rather than at preflight.
- name: Wait for memory
run: mulga/spinifex/.github/scripts/wait-for-memory.sh 8192
# Boots the ISO, installs unattended, reads the lease off the serial
# console, runs the in-guest smoke test, and installs the key for the
# next step.
@@ -1197,7 +1094,7 @@ jobs:
/tmp/smoke-$GITHUB_RUN_ID.log | tail -1)
[ -n "$qemu" ] && cp "$qemu" /tmp/cell21-artifacts-$GITHUB_RUN_ID/qemu.log || true
- uses: actions/upload-artifact@v7.0.2
- uses: actions/upload-artifact@v7
if: always()
with:
name: e2e-artifacts-cell-21-${{ github.run_id }}
@@ -1227,16 +1124,6 @@ jobs:
sleep 2
rm -f "$TMPDIR"/tmp.*.qcow2 "$TMPDIR"/tmp.*.log "$TMPDIR"/tmp.*-node.log
# The ISO build runs as root inside the workspace, and the next job on
# this runner, from any workflow, cannot remove what it leaves there.
- name: Release the workspace
if: always()
run: |
sudo umount -R -l "$GITHUB_WORKSPACE"/mulga/scripts/iso-builder/.cache/rootfs 2>/dev/null || true
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/mulga/scripts/iso-builder/.cache \
"$GITHUB_WORKSPACE"/mulga/scripts/iso-builder/dist
sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE"
iso-multi:
name: cell-22 (iso/dhcp/three-node)
# After cell-21 rather than beside it: same concurrency group, and running
@@ -1276,20 +1163,20 @@ jobs:
# disks, which is what the tmpfs assertion below is about.
echo "WORK_DIR=$E2E_ROOT/work/cell22-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
- uses: actions/create-github-app-token@v3.3.0
- uses: actions/create-github-app-token@v3
id: app-token
with:
client-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
repositories: mulga
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
repository: mulgadc/mulga
token: ${{ steps.app-token.outputs.token }}
path: mulga
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
ref: ${{ env.GIT_BRANCH }}
path: mulga/spinifex
@@ -1298,37 +1185,42 @@ jobs:
id: siblings
env:
BRANCH: ${{ env.GIT_BRANCH }}
run: bash .github/scripts/resolve-sibling-refs.sh
run: bash .github/resolve-sibling-refs.sh
working-directory: mulga/spinifex
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
repository: mulgadc/viperblock
ref: ${{ steps.siblings.outputs.viperblock }}
path: mulga/viperblock
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
repository: mulgadc/predastore
ref: ${{ steps.siblings.outputs.predastore }}
path: mulga/predastore
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
repository: mulgadc/northstar
ref: ${{ steps.siblings.outputs.northstar }}
path: mulga/northstar
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
repository: mulgadc/bluebottle
ref: ${{ steps.siblings.outputs.bluebottle }}
path: mulga/bluebottle
# 3 x 8 GiB plus 9 sparse 40G disks, before the instances the cluster
# launches on top.
- name: Preflight
run: |
set -euo pipefail
mkdir -p "$TMPDIR"
free_mb=$(free -m | awk '/^Mem:/ {print $7}')
[ "$free_mb" -ge 26624 ] ||
{ echo "::error::need 26 GiB available, have ${free_mb} MiB"; exit 1; }
fstype=$(df --output=fstype "$TMPDIR" | tail -1 | tr -d ' ')
[ "$fstype" != tmpfs ] ||
{ echo "::error::$TMPDIR is tmpfs — nine node disks would be in RAM"; exit 1; }
@@ -1351,11 +1243,6 @@ jobs:
# formation port. Both land with mulga-yxtda and the bridged WAN mode;
# until then this cell proves the cluster forms and survives, not that a
# customer can use it.
# 3 x 8 GiB plus headroom for the instances the cluster launches on top,
# claimed when the nodes boot rather than at preflight.
- name: Wait for memory
run: mulga/spinifex/.github/scripts/wait-for-memory.sh 26624
- name: Install, form and verify the cluster
working-directory: mulga/scripts/iso-builder
run: |
@@ -1373,7 +1260,7 @@ jobs:
cp "$WORK_DIR"/*-serial.log /tmp/cell22-artifacts-$GITHUB_RUN_ID/ 2>/dev/null ||
echo "no per-node serial logs — the run passed and cleaned up after itself"
- uses: actions/upload-artifact@v7.0.2
- uses: actions/upload-artifact@v7
if: always()
with:
name: e2e-artifacts-cell-22-${{ github.run_id }}
@@ -1388,16 +1275,6 @@ jobs:
rm -rf /tmp/cell22-artifacts-$GITHUB_RUN_ID /tmp/cell22-$GITHUB_RUN_ID.log \
"$WORK_DIR"
# The ISO build runs as root inside the workspace, and the next job on
# this runner, from any workflow, cannot remove what it leaves there.
- name: Release the workspace
if: always()
run: |
sudo umount -R -l "$GITHUB_WORKSPACE"/mulga/scripts/iso-builder/.cache/rootfs 2>/dev/null || true
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/mulga/scripts/iso-builder/.cache \
"$GITHUB_WORKSPACE"/mulga/scripts/iso-builder/dist
sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE"
# Predastore throughput ("cell 23"). Predastore has no runners of its own and
# spinifex depends on it, so the benchmark lives here and checks predastore
# out as a sibling.
@@ -1435,7 +1312,7 @@ jobs:
echo "PERF_RESULTS_ROOT=$E2E_ROOT/predastore-perf-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
- name: Checkout Spinifex
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
path: spinifex
@@ -1446,11 +1323,11 @@ jobs:
id: siblings
env:
BRANCH: ${{ github.ref_name }}
run: bash .github/scripts/resolve-sibling-refs.sh
run: bash .github/resolve-sibling-refs.sh
working-directory: spinifex
- name: Checkout Predastore
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/predastore
ref: ${{ steps.siblings.outputs.predastore }}
@@ -1503,7 +1380,7 @@ jobs:
# Report-only. The baseline comparison and the -10% gate are mulga-wpjsx;
# until the run-to-run spread is known from real nights, a threshold would
# be a guess, and a gate that cries wolf gets muted.
- uses: actions/upload-artifact@v7.0.2
- uses: actions/upload-artifact@v7
if: always()
with:
name: predastore-perf-${{ github.run_id }}
@@ -1562,7 +1439,7 @@ jobs:
echo "STRESS_RESULTS_ROOT=$E2E_ROOT/predastore-stress-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
- name: Checkout Spinifex
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
path: spinifex
@@ -1570,11 +1447,11 @@ jobs:
id: siblings
env:
BRANCH: ${{ github.ref_name }}
run: bash .github/scripts/resolve-sibling-refs.sh
run: bash .github/resolve-sibling-refs.sh
working-directory: spinifex
- name: Checkout Predastore
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/predastore
ref: ${{ steps.siblings.outputs.predastore }}
@@ -1629,7 +1506,7 @@ jobs:
mkdir -p "$STRESS_RESULTS_ROOT"
make e2e-stress 2>&1 | tee "$STRESS_RESULTS_ROOT/e2e-stress.log"
- uses: actions/upload-artifact@v7.0.2
- uses: actions/upload-artifact@v7
if: always()
with:
name: predastore-stress-${{ github.run_id }}
@@ -1666,13 +1543,13 @@ jobs:
if: always()
runs-on: ubuntu-26.04
steps:
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
sparse-checkout: .github/scripts/e2e-run-summary.py
sparse-checkout-cone-mode: false
- name: Collect junit
uses: actions/download-artifact@v8.0.2
uses: actions/download-artifact@v8
with:
pattern: junit-cell-*-${{ github.run_id }}
path: junit
+31 -33
View File
@@ -103,25 +103,25 @@ jobs:
| sort -r | xargs -r -I{} sudo umount -l -R {} || true
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
- uses: actions/create-github-app-token@v3.3.0
- uses: actions/create-github-app-token@v3
id: app-token
with:
client-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
repositories: mulga
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
repository: mulgadc/mulga
token: ${{ steps.app-token.outputs.token }}
sparse-checkout: scripts/tofu-cluster
path: mulga
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
path: spinifex
- uses: actions/setup-go@v7.0.0
- uses: actions/setup-go@v7
with:
go-version-file: spinifex/go.mod
cache: false
@@ -157,7 +157,7 @@ jobs:
# the env: the suite legs share this env and its baked telemetry.env, so
# the provisioner is the only job the daemon telemetry can belong to.
- name: Resolve CI job id
uses: $/.github/actions/otel-job-id
uses: ./spinifex/.github/actions/otel-job-id
with:
job-name: Single env · provision
@@ -300,7 +300,7 @@ jobs:
- name: Upload tofu log
if: always()
uses: actions/upload-artifact@v7.0.2
uses: actions/upload-artifact@v7
with:
name: tofu-log-single-${{ github.run_id }}
path: ${{ env.TOFU_LOG }}
@@ -323,19 +323,18 @@ jobs:
SSH_KEY: ${{ needs.single_provision.outputs.ssh_key }}
ARTIFACT_DIR: /tmp/spinifex-e2e-single-${{ github.run_id }}-${{ matrix.suite }}
steps:
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
path: spinifex
- uses: actions/setup-go@v7.0.0
- uses: actions/setup-go@v7
with:
go-version-file: spinifex/go.mod
cache: false
- name: Install go-junit-report
run: |
# Version from go.mod's tool directive, checked against go.sum.
GOWORK=off go -C spinifex install github.com/jstemmer/go-junit-report/v2
go install github.com/jstemmer/go-junit-report/v2@latest
echo "$HOME/go/bin" >> "$GITHUB_PATH"
- name: Run ${{ matrix.suite }}
@@ -391,7 +390,7 @@ jobs:
- name: Publish test summary
if: always()
uses: $/.github/actions/junit-report
uses: ./spinifex/.github/actions/junit-report
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
title: "E2E ${{ matrix.suite }} results"
@@ -401,20 +400,20 @@ jobs:
- name: Ship test results to Elasticsearch
if: always()
continue-on-error: true
uses: $/.github/actions/junit-to-es
uses: ./spinifex/.github/actions/junit-to-es
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
cell: single
- name: Analyze e2e failures
if: always()
uses: $/.github/actions/e2e-analyze
uses: ./spinifex/.github/actions/e2e-analyze
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
log-dir: ${{ env.ARTIFACT_DIR }}
title: "E2E ${{ matrix.suite }} failure analysis"
- uses: actions/upload-artifact@v7.0.2
- uses: actions/upload-artifact@v7
if: always()
with:
name: e2e-single-${{ matrix.suite }}-${{ github.run_id }}
@@ -445,7 +444,7 @@ jobs:
# Reuse the persisted mulga/ tofu-cluster workspace from single_provision.
# Re-checkout only spinifex (for ci-fmt + diag scripts); checking out mulga
# again would wipe the local tofu state and orphan the VM.
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
path: spinifex
@@ -468,7 +467,7 @@ jobs:
| tar -C "${ARTIFACT_DIR}/lb-console" -xf - 2>/dev/null
true
- uses: actions/upload-artifact@v7.0.2
- uses: actions/upload-artifact@v7
if: always()
with:
name: e2e-single-diag-${{ github.run_id }}
@@ -524,7 +523,7 @@ jobs:
- name: Upload tofu log
if: always()
uses: actions/upload-artifact@v7.0.2
uses: actions/upload-artifact@v7
with:
name: tofu-log-single-td-${{ github.run_id }}
path: ${{ env.TOFU_LOG }}
@@ -572,25 +571,25 @@ jobs:
| sort -r | xargs -r -I{} sudo umount -l -R {} || true
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
- uses: actions/create-github-app-token@v3.3.0
- uses: actions/create-github-app-token@v3
id: app-token
with:
client-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
repositories: mulga
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
repository: mulgadc/mulga
token: ${{ steps.app-token.outputs.token }}
sparse-checkout: scripts/tofu-cluster
path: mulga
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
path: spinifex
- uses: actions/setup-go@v7.0.0
- uses: actions/setup-go@v7
with:
go-version-file: spinifex/go.mod
cache: false
@@ -607,7 +606,7 @@ jobs:
# See the matching note on single_provision.
- name: Resolve CI job id
uses: $/.github/actions/otel-job-id
uses: ./spinifex/.github/actions/otel-job-id
with:
job-name: Multi env · provision
@@ -715,7 +714,7 @@ jobs:
- name: Upload tofu log
if: always()
uses: actions/upload-artifact@v7.0.2
uses: actions/upload-artifact@v7
with:
name: tofu-log-multi-${{ github.run_id }}
path: ${{ env.TOFU_LOG }}
@@ -738,19 +737,18 @@ jobs:
NODES_CSV: ${{ needs.multi_provision.outputs.nodes_csv }}
ARTIFACT_DIR: /tmp/spinifex-e2e-multi-${{ github.run_id }}-${{ matrix.suite }}
steps:
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
path: spinifex
- uses: actions/setup-go@v7.0.0
- uses: actions/setup-go@v7
with:
go-version-file: spinifex/go.mod
cache: false
- name: Install go-junit-report
run: |
# Version from go.mod's tool directive, checked against go.sum.
GOWORK=off go -C spinifex install github.com/jstemmer/go-junit-report/v2
go install github.com/jstemmer/go-junit-report/v2@latest
echo "$HOME/go/bin" >> "$GITHUB_PATH"
- name: Run ${{ matrix.suite }}
@@ -806,7 +804,7 @@ jobs:
- name: Publish test summary
if: always()
uses: $/.github/actions/junit-report
uses: ./spinifex/.github/actions/junit-report
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
title: "Multi-Node E2E ${{ matrix.suite }} results"
@@ -816,20 +814,20 @@ jobs:
- name: Ship test results to Elasticsearch
if: always()
continue-on-error: true
uses: $/.github/actions/junit-to-es
uses: ./spinifex/.github/actions/junit-to-es
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
cell: multi
- name: Analyze e2e failures
if: always()
uses: $/.github/actions/e2e-analyze
uses: ./spinifex/.github/actions/e2e-analyze
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
log-dir: ${{ env.ARTIFACT_DIR }}
title: "Multi-Node E2E ${{ matrix.suite }} failure analysis"
- uses: actions/upload-artifact@v7.0.2
- uses: actions/upload-artifact@v7
if: always()
with:
name: e2e-multi-${{ matrix.suite }}-${{ github.run_id }}
@@ -856,7 +854,7 @@ jobs:
SSH_KEY: ${{ needs.multi_provision.outputs.ssh_key }}
NODES_CSV: ${{ needs.multi_provision.outputs.nodes_csv }}
steps:
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
path: spinifex
@@ -910,7 +908,7 @@ jobs:
- name: Upload tofu log
if: always()
uses: actions/upload-artifact@v7.0.2
uses: actions/upload-artifact@v7
with:
name: tofu-log-multi-td-${{ github.run_id }}
path: ${{ env.TOFU_LOG }}
+104 -166
View File
@@ -12,15 +12,13 @@ on:
- go.sum
- Makefile
- tests/e2e/**
- scripts/images/**
- scripts/build-system-image.sh
- .github/workflows/e2e.yml
workflow_dispatch:
inputs:
suites:
description: "Space-separated Make targets to run (e.g. 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-eks e2e-iam e2e-rds')"
required: false
default: "e2e-single e2e-multinode e2e-cert e2e-iam"
default: "e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam"
test_run:
description: "Optional regex for -test.run (e.g. '^TestMultinodeIPSec$'). Empty = run all tests in suite."
required: false
@@ -33,75 +31,86 @@ permissions:
env:
GIT_BRANCH: ${{ github.ref_name }}
SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam e2e-partialblock' }}
TEST_RUN: ${{ inputs.test_run || '' }}
SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
jobs:
# A push runs the base set plus the suites its changed paths map to in
# suite-sets.sh; a dispatch runs its suites input, or the base set if empty.
# Budgets come from the suites' own timeouts there.
# The inner timeout must fire before the outer one. `go test` reports where it
# stopped and the suite's own failure; a job cancelled on timeout reports only
# "The operation was canceled" and takes the artifacts with it.
#
# A job budget cannot be a literal here because the suite set is a dispatch
# input: storagefault asks for 90m and diskperf for 180m, both eligible on one
# node, so any fixed number is either wrong for them or wasteful for everyone
# else. Resolving it from suite-sets.sh keeps the one source of truth and
# leaves the default push set at exactly the 60m it has always had.
plan:
name: Select Suites
name: Resolve Job Budgets
runs-on: ubuntu-26.04
timeout-minutes: 5
outputs:
single_legs: ${{ steps.select.outputs.single_legs }}
multi_suites: ${{ steps.select.outputs.multi_suites }}
multi_timeout: ${{ steps.select.outputs.multi_timeout }}
single_timeout: ${{ steps.budgets.outputs.single_timeout }}
multi_timeout: ${{ steps.budgets.outputs.multi_timeout }}
steps:
# Full history without file contents: the diff needs both commits' trees.
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
path: spinifex
fetch-depth: 0
filter: blob:none
sparse-checkout: .github/scripts
- name: Select suites
id: select
working-directory: spinifex
- name: Resolve Budgets
id: budgets
env:
EVENT: ${{ github.event_name }}
BEFORE: ${{ github.event.before }}
IN_SUITES: ${{ inputs.suites }}
# Provision, install and teardown around the suites, plus the headroom
# a multi-suite job needs for whatever ran before the one that hangs.
SETUP_ALLOWANCE_MIN: 15
# What the job has always had. Kept as a floor so resolving the budget
# can only ever raise it, never quietly shorten an existing run.
FLOOR_MIN: 60
run: |
set -euo pipefail
SELECT=.github/scripts/e2e-select.sh
if [ "${EVENT}" = push ]; then
bash "${SELECT}" push "${BEFORE}" "${GITHUB_SHA}" | tee -a "$GITHUB_OUTPUT"
else
bash "${SELECT}" suites "${IN_SUITES}" | tee -a "$GITHUB_OUTPUT"
fi
. "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
# Only max() is a sound bound: the suites share one job, so a sum would
# demand hours for a set that has never taken them. Max is what makes
# the hanging suite panic on its own timeout first, which is the point.
budget() {
local want="${FLOOR_MIN}" s t
for s in $(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -xE "$1"); do
t="$(e2e_suite_timeout "${s}")"
t=$(( ${t%m} + SETUP_ALLOWANCE_MIN ))
[ "${t}" -gt "${want}" ] && want="${t}"
done
echo "${want}"
}
single=$(budget "${E2E_SUITES_SINGLE_RE}")
multi=$(budget "${E2E_SUITES_MULTI_RE}")
echo "single_timeout=${single}" >> "$GITHUB_OUTPUT"
echo "multi_timeout=${multi}" >> "$GITHUB_OUTPUT"
echo "suites='${SUITES}' -> single ${single}m, multi ${multi}m"
e2e_single:
# The base leg keeps the job and artifact names it always had: mulga's
# registry intake finds a run's results by them.
name: ${{ matrix.leg == 'base' && 'Single-Node E2E' || format('Single-Node E2E ({0})', matrix.leg) }}
# `base` runs on ci-single; `service`, on ci-pseudo, carries the matched
# single-node suites so a long one stays off the base leg's critical path.
if: needs.plan.outputs.single_legs != '[]'
name: Single-Node E2E
# The dispatch check gates on exactly the suites this job's Run step
# allow-lists, i.e. E2E_SUITES_SINGLE in .github/scripts/suite-sets.sh —
# an `if:` expression cannot source that file, so this one list is
# hand-mirrored and must be updated with it. Gating on a suite the job
# won't run — lb, which only executes in the multinode job — would
# provision a tofu VM and pay the Spinifex install only to run nothing and
# tear down. Push triggers keep running via the null-check on
# inputs.suites.
if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-iam') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-eks') || contains(github.event.inputs.suites, 'e2e-ecs') || contains(github.event.inputs.suites, 'e2e-storagegrowth') || contains(github.event.inputs.suites, 'e2e-partialblock') || contains(github.event.inputs.suites, 'e2e-rds')
needs: plan
runs-on: [self-hosted, "${{ matrix.runner }}"]
timeout-minutes: ${{ matrix.timeout }}
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON(needs.plan.outputs.single_legs) }}
runs-on: [self-hosted, ci-single]
timeout-minutes: ${{ fromJSON(needs.plan.outputs.single_timeout) }}
env:
SUITES: ${{ matrix.suites }}
# Both legs can run on banksia at once, so every host path carries the leg.
TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}-${{ matrix.leg }}.log
ARTIFACT_DIR: /tmp/spinifex-e2e-artifacts-${{ github.run_id }}-${{ matrix.leg }}
# The base leg keeps the cell name it always had, so its history in
# Elastic and telemetry continues; any other leg appends its name.
E2E_CELL: ${{ matrix.leg == 'base' && 'single' || format('single-{0}', matrix.leg) }}
TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
ARTIFACT_DIR: /tmp/spinifex-e2e-artifacts-${{ github.run_id }}
# Job-level env wins over the workflow-level default. This job and
# e2e_multi share one run id and both name their hosts node1/node2/node3,
# so without ci.cell their telemetry collapses into one bucket. This job
# is where "Bootstrap via Binary Install" bakes SPINIFEX_OTEL_ATTRS into
# the VM's telemetry.env, so the label has to be set here.
SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=${{ matrix.leg == 'base' && 'single' || format('single-{0}', matrix.leg) }}
SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=single
steps:
- name: Reap Orphan Procs
run: |
@@ -133,25 +142,25 @@ jobs:
# stale suite-status.txt / analysis dirs can't leak into this attempt.
rm -rf "${ARTIFACT_DIR}"
- uses: actions/create-github-app-token@v3.3.0
- uses: actions/create-github-app-token@v3
id: app-token
with:
client-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
repositories: mulga
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
repository: mulgadc/mulga
token: ${{ steps.app-token.outputs.token }}
sparse-checkout: scripts/tofu-cluster
path: mulga
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
path: spinifex
- uses: actions/setup-go@v7.0.0
- uses: actions/setup-go@v7
with:
go-version-file: spinifex/go.mod
cache: false
@@ -169,9 +178,9 @@ jobs:
# Adds ci.job_id alongside ci.cell, before Bootstrap bakes the attrs into
# the VM, so a telemetry doc points back at this job's log directly.
- name: Resolve CI job id
uses: $/.github/actions/otel-job-id
uses: ./spinifex/.github/actions/otel-job-id
with:
job-name: ${{ matrix.leg == 'base' && 'Single-Node E2E' || format('Single-Node E2E ({0})', matrix.leg) }}
job-name: Single-Node E2E
- name: Build Install Artifacts
working-directory: mulga/scripts/tofu-cluster
@@ -215,12 +224,6 @@ jobs:
echo "wan_ip=${WAN_IP}" >> "$GITHUB_OUTPUT"
echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
# build-system-image.sh reuses a raw image under ten minutes old from
# its build directory, and every runner on banksia shares the default
# one in /tmp, so a job could test another commit's guest agent.
- name: Use a per-job image build directory
run: echo "SYSTEM_IMAGE_BUILD_DIR=${RUNNER_TEMP}/system-image-build" >> "$GITHUB_ENV"
# The eks suite's CreateCluster boots a control-plane VM from the
# eks-node AMI, so that AMI must exist in the VM's image store before the
# suite runs. build-system-image.sh customizes the image with libguestfs
@@ -244,7 +247,7 @@ jobs:
group_open "make build-eks-node-image"
make build-eks-node-image
group_close
ok "raw: $(du -h "${SYSTEM_IMAGE_BUILD_DIR}/eks-node-image-build/eks-node-alpine.raw" | cut -f1)"
ok "raw: $(du -h /tmp/eks-node-image-build/eks-node-alpine.raw | cut -f1)"
- name: Import eks-node AMI on VM
if: contains(env.SUITES, 'e2e-eks')
@@ -256,16 +259,13 @@ jobs:
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
VM=tf-user@${{ steps.vm.outputs.wan_ip }}
banner "import eks-node AMI to VM image store"
# The VM's /tmp is a tmpfs and scp writes each raw at its full size,
# so every import removes its file before the next image arrives.
scp "${SSH_OPTS[@]}" \
"${SYSTEM_IMAGE_BUILD_DIR}/eks-node-image-build/eks-node-alpine.raw" \
/tmp/eks-node-image-build/eks-node-alpine.raw \
"$VM":/tmp/eks-node-alpine.raw
ssh "${SSH_OPTS[@]}" "$VM" \
"spx admin images import --file /tmp/eks-node-alpine.raw \
--distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
--boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}; \
rc=\$?; rm -f /tmp/eks-node-alpine.raw; exit \$rc" >/dev/null 2>&1
--boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null 2>&1
ok "eks-node AMI imported"
# The eks data-plane subtest (TestEKS/KubectlGetNodes) drives the full
@@ -299,40 +299,6 @@ jobs:
EOF
ok "kubectl installed on VM"
# The ecs suite boots its container instance from the ecs-node AMI, found
# by its managed-by tag, so the tag matters. The build directory is named
# for the manifest's IMAGE_NAME (ecs-agent), not for the make target.
- name: Build ecs-node AMI (libguestfs, runner)
if: contains(env.SUITES, 'e2e-ecs')
working-directory: spinifex
run: |
set -euo pipefail
# direct backend = no libvirt. Set here rather than in the script:
# it has to be in the environment the build itself runs in.
export LIBGUESTFS_BACKEND=direct
"${GITHUB_WORKSPACE}/spinifex/.github/scripts/ensure-libguestfs.sh"
make build-ecs-node-image
- name: Import ecs-node AMI on VM
if: contains(env.SUITES, 'e2e-ecs')
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
spinifex/scripts/images/ecs-agent/manifest.conf)"
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
VM=tf-user@${{ steps.vm.outputs.wan_ip }}
banner "import ecs-node AMI to VM image store"
scp "${SSH_OPTS[@]}" \
"${SYSTEM_IMAGE_BUILD_DIR}/ecs-agent-image-build/ecs-agent-alpine.raw" \
"$VM":/tmp/ecs-agent-alpine.raw
ssh "${SSH_OPTS[@]}" "$VM" \
"spx admin images import --file /tmp/ecs-agent-alpine.raw \
--distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
--boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}; \
rc=\$?; rm -f /tmp/ecs-agent-alpine.raw; exit \$rc" >/dev/null 2>&1
ok "ecs-node AMI imported"
# CreateDBInstance boots the DB VM from an RDS AMI, so both engines' images
# must be in the VM's image store before the suite runs — same split as the
# eks-node pair above: build on the runner (needs source + network), import
@@ -355,7 +321,7 @@ jobs:
group_open "make build-${IMAGE}-image"
make "build-${IMAGE}-image"
group_close
ok "raw: $(du -h "${SYSTEM_IMAGE_BUILD_DIR}/${IMAGE}-image-build/${IMAGE}-alpine.raw" | cut -f1)"
ok "raw: $(du -h "/tmp/${IMAGE}-image-build/${IMAGE}-alpine.raw" | cut -f1)"
done
- name: Import RDS AMIs on VM
@@ -377,13 +343,12 @@ jobs:
for tag in ${SYSTEM_TAG}; do TAG_ARGS="${TAG_ARGS} --tag ${tag}"; done
banner "import ${IMAGE} AMI to VM image store"
scp "${SSH_OPTS[@]}" \
"${SYSTEM_IMAGE_BUILD_DIR}/${IMAGE}-image-build/${IMAGE}-alpine.raw" \
"/tmp/${IMAGE}-image-build/${IMAGE}-alpine.raw" \
"$VM":"/tmp/${IMAGE}-alpine.raw"
ssh "${SSH_OPTS[@]}" "$VM" \
"spx admin images import --file /tmp/${IMAGE}-alpine.raw \
--distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
--boot-mode bios --ami-name ${AMI_NAME}${TAG_ARGS}; \
rc=\$?; rm -f /tmp/${IMAGE}-alpine.raw; exit \$rc" >/dev/null 2>&1
--boot-mode bios --ami-name ${AMI_NAME}${TAG_ARGS}" >/dev/null 2>&1
ok "${IMAGE} AMI imported"
done
@@ -407,8 +372,7 @@ jobs:
- name: Install go-junit-report on runner
run: |
# Version from go.mod's tool directive, checked against go.sum.
GOWORK=off go -C spinifex install github.com/jstemmer/go-junit-report/v2
go install github.com/jstemmer/go-junit-report/v2@latest
echo "$HOME/go/bin" >> "$GITHUB_PATH"
- name: Ship test binaries to VM
@@ -548,7 +512,7 @@ jobs:
- name: Publish test summary
if: always()
uses: $/.github/actions/junit-report
uses: ./spinifex/.github/actions/junit-report
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
title: "E2E results"
@@ -558,30 +522,23 @@ jobs:
- name: Ship test results to Elasticsearch
if: always()
continue-on-error: true
uses: $/.github/actions/junit-to-es
uses: ./spinifex/.github/actions/junit-to-es
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
cell: ${{ env.E2E_CELL }}
cell: single
- name: Analyze e2e failures
if: always()
uses: $/.github/actions/e2e-analyze
uses: ./spinifex/.github/actions/e2e-analyze
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
log-dir: ${{ env.ARTIFACT_DIR }}
title: "E2E failure analysis"
# The WAN capture is most of the artifact and only matters when a run
# failed. Anything short of a clean pass (suite rc, a failed setup or
# collect step, a missing rc) keeps it.
- name: Drop the packet capture from a passing single-node run
if: always() && steps.e2e.outputs.rc == '0' && job.status == 'success'
run: rm -f "${ARTIFACT_DIR}"/net-diag/wan.pcap
- uses: actions/upload-artifact@v7.0.2
- uses: actions/upload-artifact@v7
if: always()
with:
name: ${{ matrix.leg == 'base' && format('e2e-artifacts-{0}', github.run_id) || format('e2e-artifacts-{0}-{1}', github.run_id, matrix.leg) }}
name: e2e-artifacts-${{ github.run_id }}
path: ${{ env.ARTIFACT_DIR }}
if-no-files-found: warn
retention-days: 14
@@ -622,9 +579,9 @@ jobs:
- name: Upload tofu log
if: always()
uses: actions/upload-artifact@v7.0.2
uses: actions/upload-artifact@v7
with:
name: ${{ matrix.leg == 'base' && format('tofu-log-single-{0}', github.run_id) || format('tofu-log-single-{0}-{1}', github.run_id, matrix.leg) }}
name: tofu-log-single-${{ github.run_id }}
path: ${{ env.TOFU_LOG }}
if-no-files-found: ignore
retention-days: 14
@@ -640,13 +597,15 @@ jobs:
e2e_multi:
name: Multi-Node E2E
# Runs when the plan job selected any multi-node suite, and only then.
if: needs.plan.outputs.multi_suites != ''
# The dispatch check gates on exactly the suites this job's Run step
# allow-lists (multinode|lb|cert) so lb — which only runs here — still
# provisions the 3-node env when requested on its own, and the env stays
# unprovisioned when none of them are asked for.
if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-multinode') || contains(github.event.inputs.suites, 'e2e-lb') || contains(github.event.inputs.suites, 'e2e-cert')
needs: plan
runs-on: [self-hosted, ci-multi]
timeout-minutes: ${{ fromJSON(needs.plan.outputs.multi_timeout) }}
env:
SUITES: ${{ needs.plan.outputs.multi_suites }}
TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
ARTIFACT_DIR: /tmp/spinifex-e2e-multi-artifacts-${{ github.run_id }}
# See the matching note on e2e_single: ci.cell is what separates this
@@ -683,25 +642,25 @@ jobs:
# stale suite-status.txt / analysis dirs can't leak into this attempt.
rm -rf "${ARTIFACT_DIR}"
- uses: actions/create-github-app-token@v3.3.0
- uses: actions/create-github-app-token@v3
id: app-token
with:
client-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
repositories: mulga
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
repository: mulgadc/mulga
token: ${{ steps.app-token.outputs.token }}
sparse-checkout: scripts/tofu-cluster
path: mulga
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
path: spinifex
- uses: actions/setup-go@v7.0.0
- uses: actions/setup-go@v7
with:
go-version-file: spinifex/go.mod
cache: false
@@ -718,7 +677,7 @@ jobs:
# See the matching note on e2e_single.
- name: Resolve CI job id
uses: $/.github/actions/otel-job-id
uses: ./spinifex/.github/actions/otel-job-id
with:
job-name: Multi-Node E2E
@@ -799,8 +758,7 @@ jobs:
- name: Install go-junit-report on runner
run: |
# Version from go.mod's tool directive, checked against go.sum.
GOWORK=off go -C spinifex install github.com/jstemmer/go-junit-report/v2
go install github.com/jstemmer/go-junit-report/v2@latest
echo "$HOME/go/bin" >> "$GITHUB_PATH"
- name: Ship test binaries to node1
@@ -1007,7 +965,7 @@ jobs:
- name: Publish test summary
if: always()
uses: $/.github/actions/junit-report
uses: ./spinifex/.github/actions/junit-report
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
title: "Multi-Node E2E results"
@@ -1017,27 +975,20 @@ jobs:
- name: Ship test results to Elasticsearch
if: always()
continue-on-error: true
uses: $/.github/actions/junit-to-es
uses: ./spinifex/.github/actions/junit-to-es
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
cell: multi
- name: Analyze e2e failures
if: always()
uses: $/.github/actions/e2e-analyze
uses: ./spinifex/.github/actions/e2e-analyze
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
log-dir: ${{ env.ARTIFACT_DIR }}
title: "Multi-Node E2E failure analysis"
# The WAN captures are 90% of the artifact and only matter when a run
# failed. Anything short of a clean pass (suite rc, a failed setup or
# collect step, a missing rc) keeps them.
- name: Drop the packet captures from a passing multi-node run
if: always() && steps.e2e.outputs.rc == '0' && job.status == 'success'
run: rm -f "${ARTIFACT_DIR}"/net-diag-node*/wan.pcap
- uses: actions/upload-artifact@v7.0.2
- uses: actions/upload-artifact@v7
if: always()
with:
name: e2e-multi-artifacts-${{ github.run_id }}
@@ -1084,7 +1035,7 @@ jobs:
- name: Upload tofu log
if: always()
uses: actions/upload-artifact@v7.0.2
uses: actions/upload-artifact@v7
with:
name: tofu-log-multi-${{ github.run_id }}
path: ${{ env.TOFU_LOG }}
@@ -1103,10 +1054,7 @@ jobs:
e2e_status:
name: Report E2E Status
if: always()
# plan is a dependency because a failed plan skips both e2e jobs, and a
# skipped job counts as passed below: without it a broken selection would
# report green having run nothing.
needs: [plan, e2e_single, e2e_multi]
needs: [e2e_single, e2e_multi]
runs-on: ubuntu-26.04
timeout-minutes: 5
steps:
@@ -1115,16 +1063,15 @@ jobs:
GH_TOKEN: ${{ github.token }}
REPORT_SHA: ${{ github.sha }}
run: |
plan_ok=true; single_ok=true; multi_ok=true
[ "${{ needs.plan.result }}" = success ] || plan_ok=false
single_ok=true; multi_ok=true
case "${{ needs.e2e_single.result }}" in success|skipped) ;; *) single_ok=false ;; esac
case "${{ needs.e2e_multi.result }}" in success|skipped) ;; *) multi_ok=false ;; esac
if $plan_ok && $single_ok && $multi_ok; then
if $single_ok && $multi_ok; then
STATE="success"
DESC="E2E suites passed"
else
STATE="failure"
DESC="E2E failed: plan=${{ needs.plan.result }} single=${{ needs.e2e_single.result }} multi=${{ needs.e2e_multi.result }}"
DESC="E2E failed: single=${{ needs.e2e_single.result }} multi=${{ needs.e2e_multi.result }}"
fi
gh api "repos/${{ github.repository }}/statuses/${REPORT_SHA}" \
-f state="$STATE" \
@@ -1141,7 +1088,7 @@ jobs:
# status-report check.
e2e_es_summary:
name: ES CI Run Summary
needs: [plan, e2e_single, e2e_multi]
needs: [e2e_single, e2e_multi]
if: always()
runs-on: [self-hosted, ci-single]
timeout-minutes: 5
@@ -1153,24 +1100,19 @@ jobs:
# this host, so no credentials are needed for the POST.
MULGA_ES_SINK: ${{ vars.MULGA_ES_SINK || '192.168.1.13:9200' }}
steps:
# The workspace outlives the run on a self-hosted runner, and the status
# files are collected recursively, so an earlier run's would count here.
- name: Clear previous summary downloads
run: rm -rf "${GITHUB_WORKSPACE}/es-summary-single" "${GITHUB_WORKSPACE}/es-summary-multi"
# Either leg may not have run (e.g. a workflow_dispatch that only
# requested single-node suites), so a missing artifact is tolerated —
# the aggregation script below treats an absent suite-status.txt as
# "no failures from that leg".
- name: Download single-node artifacts
uses: actions/download-artifact@v8.0.2
uses: actions/download-artifact@v8
continue-on-error: true
with:
pattern: e2e-artifacts-${{ github.run_id }}*
name: e2e-artifacts-${{ github.run_id }}
path: ${{ github.workspace }}/es-summary-single
- name: Download multi-node artifacts
uses: actions/download-artifact@v8.0.2
uses: actions/download-artifact@v8
continue-on-error: true
with:
name: e2e-multi-artifacts-${{ github.run_id }}
@@ -1186,7 +1128,6 @@ jobs:
BRANCH: ${{ github.head_ref || github.ref_name }}
SHA: ${{ github.sha }}
SERVER_URL: ${{ github.server_url }}
PLAN_RESULT: ${{ needs.plan.result }}
SINGLE_RESULT: ${{ needs.e2e_single.result }}
MULTI_RESULT: ${{ needs.e2e_multi.result }}
SINGLE_DIR: ${{ github.workspace }}/es-summary-single
@@ -1194,8 +1135,7 @@ jobs:
run: |
set -euo pipefail
# One subdirectory per single-node leg's artifact.
SINGLE_STATUS=$(find "${SINGLE_DIR}" -name suite-status.txt 2>/dev/null | sort || true)
SINGLE_STATUS="${SINGLE_DIR}/suite-status.txt"
MULTI_STATUS="${MULTI_DIR}/suite-status.txt"
# Collect "<suite>: FAILED" lines from both legs' suite-status.txt
@@ -1204,9 +1144,7 @@ jobs:
# rather than a bash array so this stays correct under `set -u`
# even when nothing failed.
FAILED_SUITES=""
# SINGLE_STATUS is a newline-separated list, so it is split here.
# shellcheck disable=SC2086
for f in ${SINGLE_STATUS} "$MULTI_STATUS"; do
for f in "$SINGLE_STATUS" "$MULTI_STATUS"; do
[ -f "$f" ] || continue
while IFS= read -r line; do
case "$line" in
@@ -1226,7 +1164,7 @@ jobs:
case "$MULTI_RESULT" in success | skipped) ;; *) multi_ok=false ;; esac
STATE=success
if [ "$PLAN_RESULT" != success ] || [ "$single_ok" != true ] || [ "$multi_ok" != true ] || [ "$FAILED_COUNT" -gt 0 ]; then
if [ "$single_ok" != true ] || [ "$multi_ok" != true ] || [ "$FAILED_COUNT" -gt 0 ]; then
STATE=failure
fi
+7 -8
View File
@@ -95,10 +95,10 @@ jobs:
if: steps.gate.outputs.paused != 'true'
run: sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
if: steps.gate.outputs.paused != 'true'
- uses: actions/setup-go@v7.0.0
- uses: actions/setup-go@v7
if: steps.gate.outputs.paused != 'true'
with:
go-version-file: go.mod
@@ -187,8 +187,7 @@ jobs:
- name: Install go-junit-report on runner
if: steps.gate.outputs.paused != 'true'
run: |
# Version from go.mod's tool directive, checked against go.sum.
GOWORK=off go install github.com/jstemmer/go-junit-report/v2
go install github.com/jstemmer/go-junit-report/v2@latest
echo "$HOME/go/bin" >> "$GITHUB_PATH"
- name: Ship GPU e2e test binary to wattle
@@ -244,7 +243,7 @@ jobs:
- name: Publish test summary
if: always() && steps.gate.outputs.paused != 'true'
uses: $/.github/actions/junit-report
uses: ./.github/actions/junit-report
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
title: "GPU E2E results"
@@ -252,20 +251,20 @@ jobs:
- name: Ship test results to Elasticsearch
if: always() && steps.gate.outputs.paused != 'true'
continue-on-error: true
uses: $/.github/actions/junit-to-es
uses: ./.github/actions/junit-to-es
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
cell: gpu
- name: Analyze e2e failures
if: always() && steps.gate.outputs.paused != 'true'
uses: $/.github/actions/e2e-analyze
uses: ./.github/actions/e2e-analyze
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
log-dir: ${{ env.ARTIFACT_DIR }}
title: "GPU E2E failure analysis"
- uses: actions/upload-artifact@v7.0.2
- uses: actions/upload-artifact@v7
if: always() && steps.gate.outputs.paused != 'true'
with:
name: gpu-e2e-artifacts-${{ github.run_id }}
+3 -6
View File
@@ -29,16 +29,13 @@ jobs:
steps:
- name: Checkout Code
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
- name: Run image script tests + shellcheck
run: make test-images
eks_mkosi_image:
name: Build mkosi EKS image
# A fork PR's code must never run on a persistent self-hosted runner: it
# shares disk, caches and credentials with the release jobs.
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
runs-on: [self-hosted, ci-single]
concurrency:
group: eks-mkosi-image-${{ github.ref }}
@@ -46,10 +43,10 @@ jobs:
steps:
- name: Checkout Code
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
- name: Set up Go
uses: actions/setup-go@v7.0.0
uses: actions/setup-go@v7
with:
go-version-file: "go.mod"
# Self-hosted: the module + build cache already persists on local
+8 -8
View File
@@ -80,25 +80,25 @@ jobs:
echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
echo "PERF_RESULTS_ROOT=$E2E_ROOT/predastore-perf-3node-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
- uses: actions/create-github-app-token@v3.3.0
- uses: actions/create-github-app-token@v3
id: app-token
with:
client-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
repositories: mulga
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
repository: mulgadc/mulga
token: ${{ steps.app-token.outputs.token }}
sparse-checkout: scripts/tofu-cluster
path: mulga
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
path: spinifex
- uses: actions/setup-go@v7.0.0
- uses: actions/setup-go@v7
with:
go-version-file: spinifex/go.mod
cache: false
@@ -113,10 +113,10 @@ jobs:
id: siblings
env:
BRANCH: ${{ env.GIT_BRANCH }}
run: bash .github/scripts/resolve-sibling-refs.sh
run: bash .github/resolve-sibling-refs.sh
working-directory: spinifex
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
id: predastore
with:
repository: mulgadc/predastore
@@ -366,7 +366,7 @@ jobs:
# Report-only, matching cell-23: the baseline comparison and any
# regression gate are future work once run-to-run spread is known from
# real bare-metal runs.
- uses: actions/upload-artifact@v7.0.2
- uses: actions/upload-artifact@v7
if: always()
with:
name: predastore-perf-3node-${{ github.run_id }}
@@ -407,7 +407,7 @@ jobs:
- name: Upload tofu log
if: always()
uses: actions/upload-artifact@v7.0.2
uses: actions/upload-artifact@v7
with:
name: tofu-log-perf3node-${{ github.run_id }}
path: ${{ env.TOFU_LOG }}
+30 -18
View File
@@ -17,12 +17,28 @@ name: Publish System AMIs
# spx admin images import --name spinifex-rds-postgres
# spx admin images import --name spinifex-rds-mariadb
#
# Publishing overwrites the mirror artifact, so it runs only from release.yml
# once a tag has passed its gates, with siblings at the tags go.mod requires.
# The `variant` input scopes which images build — the Ubuntu GPU legs are 16G
# and much slower (NVIDIA driver download + DKMS build).
# Publishing overwrites the mirror artifact, so it is a deliberate action, not
# something every push should trigger: it runs manually, or from release.yml
# once a tag has passed its gates. The `variant` input scopes which images
# build — the Ubuntu GPU legs are 16G and much slower (NVIDIA driver download +
# DKMS build), so publishing only the CPU AMIs stays cheap.
on:
workflow_dispatch:
inputs:
variant:
description: "Which images to build"
type: choice
default: all
options:
- all
- cpu
- gpu
- rds
dry_run:
description: "Build + convert only; skip the R2 upload"
type: boolean
default: false
workflow_call:
inputs:
variant:
@@ -78,44 +94,40 @@ jobs:
steps:
- name: Checkout Spinifex
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
path: spinifex
- name: Resolve sibling release tags from go.mod
id: siblings
run: bash spinifex/.github/scripts/resolve-release-sibling-refs.sh spinifex/go.mod
- name: Checkout Viperblock
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/viperblock
ref: ${{ steps.siblings.outputs.viperblock }}
ref: dev
path: viperblock
- name: Checkout Predastore
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/predastore
ref: ${{ steps.siblings.outputs.predastore }}
ref: dev
path: predastore
- name: Checkout Northstar
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/northstar
ref: ${{ steps.siblings.outputs.northstar }}
ref: dev
path: northstar
- name: Checkout Bluebottle
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/bluebottle
ref: ${{ steps.siblings.outputs.bluebottle }}
ref: dev
path: bluebottle
- name: Setup Go
uses: actions/setup-go@v7.0.0
uses: actions/setup-go@v7
with:
go-version-file: "spinifex/go.mod"
# Self-hosted runner: the Go module + build cache already persists on
+35 -69
View File
@@ -5,60 +5,51 @@ on:
tags: [ "v*" ]
permissions:
contents: read
contents: write
jobs:
distro-amd64:
runs-on: ubuntu-26.04
steps:
- name: Checkout Spinifex
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
path: spinifex
- name: Resolve sibling release tags from go.mod
id: siblings
run: bash spinifex/.github/scripts/resolve-release-sibling-refs.sh spinifex/go.mod
- name: Checkout Predastore
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/predastore
ref: ${{ steps.siblings.outputs.predastore }}
path: predastore
- name: Checkout Viperblock
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/viperblock
ref: ${{ steps.siblings.outputs.viperblock }}
path: viperblock
- name: Checkout Northstar
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/northstar
ref: ${{ steps.siblings.outputs.northstar }}
path: northstar
- name: Checkout Bluebottle
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/bluebottle
ref: ${{ steps.siblings.outputs.bluebottle }}
path: bluebottle
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4.4.1
uses: docker/setup-buildx-action@v4
- name: Set up Go
uses: actions/setup-go@v7.0.0
uses: actions/setup-go@v7
with:
go-version-file: spinifex/go.mod
cache: false
go-version: "1.27.0"
- name: Build microVM artifacts (lb-agent + kernel + initramfs + qboot + SQS store and worker images)
run: cd spinifex && make build-microvm-image build-sqs-worker-microvm-image
- name: Build microVM artifacts (lb-agent + kernel + initramfs)
run: cd spinifex && make build-microvm-image
- name: Build amd64 Tarball
env:
@@ -66,57 +57,48 @@ jobs:
run: cd spinifex && make distro-amd64 VERSION="$VERSION"
- name: Upload amd64 Artifacts
uses: actions/upload-artifact@v7.0.2
uses: actions/upload-artifact@v7
with:
name: distro-amd64
path: |
spinifex/dist/spinifex-${{ github.ref_name }}-linux-amd64.tar.gz
spinifex/dist/spinifex-${{ github.ref_name }}-linux-amd64.tar.gz.sha256
spinifex/dist/qboot-*.tar.gz
spinifex/scripts/setup.sh
distro-arm64:
runs-on: ubuntu-26.04-arm
steps:
- name: Checkout Spinifex
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
path: spinifex
- name: Resolve sibling release tags from go.mod
id: siblings
run: bash spinifex/.github/scripts/resolve-release-sibling-refs.sh spinifex/go.mod
- name: Checkout Predastore
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/predastore
ref: ${{ steps.siblings.outputs.predastore }}
path: predastore
- name: Checkout Viperblock
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/viperblock
ref: ${{ steps.siblings.outputs.viperblock }}
path: viperblock
- name: Checkout Northstar
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/northstar
ref: ${{ steps.siblings.outputs.northstar }}
path: northstar
- name: Checkout Bluebottle
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/bluebottle
ref: ${{ steps.siblings.outputs.bluebottle }}
path: bluebottle
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4.4.1
uses: docker/setup-buildx-action@v4
- name: Build arm64 Tarball
env:
@@ -124,7 +106,7 @@ jobs:
run: cd spinifex && make distro-arm64 VERSION="$VERSION"
- name: Upload arm64 Artifacts
uses: actions/upload-artifact@v7.0.2
uses: actions/upload-artifact@v7
with:
name: distro-arm64
path: |
@@ -134,23 +116,21 @@ jobs:
release:
needs: [ distro-amd64, distro-arm64 ]
runs-on: ubuntu-26.04
permissions:
contents: write
steps:
- name: Download amd64 Artifacts
uses: actions/download-artifact@v8.0.2
uses: actions/download-artifact@v8
with:
name: distro-amd64
path: artifacts
- name: Download arm64 Artifacts
uses: actions/download-artifact@v8.0.2
uses: actions/download-artifact@v8
with:
name: distro-arm64
path: artifacts
- name: Create Draft GitHub Release
uses: softprops/action-gh-release@v3.0.3
uses: softprops/action-gh-release@v3
with:
draft: true
files: |
@@ -160,7 +140,7 @@ jobs:
generate_release_notes: true
- name: Upload Tarball for Verify Job
uses: actions/upload-artifact@v7.0.2
uses: actions/upload-artifact@v7
with:
name: release-assets
path: artifacts/
@@ -187,14 +167,14 @@ jobs:
| sort -r | xargs -r -I{} sudo umount -l -R {} || true
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
- uses: actions/create-github-app-token@v3.3.0
- uses: actions/create-github-app-token@v3
id: app-token
with:
client-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
repositories: mulga
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
repository: mulgadc/mulga
token: ${{ steps.app-token.outputs.token }}
@@ -203,13 +183,13 @@ jobs:
# Only the shared shell helpers: this job builds nothing from source, but
# it drives the same libvirt provider the e2e cells retry around.
- uses: actions/checkout@v7.0.1
- uses: actions/checkout@v7
with:
sparse-checkout: .github/scripts
path: spinifex
- name: Download Release Assets
uses: actions/download-artifact@v8.0.2
uses: actions/download-artifact@v8
with:
name: release-assets
path: release-assets
@@ -251,8 +231,6 @@ jobs:
promote:
needs: [ verify, baremetal ]
runs-on: ubuntu-26.04
permissions:
contents: write
env:
TAG: ${{ github.ref_name }}
REPO: ${{ github.repository }}
@@ -285,8 +263,6 @@ jobs:
needs: [ verify, baremetal ]
if: failure()
runs-on: ubuntu-26.04
permissions:
contents: write
env:
TAG: ${{ github.ref_name }}
REPO: ${{ github.repository }}
@@ -304,8 +280,6 @@ jobs:
iso:
needs: release
runs-on: [ self-hosted, ci-single ]
permissions:
contents: write
# ISO build (~60-90m) overlaps the parallel baremetal run (~120m); the
# upload gate then waits for baremetal to finish, so the job must outlast it.
timeout-minutes: 180
@@ -320,7 +294,7 @@ jobs:
sudo umount -l "${ROOTFS}/proc" 2>/dev/null || true
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
- uses: actions/create-github-app-token@v3.3.0
- uses: actions/create-github-app-token@v3
id: app-token
with:
client-id: ${{ secrets.APP_ID }}
@@ -328,45 +302,37 @@ jobs:
repositories: mulga
- name: Checkout Spinifex
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
path: spinifex
ref: ${{ github.ref }}
- name: Resolve sibling release tags from go.mod
id: siblings
run: bash spinifex/.github/scripts/resolve-release-sibling-refs.sh spinifex/go.mod
- name: Checkout Predastore
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/predastore
ref: ${{ steps.siblings.outputs.predastore }}
path: predastore
- name: Checkout Viperblock
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/viperblock
ref: ${{ steps.siblings.outputs.viperblock }}
path: viperblock
- name: Checkout Northstar
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/northstar
ref: ${{ steps.siblings.outputs.northstar }}
path: northstar
- name: Checkout Bluebottle
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/bluebottle
ref: ${{ steps.siblings.outputs.bluebottle }}
path: bluebottle
- name: Checkout Mulga iso-builder
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/mulga
token: ${{ steps.app-token.outputs.token }}
@@ -377,7 +343,7 @@ jobs:
run: sudo make -C mulga/scripts/iso-builder clean
- name: Set up Go
uses: actions/setup-go@v7.0.0
uses: actions/setup-go@v7
with:
go-version-file: spinifex/go.mod
cache: false
+10 -10
View File
@@ -3,7 +3,7 @@
#
# tests/integration/segscan_oracle_test.go needs the mulga umbrella repo's
# scripts/segscan checked out alongside spinifex (it is a separate Go module,
# `package main`, and cannot be imported — see internal/testkit/segscanoracle).
# `package main`, and cannot be imported — see spinifex/testutil/segscanoracle).
# spinifex.yml's jobs check out only spinifex/viperblock/predastore and run on
# every PR; adding a second-repo checkout there would make every spinifex PR
# depend on mulga main being healthy and pay that checkout cost on every run,
@@ -25,7 +25,7 @@ jobs:
steps:
- name: Checkout Spinifex
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
path: spinifex
fetch-depth: 2
@@ -34,38 +34,38 @@ jobs:
id: siblings
env:
BRANCH: ${{ github.head_ref || github.ref_name }}
run: bash .github/scripts/resolve-sibling-refs.sh
run: bash .github/resolve-sibling-refs.sh
working-directory: spinifex
- name: Checkout Viperblock
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/viperblock
ref: ${{ steps.siblings.outputs.viperblock }}
path: viperblock
- name: Checkout Predastore
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/predastore
ref: ${{ steps.siblings.outputs.predastore }}
path: predastore
- name: Checkout Northstar
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/northstar
ref: ${{ steps.siblings.outputs.northstar }}
path: northstar
- name: Checkout Bluebottle
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/bluebottle
ref: ${{ steps.siblings.outputs.bluebottle }}
path: bluebottle
- uses: actions/create-github-app-token@v3.3.0
- uses: actions/create-github-app-token@v3
id: app-token
with:
client-id: ${{ secrets.APP_ID }}
@@ -73,7 +73,7 @@ jobs:
repositories: mulga
- name: Checkout Mulga (scripts/segscan only)
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/mulga
token: ${{ steps.app-token.outputs.token }}
@@ -81,7 +81,7 @@ jobs:
path: mulga
- name: Setup Go
uses: actions/setup-go@v7.0.0
uses: actions/setup-go@v7
with:
cache-dependency-path: "spinifex/go.sum"
go-version-file: "spinifex/go.mod"
+65 -79
View File
@@ -20,12 +20,6 @@ on:
permissions:
contents: read
# A newer push to the same pull request supersedes its run. Pushes and dispatches
# get a group of their own, so every commit on dev and main keeps its result.
concurrency:
group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.run_id }}
cancel-in-progress: true
jobs:
build_and_test:
name: Build and Test
@@ -33,7 +27,7 @@ jobs:
steps:
- name: Checkout Spinifex
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
path: spinifex
fetch-depth: 2
@@ -42,32 +36,32 @@ jobs:
id: siblings
env:
BRANCH: ${{ github.head_ref || github.ref_name }}
run: bash .github/scripts/resolve-sibling-refs.sh
run: bash .github/resolve-sibling-refs.sh
working-directory: spinifex
- name: Checkout Viperblock
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/viperblock
ref: ${{ steps.siblings.outputs.viperblock }}
path: viperblock
- name: Checkout Predastore
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/predastore
ref: ${{ steps.siblings.outputs.predastore }}
path: predastore
- name: Checkout Northstar
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/northstar
ref: ${{ steps.siblings.outputs.northstar }}
path: northstar
- name: Checkout Bluebottle
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/bluebottle
ref: ${{ steps.siblings.outputs.bluebottle }}
@@ -77,7 +71,7 @@ jobs:
# the last go.sum change and never refreshed it. Both caches are managed
# below instead.
- name: Setup Go
uses: actions/setup-go@v7.0.0
uses: actions/setup-go@v7
with:
go-version-file: "spinifex/go.mod"
cache: false
@@ -86,7 +80,7 @@ jobs:
# setup-go hashed spinifex/go.sum alone, which left the sibling modules'
# dependencies out of the key and re-downloading on every run.
- name: Cache Go modules
uses: actions/cache@v6.1.0
uses: actions/cache@v6
with:
path: ~/go/pkg/mod
key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
@@ -99,19 +93,12 @@ jobs:
# a PR reads the base branch's entry without spending a minute uploading
# one only that PR would ever read.
- name: Restore Go build cache
uses: actions/cache/restore@v6.1.0
uses: actions/cache/restore@v6
with:
path: ~/.cache/go-build
key: gobuild-${{ github.job }}-${{ github.run_id }}
restore-keys: gobuild-${{ github.job }}-
# Go refreshes an entry's mtime on use only when it is over an hour old, so
# restored entries are backdated past that before any Go command runs and
# the trim below can tell this run's working set from the last run's.
- name: Mark Go build cache
if: ${{ github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main' }}
run: spinifex/.github/scripts/go-build-cache.sh mark
- name: Setup Go Workspace
run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
@@ -132,6 +119,9 @@ jobs:
- name: Build Spinifex
run: make -C spinifex build
- name: Manifest check + drift lint
run: make -C spinifex manifest-check manifest-lint
- name: Run Tests with Coverage
env:
SPINIFEX_REQUIRE_CONFORMANCE_TOOLS: "1"
@@ -144,28 +134,34 @@ jobs:
- name: Check Diff Coverage
run: cd spinifex && scripts/diff-coverage.sh coverage.out --base HEAD~1
# Keeps what this run read or wrote and drops the rest, so the saved entry
# is one run's working set however often dev is pushed. A red run may have
# stopped early and used only part of the set, so it saves untrimmed.
# Compiled output stays valid whatever the tests said, and skipping the
# save on a red run would leave the next one rebuilding it, so this is
# gated on the branch rather than on success.
# Go bumps an entry's mtime when a run uses it, so this run's working set
# survives and objects superseded by later commits do not. Runs after the
# work, so it costs this job nothing and only shrinks what the next reads.
- name: Trim Go build cache
if: ${{ success() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
run: spinifex/.github/scripts/go-build-cache.sh trim
if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
run: |
C="$(go env GOCACHE)"
before=$(du -sm "$C" | cut -f1)
find "$C" -type f -mmin +240 -delete
echo "trimmed ${before}MB -> $(du -sm "$C" | cut -f1)MB"
- name: Save Go build cache
if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
uses: actions/cache/save@v6.1.0
uses: actions/cache/save@v6
with:
path: ~/.cache/go-build
key: gobuild-${{ github.job }}-${{ github.run_id }}
race_detection:
name: Race Detection
if: ${{ github.event_name != 'pull_request' }}
runs-on: ubuntu-26.04
steps:
- name: Checkout Spinifex
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
path: spinifex
@@ -173,32 +169,32 @@ jobs:
id: siblings
env:
BRANCH: ${{ github.head_ref || github.ref_name }}
run: bash .github/scripts/resolve-sibling-refs.sh
run: bash .github/resolve-sibling-refs.sh
working-directory: spinifex
- name: Checkout Viperblock
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/viperblock
ref: ${{ steps.siblings.outputs.viperblock }}
path: viperblock
- name: Checkout Predastore
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/predastore
ref: ${{ steps.siblings.outputs.predastore }}
path: predastore
- name: Checkout Northstar
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/northstar
ref: ${{ steps.siblings.outputs.northstar }}
path: northstar
- name: Checkout Bluebottle
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/bluebottle
ref: ${{ steps.siblings.outputs.bluebottle }}
@@ -208,7 +204,7 @@ jobs:
# the last go.sum change and never refreshed it. Both caches are managed
# below instead.
- name: Setup Go
uses: actions/setup-go@v7.0.0
uses: actions/setup-go@v7
with:
go-version-file: "spinifex/go.mod"
cache: false
@@ -217,7 +213,7 @@ jobs:
# setup-go hashed spinifex/go.sum alone, which left the sibling modules'
# dependencies out of the key and re-downloading on every run.
- name: Cache Go modules
uses: actions/cache@v6.1.0
uses: actions/cache@v6
with:
path: ~/go/pkg/mod
key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
@@ -230,35 +226,32 @@ jobs:
# a PR reads the base branch's entry without spending a minute uploading
# one only that PR would ever read.
- name: Restore Go build cache
uses: actions/cache/restore@v6.1.0
uses: actions/cache/restore@v6
with:
path: ~/.cache/go-build
key: gobuild-${{ github.job }}-${{ github.run_id }}
restore-keys: gobuild-${{ github.job }}-
# Go refreshes an entry's mtime on use only when it is over an hour old, so
# restored entries are backdated past that before any Go command runs and
# the trim below can tell this run's working set from the last run's.
- name: Mark Go build cache
if: ${{ github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main' }}
run: spinifex/.github/scripts/go-build-cache.sh mark
- name: Setup Go Workspace
run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
- name: Run Tests with Race Detector
run: make -C spinifex test-race GOTESTFLAGS=-count=1
# Keeps what this run read or wrote and drops the rest, so the saved entry
# is one run's working set however often dev is pushed. A red run may have
# stopped early and used only part of the set, so it saves untrimmed.
# Go bumps an entry's mtime when a run uses it, so this run's working set
# survives and objects superseded by later commits do not. Runs after the
# work, so it costs this job nothing and only shrinks what the next reads.
- name: Trim Go build cache
if: ${{ success() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
run: spinifex/.github/scripts/go-build-cache.sh trim
if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
run: |
C="$(go env GOCACHE)"
before=$(du -sm "$C" | cut -f1)
find "$C" -type f -mmin +240 -delete
echo "trimmed ${before}MB -> $(du -sm "$C" | cut -f1)MB"
- name: Save Go build cache
if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
uses: actions/cache/save@v6.1.0
uses: actions/cache/save@v6
with:
path: ~/.cache/go-build
key: gobuild-${{ github.job }}-${{ github.run_id }}
@@ -269,7 +262,7 @@ jobs:
steps:
- name: Checkout Spinifex
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
path: spinifex
fetch-depth: 2
@@ -278,32 +271,32 @@ jobs:
id: siblings
env:
BRANCH: ${{ github.head_ref || github.ref_name }}
run: bash .github/scripts/resolve-sibling-refs.sh
run: bash .github/resolve-sibling-refs.sh
working-directory: spinifex
- name: Checkout Viperblock
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/viperblock
ref: ${{ steps.siblings.outputs.viperblock }}
path: viperblock
- name: Checkout Predastore
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/predastore
ref: ${{ steps.siblings.outputs.predastore }}
path: predastore
- name: Checkout Northstar
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/northstar
ref: ${{ steps.siblings.outputs.northstar }}
path: northstar
- name: Checkout Bluebottle
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
repository: mulgadc/bluebottle
ref: ${{ steps.siblings.outputs.bluebottle }}
@@ -313,7 +306,7 @@ jobs:
# the last go.sum change and never refreshed it. Both caches are managed
# below instead.
- name: Setup Go
uses: actions/setup-go@v7.0.0
uses: actions/setup-go@v7
with:
go-version-file: "spinifex/go.mod"
cache: false
@@ -322,7 +315,7 @@ jobs:
# setup-go hashed spinifex/go.sum alone, which left the sibling modules'
# dependencies out of the key and re-downloading on every run.
- name: Cache Go modules
uses: actions/cache@v6.1.0
uses: actions/cache@v6
with:
path: ~/go/pkg/mod
key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
@@ -335,26 +328,19 @@ jobs:
# a PR reads the base branch's entry without spending a minute uploading
# one only that PR would ever read.
- name: Restore Go build cache
uses: actions/cache/restore@v6.1.0
uses: actions/cache/restore@v6
with:
path: ~/.cache/go-build
key: gobuild-${{ github.job }}-${{ github.run_id }}
restore-keys: gobuild-${{ github.job }}-
# Go refreshes an entry's mtime on use only when it is over an hour old, so
# restored entries are backdated past that before any Go command runs and
# the trim below can tell this run's working set from the last run's.
- name: Mark Go build cache
if: ${{ github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main' }}
run: spinifex/.github/scripts/go-build-cache.sh mark
- name: Setup Go Workspace
run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
- name: Lint
uses: golangci/golangci-lint-action@v9.3.0
uses: golangci/golangci-lint-action@v9
with:
version: v2.14.0
version: v2.13.0
working-directory: spinifex
- name: Govulncheck
@@ -364,20 +350,20 @@ jobs:
if: ${{ !cancelled() }}
run: make -C spinifex test-integration
- name: Run Gated Suites
if: ${{ !cancelled() }}
run: make -C spinifex test-gated GOTESTFLAGS=-count=1
# Keeps what this run read or wrote and drops the rest, so the saved entry
# is one run's working set however often dev is pushed. A red run may have
# stopped early and used only part of the set, so it saves untrimmed.
# Go bumps an entry's mtime when a run uses it, so this run's working set
# survives and objects superseded by later commits do not. Runs after the
# work, so it costs this job nothing and only shrinks what the next reads.
- name: Trim Go build cache
if: ${{ success() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
run: spinifex/.github/scripts/go-build-cache.sh trim
if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
run: |
C="$(go env GOCACHE)"
before=$(du -sm "$C" | cut -f1)
find "$C" -type f -mmin +240 -delete
echo "trimmed ${before}MB -> $(du -sm "$C" | cut -f1)MB"
- name: Save Go build cache
if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
uses: actions/cache/save@v6.1.0
uses: actions/cache/save@v6
with:
path: ~/.cache/go-build
key: gobuild-${{ github.job }}-${{ github.run_id }}
+12 -119
View File
@@ -6,61 +6,45 @@ on:
- main
- dev
paths:
- spinifex/runtime/roles/spinifexui/frontend/**
- spinifex/runtime/roles/spinifexui/*.go
- spinifex/operator/admin/instance_metrics*.go
- spinifex/operator/admin/live_streams*.go
- spinifex/runtime/roles/qmpcollector/**
- contracts/telemetry/**
- spinifex/services/spinifexui/frontend/**
- .github/workflows/ui.yml
pull_request:
branches:
- main
- dev
paths:
- spinifex/runtime/roles/spinifexui/frontend/**
- spinifex/runtime/roles/spinifexui/*.go
- spinifex/operator/admin/instance_metrics*.go
- spinifex/operator/admin/live_streams*.go
- spinifex/runtime/roles/qmpcollector/**
- contracts/telemetry/**
- spinifex/services/spinifexui/frontend/**
- .github/workflows/ui.yml
workflow_dispatch:
permissions:
contents: read
# A newer push to the same pull request supersedes its run. Pushes and dispatches
# get a group of their own, so every commit on dev and main keeps its result.
concurrency:
group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.run_id }}
cancel-in-progress: true
jobs:
ui_build_and_test:
name: UI Build and Test
runs-on: ubuntu-26.04
defaults:
run:
working-directory: spinifex/runtime/roles/spinifexui/frontend
working-directory: spinifex/services/spinifexui/frontend
steps:
- name: Checkout Code
uses: actions/checkout@v7.0.1
uses: actions/checkout@v7
with:
sparse-checkout: spinifex/runtime/roles/spinifexui/frontend
sparse-checkout: spinifex/services/spinifexui/frontend
- name: Setup pnpm
uses: pnpm/action-setup@v6.1.0
uses: pnpm/action-setup@v6
with:
package_json_file: spinifex/runtime/roles/spinifexui/frontend/package.json
package_json_file: spinifex/services/spinifexui/frontend/package.json
- name: Setup Node.js
uses: actions/setup-node@v7.1.0
uses: actions/setup-node@v7
with:
cache: "pnpm"
cache-dependency-path: spinifex/runtime/roles/spinifexui/frontend/pnpm-lock.yaml
node-version-file: spinifex/runtime/roles/spinifexui/frontend/.nvmrc
cache-dependency-path: spinifex/services/spinifexui/frontend/pnpm-lock.yaml
node-version-file: spinifex/services/spinifexui/frontend/.nvmrc
- name: Install Dependencies
run: pnpm install --frozen-lockfile
@@ -74,96 +58,5 @@ jobs:
- name: Build Frontend
run: pnpm build
# spx embeds the committed dist/, so a bundle that differs from a clean
# build of this source would ship JavaScript CI never built. Porcelain
# rather than git diff, so added and deleted chunks count too.
- name: Check for generated file drift
run: |
drift=$(git status --porcelain -- src/routeTree.gen.ts src/lib/cn-tables.gen.ts dist)
if [ -n "$drift" ]; then
echo "::error::Generated files differ from a clean build. Run pnpm build and commit the result."
echo "$drift"
exit 1
fi
# The pinned image lives in one file that scripts/ui-playwright.sh also reads,
# so make ui-baselines and the job below draw screenshots in the same image.
ui_e2e_image:
name: UI End-to-End Image
runs-on: ubuntu-26.04
outputs:
image: ${{ steps.read.outputs.image }}
steps:
- name: Checkout Code
uses: actions/checkout@v7.0.1
with:
sparse-checkout: spinifex/runtime/roles/spinifexui/frontend/playwright.image
sparse-checkout-cone-mode: false
- name: Read Playwright Image
id: read
run: echo "image=$(tr -d '[:space:]' < spinifex/runtime/roles/spinifexui/frontend/playwright.image)" >> "$GITHUB_OUTPUT"
# The browser suite: the console against a faked API on Vite dev, plus the
# built dist under the production CSP. It runs in Playwright's own image, which
# carries the browsers, so screenshots rasterise as they did for the baselines.
ui_e2e:
name: UI End-to-End
needs: ui_e2e_image
runs-on: ubuntu-26.04
timeout-minutes: 20
container:
image: ${{ needs.ui_e2e_image.outputs.image }}
options: --ipc=host
defaults:
run:
working-directory: spinifex/runtime/roles/spinifexui/frontend
steps:
- name: Checkout Code
uses: actions/checkout@v7.0.1
with:
sparse-checkout: spinifex/runtime/roles/spinifexui/frontend
- name: Setup pnpm
uses: pnpm/action-setup@v6.1.0
with:
package_json_file: spinifex/runtime/roles/spinifexui/frontend/package.json
- name: Setup Node.js
uses: actions/setup-node@v7.1.0
with:
cache: "pnpm"
cache-dependency-path: spinifex/runtime/roles/spinifexui/frontend/pnpm-lock.yaml
node-version-file: spinifex/runtime/roles/spinifexui/frontend/.nvmrc
- name: Install Dependencies
run: pnpm install --frozen-lockfile
# The image's browsers belong to one Playwright release, so a lockfile
# bump without an image bump would otherwise fail on a missing executable.
- name: Check Playwright Matches the Image
env:
IMAGE: ${{ needs.ui_e2e_image.outputs.image }}
run: |
version=$(pnpm exec playwright --version | awk '{print $2}')
case "$IMAGE" in
*":v$version-"*) ;;
*) echo "::error::Lockfile has Playwright $version but the image is $IMAGE. Update playwright.image."; exit 1 ;;
esac
- name: Build Frontend
run: pnpm build
- name: Run End-to-End Suite
run: pnpm e2e
- name: Upload Screenshot Diffs
if: failure()
uses: actions/upload-artifact@v7.0.2
with:
name: ui-e2e-test-results
path: spinifex/runtime/roles/spinifexui/frontend/test-results/
if-no-files-found: ignore
retention-days: 14
- name: Check for generated route tree drift
run: git diff --exit-code -- src/routeTree.gen.ts
+1 -11
View File
@@ -19,17 +19,7 @@ bin/
dist/
build/microvm/vmlinuz
build/microvm/initramfs.cpio.gz
build/microvm/qboot.rom
build/microvm/qboot.LICENSE
build/microvm/sqs-worker/vmlinuz
build/microvm/sqs-worker/initramfs.cpio.gz
build/microvm/sqs-worker/.stage.*/
build/microvm/lambda-runtime/vmlinuz
build/microvm/lambda-runtime/initramfs.cpio.gz
build/microvm/lambda-runtime/.stage.*/
build/microvm/lambda-testlink-runtime/
build/qboot/
!spinifex/runtime/roles/spinifexui/frontend/dist/
!spinifex/services/spinifexui/frontend/dist/
tmp/
*.test
*.out
+7 -7
View File
@@ -1,8 +1,5 @@
version: "2"
run:
allow-serial-runners: true
issues:
max-same-issues: 10
@@ -71,18 +68,17 @@ linters:
- pkg: math/rand$
desc: Use math/rand/v2 instead, see https://go.dev/blog/randv2
# The control plane reaches block storage through the EBS provider, not
# The EC2 control plane reaches block storage through ebsprovider, not
# by importing the engine. viperblock.New starts a chunk uploader, so
# every construction is a writer against a volume viperblockd serves.
# No domain or legacy handler may import viperblock, in any form, tests
# Nothing under handlers/ may import viperblock, in any form, tests
# included. A types-only import counts: one became an engine import.
ec2-control-plane:
files:
- '**/spinifex/domains/**'
- '**/spinifex/handlers/**'
deny:
- pkg: github.com/mulgadc/viperblock
desc: Use the providers/ebs EBSProvider interface; the control plane must not construct or name viperblock types.
desc: Use the ebsprovider.EBSProvider interface; the control plane must not construct or name viperblock types.
embeddedstructfieldcheck:
forbid-mutex: true
@@ -146,10 +142,14 @@ linters:
staticcheck:
checks:
- all
- -ST1000 # package comment
- -ST1003 # naming conventions (Id→ID etc.) — too noisy to retrofit
- -ST1012 # error var naming — errname covers this + error type suffixes
- -ST1013 # HTTP status constants — usestdlibvars is broader
- -ST1016 # receiver name consistency (too noisy to retrofit)
- -ST1020 # godoc on exported types — not enforcing yet
- -ST1021 # godoc on exported type groups — not enforcing yet
- -ST1022 # godoc on exported vars/consts — not enforcing yet
- -SA4006 # value overwritten — ineffassign catches strictly more of these
- -SA9005 # marshaling structs with no exported fields — intended {} for empty API-response types
-97
View File
@@ -1,97 +0,0 @@
# How to Contribute
We welcome contributions to Spinifex!
Report security vulnerabilities privately through [SECURITY.md](SECURITY.md), not as a public issue.
**AWS parity is the design rule.** When Spinifex's behaviour is in question, the answer is what AWS does. If you believe Spinifex diverges from AWS, state what AWS does and show it with a test or a reproduction against a running cluster.
## Understanding the Codebase
Every AWS call enters through a SigV4-authenticated gateway, is published to NATS, and is answered by the daemon that claims it. Start with these:
| Doc | Covers |
|---|---|
| [`docs/DESIGN.md`](docs/DESIGN.md) | Architecture, request flow, daemons, storage integration, file layout. |
| [`docs/COMMANDS.md`](docs/COMMANDS.md) | The `spx admin` CLI and the systemd units it manages. |
| [`docs/coverage/`](docs/coverage/README.md) | Which AWS API operations each service implements. |
| [`scripts/README.md`](scripts/README.md) | Install, dev-environment, and image-build scripts. |
| [`docs/PACKAGE_BOUNDARY_MIGRATION.md`](docs/PACKAGE_BOUNDARY_MIGRATION.md) | ADR-0001 implementation inventory and incremental source moves. |
User-facing docs are published at [docs.mulgadc.com](https://docs.mulgadc.com).
## Development Setup
These steps build Spinifex from source and run it as a single-node cluster, following the [Source Install](https://docs.mulgadc.com/docs/install-source) guide.
### Prerequisites
- **Ubuntu 26.04** or **Debian 13**, on x86_64 or aarch64
- A host with hardware virtualisation (KVM) and `sudo` access
- The host's WAN interface enslaved to a Linux bridge named `br-wan`, with the host IP, default route and DHCP on the bridge rather than the bare NIC (see [VPC Networking → Bridge Setup](https://docs.mulgadc.com/docs/vpc-networking#bridge-setup-physical-network-wiring))
Check the bridge before continuing:
```bash
ip -br link show br-wan # bridge exists and is UP
ip route # default route's dev is br-wan
```
### Install and Run
```bash
mkdir -p ~/Development/mulga && cd ~/Development/mulga
git clone https://github.com/mulgadc/spinifex.git
sudo make -C spinifex quickinstall # system packages, Go 1.27, AWS CLI v2
export PATH=$PATH:/usr/local/go/bin
cd spinifex
./scripts/clone-deps.sh # sibling repos: viperblock, predastore, northstar, bluebottle
./scripts/dev-install.sh # build, install, init a one-node cluster, start services
AWS_PROFILE=spinifex aws ec2 describe-instance-types
```
A list of instance types means the cluster is up.
`make preflight` also needs `shellcheck` and `golangci-lint`
```bash
sudo apt-get install -y shellcheck
curl -sSfL https://golangci-lint.run/install.sh | sh -s -- -b "$(go env GOPATH)/bin" v2.14.0
```
The web console additionally needs `pnpm` and the Node.js version pinned in `spinifex/runtime/roles/spinifexui/frontend/.nvmrc`.
## Development Workflow
| Command | Use it to |
|---|---|
| `make deploy` | Rebuild, install `spx`, and restart `spinifex.target` after a code change. |
| `make reinstall` | Re-run `dev-install.sh` after changing systemd units, helper scripts, or logrotate config. |
| `./scripts/reset-dev-env.sh` | Wipe all cluster state and reinstall from scratch, keeping your network topology. |
| `make test` | Run unit tests (`make test-race` for the race detector). |
| `make test-integration` | Run the real gateway router against embedded NATS JetStream. |
| `make preflight` | Run the pre-commit gate: lint, `govulncheck`, coverage, and manifest and script checks. Must pass. |
Follow the services with `journalctl -u 'spinifex-*' -f`.
The web console lives in `spinifex/runtime/roles/spinifexui/frontend/` (`pnpm install`, `pnpm dev`, `pnpm lint`, `pnpm test`). `spx` embeds its checked-in `dist/`, so run `make build-ui` and commit the rebuilt `dist/` with your change.
## Creating a Pull Request
1. Fork [mulgadc/spinifex](https://github.com/mulgadc/spinifex) and clone your fork into the same parent directory as the sibling repositories, branching off `dev`:
```bash
cd ~/Development/mulga
git clone git@github.com:YOUR_USERNAME/spinifex.git && cd spinifex
git remote add upstream git@github.com:mulgadc/spinifex.git
git fetch upstream && git checkout -b feat/my-feature upstream/dev
```
2. Make your change, then verify it:
- `make preflight` passes.
- `go mod tidy` leaves only intended changes in `go.mod` and `go.sum`, with none of the `clone-deps.sh` `replace` lines.
- Runtime changes are exercised on your dev cluster with `make deploy`.
3. Commit using [Conventional Commits](https://www.conventionalcommits.org/) (`feat:`, `fix:`, `refactor:`, `docs:`, `chore:`), push, and open a pull request against `dev`, filling in the template.
+59 -224
View File
@@ -42,15 +42,11 @@ ifdef QUIET
_COVQ = 2>&1 | { grep -Ev '^\s*(ok|PASS|\?|=== RUN|--- PASS:)\s' | grep -v 'coverage: 0\.0%' || true; }; exit $${PIPESTATUS[0]}
_RACEQ = 2>&1 | { grep -Ev '^\s*(ok|PASS|\?|=== RUN|--- PASS:)\s' || true; }; exit $${PIPESTATUS[0]}
_SECQ = >
_HDR = :
_RUNSH = sh "$$t" >"$$log" 2>&1 || { echo "-- $$t"; cat "$$log"; exit 1; }
else
_Q =
_COVQ =
_RACEQ =
_SECQ = 2>&1 | tee
_HDR = echo -e
_RUNSH = echo "-- $$t"; sh "$$t" || exit 1
endif
build: go_build build-installer build-lb-agent generate-aws-model-coverage
@@ -58,60 +54,17 @@ build: go_build build-installer build-lb-agent generate-aws-model-coverage
# Build spinifex-ui frontend (requires pnpm)
build-ui:
@echo -e "\n....Building spinifex-ui frontend...."
cd spinifex/runtime/roles/spinifexui/frontend && pnpm build
# Playwright suite for the console, against Vite dev and the built dist under
# the console's CSP. Not part of preflight; needs `pnpm exec playwright install chromium`.
test-ui-e2e: build-ui
cd spinifex/runtime/roles/spinifexui/frontend && pnpm e2e
# The same suite inside the pinned Playwright image CI uses (podman or docker),
# so screenshots compare as they will on the runner. ui-baselines redraws them.
ui-e2e-container:
scripts/ui-playwright.sh compare
ui-baselines:
scripts/ui-playwright.sh baselines
cd spinifex/services/spinifexui/frontend && pnpm build
# GO commands
VERSION ?= $(shell git describe --tags --always --dirty --exclude '*-dev')
VERSION ?= $(shell git describe --tags --always --dirty)
COMMIT ?= $(shell git rev-parse --short HEAD)
LDFLAGS := -s -w -X github.com/mulgadc/spinifex/spinifex/operator/cli.Version=$(VERSION) -X github.com/mulgadc/spinifex/spinifex/operator/cli.Commit=$(COMMIT)
LDFLAGS := -s -w -X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Version=$(VERSION) -X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Commit=$(COMMIT)
go_build:
@echo -e "\n....Building $(GO_PROJECT_NAME)"
go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go
# Controlled-test only: builds the tagged daemon and its hard-kill initializer
# together, pinning the initializer's digest into that daemon build. The daemon
# keeps its symbols so the pin can be read back from the binary.
# LAMBDA_IMAGE_DIGEST is the test-link runtime image digest that
# build-lambda-testlink-image prints; it is read raw and validated before use.
LAMBDA_TESTLINK_DIR := ./bin/lambda-testlink
LAMBDA_TESTLINK_PKG := github.com/mulgadc/spinifex/spinifex/runtime/daemon
LAMBDA_TESTLINK_PIN := $(LAMBDA_TESTLINK_PKG).lambdaHardKillSHA256
LAMBDA_TESTLINK_GW := github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw
LAMBDA_DIGEST_RE := ^[0-9a-f]{64}$$
build-lambda-testlink: override export LAMBDA_IMAGE_DIGEST := $(value LAMBDA_IMAGE_DIGEST)
build-lambda-testlink:
@[[ "$$LAMBDA_IMAGE_DIGEST" =~ $(LAMBDA_DIGEST_RE) ]] || { echo "LAMBDA_IMAGE_DIGEST must be 64 lowercase hex digits"; exit 1; }
@host=$$(uname -m); goos=$$(go env GOOS); goarch=$$(go env GOARCH); \
test "$$host" = x86_64 && test "$$goos/$$goarch" = linux/amd64 || { echo "the test link supports only an x86_64 host building linux/amd64, not $$host building $$goos/$$goarch"; exit 1; }
@rm -f $(LAMBDA_TESTLINK_DIR)/hardkill.so $(LAMBDA_TESTLINK_DIR)/$(GO_PROJECT_NAME)
@mkdir -p $(LAMBDA_TESTLINK_DIR).tmp
@rm -f $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so $(LAMBDA_TESTLINK_DIR).tmp/$(GO_PROJECT_NAME)
cc -std=c17 -O2 -Wall -Wextra -Werror -shared -fPIC -o $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so spinifex/runtime/compute/lambda/testlink/hardkill/hardkill.c -lrt
scripts/check-glibc-ceiling.sh 2.34 $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so
@digest=$$(sha256sum $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so | cut -d' ' -f1); \
test $${#digest} -eq 64 || { echo "initializer digest missing"; exit 1; }; \
echo "initializer sha256 $$digest"; \
pin="-X $(LAMBDA_TESTLINK_PIN)=$$digest -X $(LAMBDA_TESTLINK_PKG).lambdaImageDigest=$$LAMBDA_IMAGE_DIGEST -X $(LAMBDA_TESTLINK_GW).lambdaImageDigest=$$LAMBDA_IMAGE_DIGEST"; \
LAMBDA_TESTLINK_INITIALIZER=$(abspath $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so) go test -count=1 -tags lambdatestlink -ldflags "$$pin" -run '^TestLambda(HardKillPin|ImageDigestPin)$$' ./spinifex/runtime/daemon/ ./spinifex/runtime/roles/awsgw/ && \
go build $(GO_BUILD_MOD) -tags lambdatestlink -ldflags "$(subst -s -w ,,$(LDFLAGS)) $$pin" -o $(LAMBDA_TESTLINK_DIR).tmp/$(GO_PROJECT_NAME) cmd/spinifex/main.go
@mkdir -p $(LAMBDA_TESTLINK_DIR)
mv $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so $(LAMBDA_TESTLINK_DIR).tmp/$(GO_PROJECT_NAME) $(LAMBDA_TESTLINK_DIR)/
@rmdir $(LAMBDA_TESTLINK_DIR).tmp
build-installer:
@echo -e "\n....Building spinifex-installer"
go build -ldflags "-s -w" -o ./bin/spinifex-installer cmd/installer/main.go
@@ -187,135 +140,35 @@ $(MICROVM_ARTIFACTS) &: $(MICROVM_INPUTS)
bin/lb-agent:
$(MAKE) build-lb-agent
# The dedicated SQS server, built from the go.mod-pinned module so the image
# ships exactly the reviewed version. Static for the musl guest.
bin/nats-server: go.mod go.sum
@mkdir -p bin
GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags '-s -w' -o $@ github.com/nats-io/nats-server/v2
build-microvm-image: $(MICROVM_ARTIFACTS) ## Build microVM kernel + initramfs (incremental — skips when up to date)
# go_local_srcs lists the main module package dirs and files guest package $(1)
# links, scanned with the guest build settings; a dir's mtime catches a deleted
# or renamed file. A failed scan names a missing prerequisite, so make stops.
go_local_srcs = $(shell GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go list -deps -f '{{if and .Module .Module.Main}}{{.Dir}} {{range .GoFiles}}{{$$.Dir}}/{{.}} {{end}}{{range .EmbedFiles}}{{$$.Dir}}/{{.}} {{end}}{{end}}' $(1) || echo go-list-failed-for-$(1))
.SECONDEXPANSION:
# The SQS microVM image: one guest running its own SQS nats-server and the SQS
# worker as separate processes. A separate artifact from the LB image above.
SQS_WORKER_MICROVM_OUT_DIR := $(MICROVM_OUT_DIR)/sqs-worker
SQS_WORKER_MICROVM_ARTIFACTS := $(SQS_WORKER_MICROVM_OUT_DIR)/vmlinuz $(SQS_WORKER_MICROVM_OUT_DIR)/initramfs.cpio.gz
SQS_WORKER_MICROVM_INPUTS := scripts/build-sqs-store-microvm-image.sh scripts/lib/microvm-image.sh scripts/lib/sqs-store-microvm-publish.sh $(SQS_WORKER_MICROVM_OUT_DIR)/init.sh $(SQS_WORKER_MICROVM_OUT_DIR)/inittab $(addprefix $(SQS_WORKER_MICROVM_OUT_DIR)/,sqs-worker-lib.sh sqs-worker-prepare sqs-worker-store sqs-worker-ready sqs-worker-stop) bin/nats-server bin/sqs-worker
$(SQS_WORKER_MICROVM_ARTIFACTS) &: $(SQS_WORKER_MICROVM_INPUTS)
./scripts/build-sqs-store-microvm-image.sh
bin/sqs-worker: go.mod go.sum $$(call go_local_srcs,./cmd/sqs-worker)
@mkdir -p bin
GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags '-s -w' -o $@ ./cmd/sqs-worker
build-sqs-worker-microvm-image: $(SQS_WORKER_MICROVM_ARTIFACTS) ## Build the SQS microVM kernel and initramfs
# The Lambda runtime microVM image. Built on demand; not shipped or installed.
LAMBDA_RUNTIME_MICROVM_OUT_DIR := $(MICROVM_OUT_DIR)/lambda-runtime
LAMBDA_RUNTIME_MICROVM_ARTIFACTS := $(LAMBDA_RUNTIME_MICROVM_OUT_DIR)/vmlinuz $(LAMBDA_RUNTIME_MICROVM_OUT_DIR)/initramfs.cpio.gz
LAMBDA_RUNTIME_MICROVM_INPUTS := scripts/build-lambda-runtime-microvm-image.sh scripts/lib/microvm-image.sh $(addprefix $(LAMBDA_RUNTIME_MICROVM_OUT_DIR)/,init.sh inittab run-agent) bin/lambda-agent
$(LAMBDA_RUNTIME_MICROVM_ARTIFACTS) &: $(LAMBDA_RUNTIME_MICROVM_INPUTS)
./scripts/build-lambda-runtime-microvm-image.sh
bin/lambda-agent: go.mod go.sum $$(call go_local_srcs,./cmd/lambda-agent)
@mkdir -p bin
GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags '-s -w' -o $@ ./cmd/lambda-agent
build-lambda-runtime-microvm-image: $(LAMBDA_RUNTIME_MICROVM_ARTIFACTS) ## Build the Lambda runtime microVM kernel and initramfs (not shipped)
GOWORK=off go run ./cmd/lambda-image-agent-tag $(LAMBDA_RUNTIME_MICROVM_OUT_DIR)/initramfs.cpio.gz absent
# Controlled-test only: a separately built guest whose agent has the
# lambdatestlink tag, in its own outputs, never touching bin/lambda-agent or the
# default image. It prints the digest to pass as LAMBDA_IMAGE_DIGEST.
LAMBDA_TESTLINK_IMAGE_DIR := $(MICROVM_OUT_DIR)/lambda-testlink-runtime
build-lambda-testlink-image: ## Build the test-link Lambda guest image and print its digest (not shipped)
@host=$$(uname -m); goos=$$(go env GOOS); goarch=$$(go env GOARCH); \
test "$$host" = x86_64 && test "$$goos/$$goarch" = linux/amd64 || { echo "the test link supports only an x86_64 host building linux/amd64, not $$host building $$goos/$$goarch"; exit 1; }
$(MAKE) --no-print-directory $(MICROVM_OUT_DIR)/qboot.rom
@mkdir -p bin $(LAMBDA_TESTLINK_IMAGE_DIR)
GOWORK=off GOFLAGS= CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -tags lambdatestlink -ldflags '-s -w' -o bin/lambda-agent-testlink ./cmd/lambda-agent
./scripts/build-lambda-runtime-microvm-image.sh bin/lambda-agent-testlink $(LAMBDA_TESTLINK_IMAGE_DIR)
GOWORK=off go run ./cmd/lambda-image-agent-tag $(LAMBDA_TESTLINK_IMAGE_DIR)/initramfs.cpio.gz present
install -m 0644 $(MICROVM_OUT_DIR)/qboot.rom $(LAMBDA_TESTLINK_IMAGE_DIR)/qboot.rom
@digest=$$(GOWORK=off go run -tags lambdatestlink ./cmd/lambda-image-digest $(LAMBDA_TESTLINK_IMAGE_DIR)) && \
[[ "$$digest" =~ $(LAMBDA_DIGEST_RE) ]] || { echo "image digest not computed"; exit 1; }; \
echo "LAMBDA_IMAGE_DIGEST=$$digest"
# qboot is the microvm firmware. The source tarball ships with each release
# because qboot is GPL-2.0.
QBOOT_COMMIT := $(shell sed -n 's/^QBOOT_COMMIT="\(.*\)"$$/\1/p' scripts/build-qboot.sh)
QBOOT_SOURCE := build/qboot/qboot-$(QBOOT_COMMIT).tar.gz
QBOOT_ARTIFACTS := $(MICROVM_OUT_DIR)/qboot.rom $(MICROVM_OUT_DIR)/qboot.LICENSE $(QBOOT_SOURCE)
$(QBOOT_ARTIFACTS) &: scripts/build-qboot.sh
./scripts/build-qboot.sh
build-microvm-image: $(MICROVM_ARTIFACTS) $(QBOOT_ARTIFACTS) ## Build microVM kernel, initramfs and qboot firmware (incremental — skips when up to date)
install-sqs-worker-microvm: $(SQS_WORKER_MICROVM_ARTIFACTS) ## Install the SQS microVM image to /usr/share/spinifex/microvm/sqs-worker/
sudo install -d /usr/share/spinifex/microvm/sqs-worker
sudo rm -f /usr/share/spinifex/microvm/sqs-worker/initramfs.cpio.gz
sudo install -m 0644 $(SQS_WORKER_MICROVM_OUT_DIR)/vmlinuz /usr/share/spinifex/microvm/sqs-worker/vmlinuz
sudo install -m 0644 $(SQS_WORKER_MICROVM_OUT_DIR)/initramfs.cpio.gz /usr/share/spinifex/microvm/sqs-worker/initramfs.cpio.gz
install-microvm: $(MICROVM_ARTIFACTS) $(QBOOT_ARTIFACTS) ## Install microVM artifacts to /usr/share/spinifex/microvm/
install-microvm: $(MICROVM_ARTIFACTS) ## Install microVM artifacts to /usr/share/spinifex/microvm/
sudo install -d /usr/share/spinifex/microvm
sudo install -m 0644 $(MICROVM_OUT_DIR)/vmlinuz /usr/share/spinifex/microvm/vmlinuz
sudo install -m 0644 $(MICROVM_OUT_DIR)/initramfs.cpio.gz /usr/share/spinifex/microvm/initramfs.cpio.gz
sudo install -m 0644 $(MICROVM_OUT_DIR)/qboot.rom /usr/share/spinifex/microvm/qboot.rom
sudo install -m 0644 $(MICROVM_OUT_DIR)/qboot.LICENSE /usr/share/spinifex/microvm/qboot.LICENSE
# the pre-commit gate: lint, vuln, and the unit and e2e-harness tiers.
# the pre-commit gate: manifest checks, lint, vuln, and the unit and e2e-harness tiers.
# integration and race tests skipped to keep quick, they run in CI.
#
# -j2 overlaps the only two heavy gates, lint and test-cover: alone each leaves
# most of the machine idle, so pairing them cuts a cold run by about a quarter.
# -Otarget keeps each gate's output in one block instead of interleaving them.
preflight:
@$(MAKE) --no-print-directory -j2 -Otarget QUIET=1 lint govulncheck test-cover diff-coverage test-harness test-build-scripts
@$(MAKE) --no-print-directory -j2 -Otarget QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-package-check test-harness test-build-scripts
@echo -e "\n ✅ Preflight passed — safe to commit."
# Shell suites + shellcheck for build/scripts/, the systemd-unit helpers that
# ship on every node (unlike scripts/images/, kept in preflight: a wrong
# restart decision here is an outage, not asset churn).
#
# NODE_SCRIPTS are the four that setup.sh installs into
# /usr/local/share/spinifex on every node. They are covered here for the same
# reason: between them they install the software, wire OVN, erase the state and
# remove the install, so a shellcheck warning in one is an outage or a wrong
# deletion, not a style note.
NODE_SCRIPTS := scripts/setup.sh scripts/setup-ovn.sh scripts/install-node.sh scripts/smoke-test.sh scripts/node-reset.sh scripts/uninstall-spx.sh scripts/lib/distro.sh scripts/lib/stage-sqs-store-microvm.sh
test-build-scripts:
@$(_HDR) "\n....Running build/scripts/**/*_test.sh...."
@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$(find build/scripts -name '*_test.sh' | sort); do \
$(_RUNSH); \
@echo -e "\n....Running build/scripts/**/*_test.sh...."
@for t in $$(find build/scripts -name '*_test.sh' | sort); do \
echo "-- $$t"; \
sh "$$t" || exit 1; \
done
@$(_HDR) "\n....Running scripts/*_test.sh, scripts/lib/*_test.sh and scripts/appliance/*_test.sh...."
@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$( (find scripts -maxdepth 1 -name '*_test.sh'; find scripts/lib scripts/appliance -name '*_test.sh') | sort); do \
$(_RUNSH); \
done
@$(_HDR) "\n....Running scripts/terraform/latitude-spx/*_test.sh...."
@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$(find scripts/terraform/latitude-spx -maxdepth 1 -name '*_test.sh' | sort); do \
$(_RUNSH); \
done
@$(_HDR) "\n....Running .github/scripts/*_test.sh...."
@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$(find .github/scripts -maxdepth 1 -name '*_test.sh' | sort); do \
$(_RUNSH); \
done
$(_Q)shellcheck -S warning .github/scripts/hypervisor-disk-preflight.sh \
.github/scripts/wait-for-memory.sh \
.github/scripts/go-build-cache.sh \
.github/scripts/e2e-select.sh
@$(_HDR) "\n....Running shellcheck over build/scripts/**/*.sh + the node scripts...."
$(_Q)shellcheck -S warning $$(find build/scripts -name '*.sh' | sort) $(NODE_SCRIPTS)
$(_Q)shellcheck -S warning -x scripts/build-oracle-appliance.sh $$(find scripts/appliance -name '*.sh' | sort)
@$(_HDR) " test-build-scripts ok"
@echo -e "\n....Running shellcheck over build/scripts/**/*.sh...."
shellcheck -S warning $$(find build/scripts -name '*.sh' | sort)
@echo " test-build-scripts ok"
# E2E harness unit tests. Build-tagged `e2e` so they're skipped by the
# default `go test ./spinifex/...`. Runs with mocked AWS clients — no
@@ -336,26 +189,14 @@ AWS_MODEL_OPERATION_COVERAGE_DIR ?= $(CURDIR)/docs/coverage
# -count=1 is load-bearing: the conformance report is written by the test binary,
# so a cached pass skips the run, leaves no report, and the cat below fails the
# target. It also keeps the conformance gate honest — a cached result would mean
# the check never ran against this commit. The generated request sweep adds
# about 15s, most of it EC2 fan-outs waiting out their window for instances
# no daemon-lite owns.
# the check never ran against this commit.
test-integration:
@echo -e "\n....Running in-process integration tests...."
$(_Q)LOG_IGNORE=1 AWS_MODEL_CONFORMANCE_MODE=$(AWS_MODEL_CONFORMANCE_MODE) AWS_MODEL_CONFORMANCE_REPORT=$(AWS_MODEL_CONFORMANCE_REPORT) go test -count=1 -tags=integration -timeout 120s ./tests/integration/... $(_RACEQ)
$(_Q)LOG_IGNORE=1 AWS_MODEL_CONFORMANCE_MODE=$(AWS_MODEL_CONFORMANCE_MODE) AWS_MODEL_CONFORMANCE_REPORT=$(AWS_MODEL_CONFORMANCE_REPORT) go test -count=1 -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ)
@cat $(AWS_MODEL_CONFORMANCE_REPORT)
@$(MAKE) --no-print-directory generate-aws-model-coverage
@echo "AWS operation coverage: $(AWS_MODEL_OPERATION_COVERAGE_DIR)"
# Suites the unit tiers skip: the three-node SQS R3 scenarios behind
# SPINIFEX_SQS_R3 and the lambdatestlink-tagged tests. CI runs these on every PR.
test-gated:
@echo -e "\n....Running env- and tag-gated suites...."
$(_Q)LOG_IGNORE=1 SPINIFEX_SQS_R3=1 go test $(GOTESTFLAGS) -timeout 180s -run '^TestR3$$' ./spinifex/domains/sqs/message/jetstream/ $(_RACEQ)
$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -tags=lambdatestlink -timeout 120s ./cmd/lambda-agent/ ./spinifex/agents/lambda/hostlink/ ./spinifex/runtime/compute/lambda/guestconn/ ./spinifex/runtime/compute/lambda/testlink/ ./spinifex/domains/network/host/ $(_RACEQ)
$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -tags=lambdatestlink -timeout 120s -run '^TestLambdaTestLink(Executor|Reply)_' ./spinifex/runtime/roles/awsgw/ $(_RACEQ)
$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -tags=lambdatestlink,lambdatestlinkseam -timeout 120s -run '^TestLambdaTestLinkSeam_' ./spinifex/runtime/roles/awsgw/ $(_RACEQ)
$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -tags=lambdatestlink -timeout 120s -run '^TestLambdaTestLink' ./spinifex/runtime/daemon/ $(_RACEQ)
generate-aws-model-coverage:
@go run ./cmd/aws-model-coverage -out $(AWS_MODEL_OPERATION_COVERAGE_DIR) -json $(AWS_MODEL_OPERATION_COVERAGE_DIR)/coverage.json
@@ -363,7 +204,7 @@ aws-model-coverage: generate-aws-model-coverage
@go run ./cmd/aws-model-coverage
# Segscan storage oracle: needs the mulga umbrella repo's scripts/segscan
# checked out alongside spinifex (see internal/testkit/segscanoracle), which
# checked out alongside spinifex (see spinifex/testutil/segscanoracle), which
# is not the default local or CI layout, so this is a separate target from
# test-integration rather than folded into it. Skips itself when segscan's
# source isn't found.
@@ -371,10 +212,27 @@ test-segscan-oracle:
@echo -e "\n....Running segscan storage oracle test...."
$(_Q)LOG_IGNORE=1 go test -tags=integration,segscanoracle -timeout 120s ./tests/integration/... -run TestSegscanOracle $(_RACEQ)
# Validate docs/service-interfaces.yaml. Schema check + cross-reference
# of services/suites/fixtures + on-disk path existence.
manifest-check:
@echo -e "\n....Checking service-interfaces.yaml...."
@go run ./tests/e2e/manifest-check/cmd/manifest-check -repo-root . -manifest docs/service-interfaces.yaml
# Drift guards: direct-create fixture lint + NATS subject lint,
# ratcheted against tests/e2e/manifest-lint/baseline.txt. Fails only on NEW
# drift beyond the baseline.
manifest-lint:
@echo -e "\n....Linting manifest drift (fixtures + subjects)...."
@go run ./tests/e2e/manifest-lint/cmd/manifest-lint -repo-root .
# Accept current drift into the baseline. Run after an intentional change.
manifest-lint-update:
@go run ./tests/e2e/manifest-lint/cmd/manifest-lint -repo-root . -update
# Run unit tests
test:
@echo -e "\n....Running tests for $(GO_PROJECT_NAME)...."
LOG_IGNORE=1 go test -timeout 180s ./spinifex/... ./cmd/... ./internal/... ./contracts/... ./architecture/...
LOG_IGNORE=1 go test -timeout 180s ./spinifex/... ./cmd/... ./internal/...
# Empty locally, where reusing a cached result between runs is the point.
# CI passes -count=1 so a green run means the suite executed against that commit
@@ -385,13 +243,13 @@ GOTESTFLAGS ?=
COVERPROFILE ?= coverage.out
test-cover:
@echo -e "\n....Running tests with coverage for $(GO_PROJECT_NAME)...."
$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -timeout 180s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... ./cmd/... ./internal/... ./contracts/... ./architecture/... $(_COVQ)
$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -timeout 180s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... ./cmd/... ./internal/... $(_COVQ)
@scripts/check-coverage.sh $(COVERPROFILE) $(QUIET)
# Run unit tests with race detector
test-race:
@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... ./contracts/... ./architecture/... $(_RACEQ)
$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... $(_RACEQ)
# Unit tests for in-repo GitHub Actions (e.g. .github/actions/e2e-analyze).
# Kept out of `test-cover` so coverage % isn't diluted by CI-only tooling.
@@ -444,35 +302,18 @@ bench:
@echo -e "\n....Running benchmarks for $(GO_PROJECT_NAME)...."
LOG_IGNORE=1 go test -benchmem -run=. -bench=. ./...
# Fuzz every Fuzz* target for FUZZTIME each; -fuzz takes one target per run.
# Plain `go test` already replays the seeds and testdata/fuzz on preflight.
FUZZTIME ?= 30s
fuzz:
@set -e; grep -rlE '^func Fuzz\w+\(' --include='*_test.go' spinifex cmd internal | while read -r file; do \
for name in $$(grep -oE '^func Fuzz\w+' "$$file" | cut -d' ' -f2); do \
echo -e "\n....Fuzzing $$name ($$(dirname $$file)) for $(FUZZTIME)...."; \
LOG_IGNORE=1 go test -run='^$$' -fuzz="^$$name$$" -fuzztime=$(FUZZTIME) ./$$(dirname $$file); \
done; \
done
# Fast iteration: build + install binary + restart all services.
# Microvm artifacts are reinstalled when they already exist on disk — the rule's
# input timestamps drive a rebuild only if anything actually changed. On a fresh
# checkout (no build/microvm/vmlinuz yet) the install step is skipped; run
# `make install-microvm` explicitly the first time. They go in before the
# preflight, which checks the microvm firmware.
# `make install-microvm` explicitly the first time.
deploy: build
sudo install -m 755 bin/spx /usr/local/bin/spx
@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ]; then \
$(MAKE) install-microvm; \
else \
echo "[deploy] microvm artifacts absent — run 'make install-microvm' for first-time setup"; \
fi
@if [ "$${SKIP_ASSET_PREFLIGHT:-}" != "1" ]; then \
if ! sudo /usr/local/bin/spx admin preflight; then \
echo ""; \
echo "[deploy] host-asset preflight failed: this node's helper scripts, sudoers"; \
echo " grants, QEMU or microvm firmware don't match the binary just installed. Remediate with:"; \
echo "[deploy] host-asset preflight failed: this node's helper scripts or sudoers"; \
echo " grants don't match the binary just installed. Remediate with:"; \
echo " scripts/update-nodes.sh (from the mulga root)"; \
echo " or:"; \
echo " make reinstall"; \
@@ -482,6 +323,11 @@ deploy: build
else \
echo "[deploy] SKIP_ASSET_PREFLIGHT=1 — skipping host-asset preflight"; \
fi
@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ]; then \
$(MAKE) install-microvm; \
else \
echo "[deploy] microvm artifacts absent — run 'make install-microvm' for first-time setup"; \
fi
sudo systemctl daemon-reload
sudo systemctl restart spinifex.target
@@ -492,32 +338,23 @@ reinstall:
clean:
rm -f ./bin/$(GO_PROJECT_NAME)
rm -rf spinifex/runtime/roles/spinifexui/frontend/dist
rm -rf spinifex/services/spinifexui/frontend/dist
# On dnf hosts the runtime packages and repositories come from setup.sh, so the
# two cannot drift; nbdkit-devel, for building the plugin, is only in CodeReady.
install-system:
@echo -e "\n....Installing system dependencies for $(ARCH)...."
@pm=$$(INSTALL_SPINIFEX_LIB_ONLY=1 bash -c '. scripts/setup.sh && distro_pkg_manager') || exit 1; \
if [ "$$pm" = dnf ]; then \
INSTALL_SPINIFEX_LIB_ONLY=1 bash -c '. scripts/setup.sh && setup_sudo && detect_os && install_dnf_deps' && \
sudo dnf config-manager --set-enabled ol10_codeready_builder && \
sudo dnf install -y nbdkit-devel pkgconf-pkg-config make gcc openssh-clients git sudo; \
else \
sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
-o Dpkg::Options::="--force-confdef" \
-o Dpkg::Options::="--force-confold" \
nbdkit nbdkit-plugin-dev pkg-config qemu-system-x86 qemu-system-arm qemu-utils \
ovmf qemu-efi-aarch64 \
libvirt-daemon-system libvirt-clients libvirt-dev make gcc jq curl \
iproute2 netcat-openbsd openssh-client wget git unzip sudo xz-utils file \
ovn-central ovn-host openvswitch-switch dhcpcd-base; \
fi
sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
-o Dpkg::Options::="--force-confdef" \
-o Dpkg::Options::="--force-confold" \
nbdkit nbdkit-plugin-dev pkg-config qemu-system-x86 qemu-system-arm qemu-utils \
ovmf qemu-efi-aarch64 \
libvirt-daemon-system libvirt-clients libvirt-dev make gcc jq curl \
iproute2 netcat-openbsd openssh-client wget git unzip sudo xz-utils file \
ovn-central ovn-host openvswitch-switch dhcpcd-base
install-go:
@echo -e "\n....Installing Go 1.27.3 for $(ARCH) ($(GO_ARCH))...."
@echo -e "\n....Installing Go 1.27.0 for $(ARCH) ($(GO_ARCH))...."
@if [ ! -d "/usr/local/go" ]; then \
curl -L https://go.dev/dl/go1.27.3.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
curl -L https://go.dev/dl/go1.27.0.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
else \
echo "Go already installed in /usr/local/go"; \
fi
@@ -574,16 +411,14 @@ distro-amd64:
-f build/Dockerfile.distro \
--output type=local,dest=dist/amd64/ \
../
@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ] && [ -f $(MICROVM_OUT_DIR)/initramfs.cpio.gz ] && [ -f $(MICROVM_OUT_DIR)/qboot.rom ] && [ -f $(QBOOT_SOURCE) ]; then \
@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ] && [ -f $(MICROVM_OUT_DIR)/initramfs.cpio.gz ]; then \
echo "[distro-amd64] staging microvm artifacts into tarball"; \
mkdir -p dist/amd64/microvm; \
cp $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz $(MICROVM_OUT_DIR)/qboot.rom $(MICROVM_OUT_DIR)/qboot.LICENSE dist/amd64/microvm/; \
cp $(QBOOT_SOURCE) dist/; \
cp $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz dist/amd64/microvm/; \
else \
echo "[distro-amd64] WARNING: microvm artifacts missing — tarball will not include them"; \
echo "[distro-amd64] run 'make build-microvm-image' before 'make distro-amd64'"; \
fi
./scripts/lib/stage-sqs-store-microvm.sh $(SQS_WORKER_MICROVM_OUT_DIR) dist/amd64 sqs-worker
tar -czf dist/spinifex-$(VERSION)-linux-amd64.tar.gz -C dist/amd64 .
sha256sum dist/spinifex-$(VERSION)-linux-amd64.tar.gz > dist/spinifex-$(VERSION)-linux-amd64.tar.gz.sha256
@@ -602,7 +437,7 @@ distro-arm64:
distro-clean:
rm -rf dist/
.PHONY: test-package-check build build-lambda-testlink build-lambda-testlink-image build-ui test-ui-e2e ui-e2e-container ui-baselines build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-worker-microvm-image build-lambda-runtime-microvm-image install-microvm install-sqs-worker-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration test-gated generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
.PHONY: test-package-check build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
deploy reinstall clean \
install-system install-go install-aws quickinstall \
lint fix govulncheck nilaway \
+179 -94
View File
@@ -4,138 +4,223 @@
<p align="center">
<a href="https://go.dev"><img src="https://img.shields.io/badge/Go-1.27+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
<a href="https://pkg.go.dev/github.com/mulgadc/spinifex"><img src="https://img.shields.io/badge/pkg.go.dev-Reference-007D9C?style=flat-square&logo=go&logoColor=white" alt="Go Reference"></a>
<a href="LICENSE"><img src="https://img.shields.io/badge/License-AGPL--3.0-3fb950?style=flat-square" alt="License"></a>
<a href="https://mulgadc.com"><img src="https://img.shields.io/badge/Site-mulga-orange?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTE2LjcxOCA4Ljg5MWMtMS4yODUgMS4zNy0zLjE3OCAyLjMxNy00LjY5NSAzLjQ0LS44NTQuNjMtMy4wODkgMi4yNy0xLjYxIDMuMzQ0IDEuNzg4IDEuMjk4IDYuMjQzLjE2NCA3Ljc4NS0xLjI2Ljg0LS43NzUuODE0LTEuODIyLS4zMjgtMi4yNTktMS4yMzUtLjQ3Mi0yLjY2LS4xMTEtMy45MTMuMDg3LS4wNDIuMDA3LS4xMzEuMDU1LS4xMjMtLjAyLjUyMy0uMzM2Ljk5My0uNzUgMS41MDMtMS4xMDIuNDkyLS4zNCAxLjA5Ny0uODI3IDEuNy0uODM4IDEuODcxLS4wMzQgMy43OTkuODkgNC4yODcgMi44MTUuODExIDMuMjAzLTMuMDA2IDUuNzE1LTUuNzg0IDUuOTQybDEuNjE0LS43NDdjLjYwNS0uMzYgMS4yMTctLjczNCAxLjc1Mi0xLjE5Ni4xMzMtLjExNS4yMy0uMjYzLjM0Mi0uMzczLjAyNy0uMDI2LjMwMi0uMTQ0LjE0NC0uMTUtMS40NDEuOTQ1LTMuMTI3IDEuNTcyLTQuODEzIDEuOTMyLS41OC4xMjMtMS4xOTUuMjQyLTEuNzg1LjI1Ny4wMTUuMDguMDg3LjA2NC4xNC4wOC40NzYuMTU0IDEuMDIuMjQ1IDEuNTE2LjMyLjA2OC4wNTItLjAwNi4wNzQtLjA2LjA4MS0uNDQ4LjA1OC0uOTIzLjE0My0xLjM3LjE2Ni0xLjI3LjA2NC0yLjU1LS4wNjgtMy43NzctLjM4bC0uNjktMS45NTdjLS4wNi4wMDYtLjA2My4wNi0uMDc5LjEwMy0uMDYuMTctLjMwNSAxLjQxNy0uMzg3IDEuNDM1LS42Ni0uMi0xLjI1MS0uNjQ3LTEuNzM2LTEuMTMybDEuMTUtMi4wNi0xLjgzNSAxLjAxMWMtLjI1OC0uNTgzLS4zMDUtMS4yNDMtLjIxOS0xLjg3bDIuMzc1LTEuMThjLS43MzktLjA0OS0xLjQ4Ni4wNjgtMi4yMi4xNDUuMTI1LS4zNi4yNzctLjcxOC40NjItMS4wNTEuMDY3LS4xMjIuNDI2LS43MjEuNTItLjcyNC44Ni4xNjQgMS43ODYuMjQxIDIuNjQ4LjA3NGwtMS44Ny0uNzcxLS4wNjktLjA5M2MuMzgyLS4zNi43Ny0uNzE3IDEuMTk5LTEuMDI0LjgwNC4yMzQgMS42My40MjIgMi40NzMuNDMzTDkuMzUgOS4zNDJhNyA3IDAgMCAxIC41NjctLjQwMWMuMTYtLjEwMy42MDktLjQwMi43NzItLjM4LjY1LjIyMSAxLjMyNC4zNzggMi4wMS40MzMtLjMxMy0uMzI1LS45MDktLjU2NC0xLjIxLS44NjgtLjAzMy0uMDM0LS4wNTgtLjAzNi0uMDQzLS4wOTguNzg3LS40NTIgMS42NjItLjkwMSAyLjM0LTEuNTE3LjY5LS42MjkgMS40MjEtMS42NTYuOTI0LTIuNjA1LjU3My4xODIgMS4wNjcuOTA2IDEuMDU0IDEuNTEyLS4wMzQgMS42NzYtMS44MjIgMy4xNy0yLjk0MyA0LjIyMiAxLjczMi0uNzI4IDMuNzE0LTIuMjMgMy43MS00LjMwNS0uMDAzLTEuNjg4LTEuNTQtMi4zNjUtMi45OTMtMi41MThhMy4yIDMuMiAwIDAgMS0uMzg1IDEuMDg3Yy0uNDI4LjcxOS0xLjMwMiAxLjE2OC0xLjc1OCAxLjkxNS0uMzExLjUxLS4zNyAxLjE5NS0xLjAzMSAxLjM5LS4yNy4wOC0uNjE3LjA5My0uODk3LjA5NGwuNjE1LS4zMjVjLjY4OS0uNTA2LjY3Ny0xLjQyIDEuMDk5LTIuMS0uMDUtLjA3LS41NDUtLjItLjY2LS4yMjktLjUxLS4xMjUtMS4zNzUtLjI4NS0xLjg4Ni0uMjUtLjE1Ny4wMS0uODY5LjEyMS0uOTMuMjQyLS4wODguMTc3LjM0MS45My40NjggMS4xMDEuMDI1LjAzNS4wNzcuMDI2LjA4LjAzLjAyNC4wMzEuMDA3LjEwNS0uMDYuMDhhNSA1IDAgMCAxLS40MDQtLjI4M2MtLjUwMy0uMzg4LTEuMDc4LS45NzQtMS40OTktMS40NDctLjA2OC0uMTY2LS4xNi0uMzUuMDAyLS40ODcuODg4LS41NSAxLjc2OC0xLjEyNiAyLjY3LTEuNjUxIDIuNjcxLTEuNTU4IDUuNzExLTMuMzE4IDguMjY3LS40NjUgMi4xMTkgMi4zNjYgMS41MTQgNS4yMTItLjUxMiA3LjM3MnptLTguMjI0LTUuMzhjLjY5OS0uMTIyIDIuMDE4LjU3MiAyLjIzNS0uNDU2LjAyMS0uMS0uMDM2LS4xNTcuMDItLjI1NS4wNDMtLjA3My4yODYtLjI1LjM3LS4zMTguMjctLjIxNy41NzctLjM4Ny44NDItLjYxMi0xLjAyOS4xNjYtMi4wNjUuNzU2LTIuOTY0IDEuMjc3LS4xNDkuMDg2LS4zMS4xNzYtLjQ1MS4yNzMtLjAzNy4wMjUtLjA3NS0uMDA2LS4wNTMuMDltLTQuODMgMTAuODQ0Yy0xLjcxNSAxLjg4MS0xLjMyNSA0LjU3LjQ5NCA2LjIxNyAxLjgxMSAxLjY0MSA0LjY2IDIuMjIzIDcuMDQ4IDIuMTA4IDIuMTUyLS4xMDMgNC4zMzctLjgxMiA2LjQ2LS4wOTEuODI1LjI4IDEuNTQ2LjgwNSAyLjE2IDEuNDExLS4wODMtLjQtLjMyNC0uODE5LS41NTgtMS4xNTktMi4xMy0zLjA5NS02LjI3LTEuOTM1LTkuNDI2LTIuNTUzLTEuNzExLS4zMzUtMy40OTEtMS4xMTgtNC41MzMtMi41NjctLjkwMS0xLjI1My0xLjA0Mi0yLjczLS41OTUtNC4xOTQtLjA5MS0uMDktLjk1Mi43MjEtMS4wNS44MjhtNi4zNjYtOC42NjdjLS4zODIuMzM5LS43ODcuNjYtMS4yMTIuOTQ4bC0xLjIwNy41OWMxLjAyMi4wMzkgMi4wOC0uNTQ2IDIuNDItMS41MzciLz48L3N2Zz4=" alt="mulgadc.com"></a>
<a href="https://docs.mulgadc.com"><img src="https://img.shields.io/badge/Docs-spinifex-0969da?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTYuNSAySDIwYTEgMSAwIDAgMSAxIDF2MTQuNWExIDEgMCAwIDEtMSAxSDYuNWExLjUgMS41IDAgMCAwIDAgM0gyMWExIDEgMCAwIDEgMCAySDYuNUEzLjUgMy41IDAgMCAxIDMgMjBWNS41QTMuNSAzLjUgMCAwIDEgNi41IDJ6bTEgNGExIDEgMCAwIDAgMCAyaDlhMSAxIDAgMCAwIDAtMmgtOXptMCA0YTEgMSAwIDAgMCAwIDJoOWExIDEgMCAwIDAgMC0yaC05eiIvPjwvc3ZnPg==" alt="docs.mulgadc.com"></a>
<a href="https://mulgadc.com"><img src="https://img.shields.io/badge/home-mulga-orange?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTE2LjcxOCA4Ljg5MWMtMS4yODUgMS4zNy0zLjE3OCAyLjMxNy00LjY5NSAzLjQ0LS44NTQuNjMtMy4wODkgMi4yNy0xLjYxIDMuMzQ0IDEuNzg4IDEuMjk4IDYuMjQzLjE2NCA3Ljc4NS0xLjI2Ljg0LS43NzUuODE0LTEuODIyLS4zMjgtMi4yNTktMS4yMzUtLjQ3Mi0yLjY2LS4xMTEtMy45MTMuMDg3LS4wNDIuMDA3LS4xMzEuMDU1LS4xMjMtLjAyLjUyMy0uMzM2Ljk5My0uNzUgMS41MDMtMS4xMDIuNDkyLS4zNCAxLjA5Ny0uODI3IDEuNy0uODM4IDEuODcxLS4wMzQgMy43OTkuODkgNC4yODcgMi44MTUuODExIDMuMjAzLTMuMDA2IDUuNzE1LTUuNzg0IDUuOTQybDEuNjE0LS43NDdjLjYwNS0uMzYgMS4yMTctLjczNCAxLjc1Mi0xLjE5Ni4xMzMtLjExNS4yMy0uMjYzLjM0Mi0uMzczLjAyNy0uMDI2LjMwMi0uMTQ0LjE0NC0uMTUtMS40NDEuOTQ1LTMuMTI3IDEuNTcyLTQuODEzIDEuOTMyLS41OC4xMjMtMS4xOTUuMjQyLTEuNzg1LjI1Ny4wMTUuMDguMDg3LjA2NC4xNC4wOC40NzYuMTU0IDEuMDIuMjQ1IDEuNTE2LjMyLjA2OC4wNTItLjAwNi4wNzQtLjA2LjA4MS0uNDQ4LjA1OC0uOTIzLjE0My0xLjM3LjE2Ni0xLjI3LjA2NC0yLjU1LS4wNjgtMy43NzctLjM4bC0uNjktMS45NTdjLS4wNi4wMDYtLjA2My4wNi0uMDc5LjEwMy0uMDYuMTctLjMwNSAxLjQxNy0uMzg3IDEuNDM1LS42Ni0uMi0xLjI1MS0uNjQ3LTEuNzM2LTEuMTMybDEuMTUtMi4wNi0xLjgzNSAxLjAxMWMtLjI1OC0uNTgzLS4zMDUtMS4yNDMtLjIxOS0xLjg3bDIuMzc1LTEuMThjLS43MzktLjA0OS0xLjQ4Ni4wNjgtMi4yMi4xNDUuMTI1LS4zNi4yNzctLjcxOC40NjItMS4wNTEuMDY3LS4xMjIuNDI2LS43MjEuNTItLjcyNC44Ni4xNjQgMS43ODYuMjQxIDIuNjQ4LjA3NGwtMS44Ny0uNzcxLS4wNjktLjA5M2MuMzgyLS4zNi43Ny0uNzE3IDEuMTk5LTEuMDI0LjgwNC4yMzQgMS42My40MjIgMi40NzMuNDMzTDkuMzUgOS4zNDJhNyA3IDAgMCAxIC41NjctLjQwMWMuMTYtLjEwMy42MDktLjQwMi43NzItLjM4LjY1LjIyMSAxLjMyNC4zNzggMi4wMS40MzMtLjMxMy0uMzI1LS45MDktLjU2NC0xLjIxLS44NjgtLjAzMy0uMDM0LS4wNTgtLjAzNi0uMDQzLS4wOTguNzg3LS40NTIgMS42NjItLjkwMSAyLjM0LTEuNTE3LjY5LS42MjkgMS40MjEtMS42NTYuOTI0LTIuNjA1LjU3My4xODIgMS4wNjcuOTA2IDEuMDU0IDEuNTEyLS4wMzQgMS42NzYtMS44MjIgMy4xNy0yLjk0MyA0LjIyMiAxLjczMi0uNzI4IDMuNzE0LTIuMjMgMy43MS00LjMwNS0uMDAzLTEuNjg4LTEuNTQtMi4zNjUtMi45OTMtMi41MThhMy4yIDMuMiAwIDAgMS0uMzg1IDEuMDg3Yy0uNDI4LjcxOS0xLjMwMiAxLjE2OC0xLjc1OCAxLjkxNS0uMzExLjUxLS4zNyAxLjE5NS0xLjAzMSAxLjM5LS4yNy4wOC0uNjE3LjA5My0uODk3LjA5NGwuNjE1LS4zMjVjLjY4OS0uNTA2LjY3Ny0xLjQyIDEuMDk5LTIuMS0uMDUtLjA3LS41NDUtLjItLjY2LS4yMjktLjUxLS4xMjUtMS4zNzUtLjI4NS0xLjg4Ni0uMjUtLjE1Ny4wMS0uODY5LjEyMS0uOTMuMjQyLS4wODguMTc3LjM0MS45My40NjggMS4xMDEuMDI1LjAzNS4wNzcuMDI2LjA4LjAzLjAyNC4wMzEuMDA3LjEwNS0uMDYuMDhhNSA1IDAgMCAxLS40MDQtLjI4M2MtLjUwMy0uMzg4LTEuMDc4LS45NzQtMS40OTktMS40NDctLjA2OC0uMTY2LS4xNi0uMzUuMDAyLS40ODcuODg4LS41NSAxLjc2OC0xLjEyNiAyLjY3LTEuNjUxIDIuNjcxLTEuNTU4IDUuNzExLTMuMzE4IDguMjY3LS40NjUgMi4xMTkgMi4zNjYgMS41MTQgNS4yMTItLjUxMiA3LjM3MnptLTguMjI0LTUuMzhjLjY5OS0uMTIyIDIuMDE4LjU3MiAyLjIzNS0uNDU2LjAyMS0uMS0uMDM2LS4xNTcuMDItLjI1NS4wNDMtLjA3My4yODYtLjI1LjM3LS4zMTguMjctLjIxNy41NzctLjM4Ny44NDItLjYxMi0xLjAyOS4xNjYtMi4wNjUuNzU2LTIuOTY0IDEuMjc3LS4xNDkuMDg2LS4zMS4xNzYtLjQ1MS4yNzMtLjAzNy4wMjUtLjA3NS0uMDA2LS4wNTMuMDltLTQuODMgMTAuODQ0Yy0xLjcxNSAxLjg4MS0xLjMyNSA0LjU3LjQ5NCA2LjIxNyAxLjgxMSAxLjY0MSA0LjY2IDIuMjIzIDcuMDQ4IDIuMTA4IDIuMTUyLS4xMDMgNC4zMzctLjgxMiA2LjQ2LS4wOTEuODI1LjI4IDEuNTQ2LjgwNSAyLjE2IDEuNDExLS4wODMtLjQtLjMyNC0uODE5LS41NTgtMS4xNTktMi4xMy0zLjA5NS02LjI3LTEuOTM1LTkuNDI2LTIuNTUzLTEuNzExLS4zMzUtMy40OTEtMS4xMTgtNC41MzMtMi41NjctLjkwMS0xLjI1My0xLjA0Mi0yLjczLS41OTUtNC4xOTQtLjA5MS0uMDktLjk1Mi43MjEtMS4wNS44MjhtNi4zNjYtOC42NjdjLS4zODIuMzM5LS43ODcuNjYtMS4yMTIuOTQ4bC0xLjIwNy41OWMxLjAyMi4wMzkgMi4wOC0uNTQ2IDIuNDItMS41MzciLz48L3N2Zz4=" alt="mulgadc.com"></a>
<a href="https://docs.mulgadc.com"><img src="https://img.shields.io/badge/docs-spinifex-0969da?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTYuNSAySDIwYTEgMSAwIDAgMSAxIDF2MTQuNWExIDEgMCAwIDEtMSAxSDYuNWExLjUgMS41IDAgMCAwIDAgM0gyMWExIDEgMCAwIDEgMCAySDYuNUEzLjUgMy41IDAgMCAxIDMgMjBWNS41QTMuNSAzLjUgMCAwIDEgNi41IDJ6bTEgNGExIDEgMCAwIDAgMCAyaDlhMSAxIDAgMCAwIDAtMmgtOXptMCA0YTEgMSAwIDAgMCAwIDJoOWExIDEgMCAwIDAgMC0yaC05eiIvPjwvc3ZnPg==" alt="docs.mulgadc.com"></a>
</p>
<p align="center">
<a href="#try-it-live">Live Demo</a> ·
<a href="#services">Services</a> ·
<a href="#deploy-anywhere">Deploy</a> ·
<a href="#install">Install</a> ·
<a href="#architecture">Architecture</a>
<a href="#what-is-spinifex">What is Spinifex?</a> ·
<a href="#the-platform">Platform</a> ·
<a href="#three-ways-to-deploy">Deploy</a> ·
<a href="#aws-compatibility">AWS Compatibility</a> ·
<a href="#live-demo">Live Demo</a> ·
<a href="#core-components">Components</a> ·
<a href="#architecture-at-a-glance">Architecture</a> ·
<a href="#self-hosted-installation">Installation</a> ·
<a href="https://docs.mulgadc.com">Docs</a>
</p>
---
# Spinifex: the open, AWS-compatible cloud you run yourself
Spinifex, built by [Mulga](https://mulgadc.com), recreates the AWS services your software already uses on infrastructure you control. Move workloads off the hyperscalers, run AI at a fraction of the cost, or take the cloud to the edge, without rewriting a line.
Spinifex, built by [Mulga](https://mulgadc.com), recreates the AWS services your software already uses (EC2, EBS, S3, VPC, IAM) on infrastructure you control. Move workloads off the hyperscalers, run AI at a fraction of the cost, or take the cloud to the edge, without rewriting a line. The exact build you ship to AWS, Spinifex serves on your own hardware: same APIs, same Terraform, same SDKs.
- **Your AWS tooling, unchanged.** The AWS CLI, SDKs, and Terraform work as they are.
- **Full control of the stack.** Your hardware, data, and keys, with no external control plane, running through disconnection.
- **Open core under AGPL-3.0.** Auditable, yours to keep, with no lock-in.
## What is Spinifex?
Spinifex is an open-source infrastructure platform that recreates the AWS service surface (EC2, EBS, S3, VPC, IAM) on bare-metal, edge, and on-premises environments. Run cloud-native software without a hyperscaler.
Built for teams that need:
- The AWS tooling you already use (AWS CLI, SDKs, Terraform), with nothing to rewrite
- Full control of the stack: your hardware, your network, your data, your keys
- A cloud that keeps running offline, through disconnection, with no external control plane
- An open core (AGPL-3.0), auditable and yours to keep, with no lock-in
You change *where* your software runs, not *what* it is.
## Try It Live
Get a real Spinifex endpoint in under a minute, with no card and no install. The sandbox is yours for 72 hours, with access keys, the web console, and EC2, EBS, S3, VPC, IAM, EKS, and RDS ready to call. Point your existing AWS CLI or Terraform straight at it.
**[Create your sandbox →](https://mulgadc.com/signup)**
## Same Commands, Your Cloud
```bash
export AWS_PROFILE=spinifex
aws ec2 describe-images
aws ec2 run-instances --image-id ami-xxxxxxxx --instance-type t3.micro
aws s3 mb s3://my-bucket
```
For Terraform, start from the [example workbooks](https://docs.mulgadc.com/docs/nginx-webserver).
## Why Not LocalStack or Floci?
[LocalStack](https://github.com/localstack/localstack) and [Floci](https://github.com/floci-io/floci) are emulators for local development and CI testing. An emulator returns a plausible response; Spinifex boots the machine. A call launches a real KVM guest, attaches a replicated Viperblock volume, or writes an erasure-coded object into Predastore, so Spinifex can run your production workloads on hardware you own. Test against an emulator, then run in production on Spinifex.
## Services
The platform serves **405 AWS API operations** across the services below. See exactly which ones in the [coverage report](https://docs.mulgadc.com/coverage).
| Service | What it is | Built on | Status |
| --- | --- | --- | --- |
| **EC2** | Compute | QEMU/KVM, with spot instances and GPU passthrough | Available |
| **EBS** | Block storage | [Viperblock](https://github.com/mulgadc/viperblock): replicated, with a WAL on local NVMe | Available |
| **S3** | Object storage | [Predastore](https://github.com/mulgadc/predastore): Reed-Solomon erasure coding | Available |
| **VPC** | Networking | OVN, with security groups | Available |
| **IAM** | Identity & auth | SigV4 across every service | Available |
| **ALB / NLB** | Load balancers | HAProxy (L7), nginx `stream` (L4), with ACM certificates | Available |
| **EKS** | Kubernetes | k3s, with HA control planes and managed node groups | Available |
| **ECR** | Container registry | Predastore, deduplicated per account | Available |
| **ECS** | Container service | Spinifex instances | Available |
| **RDS** | Databases | PostgreSQL and MariaDB, with automated backups | Available |
| **Route 53** | DNS | [Northstar](https://github.com/mulgadc/northstar): authoritative DNS with hosted zones | Available |
| **Bedrock** | AI deployment | vLLM for text generation and Hugging Face TEI for embeddings and reranking | Q4 2026 |
| **Lambda** | Serverless functions | QEMU `microvm` execution environments | Q1 2027 |
| **SQS** | Message queues | A dedicated NATS JetStream store in a per-Region QEMU `microvm` guest | Q1 2027 |
Roadmap services ship under the same AWS API surface, so code written for AWS today works the moment they land. [Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
## Deploy Anywhere
## The Platform
<p align="center">
<img src=".github/assets/platform.svg" alt="The Spinifex stack: your apps and AWS tooling on top, the Spinifex AWS-compatible service layer, running on standard Linux on CPU and GPU hardware, deployable to Neocloud, on-premise, or the edge." width="900">
</p>
- **Neocloud:** lift and shift onto partner GPU capacity (H100 / H200 / B200). Change an endpoint, keep the software.
- **On-premise:** a multi-node HA cluster in your own data centre, so the stack, the data, and the jurisdiction stay yours.
- **Edge:** air-gapped sites, vehicles, vessels, factories, and clinics, with compute next to the data.
From commodity hardware up to unmodified AWS tooling, every layer is replaceable and yours to own.
## Install
## Three Ways To Deploy
Spinifex runs on Ubuntu 26.04 or Debian 13. The full guides are at [docs.mulgadc.com](https://docs.mulgadc.com).
The same AWS-compatible platform runs in three shapes. Pick the one that matches your hardware.
>*Prerequisite:* Spinifex requires a Linux bridge configured on the host for VM networking. See the [single-node install guide](https://docs.mulgadc.com/docs/install#prerequisites) for setup details.
### Neocloud
**Bare metal (recommended):** download the [bootable x86 installer](https://iso.mulgadc.com/spinifex.iso) and follow the [USB install guide](https://docs.mulgadc.com/docs/install-usb).
Lift and shift onto a partner Neocloud without rewriting workloads. GPU capacity (H100 / H200 / B200), cheaper, available now. Change an endpoint, keep the software.
```bash
curl -fLO https://iso.mulgadc.com/spinifex.iso
```
### On-premise
**Single node:**
Host Spinifex on your own hardware in your own data centre. You get a multi-node HA cluster integrated with your storage and networking, so the stack, the data, and the jurisdiction stay yours.
```bash
curl -fsSL https://install.mulgadc.com | bash
sudo /usr/local/share/spinifex/setup-ovn.sh --management
sudo spx admin init --node node1 --nodes 1
sudo systemctl start spinifex.target
export AWS_PROFILE=spinifex
aws ec2 describe-instance-types
```
### Edge
**From source:** see the [Source Install](https://docs.mulgadc.com/docs/install-source) guide.
Cloud where the cloud can't reach. Air-gapped sites, vehicles, vessels, factories, and clinics. Compute next to the data, running through disconnection, on hardware you own. The same AWS APIs your software already uses.
## Web Console
## AWS Compatibility
<p align="center">
<img src=".github/assets/dashboard.webp" alt="Spinifex web console dashboard" width="900">
</p>
Spinifex speaks the AWS API surface. The AWS SDKs, AWS CLI, and Terraform all work unchanged, whether you deploy at the edge, on-premise, or on a partner Neocloud.
Spinifex ships with a built-in web console for your instances, volumes, buckets, networks, and IAM. See [Launching the Web UI](https://docs.mulgadc.com/docs/setting-up-your-cluster#7-launching-the-web-ui).
| Service | What it is | Status |
| --- | --- | --- |
| **EC2** | Compute | Available |
| **EBS** | Block Storage | Available |
| **S3** | Object Storage | Available |
| **VPC** | Networking | Available |
| **IAM** | Identity & Auth | Available |
| **ALB / NLB** | Load Balancers | Available |
| **EKS** | Kubernetes | Available |
| **ECR** | Container Registry | Available |
| **ECS** | Container Service | Available |
| **RDS** | Databases | Available |
| **Bedrock** | AI Deployment | Q4 2026 |
## Architecture
Roadmap items ship under the same AWS API surface. Code written for AWS today keeps working the moment they land. [Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
## Live Demo
Create a live sandbox in under a minute. The fastest way to experience the stack. Try Spinifex against one of your own workloads on infrastructure we run for you.
* Live in under a minute. No card, no install.
* Yours for 72 hours to test and experiment.
* Real AWS-compatible endpoint, access keys and console.
* EC2, EBS, S3, VPC, IAM, EKS and RDS, ready to call.
* Point your existing AWS CLI or Terraform straight at it.
Sign up at [https://mulgadc.com/signup](https://mulgadc.com/signup)
## Core Components
### Spinifex (Compute and Platform Services – EC2, EKS, ECS, RDS)
Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing an EC2 compatible API. It manages lifecycle operations like start, stop, and terminate through QEMU's QMP interface. Designed to be straightforward and scriptable, Spinifex lets you launch VMs with the AWS CLI, SDKs, or Terraform, without Kubernetes or a heavyweight orchestrator.
The managed services build on that same instance layer. An EKS node group, an ECS container instance, and an RDS database are Spinifex VMs on your own hardware, each driven through its own AWS API.
- **EC2** – VM lifecycle on bare metal, cloud-init and IMDS metadata, key pairs, tags, spot instances, capacity reservations, and GPU passthrough
- **EKS** – Managed Kubernetes on k3s, with HA control planes, managed node groups, cluster add-ons, an OIDC provider, and IAM access entries
- **ECS** – Task and service scheduling across container instances, with awsvpc ENIs, capacity providers, rolling deployments, and GPU-aware placement
- **ECR** – Private container registry backed by Predastore, with content-addressed blobs deduplicated per account and lifecycle policies
- **RDS** – Managed PostgreSQL and MariaDB, with parameter groups, automated backups, and DNS endpoints served by Northstar
- **ALB / NLB** – Layer 7 load balancing on HAProxy and layer 4 on nginx `stream`, with health checks and ACM-issued listener certificates
- **VPC and IAM** – OVN-backed networking and security groups, with roles, policies, and SigV4 auth applied across every service
### Viperblock (Block Storage – EBS Alternative)
[Viperblock](https://github.com/mulgadc/viperblock) is a high-performance, WAL-backed block storage service that replicates volumes across multiple nodes. It's built for reliability and speed, with support for snapshots, recovery, and direct connection to QEMU instances using NBD or virtio-blk.
- Fast, durable virtual disks
- Replication for resilience
- Exposed over NBD or embedded in VMs
- High performance WAL logs on local NVMe drives, reducing IO traffic to S3
- In-memory read/write block cache
### Predastore (Object Storage – S3-Compatible)
[Predastore](https://github.com/mulgadc/predastore) is a fully S3-compatible object storage system. It supports the AWS S3 API, including Signature V4 authentication, multipart uploads, and Terraform provisioning. Predastore chunks data and distributes it across nodes using Reed-Solomon erasure coding, so it survives node loss and suits large-scale or low-bandwidth sites.
- S3-compatible API and auth
- Multipart uploads, streaming reads/writes
- Data redundancy with Reed-Solomon encoding
### Northstar (Authoritative DNS)
[Northstar](https://github.com/mulgadc/northstar) is a lightweight authoritative DNS server that handles both internal service discovery and public-facing DNS for a Spinifex cluster. Zones are human-readable TOML files, served from the local filesystem or synced from Predastore, keeping DNS configuration in the same object storage as the rest of your infrastructure.
- Authoritative DNS over UDP, TCP, and DNS-over-TLS
- Service discovery for cluster components via SRV records
- Zone files stored locally or in S3-compatible storage
## Architecture at a Glance
<p align="center">
<img src=".github/assets/architecture.svg" alt="Spinifex message-driven architecture: standard AWS clients call the AWS Gateway over HTTPS with SigV4, which publishes to a NATS bus answered by stateless ec2/ebs/s3/vpc/iam daemons over local backends." width="900">
</p>
Every AWS API call is authenticated at the gateway, published to a NATS subject, and answered by whichever daemon claims it. Daemons are stateless, so you scale horizontally by starting more. Deep dive: **[`docs/DESIGN.md`](docs/DESIGN.md)**.
Every AWS API call is authenticated at the gateway, published to a NATS subject, and answered by whichever daemon claims it. Daemons are stateless: scale horizontally by starting more. No etcd, no Kubernetes, no external control plane. Just systemd units, a NATS cluster, and your hardware. Deep dive: **[`docs/DESIGN.md`](docs/DESIGN.md)**.
## How We Build Spinifex
## Self-hosted Installation
Spinifex is built by infrastructure engineers with deep AWS experience, including former AWS team members, drawing on decades of combined work across AWS, enterprise infrastructure, and edge computing. Engineers architect and implement it, with **Claude Code** accelerating parts of the work.
Installation requires an Ubuntu 26.04 or Debian 13 system. See the detailed documentation at [docs.mulgadc.com](https://docs.mulgadc.com) for installing Spinifex.
| Claude Code assists with | Engineers own |
| --- | --- |
| Boilerplate AWS API structures and handlers | Architecture and scalability decisions |
| Development guides and API documentation | Authentication, encryption, and threat modelling |
| Test case generation and validation scenarios | Real-world performance tuning and benchmarking |
| Large-scale restructuring and refactoring | Deployment strategy and production operations |
### Bare Metal ISO
The recommended installation is a [bootable x86 installer](https://iso.mulgadc.com/spinifex.iso) for bare-metal hardware.
```bash
curl -fLO https://iso.mulgadc.com/spinifex.iso
```
Follow the [USB install guide](https://docs.mulgadc.com/docs/install-usb) to write the ISO to USB and install on your hardware.
### Single Node Install
>*Prerequisite:* Spinifex requires a Linux bridge configured on the host for VM networking. See the [single-node install guide](https://docs.mulgadc.com/docs/install#prerequisites) for setup details.
```bash
curl -fsSL https://install.mulgadc.com | bash
sudo /usr/local/share/spinifex/setup-ovn.sh --management
sudo spx admin init --node node1 --nodes 1
sudo systemctl start spinifex.target
export AWS_PROFILE=spinifex
aws ec2 describe-instance-types
```
### Development Setup
For a complete development environment see the [Source Install](https://docs.mulgadc.com/docs/install-source) documentation.
## Spinifex UI
Spinifex ships with a built-in web console, an optional alternative to the AWS CLI, SDKs, and Terraform. If you know the AWS Management Console, the Spinifex UI fills the same role: a browser-based view of your instances, volumes, buckets, VPCs, and IAM resources, without leaving your own network.
<p align="center">
<img src=".github/assets/spinifex-ui.jpg" alt="Spinifex web console dashboard" width="900">
</p>
Each node serves the console on port `3000` over TLS, and it comes up with `spinifex.target`:
- **Same API, different surface.** The UI issues the same AWS SigV4 calls the CLI does, so RBAC, audit trails, and IAM policies apply the same way.
- **Single sign-on against your AWS credentials.** Log in with the access keys from `~/.aws/credentials` on the node where Spinifex is installed. There is no separate user database.
- **Self-hosted, works offline.** The Spinifex binary embeds the UI and each node serves it directly, with no external CDN or analytics calls.
For the full walkthrough, covering first-time TLS certificate trust, login, and a feature tour, see [**Launching the Web UI**](https://docs.mulgadc.com/docs/setting-up-your-cluster#7-launching-the-web-ui) in the cluster setup guide.
## Development Philosophy
### Built by Engineers, For Engineers
Spinifex is developed by experienced infrastructure engineers with deep AWS expertise, including former AWS team members who understand the intricacies of building production-grade cloud services. Our team brings decades of combined experience from AWS, enterprise infrastructure, and edge computing environments.
### AI-Assisted Development
Spinifex is architected and implemented by experienced engineers, with **Claude Code** accelerating certain development tasks. The split keeps the quality and reliability standards that production infrastructure demands:
**How We Use Claude Code:**
- **Code Generation**: Boilerplate AWS API structures and handlers
- **Documentation**: Comprehensive development guides and API documentation
- **Testing**: Test case generation and validation scenarios
- **Refactoring**: Large-scale code restructuring and optimization
**What Remains Human-Driven:**
- **Architecture Decisions**: Core system design and scalability choices
- **Security Implementation**: Authentication, encryption, and threat modeling
- **Performance Optimization**: Real-world performance tuning and benchmarking
- **Production Operations**: Deployment strategies and operational procedures
## Trademarks
@@ -143,4 +228,4 @@ Spinifex is built by infrastructure engineers with deep AWS experience, includin
## License
Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE), with a commercial license available.
Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE), with a commercial license available. You're free to use, modify, and deploy it anywhere you need reliable infrastructure without depending on centralized cloud platforms.
@@ -1,92 +0,0 @@
package instancestate
// allowance is one recorded production consumer of raw instance state.
type allowance struct {
File, Symbol string
Inv string // inventory item in docs/PACKAGE_BOUNDARY_MIGRATION.md
}
// allowlist mirrors the ADR-0007 slice 0 inventory in
// docs/PACKAGE_BOUNDARY_MIGRATION.md. It only shrinks: delete an entry as soon
// as its use is gone, and never add one to make a new consumer pass.
var allowlist = []allowance{
{File: "spinifex/runtime/daemon/daemon.go", Symbol: "daemon.LocalStatePath", Inv: "INV-01"},
{File: "spinifex/runtime/daemon/daemon.go", Symbol: "daemon.ReadLocalState", Inv: "INV-01"},
{File: "spinifex/runtime/daemon/daemon.go", Symbol: "daemon.MarshalLocalState", Inv: "INV-01"},
{File: "spinifex/runtime/daemon/daemon.go", Symbol: "daemon.WriteLocalStateBytes", Inv: "INV-01"},
{File: "spinifex/runtime/daemon/daemon.go", Symbol: "JetStreamManager.InitKVBucket", Inv: "INV-02"},
{File: "spinifex/runtime/daemon/daemon.go", Symbol: "JetStreamManager.InitTerminatedInstanceBucket", Inv: "INV-02"},
{File: "spinifex/runtime/daemon/daemon.go", Symbol: "JetStreamManager.WriteNodeMarkerBestEffort", Inv: "INV-03"},
{File: "spinifex/runtime/daemon/daemon.go", Symbol: "JetStreamManager.WriteRunningSet", Inv: "INV-03"},
{File: "spinifex/runtime/daemon/instance_record_repair.go", Symbol: "JetStreamManager.WriteRunningSet", Inv: "INV-03"},
{File: "spinifex/runtime/daemon/daemon.go", Symbol: "JetStreamManager.ListTerminatedInstances", Inv: "INV-04"},
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.LoadState", Inv: "INV-05"},
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.WriteStoppedInstance", Inv: "INV-05"},
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.LoadStoppedInstance", Inv: "INV-05"},
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.DeleteStoppedInstance", Inv: "INV-05"},
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.UpdateStoppedInstance", Inv: "INV-05"},
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.ClaimStoppedInstance", Inv: "INV-05"},
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.ListStoppedInstances", Inv: "INV-05"},
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.WriteTerminatedInstance", Inv: "INV-05"},
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.UpdateTerminatedInstance", Inv: "INV-05"},
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.ListTerminatedInstances", Inv: "INV-05"},
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.DeleteTerminatedInstance", Inv: "INV-05"},
{File: "spinifex/runtime/compute/vm/migrate.go", Symbol: "JetStreamManager.WriteStoppedInstance", Inv: "INV-06"},
{File: "spinifex/runtime/compute/vm/migrate.go", Symbol: "JetStreamManager.WriteTerminatedInstance", Inv: "INV-06"},
{File: "spinifex/runtime/compute/vm/shutdown.go", Symbol: "JetStreamManager.WriteTerminatedInstance", Inv: "INV-06"},
{File: "spinifex/runtime/compute/vm/orphan_qemu_reaper.go", Symbol: "JetStreamManager.ListTerminatedInstances", Inv: "INV-06"},
{File: "spinifex/runtime/compute/vm/teardown_reaper.go", Symbol: "JetStreamManager.ListTerminatedInstances", Inv: "INV-06"},
{File: "spinifex/runtime/compute/vm/teardown_reaper.go", Symbol: "JetStreamManager.UpdateTerminatedInstance", Inv: "INV-06"},
{File: "spinifex/runtime/compute/vm/teardown_reaper.go", Symbol: "JetStreamManager.DeleteTerminatedInstance", Inv: "INV-06"},
{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.LoadStoppedInstance", Inv: "INV-07"},
{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.UpdateStoppedInstance", Inv: "INV-07"},
{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.ListStoppedInstances", Inv: "INV-07"},
{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.ListTerminatedInstances", Inv: "INV-07"},
{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.WriteStoppedInstance", Inv: "INV-07"},
{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.ClaimStoppedInstance", Inv: "INV-07"},
{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.WriteTerminatedInstance", Inv: "INV-07"},
{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.DeleteStoppedInstance", Inv: "INV-07"},
{File: "spinifex/runtime/daemon/daemon_handlers_image.go", Symbol: "JetStreamManager.LoadStoppedInstance", Inv: "INV-08"},
{File: "spinifex/runtime/daemon/daemon_handlers_image.go", Symbol: "JetStreamManager.UpdateStoppedInstance", Inv: "INV-08"},
{File: "spinifex/runtime/daemon/instance_recovery.go", Symbol: "vm.InstanceRecord", Inv: "INV-10"},
{File: "spinifex/runtime/daemon/instance_recovery.go", Symbol: "JetStreamManager.ListInstanceRecords", Inv: "INV-10"},
{File: "spinifex/runtime/daemon/instance_recovery.go", Symbol: "JetStreamManager.ClaimRecoverableInstance", Inv: "INV-10"},
{File: "spinifex/runtime/daemon/instance_recovery.go", Symbol: "JetStreamManager.ReleaseRecoveredInstance", Inv: "INV-10"},
{File: "spinifex/runtime/daemon/instance_recovery.go", Symbol: "JetStreamManager.AbandonRecovery", Inv: "INV-10"},
{File: "spinifex/runtime/daemon/instance_recovery.go", Symbol: "JetStreamManager.LoadInstanceRecord", Inv: "INV-10"},
{File: "spinifex/runtime/daemon/dns_reconcile.go", Symbol: "daemon.InstanceRecordPrefix", Inv: "INV-11"},
{File: "spinifex/runtime/daemon/dns_reconcile.go", Symbol: "daemon.InstanceStateBucket", Inv: "INV-11"},
{File: "spinifex/runtime/daemon/dns_reconcile.go", Symbol: "JetStreamManager.ListInstanceRecords", Inv: "INV-11"},
{File: "spinifex/runtime/daemon/eni_orphan_reaper.go", Symbol: "vm.InstanceRecord", Inv: "INV-12"},
{File: "spinifex/runtime/daemon/eni_orphan_reaper.go", Symbol: "JetStreamManager.ListInstanceRecords", Inv: "INV-12"},
{File: "spinifex/runtime/daemon/eni_orphan_reaper.go", Symbol: "JetStreamManager.ListTerminatedInstanceRecords", Inv: "INV-12"},
{File: "spinifex/domains/ec2/instance/state/imds_readers.go", Symbol: "daemon.LocalState", Inv: "INV-13"},
{File: "spinifex/domains/ec2/instance/state/imds_readers.go", Symbol: "daemon.LocalStatePath", Inv: "INV-13"},
{File: "spinifex/domains/ec2/instance/state/imds_readers.go", Symbol: "daemon.ReadLocalState", Inv: "INV-13"},
{File: "spinifex/domains/ec2/instance/state/imds_readers.go", Symbol: "daemon.InstanceStateBucket", Inv: "INV-13"},
{File: "spinifex/domains/ec2/instance/state/imds_readers.go", Symbol: "JetStreamManager.InitKVBucket", Inv: "INV-13"},
{File: "spinifex/domains/ec2/guestmetadata/instance_lookup.go", Symbol: "vm.InstanceRecord", Inv: "INV-14"},
{File: "spinifex/domains/ec2/guestmetadata/instance_lookup.go", Symbol: "vm.VMFromRecord", Inv: "INV-14"},
{File: "spinifex/domains/ec2/guestmetadata/instance_lookup.go", Symbol: "JetStreamManager.LoadInstanceRecord", Inv: "INV-14"},
{File: "spinifex/domains/network/reconcile/intent.go", Symbol: `literal "spinifex-instance-state"`, Inv: "INV-15"},
{File: "spinifex/domains/network/reconcile/intent.go", Symbol: "vm.InstanceRecord", Inv: "INV-15"},
{File: "spinifex/runtime/roles/awsgw/awsgw.go", Symbol: "daemon.InstanceStateBucket", Inv: "INV-16"},
{File: "spinifex/runtime/roles/awsgw/awsgw.go", Symbol: "daemon.InstanceRecordPrefix", Inv: "INV-16"},
{File: "spinifex/runtime/roles/awsgw/awsgw.go", Symbol: "vm.InstanceRecord", Inv: "INV-16"},
{File: "spinifex/runtime/compute/cache/cache.go", Symbol: "vm.InstanceRecord", Inv: "INV-17"},
{File: "spinifex/runtime/compute/cache/cache.go", Symbol: "vm.VMFromRecord", Inv: "INV-17"},
{File: "spinifex/domains/admission/quota/records.go", Symbol: "vm.InstanceRecord", Inv: "INV-18"},
}
@@ -1,200 +0,0 @@
package instancestate
import (
"fmt"
"go/parser"
"go/token"
"os"
"path/filepath"
"testing"
)
type report struct {
unrecorded []use
stale []allowance
malformed []string
}
// check compares the scanned uses with allow.
func check(s scan, allow []allowance) report {
var rep report
allowed := map[use]bool{}
for _, a := range allow {
u := use{File: a.File, Symbol: a.Symbol}
switch {
case allowed[u]:
rep.malformed = append(rep.malformed, "duplicate entry "+u.String())
case a.Inv == "":
rep.malformed = append(rep.malformed, "entry "+u.String()+" cites no inventory item")
}
allowed[u] = true
}
for _, u := range s.sorted() {
if !allowed[u] {
rep.unrecorded = append(rep.unrecorded, u)
}
}
for _, a := range allow {
if _, ok := s[use{File: a.File, Symbol: a.Symbol}]; !ok {
rep.stale = append(rep.stale, a)
}
}
return rep
}
// TestADR0007_S6_NoNewRawInstanceStateConsumers is the blocking ratchet: no
// production file may start naming the daemon's instance-state bucket, keys,
// local file, raw record or JetStreamManager instance methods.
func TestADR0007_S6_NoNewRawInstanceStateConsumers(t *testing.T) {
root, err := moduleRoot()
if err != nil {
t.Fatal(err)
}
s, files, err := scanModule(root)
if err != nil {
t.Fatal(err)
}
// Guards against a walk that silently finds nothing and passes vacuously.
if files < 500 || len(s) < 20 {
t.Fatalf("scanned %d files and found %d uses under %s; the walk has lost the tree", files, len(s), root)
}
if len(allowlist) == 0 {
t.Fatal("the allowlist is empty, so the stale check would pass vacuously")
}
rep := check(s, allowlist)
t.Run("NoUnrecordedConsumer", func(t *testing.T) {
for _, u := range rep.unrecorded {
t.Errorf("ADR-0007 S6 (consumers receive narrow state capabilities rather than raw stores): %s at %s; "+
"consume an instance-domain capability instead of the daemon's raw instance state", u, s[u])
}
})
t.Run("Allowlist_NoStaleEntries", func(t *testing.T) {
for _, a := range rep.stale {
t.Errorf("allowlist entry %s uses %s (%s) no longer matches; delete it and update the inventory",
a.File, a.Symbol, a.Inv)
}
})
t.Run("Allowlist_WellFormed", func(t *testing.T) {
for _, m := range rep.malformed {
t.Errorf("allowlist_test.go: %s", m)
}
t.Logf("%d allow-listed uses", len(allowlist))
})
t.Run("OwnerFilesExist", func(t *testing.T) {
for _, f := range ownerFiles {
if _, err := os.Stat(filepath.Join(root, f)); err != nil {
t.Errorf("owner file %s is gone; remove it from ownerFiles and re-scan its consumers: %v", f, err)
}
}
})
}
const faultSource = `package %s
import (
"encoding/json"
spxd "github.com/mulgadc/spinifex/spinifex/runtime/daemon"
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
)
type holder struct {
LoadInstanceRecord func() // a field name, not a use
m *spxd.JetStreamManager
}
func (h holder) WriteRunningSet() {} // a declaration, not a use
func use(h holder, raw []byte, jsManager *spxd.JetStreamManager, other interface{ LoadState() }) {
_ = spxd.InstanceStateBucket
_ = spxd.LocalStatePath("")
var r vm.InstanceRecord
_ = json.Unmarshal(raw, &r)
_ = vm.VMFromRecord(&r)
_ = (&vm.VM{}).Record()
_, _ = h.m.LoadInstanceRecord("i-1")
_, _, _ = jsManager.LoadState("node")
other.LoadState()
_ = "spinifex-instance-state"
cluster, _ := spxd.NewJetStreamManager(nil)
_ = cluster.InitClusterStateBucket()
_ = cluster.WriteRunningSet("node", nil)
}
`
func parseFault(t *testing.T, rel, pkg string) scan {
t.Helper()
fset := token.NewFileSet()
f, err := parser.ParseFile(fset, rel, fmt.Sprintf(faultSource, pkg), parser.SkipObjectResolution)
if err != nil {
t.Fatal(err)
}
out := scan{}
scanFile(fset, rel, f, out)
return out
}
// TestADR0007_S6_RatchetDetectsFaults injects a new consumer, a stale entry
// and a malformed entry, and checks the matcher's declaration exclusions.
func TestADR0007_S6_RatchetDetectsFaults(t *testing.T) {
const rel = "spinifex/domains/widget/w.go"
s := parseFault(t, rel, "widget")
want := []string{
"daemon.InstanceStateBucket", "daemon.LocalStatePath",
"vm.InstanceRecord", "vm.VMFromRecord", "VM.Record()",
"JetStreamManager.LoadInstanceRecord", "JetStreamManager.LoadState",
"JetStreamManager.WriteRunningSet",
`literal "spinifex-instance-state"`,
}
for _, sym := range want {
if _, ok := s[use{File: rel, Symbol: sym}]; !ok {
t.Errorf("injected use of %s was not detected; got %v", sym, s.sorted())
}
}
if len(s) != len(want) {
t.Errorf("want exactly %d uses (the manager type, its constructor, cluster-state methods, field names, method declarations and an unrelated LoadState excluded), got %v",
len(want), s.sorted())
}
stale := allowance{File: "spinifex/domains/gone/g.go", Symbol: "vm.InstanceRecord", Inv: "INV-00"}
recorded := allowance{File: rel, Symbol: "vm.InstanceRecord", Inv: "INV-00"}
rep := check(s, []allowance{recorded, stale, {File: rel, Symbol: "vm.VMFromRecord"}, recorded})
if len(rep.unrecorded) != len(want)-2 {
t.Errorf("want %d unrecorded uses, got %v", len(want)-2, rep.unrecorded)
}
if len(rep.stale) != 1 || rep.stale[0] != stale {
t.Errorf("want stale entry %+v, got %+v", stale, rep.stale)
}
if len(rep.malformed) != 2 {
t.Errorf("want a missing-inventory and a duplicate entry reported, got %v", rep.malformed)
}
}
// Inside the owning packages the symbols are bare identifiers, and the scan
// must see those too.
func TestADR0007_S6_RatchetSeesBareIdentifiersInOwnerPackages(t *testing.T) {
src := map[string]string{
"spinifex/runtime/daemon/new_consumer.go": "package daemon\n\nfunc f() string { return InstanceStateBucket + LocalStatePath(\"\") }\n",
"spinifex/runtime/compute/vm/new_reader.go": "package vm\n\nfunc g(r *InstanceRecord) *VM { return VMFromRecord(r) }\n",
}
want := map[string][]string{
"spinifex/runtime/daemon/new_consumer.go": {"daemon.InstanceStateBucket", "daemon.LocalStatePath"},
"spinifex/runtime/compute/vm/new_reader.go": {"vm.InstanceRecord", "vm.VMFromRecord"},
}
for rel, code := range src {
fset := token.NewFileSet()
f, err := parser.ParseFile(fset, rel, code, parser.SkipObjectResolution)
if err != nil {
t.Fatal(err)
}
out := scan{}
scanFile(fset, rel, f, out)
for _, sym := range want[rel] {
if _, ok := out[use{File: rel, Symbol: sym}]; !ok {
t.Errorf("%s: bare use of %s not detected; got %v", rel, sym, out.sorted())
}
}
}
}
-302
View File
@@ -1,302 +0,0 @@
// Package instancestate freezes the production consumers of the daemon's raw
// instance-state persistence, so ADR-0007 S6 and S7 start from a known set
// that can only shrink.
package instancestate
import (
"errors"
"fmt"
"go/ast"
"go/build/constraint"
"go/parser"
"go/token"
"io/fs"
"os"
"path/filepath"
"runtime"
"sort"
"strconv"
"strings"
)
const (
daemonPath = "github.com/mulgadc/spinifex/spinifex/runtime/daemon"
vmPath = "github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
daemonDir = "spinifex/runtime/daemon"
vmDir = "spinifex/runtime/compute/vm"
)
// daemonSymbols are the daemon's instance-state bucket, key and local-file
// identifiers. The JetStreamManager type and constructor are not listed: they
// also serve cluster state, so only the instance-state methods below count.
var daemonSymbols = set(
"InstanceStateBucket", "InstanceStateBucketVersion",
"TerminatedInstanceBucket", "TerminatedInstanceBucketVersion",
"InstanceRecordPrefix", "InstanceStatePrefix", "NodePresencePrefix",
"StoppedInstancePrefix", "TerminatedInstancePrefix",
"LocalState", "LocalStateSchemaVersion", "LocalStatePath", "ReadLocalState",
"MarshalLocalState", "WriteLocalStateBytes", "LocalStateFileName",
"LocalStateFileMode", "DefaultLocalStateDir",
)
// vmSymbols are the raw persisted record and its conversion.
var vmSymbols = set("InstanceRecord", "InstanceSpec", "InstanceStatus", "VMFromRecord")
// jsmMethods are JetStreamManager's instance-state methods. Matched by name, so
// a call through an interface that mirrors them counts as a consumer too.
var jsmMethods = set(
"WriteInstanceRecord", "LoadInstanceRecord", "UpdateInstanceRecord",
"DeleteInstanceRecord", "ListInstanceRecords",
"WriteTerminatedInstanceRecord", "LoadTerminatedInstanceRecord",
"UpdateTerminatedInstanceRecord", "DeleteTerminatedInstanceRecord",
"ListTerminatedInstanceRecords",
"WriteNodeMarker", "WriteNodeMarkerBestEffort", "WriteRunningSet",
"WriteStoppedInstance", "LoadStoppedInstance", "DeleteStoppedInstance",
"ClaimStoppedInstance", "ClaimRecoverableInstance", "ReleaseRecoveredInstance",
"AbandonRecovery", "UpdateStoppedInstance", "ListStoppedInstances",
"WriteTerminatedInstance", "UpdateTerminatedInstance", "ListTerminatedInstances",
"DeleteTerminatedInstance", "LoadTerminatedInstance",
"InitKVBucket", "InitTerminatedInstanceBucket",
)
// ambiguousJSMMethods share a name with unrelated methods, so they count only
// when called on a field or variable named like a JetStreamManager handle.
var ambiguousJSMMethods = set("LoadState", "DeleteState")
// literals duplicate a bucket or file name instead of naming the owner's symbol.
var literals = set(`"spinifex-instance-state"`, `"spinifex-terminated-instances"`, `"instance-state.json"`)
// ownerFiles define the symbols above. They are the implementation being
// migrated, not consumers of it, and are listed so a deleted one is noticed.
var ownerFiles = []string{
"spinifex/runtime/daemon/instance_membership.go",
"spinifex/runtime/daemon/instance_records.go",
"spinifex/runtime/daemon/instance_records_migrate.go",
"spinifex/runtime/daemon/instance_running_set.go",
"spinifex/runtime/daemon/jetstream.go",
"spinifex/runtime/daemon/local_state.go",
"spinifex/runtime/compute/vm/record.go",
}
func set(names ...string) map[string]bool {
m := make(map[string]bool, len(names))
for _, n := range names {
m[n] = true
}
return m
}
// use is one consumer of one symbol, keyed by file so a new file is a new use.
type use struct {
File, Symbol string
}
func (u use) String() string { return u.File + " uses " + u.Symbol }
// scan maps each use to the first file:line it appears at.
type scan map[use]string
func moduleRoot() (string, error) {
_, file, _, ok := runtime.Caller(0)
if !ok {
return "", errors.New("cannot locate test source")
}
return filepath.Abs(filepath.Join(filepath.Dir(file), "..", ".."))
}
var excludedTopDirs = map[string]bool{"tests": true, "docs": true, "architecture": true}
// scanModule parses every non-test production Go file under root.
func scanModule(root string) (scan, int, error) {
out := scan{}
files := 0
fset := token.NewFileSet()
owners := set(ownerFiles...)
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
rel, err := filepath.Rel(root, path)
if err != nil {
return err
}
rel = filepath.ToSlash(rel)
if d.IsDir() {
if rel == "." {
return nil
}
name := d.Name()
if strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_") ||
name == "testdata" || name == "vendor" || name == "node_modules" || excludedTopDirs[rel] {
return filepath.SkipDir
}
if _, err := os.Stat(filepath.Join(path, "go.mod")); err == nil {
return filepath.SkipDir
}
return nil
}
if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
return nil
}
src, err := os.ReadFile(path)
if err != nil {
return err
}
f, err := parser.ParseFile(fset, path, src, parser.ParseComments|parser.SkipObjectResolution)
if err != nil {
return err
}
if ignoredByBuildTag(f.Comments, f.Package) {
return nil
}
files++
if owners[rel] {
return nil
}
scanFile(fset, rel, f, out)
return nil
})
return out, files, err
}
// scanFile records every use in f, which lives at module-relative path rel.
func scanFile(fset *token.FileSet, rel string, f *ast.File, out scan) {
dir := filepath.ToSlash(filepath.Dir(rel))
daemonName, vmName := importName(f, daemonPath, "daemon"), importName(f, vmPath, "vm")
inDaemon, inVM := dir == daemonDir, dir == vmDir
seesVM := inDaemon || inVM || daemonName != "" || vmName != ""
record := func(symbol string, pos token.Pos) {
u := use{File: rel, Symbol: symbol}
if _, ok := out[u]; !ok {
out[u] = fmt.Sprintf("%s:%d", rel, fset.Position(pos).Line)
}
}
var visit func(n ast.Node) bool
visit = func(n ast.Node) bool {
switch n := n.(type) {
case *ast.ImportSpec:
return false
case *ast.BasicLit:
if n.Kind == token.STRING && literals[n.Value] {
record("literal "+n.Value, n.Pos())
}
case *ast.CallExpr:
if sel, ok := n.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "Record" && len(n.Args) == 0 && seesVM {
record("VM.Record()", sel.Sel.Pos())
}
case *ast.SelectorExpr:
if x, ok := n.X.(*ast.Ident); ok {
switch {
case daemonName != "" && x.Name == daemonName && daemonSymbols[n.Sel.Name]:
record("daemon."+n.Sel.Name, n.Sel.Pos())
case vmName != "" && x.Name == vmName && vmSymbols[n.Sel.Name]:
record("vm."+n.Sel.Name, n.Sel.Pos())
}
}
if jsmMethods[n.Sel.Name] || (ambiguousJSMMethods[n.Sel.Name] && jsmReceivers[receiverName(n.X)]) {
record("JetStreamManager."+n.Sel.Name, n.Sel.Pos())
}
// The selected name is a field or method, never a bare package symbol.
ast.Inspect(n.X, visit)
return false
case *ast.FuncDecl:
// A method declared with one of these names is an implementation,
// not a use; its body is still scanned.
if n.Recv != nil {
ast.Inspect(n.Recv, visit)
}
if n.Body != nil {
ast.Inspect(n.Body, visit)
}
if n.Type != nil {
ast.Inspect(n.Type, visit)
}
return false
case *ast.Field:
// Interface method and struct field names are declarations.
if n.Type != nil {
ast.Inspect(n.Type, visit)
}
if n.Tag != nil {
ast.Inspect(n.Tag, visit)
}
return false
case *ast.Ident:
if inDaemon && daemonSymbols[n.Name] {
record("daemon."+n.Name, n.Pos())
}
if inVM && vmSymbols[n.Name] {
record("vm."+n.Name, n.Pos())
}
}
return true
}
ast.Inspect(f, visit)
}
var jsmReceivers = set("jsManager", "jsm", "js")
// receiverName is the last name in x: "js" for both js and a.js.
func receiverName(x ast.Expr) string {
switch x := x.(type) {
case *ast.Ident:
return x.Name
case *ast.SelectorExpr:
return x.Sel.Name
}
return ""
}
// importName returns the name f refers to path by, or "" if f does not import it.
func importName(f *ast.File, path, def string) string {
for _, imp := range f.Imports {
p, err := strconv.Unquote(imp.Path.Value)
if err != nil || p != path {
continue
}
if imp.Name != nil {
if imp.Name.Name == "_" || imp.Name.Name == "." {
return ""
}
return imp.Name.Name
}
return def
}
return ""
}
// ignoredByBuildTag reports a `//go:build ignore` file, which is a standalone
// program rather than part of its directory's package.
func ignoredByBuildTag(groups []*ast.CommentGroup, pkgPos token.Pos) bool {
for _, cg := range groups {
if cg.Pos() >= pkgPos {
break
}
for _, c := range cg.List {
if !constraint.IsGoBuild(c.Text) {
continue
}
expr, err := constraint.Parse(c.Text)
if err == nil && expr.String() == "ignore" {
return true
}
}
}
return false
}
func (s scan) sorted() []use {
out := make([]use, 0, len(s))
for u := range s {
out = append(out, u)
}
sort.Slice(out, func(i, j int) bool {
if out[i].File != out[j].File {
return out[i].File < out[j].File
}
return out[i].Symbol < out[j].Symbol
})
return out
}
-136
View File
@@ -1,136 +0,0 @@
package layering
// allowance is one recorded forbidden import, allowed until its debt clears.
type allowance struct {
From, To string
Debt string // subject of the migration record's debt entry or close-out row
Reason string
}
// allowlist mirrors the dependency-rule debt and close-out table in
// docs/PACKAGE_BOUNDARY_MIGRATION.md. It only shrinks: add an edge only with
// recorded debt, and delete an entry as soon as its import is gone.
var allowlist = []allowance{
{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/eip", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/igw", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/image", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/instance", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/routetable", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/snapshot", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/vpc", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
{From: "agents/ecs/agent", To: "domains/ecs", Debt: "agents/ecs/agent", Reason: "assignment/state wire types from the ECS implementation; needs a guest/controller contract"},
{From: "agents/eks/tokenwebhook", To: "domains/eks/cluster", Debt: "agents/eks/tokenwebhook", Reason: "WebhookTokenReviewResult from the EKS implementation; needs an EKS contract or projection"},
{From: "agents/rds/agent", To: "domains/rds/instance", Debt: "agents/rds/agent (dependency, narrowed)", Reason: "command protocol, wire and record types; needs a guest/controller wire contract"},
{From: "cmd/aws-model-coverage", To: "domains/acm/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
{From: "cmd/aws-model-coverage", To: "domains/ecr/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
{From: "cmd/aws-model-coverage", To: "domains/route53/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
{From: "cmd/aws-model-coverage", To: "domains/elbv2/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
{From: "cmd/aws-model-coverage", To: "domains/rds/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
{From: "cmd/aws-model-coverage", To: "domains/ec2/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
{From: "cmd/aws-model-coverage", To: "domains/ecs/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
{From: "cmd/aws-model-coverage", To: "domains/eks/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
{From: "cmd/aws-model-coverage", To: "domains/iam/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
{From: "cmd/aws-model-coverage", To: "domains/sqs/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
{From: "cmd/aws-model-coverage", To: "domains/sts/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
{From: "runtime/daemon", To: "domains/ec2/awsapi/instance", Debt: "Instance adapter consumers outside EC2", Reason: "runtime/daemon/eks_cp_control.go and rds_deps.go call the instance adapter; needs an EC2 capability"},
{From: "domains/acm", To: "domains/iam", Debt: "ACM: the IAM AES-GCM helper import (ACM relocation residual)", Reason: "IAM AES-GCM helpers; needs a shared crypto boundary"},
{From: "domains/admission/quota", To: "domains/ec2/awsapi/eip", Debt: "domains/admission/quota (dependency, moved from handlers/quota unchanged)", Reason: "describe adapter used as a counter; needs a reader capability"},
{From: "domains/admission/quota", To: "domains/ec2/awsapi/instance", Debt: "domains/admission/quota (dependency, moved from handlers/quota unchanged)", Reason: "vcpu.go counts instances through the instance adapter; needs an EC2 reader capability"},
{From: "domains/admission/quota", To: "domains/ec2/awsapi/volume", Debt: "domains/admission/quota (dependency, moved from handlers/quota unchanged)", Reason: "describe adapter used as a counter; needs a reader capability"},
{From: "domains/admission/quota", To: "domains/ec2/awsapi/vpc", Debt: "domains/admission/quota (dependency, moved from handlers/quota unchanged)", Reason: "describe adapter used as a counter; needs a reader capability"},
{From: "domains/admission/quota", To: "domains/ec2/instancetypes", Debt: "domains/admission/quota → domains/ec2/instancetypes", Reason: "DefaultVCPUs sizing; needs a consumer-owned capability or EC2 projection"},
{From: "domains/admission/quota", To: "domains/elbv2/awsapi", Debt: "domains/admission/quota (dependency, moved from handlers/quota unchanged)", Reason: "live ELBv2 counter; needs a reader capability"},
{From: "domains/admission/quota", To: "domains/rds/instance", Debt: "domains/admission/quota (dependency, moved from handlers/quota unchanged)", Reason: "live RDS counter; needs a reader capability"},
{From: "domains/admission/quota", To: "runtime/compute/vm", Debt: "domains/admission/quota (dependency, moved from handlers/quota unchanged)", Reason: "instance-record vCPU sweep; needs a reader capability"},
{From: "domains/dns", To: "domains/network/reconcile", Debt: "domains/dns", Reason: "takes the reconcile lock from network's implementation"},
{From: "domains/dns", To: "runtime/compute/vm", Debt: "domains/dns", Reason: "domain-to-runtime import of VM state"},
{From: "domains/ec2/awsapi/instance", To: "domains/iam", Debt: "domains/ec2/awsapi/instance (dependency)", Reason: "IAM edge needs an IAM capability"},
{From: "domains/ec2/awsapi/instance", To: "runtime/compute/cache", Debt: "domains/ec2/awsapi/instance (dependency)", Reason: "node liveness and VM state; needs an EC2-owned projection"},
{From: "domains/ec2/awsapi/instance", To: "runtime/compute/vm", Debt: "domains/ec2/awsapi/instance (dependency)", Reason: "node liveness and VM state; needs an EC2-owned projection"},
{From: "domains/ec2/eip", To: "domains/network/topology", Debt: "EC2 → network implementation", Reason: "EC2 reads network implementation; clears with the EC2-owned projection"},
{From: "domains/ec2/guestmetadata", To: "domains/dns", Debt: "Guest metadata: the cross-domain and runtime imports above", Reason: "guest-metadata relocation residual; needs capabilities, projections or contracts"},
{From: "domains/ec2/guestmetadata", To: "domains/iam", Debt: "Guest metadata: the cross-domain and runtime imports above", Reason: "guest-metadata relocation residual; needs capabilities, projections or contracts"},
{From: "domains/ec2/guestmetadata", To: "domains/sts", Debt: "Guest metadata: the cross-domain and runtime imports above", Reason: "guest-metadata relocation residual; needs capabilities, projections or contracts"},
{From: "domains/ec2/guestmetadata", To: "runtime/compute/vm", Debt: "Guest metadata: the cross-domain and runtime imports above", Reason: "guest-metadata relocation residual; needs capabilities, projections or contracts"},
{From: "domains/ec2/instance", To: "domains/dns", Debt: "domains/ec2/instance", Reason: "cross-domain implementation import of DNS"},
{From: "domains/ec2/instance", To: "domains/network/topology", Debt: "EC2 → network implementation", Reason: "EC2 reads network implementation; clears with the EC2-owned projection"},
{From: "domains/ec2/instance", To: "runtime/compute/gpu", Debt: "domains/ec2/instance", Reason: "VM/GPU runtime state read directly; needs a domain-owned projection"},
{From: "domains/ec2/instance", To: "runtime/compute/vm", Debt: "domains/ec2/instance", Reason: "VM/GPU runtime state read directly; needs a domain-owned projection"},
{From: "domains/ec2/instance/state", To: "runtime/daemon", Debt: "domains/ec2/instance/state (dependency, IMDS readers)", Reason: "wraps the daemon's local-file reader and JetStreamManager bucket open unchanged for vpcd's IMDS lookup"},
{From: "domains/ec2/instance/state", To: "runtime/compute/vm", Debt: "domains/ec2/instance/state (dependency)", Reason: "the vpcd IMDS local reader returns vm.VM"},
{From: "domains/ec2/volume", To: "runtime/compute/vm", Debt: "Domain → runtime", Reason: "VM runtime state read directly; needs a domain-owned projection"},
{From: "domains/ec2/vpc", To: "domains/network/external", Debt: "EC2 → network implementation", Reason: "EC2 reads network implementation; clears with the EC2-owned projection"},
{From: "domains/ec2/vpc", To: "domains/network/external/dhcp", Debt: "EC2 → network implementation", Reason: "EC2 reads network implementation; clears with the EC2-owned projection"},
{From: "domains/ec2/vpc", To: "domains/network/identifiers", Debt: "EC2 → network implementation", Reason: "EC2 reads network implementation; clears with the EC2-owned projection"},
{From: "domains/ec2/vpc", To: "domains/network/topology", Debt: "EC2 → network implementation", Reason: "EC2 reads network implementation; clears with the EC2-owned projection"},
{From: "domains/ecr/auth", To: "domains/iam", Debt: "Cross-domain", Reason: "IAM implementation import; needs an IAM capability"},
{From: "domains/network/host", To: "runtime/compute/vm", Debt: "domains/network", Reason: "domain-to-runtime import"},
{From: "domains/network/host", To: "runtime/host/command", Debt: "domains/network", Reason: "domain-to-runtime import"},
{From: "domains/network/reconcile", To: "domains/ec2/eip", Debt: "domains/network", Reason: "reconcile reads EC2 private records; clears with the EC2-owned projection"},
{From: "domains/network/reconcile", To: "domains/ec2/igw", Debt: "domains/network", Reason: "reconcile reads EC2 private records; clears with the EC2-owned projection"},
{From: "domains/network/reconcile", To: "domains/ec2/natgw", Debt: "domains/network", Reason: "reconcile reads EC2 private records; clears with the EC2-owned projection"},
{From: "domains/network/reconcile", To: "domains/ec2/routetable", Debt: "domains/network", Reason: "reconcile reads EC2 private records; clears with the EC2-owned projection"},
{From: "domains/network/reconcile", To: "domains/ec2/vpc", Debt: "domains/network", Reason: "reconcile reads EC2 private records; clears with the EC2-owned projection"},
{From: "domains/network/reconcile", To: "runtime/compute/vm", Debt: "domains/network", Reason: "domain-to-runtime import"},
{From: "domains/ochre", To: "domains/ec2/systeminstance", Debt: "Cross-domain", Reason: "consumes another domain's implementation rather than a capability"},
{From: "domains/ochre", To: "domains/iam", Debt: "IAM implementation consumers in STS, Ochre and ELBv2", Reason: "secret encrypt/decrypt helpers; needs a shared crypto boundary"},
{From: "domains/ochre", To: "domains/network/systemvpc", Debt: "Cross-domain", Reason: "consumes another domain's implementation rather than a capability"},
{From: "domains/ochre", To: "runtime/compute/gpu", Debt: "Domain → runtime", Reason: "GPU runtime state read directly; needs a domain-owned projection"},
{From: "domains/ochre/vector", To: "domains/iam", Debt: "Cross-domain", Reason: "consumes another domain's implementation rather than a capability"},
{From: "domains/rds/awsapi", To: "domains/ec2/awsapi/instance", Debt: "Instance adapter consumers outside EC2", Reason: "enginecatalog.go calls the instance adapter; needs an EC2 capability"},
{From: "domains/sts/awsapi", To: "domains/iam", Debt: "IAM implementation consumers in STS, Ochre and ELBv2", Reason: "GetCallerIdentity takes IAMService; needs an IAM capability"},
{From: "domains/ecs", To: "domains/ec2/ebs/metadata", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
{From: "domains/ecs", To: "domains/ec2/eip", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
{From: "domains/ecs", To: "domains/ec2/instancetypes", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
{From: "domains/ecs", To: "domains/elbv2", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
{From: "domains/ecs", To: "domains/iam", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
{From: "domains/eks/cluster", To: "domains/dns", Debt: "close-out: handlers/eks (ADR-0004 blocked-dependency list; inventory-eks boundary imports)", Reason: "legacy EKS calls another domain's or runtime implementation directly"},
{From: "domains/eks/cluster", To: "domains/ec2/ebs/metadata", Debt: "close-out: handlers/eks (ADR-0004 blocked-dependency list; inventory-eks boundary imports)", Reason: "legacy EKS calls another domain's or runtime implementation directly"},
{From: "domains/eks/cluster", To: "domains/ec2/instancetypes", Debt: "close-out: handlers/eks (ADR-0004 blocked-dependency list; inventory-eks boundary imports)", Reason: "legacy EKS calls another domain's or runtime implementation directly"},
{From: "domains/eks/cluster", To: "domains/ec2/placementgroup", Debt: "close-out: handlers/eks (ADR-0004 blocked-dependency list; inventory-eks boundary imports)", Reason: "legacy EKS calls another domain's or runtime implementation directly"},
{From: "domains/eks/cluster", To: "domains/ec2/systeminstance", Debt: "close-out: handlers/eks (ADR-0004 blocked-dependency list; inventory-eks boundary imports)", Reason: "legacy EKS calls another domain's or runtime implementation directly"},
{From: "domains/eks/cluster", To: "domains/network/systemvpc", Debt: "close-out: handlers/eks (ADR-0004 blocked-dependency list; inventory-eks boundary imports)", Reason: "legacy EKS calls another domain's or runtime implementation directly"},
{From: "domains/eks/cluster", To: "domains/iam", Debt: "close-out: handlers/eks (ADR-0004 blocked-dependency list; inventory-eks boundary imports)", Reason: "legacy EKS calls another domain's or runtime implementation directly"},
{From: "domains/eks/cluster", To: "runtime/compute/vm", Debt: "close-out: handlers/eks (ADR-0004 blocked-dependency list; inventory-eks boundary imports)", Reason: "legacy EKS calls another domain's or runtime implementation directly"},
{From: "domains/elbv2", To: "domains/acm", Debt: "domains/elbv2 (cross-domain)", Reason: "ELBv2 cross-domain import; clears when ELBv2 is extracted behind consumer-owned capabilities"},
{From: "domains/elbv2", To: "domains/dns", Debt: "domains/elbv2 (cross-domain)", Reason: "ELBv2 cross-domain import; clears when ELBv2 is extracted behind consumer-owned capabilities"},
{From: "domains/elbv2", To: "domains/ec2/systeminstance", Debt: "domains/elbv2 (cross-domain)", Reason: "ELBv2 cross-domain import; clears when ELBv2 is extracted behind consumer-owned capabilities"},
{From: "domains/elbv2", To: "domains/ec2/vpc", Debt: "domains/elbv2 (cross-domain)", Reason: "ELBv2 cross-domain import; clears when ELBv2 is extracted behind consumer-owned capabilities"},
{From: "domains/elbv2", To: "domains/network/topology", Debt: "domains/elbv2 (cross-domain)", Reason: "ELBv2 cross-domain import; clears when ELBv2 is extracted behind consumer-owned capabilities"},
{From: "domains/elbv2", To: "domains/iam", Debt: "IAM implementation consumers in STS, Ochre and ELBv2", Reason: "system-instance role ensurer; needs an IAM capability"},
{From: "domains/elbv2", To: "agents/elbv2", Debt: "domains/elbv2/{agent_types,haproxy,health_checker,nginx}.go import lbagent", Reason: "health-report and cert/PID-path vocabulary; needs a guest/controller contract"},
{From: "domains/rds/instance", To: "domains/dns", Debt: "close-out: handlers/rds (ADR-0004 blocked-dependency list; inventory-rds boundary imports)", Reason: "legacy RDS calls another domain's or runtime implementation directly"},
{From: "domains/rds/instance", To: "domains/ec2/instance", Debt: "close-out: handlers/rds (ADR-0004 blocked-dependency list; inventory-rds boundary imports)", Reason: "legacy RDS calls another domain's or runtime implementation directly"},
{From: "domains/rds/instance", To: "domains/ec2/instancetypes", Debt: "close-out: handlers/rds (ADR-0004 blocked-dependency list; inventory-rds boundary imports)", Reason: "legacy RDS calls another domain's or runtime implementation directly"},
{From: "domains/rds/instance", To: "domains/ec2/systeminstance", Debt: "close-out: handlers/rds (ADR-0004 blocked-dependency list; inventory-rds boundary imports)", Reason: "legacy RDS calls another domain's or runtime implementation directly"},
{From: "domains/rds/instance", To: "domains/network/systemvpc", Debt: "close-out: handlers/rds (ADR-0004 blocked-dependency list; inventory-rds boundary imports)", Reason: "legacy RDS calls another domain's or runtime implementation directly"},
{From: "domains/rds/instance", To: "domains/iam", Debt: "close-out: handlers/rds (ADR-0004 blocked-dependency list; inventory-rds boundary imports)", Reason: "legacy RDS calls another domain's or runtime implementation directly"},
{From: "domains/rds/instance", To: "runtime/compute/vm", Debt: "close-out: handlers/rds (ADR-0004 blocked-dependency list; inventory-rds boundary imports)", Reason: "legacy RDS calls another domain's or runtime implementation directly"},
{From: "domains/sts", To: "domains/iam", Debt: "IAM implementation consumers in STS, Ochre and ELBv2", Reason: "STS reads IAM service, trust-policy and OIDC-provider state; needs an IAM capability"},
}
-163
View File
@@ -1,163 +0,0 @@
// Package layering enforces ADR-0001's package-boundary dependency rules over
// the module's production import graph, ratcheted by a recorded-debt allowlist.
package layering
import (
"errors"
"fmt"
"go/ast"
"go/build/constraint"
"go/parser"
"go/token"
"io/fs"
"os"
"path/filepath"
"runtime"
"sort"
"strconv"
"strings"
)
const modulePath = "github.com/mulgadc/spinifex"
// edge is one package-to-package import, both sides named as in the allowlist:
// application packages relative to spinifex/, module-root ones (cmd/,
// contracts/, internal/) relative to the module root.
type edge struct {
From, To string
}
func (e edge) String() string { return e.From + " -> " + e.To }
// graph is the production import graph restricted to this module.
type graph struct {
// pkgs maps a package name to its module-relative directory.
pkgs map[string]string
// imports records the first file:line at which each edge appears.
imports map[edge]string
}
func newGraph() *graph {
return &graph{pkgs: map[string]string{}, imports: map[edge]string{}}
}
// shortName drops the application-root prefix so names read like the ADR and
// the migration record; classify rejects app-root dirs that would collide.
func shortName(full string) string {
if rest, ok := strings.CutPrefix(full, "spinifex/"); ok {
return rest
}
return full
}
func moduleRoot() (string, error) {
_, file, _, ok := runtime.Caller(0)
if !ok {
return "", errors.New("cannot locate test source")
}
// architecture/layering/graph_test.go -> module root.
return filepath.Abs(filepath.Join(filepath.Dir(file), "..", ".."))
}
// excludedTopDirs hold no production code: tests/ is test-only, docs/ only
// embeds documentation and architecture/ is repository governance tooling.
var excludedTopDirs = map[string]bool{"tests": true, "docs": true, "architecture": true}
// loadGraph parses the imports of every non-test Go file under root, so it
// sees every build-tag variant and needs no Go toolchain or workspace.
func loadGraph(root string) (*graph, error) {
g := newGraph()
fset := token.NewFileSet()
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
rel, err := filepath.Rel(root, path)
if err != nil {
return err
}
rel = filepath.ToSlash(rel)
if d.IsDir() {
if rel == "." {
return nil
}
name := d.Name()
if strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_") ||
name == "testdata" || name == "vendor" || name == "node_modules" || excludedTopDirs[rel] {
return filepath.SkipDir
}
if _, err := os.Stat(filepath.Join(path, "go.mod")); err == nil {
return filepath.SkipDir
}
return nil
}
if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
return nil
}
f, err := parser.ParseFile(fset, path, nil, parser.ImportsOnly|parser.ParseComments)
if err != nil {
return err
}
if ignoredByBuildTag(f.Comments, f.Package) {
return nil
}
dir := filepath.ToSlash(filepath.Dir(rel))
if dir == "." {
dir = ""
}
from := shortName(dir)
g.pkgs[from] = dir
for _, imp := range f.Imports {
p, err := strconv.Unquote(imp.Path.Value)
if err != nil {
return err
}
target, ok := strings.CutPrefix(p, modulePath+"/")
if !ok {
continue
}
e := edge{From: from, To: shortName(target)}
if _, seen := g.imports[e]; !seen {
pos := fset.Position(imp.Pos())
g.imports[e] = fmt.Sprintf("%s:%d", rel, pos.Line)
}
}
return nil
})
return g, err
}
// ignoredByBuildTag reports a `//go:build ignore` file, which is a standalone
// program (a generator) rather than part of its directory's package.
func ignoredByBuildTag(groups []*ast.CommentGroup, pkgPos token.Pos) bool {
for _, cg := range groups {
if cg.Pos() >= pkgPos {
break
}
for _, c := range cg.List {
if !constraint.IsGoBuild(c.Text) {
continue
}
expr, err := constraint.Parse(c.Text)
if err == nil && expr.String() == "ignore" {
return true
}
}
}
return false
}
// sortedEdges returns the graph's edges in a stable order.
func (g *graph) sortedEdges() []edge {
out := make([]edge, 0, len(g.imports))
for e := range g.imports {
out = append(out, e)
}
sort.Slice(out, func(i, j int) bool {
if out[i].From != out[j].From {
return out[i].From < out[j].From
}
return out[i].To < out[j].To
})
return out
}
-238
View File
@@ -1,238 +0,0 @@
package layering
import (
"fmt"
"sort"
"strings"
"testing"
)
type violation struct {
rule string
edge edge
pos string
}
type report struct {
unmapped []string
violations []violation // every rule hit, allowlisted or not
unrecorded []violation // hits whose edge has no allowlist entry
stale []allowance // entries whose edge no rule rejects any more
malformed []string
}
// check classifies g, applies every rule and compares the result with allow.
func check(g *graph, allow []allowance) report {
var rep report
classes := map[string]class{}
for name, dir := range g.pkgs {
c, ok := classify(dir)
if !ok || c.layer == layerTests {
rep.unmapped = append(rep.unmapped, fmt.Sprintf("%s (directory %s)", name, dir))
continue
}
classes[name] = c
}
allowed := map[edge]bool{}
for _, a := range allow {
e := edge{From: a.From, To: a.To}
switch {
case allowed[e]:
rep.malformed = append(rep.malformed, "duplicate entry "+e.String())
case a.Debt == "" || a.Reason == "":
rep.malformed = append(rep.malformed, "entry "+e.String()+" needs both Debt and Reason")
}
allowed[e] = true
}
hit := map[edge]bool{}
for _, e := range g.sortedEdges() {
from, ok := classes[e.From]
if !ok {
continue
}
to, ok := classify(fullPath(e.To))
if !ok {
rep.unmapped = append(rep.unmapped, fmt.Sprintf("%s (imported by %s at %s)", e.To, e.From, g.imports[e]))
continue
}
for _, r := range rules {
if !r.subject(from, e.From) || !r.forbids(from, to, e.From) {
continue
}
v := violation{rule: r.id, edge: e, pos: g.imports[e]}
rep.violations = append(rep.violations, v)
hit[e] = true
if !allowed[e] {
rep.unrecorded = append(rep.unrecorded, v)
}
}
}
for _, a := range allow {
if !hit[edge{From: a.From, To: a.To}] {
rep.stale = append(rep.stale, a)
}
}
sort.Strings(rep.unmapped)
return rep
}
func ruleByID(id string) rule {
for _, r := range rules {
if r.id == id {
return r
}
}
panic("unknown rule " + id)
}
// TestADR0001_Layering is the blocking ratchet over the real import graph.
func TestADR0001_Layering(t *testing.T) {
root, err := moduleRoot()
if err != nil {
t.Fatal(err)
}
g, err := loadGraph(root)
if err != nil {
t.Fatal(err)
}
// Guards against a walk that silently finds nothing and passes vacuously.
if len(g.pkgs) < 100 || len(g.imports) < 500 {
t.Fatalf("scanned %d packages and %d module imports under %s; the walk has lost the tree and every rule would pass vacuously",
len(g.pkgs), len(g.imports), root)
}
rep := check(g, allowlist)
t.Run("Classification_EveryPackageMapped", func(t *testing.T) {
for _, u := range rep.unmapped {
t.Errorf("ADR-0001 current-to-target map: package %s has no layer; place it under an ADR-0001 home or classify it deliberately in layers_test.go", u)
}
})
for _, r := range rules {
t.Run(r.id, func(t *testing.T) {
subjects := 0
for name, dir := range g.pkgs {
if c, ok := classify(dir); ok && r.subject(c, name) {
subjects++
}
}
if subjects == 0 {
t.Fatalf("no scanned package is governed by %s; the rule would pass vacuously", r.id)
}
var snippets []string
for _, v := range rep.unrecorded {
if v.rule != r.id {
continue
}
t.Errorf("ADR-0001 %s: %s imports %s (%s); %s",
r.clause, v.edge.From, v.edge.To, v.pos, r.hint)
snippets = append(snippets, fmt.Sprintf("{From: %q, To: %q, Debt: \"<migration-record debt subject>\", Reason: \"<why>\"},", v.edge.From, v.edge.To))
}
if len(snippets) > 0 {
t.Logf("fix the import; only debt already recorded in docs/PACKAGE_BOUNDARY_MIGRATION.md may be allow-listed:\n%s",
strings.Join(snippets, "\n"))
}
})
}
t.Run("Allowlist_NoStaleEntries", func(t *testing.T) {
for _, a := range rep.stale {
t.Errorf("allowlist entry %s -> %s (%s) no longer matches a forbidden import; the debt is paid, so delete the entry from allowlist_test.go",
a.From, a.To, a.Debt)
}
})
t.Run("Allowlist_WellFormed", func(t *testing.T) {
for _, m := range rep.malformed {
t.Errorf("allowlist_test.go: %s", m)
}
t.Logf("%d allow-listed edges", len(allowlist))
})
}
// TestADR0001_LayeringCheckerDetectsFaults injects a forbidden edge, a stale
// allowlist entry and an unmapped package into a synthetic graph.
func TestADR0001_LayeringCheckerDetectsFaults(t *testing.T) {
g := newGraph()
for _, dir := range []string{
"spinifex/foundation/widget", "spinifex/domains/alpha/thing",
"spinifex/domains/beta/store", "spinifex/misc",
} {
g.pkgs[shortName(dir)] = dir
}
forbidden := edge{From: "foundation/widget", To: "domains/alpha/thing"}
g.imports[forbidden] = "spinifex/foundation/widget/w.go:7"
allowedEdge := edge{From: "domains/alpha/thing", To: "domains/beta/store"}
g.imports[allowedEdge] = "spinifex/domains/alpha/thing/t.go:9"
staleEntry := allowance{From: "domains/beta/store", To: "domains/alpha/thing", Debt: "synthetic", Reason: "stale"}
rep := check(g, []allowance{
{From: allowedEdge.From, To: allowedEdge.To, Debt: "synthetic", Reason: "recorded"},
staleEntry,
})
if len(rep.unrecorded) != 1 || rep.unrecorded[0].edge != forbidden ||
rep.unrecorded[0].rule != ruleByID("R1_FoundationImportsOnlyFoundation").id {
t.Errorf("want only %s reported under R1, got %+v", forbidden, rep.unrecorded)
}
if len(rep.stale) != 1 || rep.stale[0] != staleEntry {
t.Errorf("want stale entry %+v, got %+v", staleEntry, rep.stale)
}
if len(rep.unmapped) != 1 || !strings.HasPrefix(rep.unmapped[0], "misc ") {
t.Errorf("want unmapped package misc, got %v", rep.unmapped)
}
var allowedHit bool
for _, v := range rep.violations {
allowedHit = allowedHit || v.edge == allowedEdge
}
if !allowedHit {
t.Errorf("allow-listed cross-domain edge %s was not detected by R3, so the allowlist entry would read as stale", allowedEdge)
}
}
// TestADR0001_R2_IngressRejectsRuntimeOperatorAndDomains: the request pipeline
// lives under ingress/aws, so it must not import the daemon, any runtime or
// operator package, or a domain implementation.
func TestADR0001_R2_IngressRejectsRuntimeOperatorAndDomains(t *testing.T) {
r := ruleByID("R2_IngressImportsNoImplementation")
from := "spinifex/ingress/aws/pipeline"
f, ok := classify(from)
if !ok || !r.subject(f, from) {
t.Fatalf("%s is not governed by R2", from)
}
for _, to := range []string{
"spinifex/runtime/daemon", "spinifex/runtime/host/process", "spinifex/runtime/roles/awsgw",
"spinifex/operator/admin", "spinifex/domains/ecs", "spinifex/domains/iam/awsapi",
} {
target, _ := classify(to)
if !r.forbids(f, target, from) {
t.Errorf("%s -> %s is not rejected by R2", from, to)
}
}
}
// TestADR0002_S7_RequestRecordRejectsOutsiders: the request record is readable
// by packages under ingress/aws only.
func TestADR0002_S7_RequestRecordRejectsOutsiders(t *testing.T) {
record, ok := classify("spinifex/ingress/aws/reqctx")
if !ok || !record.requestRecord {
t.Fatalf("ingress/aws/reqctx is not classified as the request record: %+v", record)
}
r := ruleByID("S7_RequestRecordPrivateToIngress")
for name, wantRejected := range map[string]bool{
"domains/ecs/awsapi": true,
"runtime/roles/awsgw": true,
"operator/admin": true,
"ingress/aws/dispatch": false,
} {
from, ok := classify(fullPath(name))
if !ok {
t.Fatalf("%s is unclassified", name)
}
if got := r.forbids(from, record, name); got != wantRejected {
t.Errorf("%s -> ingress/aws/reqctx rejected=%v, want %v", name, got, wantRejected)
}
}
}
-147
View File
@@ -1,147 +0,0 @@
package layering
import (
"slices"
"strings"
)
type layer string
const (
layerFoundation layer = "foundation"
layerContracts layer = "contracts"
layerDomain layer = "domains"
layerIngress layer = "ingress"
layerRuntime layer = "runtime"
layerOperator layer = "operator"
layerAgents layer = "agents"
layerProviders layer = "providers"
layerBootstrap layer = "bootstrap"
layerCmd layer = "cmd"
layerInternal layer = "internal"
layerTestkit layer = "testkit"
layerLegacy layer = "legacy"
// layerTests is valid only as an import target; tests/ is never scanned.
layerTests layer = "tests"
)
// domainKind distinguishes the parts of a domain the rules treat differently.
type domainKind int
const (
notDomain domainKind = iota
domainImpl // domains/<d>/... resource implementation
domainAWSAPI // domains/<d>/awsapi/... AWS action adapters
domainProj // domains/<d>/projection/... authorised view
legacyImpl // handlers/<d>/... legacy domain implementation
)
type class struct {
layer layer
legacy string // legacy root name when layer is layerLegacy
domain string // owning domain for domain-shaped and agent packages
kind domainKind
daemon bool // runtime/daemon, the interim daemon -> gateway -> handlers root
// requestRecord marks ingress/aws/reqctx, the request record ADR-0002:S7 keeps private to ingress.
requestRecord bool
}
func (c class) domainShaped() bool { return c.kind != notDomain }
// isLegacy reports whether c sits under one of the named legacy roots.
func (c class) isLegacy(roots ...string) bool {
if c.layer != layerLegacy {
return false
}
if len(roots) == 0 {
return true
}
return slices.Contains(roots, c.legacy)
}
// legacyRoots are pre-ADR-0001 application roots still standing. None remain.
var legacyRoots = map[string]bool{}
// classify maps a module-relative package directory to its layer. ok is false
// for a package that has no deliberate classification.
func classify(full string) (class, bool) {
parts := strings.Split(full, "/")
switch parts[0] {
case "cmd":
return class{layer: layerCmd}, len(parts) >= 2
case "contracts":
return class{layer: layerContracts}, len(parts) >= 2
case "internal":
if len(parts) < 2 {
return class{}, false
}
if parts[1] == "testkit" || parts[1] == "tooling" {
return class{layer: layerTestkit}, true
}
return class{layer: layerInternal}, true
case "tests":
return class{layer: layerTests}, true
case "spinifex":
return classifyApp(parts[1:])
}
return class{}, false
}
func classifyApp(app []string) (class, bool) {
if len(app) == 0 {
return class{}, false
}
switch app[0] {
case "foundation":
return class{layer: layerFoundation}, true
case "ingress":
return class{layer: layerIngress, requestRecord: len(app) >= 3 && app[1] == "aws" && app[2] == "reqctx"}, true
case "runtime":
return class{layer: layerRuntime, daemon: len(app) >= 2 && app[1] == "daemon"}, true
case "operator":
return class{layer: layerOperator}, true
case "providers":
return class{layer: layerProviders}, true
case "bootstrap":
return class{layer: layerBootstrap}, true
case "agents":
c := class{layer: layerAgents}
if len(app) >= 2 {
c.domain = app[1]
}
return c, true
case "domains":
if len(app) < 2 {
return class{}, false
}
c := class{layer: layerDomain, domain: app[1], kind: domainImpl}
if len(app) >= 3 {
switch app[2] {
case "awsapi":
c.kind = domainAWSAPI
case "projection":
c.kind = domainProj
}
}
return c, true
}
if !legacyRoots[app[0]] {
return class{}, false
}
c := class{layer: layerLegacy, legacy: app[0]}
switch {
case app[0] == "handlers" && len(app) >= 2:
c.domain, c.kind = app[1], legacyImpl
}
return c, true
}
// fullPath reverses shortName for classification.
func fullPath(name string) string {
top, _, _ := strings.Cut(name, "/")
switch top {
case "cmd", "contracts", "internal", "tests", "docs":
return name
}
return "spinifex/" + name
}
-58
View File
@@ -1,58 +0,0 @@
package layering
import (
"os"
"path/filepath"
"testing"
)
// frozenLegacyPackages is every production package still under a legacy root.
// It only shrinks: a new entry is a new legacy package, a missing one is a
// completed move whose line must be deleted.
var frozenLegacyPackages = []string{}
// retiredLegacyRoots were legacy roots whose every package has moved to its
// ADR-0001 target; recreating one would reopen a closed migration row.
var retiredLegacyRoots = []string{"accountteardown", "admin", "daemon", "gateway", "handlers", "lbagent", "services", "utils", "vpcd"}
// TestADR0001_LegacyRootsFrozen: the current-to-target map gives every legacy
// root a destination, so none may gain a package and no retired root may return.
func TestADR0001_LegacyRootsFrozen(t *testing.T) {
root, err := moduleRoot()
if err != nil {
t.Fatal(err)
}
g, err := loadGraph(root)
if err != nil {
t.Fatal(err)
}
present := map[string]bool{}
for name, dir := range g.pkgs {
if c, ok := classify(dir); ok && c.layer == layerLegacy {
present[name] = true
}
}
frozen := map[string]bool{}
for _, p := range frozenLegacyPackages {
frozen[p] = true
if !present[p] {
t.Errorf("legacy package %s no longer exists; delete it from frozenLegacyPackages", p)
}
}
for p := range present {
if !frozen[p] {
t.Errorf("ADR-0001 current-to-target map: %s is a new package under a legacy root; create it at its ADR-0001 target home instead", p)
}
}
for _, r := range retiredLegacyRoots {
entries, err := os.ReadDir(filepath.Join(root, "spinifex", r))
if err != nil && !os.IsNotExist(err) {
t.Fatal(err)
}
if len(entries) > 0 {
t.Errorf("ADR-0001 current-to-target map: spinifex/%s is a retired legacy root; give the code its target home instead", r)
}
}
}
-167
View File
@@ -1,167 +0,0 @@
package layering
import (
"slices"
"strings"
)
// rule is one ADR-0001 dependency rule over a classified import edge.
type rule struct {
id string // subtest name, led by the ADR-0001 clause it encodes
clause string // the ADR text the rule enforces, quoted in failures
hint string // one-line fix direction
// subject reports whether an importer is governed by the rule; the
// non-empty guard requires at least one governed package per rule.
subject func(from class, fromName string) bool
// forbids reports whether the rule rejects the edge.
forbids func(from, to class, fromName string) bool
}
// awsgwRole is the composition root ADR-0002 lets wire domain registrations.
const awsgwRole = "runtime/roles/awsgw"
// qualificationBinaries are the developer/qualification mains ADR-0001 lets
// keep reusable source in internal/testkit or internal/tooling.
var qualificationBinaries = map[string]bool{
"cmd/aws-model-coverage": true, "cmd/dhcptest": true, "cmd/spx-loadgen": true,
}
func isLayer(c class, layers ...layer) bool {
return slices.Contains(layers, c.layer)
}
// importsDomainImplementation is any domain-owned package other than a named
// projection, in its target or legacy home.
func importsDomainImplementation(to class) bool {
return to.domainShaped() && to.kind != domainProj
}
var rules = []rule{
{
id: "R1_FoundationImportsOnlyFoundation",
clause: "rule 1: foundation imports neither domains, ingress, runtime nor operator (enforced conservatively: foundation imports no module package above foundation)",
hint: "move the dependency's needed primitive into foundation or invert it through a parameter",
subject: func(f class, _ string) bool { return f.layer == layerFoundation },
forbids: func(_, t class, _ string) bool { return t.layer != layerFoundation },
},
{
id: "R2_IngressImportsNoImplementation",
clause: "rule 2: ingress/aws imports no domain implementation; domains supply their own action registration through the ingress registration contract",
hint: "take the capability through the registration contract wired by runtime/roles/awsgw",
subject: func(f class, _ string) bool { return f.layer == layerIngress },
forbids: func(_, t class, _ string) bool {
return !isLayer(t, layerFoundation, layerContracts, layerIngress)
},
},
{
id: "R3_DomainImportsNoOtherDomainImplementation",
clause: "rule 3: a domain does not read another domain's private records, migrations or repository; it uses a consumer-owned capability, a named projection, or a versioned contracts/<domain>/v1 contract",
hint: "declare a narrow capability in the consumer, or import the owner's projection or contracts/<domain>/v1",
subject: func(f class, _ string) bool { return f.domainShaped() },
forbids: func(f, t class, _ string) bool {
return f.domainShaped() && importsDomainImplementation(t) && t.domain != f.domain
},
},
{
id: "R4_ProviderImportsNoDomainImplementation",
clause: "rule 4: a provider adapter implements a provider contract and cannot acquire AWS-visible resource authority by reading a domain's private state",
hint: "pass the needed values into the provider contract instead of importing the domain",
subject: func(f class, _ string) bool { return f.layer == layerProviders },
forbids: func(_, t class, _ string) bool { return t.domainShaped() || t.isLegacy() },
},
{
id: "R5_DomainImportsNoRuntimeOrOperator",
clause: "rule 5: operator and runtime are composition consumers; they wire domain modules, so a domain does not import them",
hint: "expose the runtime or operator state through a domain-owned projection or a capability wired at the composition root",
subject: func(f class, _ string) bool { return f.domainShaped() },
forbids: func(f, t class, _ string) bool {
return f.domainShaped() && isLayer(t, layerRuntime, layerOperator)
},
},
{
id: "R5_AWSAPIAdaptersImportedOnlyByOwnerOrComposition",
clause: "rule 5: operator and runtime call public use cases; a domain's awsapi action adapters are owned by that domain and wired only by the composition root",
hint: "call an owner-provided capability instead of the AWS action adapter",
subject: func(_ class, _ string) bool { return true },
forbids: func(f, t class, fromName string) bool {
if t.kind != domainAWSAPI || fromName == awsgwRole {
return false
}
return !f.domainShaped() || f.domain != t.domain
},
},
{
id: "R5_OperatorImportsNoLegacyImplementation",
clause: "rule 5 and the cmd/spinifex map row: operator use cases reach domains through public capabilities and do not read domain-private state",
hint: "reach the domain through its public capability; do not import a legacy implementation root",
subject: func(f class, _ string) bool { return f.layer == layerOperator },
forbids: func(f, t class, _ string) bool { return f.layer == layerOperator && t.isLegacy() },
},
{
id: "R6_ProductionImportsNoTestkit",
clause: "rule 6: test support is under internal/testkit; production packages do not import it",
hint: "move the shared code to its production owner or keep the import in a _test.go file",
subject: func(f class, n string) bool { return f.layer != layerTestkit && !qualificationBinaries[n] },
forbids: func(f, t class, n string) bool {
return f.layer != layerTestkit && !qualificationBinaries[n] && isLayer(t, layerTestkit, layerTests)
},
},
{
id: "Contracts_ImportOnlyFoundationAndContracts",
clause: "decision item 1 and the area table: contracts own cross-process vocabulary, not application interfaces, domain persistence or transport implementation",
hint: "keep wire types self-contained; depend only on foundation or another contract",
subject: func(f class, _ string) bool { return f.layer == layerContracts },
forbids: func(_, t class, _ string) bool { return !isLayer(t, layerFoundation, layerContracts) },
},
{
id: "Domains_ImportNoAgentsIngressOrLegacy",
clause: "area table and domain module shape: a domain owns no other domain's implementation; only its awsapi adapters may use the generic ingress/aws machinery",
hint: "depend on a contract or capability; leave ingress use to the domain's awsapi package",
subject: func(f class, _ string) bool { return f.domainShaped() },
forbids: func(f, t class, _ string) bool {
if !f.domainShaped() {
return false
}
switch {
case t.layer == layerAgents:
return true
case t.layer == layerIngress:
return f.kind != domainAWSAPI
case t.isLegacy("handlers"):
return f.layer == layerDomain
}
return false
},
},
{
id: "Agents_ImportNoDomainOrLegacyImplementation",
clause: "area table: agents/<domain> own in-guest and service-role behaviour, not tenant API authority or another domain's private state",
hint: "move the shared wire vocabulary behind a guest/controller contract under contracts/",
subject: func(f class, _ string) bool { return f.layer == layerAgents },
forbids: func(f, t class, _ string) bool {
return f.layer == layerAgents && (t.domainShaped() || t.isLegacy())
},
},
{
id: "Q22_STSImportsNoEKSImplementation",
clause: "Q-22 prohibits the STS-to-EKS dependency in its present form; cut it through an identity-owned contract or an EKS projection",
hint: "consume an EKS projection or an identity-owned contract instead of the EKS implementation",
subject: func(f class, _ string) bool { return f.domainShaped() && f.domain == "sts" },
forbids: func(f, t class, _ string) bool {
return f.domainShaped() && f.domain == "sts" && t.domain == "eks" &&
(importsDomainImplementation(t) || t.layer == layerAgents)
},
},
{
id: "S7_RequestRecordPrivateToIngress",
clause: "ADR-0002:S7: the request record is private to ingress, so no domain, runtime or operator package reads it; only packages under ingress/aws may import ingress/aws/reqctx",
hint: "receive what you need through dispatch.Invocation instead of the request record",
subject: func(_ class, _ string) bool { return true },
forbids: func(_, t class, fromName string) bool {
if !t.requestRecord {
return false
}
return !strings.HasPrefix(fromName, "ingress/aws/")
},
},
}
+3 -10
View File
@@ -1,6 +1,6 @@
ARG VERSION=dev
FROM golang:1.27.3-bookworm@sha256:d9b6a1beb9474adc2a1ce842573639b44b2a0dd49b6669f51d3821baf9ec37c9 AS builder
FROM golang:1.27-bookworm AS builder
ARG VERSION
@@ -35,23 +35,16 @@ COPY bluebottle/ ./bluebottle/
# Build spinifex binary (spx)
RUN COMMIT=$(cd spinifex && git rev-parse --short HEAD 2>/dev/null || echo "unknown") && \
cd spinifex && GOFIPS140=v1.0.0 go build -ldflags "-s -w \
-X github.com/mulgadc/spinifex/spinifex/operator/cli.Version=${VERSION} \
-X github.com/mulgadc/spinifex/spinifex/operator/cli.Commit=${COMMIT}" \
-X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Version=${VERSION} \
-X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Commit=${COMMIT}" \
-o /out/spx cmd/spinifex/main.go
# Build nbdkit viperblock plugin (.so)
RUN cd viperblock && GOFIPS140=v1.0.0 go build -o /out/nbdkit-viperblock-plugin.so -buildmode=c-shared nbd/viperblock.go
# One tarball ships to every OS. The ceiling is what both binaries need today,
# so a rise is a decision rather than an accident; every supported OS has 2.39+.
RUN spinifex/scripts/check-glibc-ceiling.sh 2.34 /out/spx /out/nbdkit-viperblock-plugin.so
# Copy setup scripts and systemd units
RUN cp spinifex/scripts/setup.sh /out/setup.sh 2>/dev/null || true
RUN cp spinifex/scripts/setup-ovn.sh /out/setup-ovn.sh
RUN cd spinifex/scripts && cp install-node.sh smoke-test.sh node-reset.sh uninstall-spx.sh /out/
RUN mkdir -p /out/udev && cp spinifex/build/udev/* /out/udev/
RUN mkdir -p /out/lib && cp spinifex/scripts/lib/distro.sh /out/lib/distro.sh
RUN mkdir -p /out/systemd && cp spinifex/build/systemd/* /out/systemd/
RUN mkdir -p /out/scripts && cp spinifex/build/scripts/* /out/scripts/
RUN cp spinifex/build/logrotate/spinifex /out/logrotate-spinifex
-2
View File
@@ -32,8 +32,6 @@ load_mod qemu_fw_cfg || load_mod fw_cfg_sysfs || true
load_mod failover || true
load_mod net_failover || true
load_mod virtio_net || true
# lb-agent probes each data NIC's gateway on an AF_PACKET socket.
load_mod af_packet || true
NETCFG=/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw
i=0
-90
View File
@@ -1,90 +0,0 @@
#!/bin/sh
mount -t proc proc /proc
mount -t sysfs sys /sys
mount -t devtmpfs dev /dev
# modprobe needs a usable modules.dep; insmod by explicit path does not, so
# fall back to it. Handles .ko and .ko.gz.
load_mod() {
modprobe "$1" 2>/dev/null && return 0
for ko in $(find /lib/modules -name "$1.ko" -o -name "$1.ko.gz" 2>/dev/null); do
insmod "$ko" 2>/dev/null && return 0
case "$ko" in
*.gz) gunzip -c "$ko" >/tmp/mod.ko 2>/dev/null \
&& insmod /tmp/mod.ko 2>/dev/null && return 0 ;;
esac
done
return 1
}
# lambda-agent reads its identity from fw_cfg, or the configuration disk, and
# seals it before anything else runs. virtio_net needs net_failover and failover, and the
# insmod fallback resolves no dependencies, so load the chain bottom-up.
load_mod qemu_fw_cfg || echo "[init] qemu_fw_cfg did not load" >/dev/console
load_mod failover || true
load_mod net_failover || true
load_mod virtio_net || echo "[init] virtio_net did not load" >/dev/console
ip link set lo up
# Without fw_cfg data the executor attaches the same entries as a tar archive
# on the second disk; lambda-agent reads them here and erases the disk before
# the function starts. qemu_fw_cfg loads without a device, so test the entry.
CFG_ROOT=/spinifex-cfg
FWCFG_ROOT=/sys/firmware/qemu_fw_cfg
if [ ! -f "$FWCFG_ROOT/by_name/opt/spinifex/netcfg/raw" ]; then
load_mod virtio_blk || echo "[init] virtio_blk did not load" >/dev/console
i=0
while [ "$i" -lt 25 ] && [ ! -b /dev/vdb ]; do
sleep 0.04
i=$((i + 1))
done
if [ -b /dev/vdb ]; then
mkdir -m 0700 "$CFG_ROOT"
if ! mount -t tmpfs -o mode=0700,size=1m tmpfs "$CFG_ROOT" ||
! tar -xf /dev/vdb -C "$CFG_ROOT"; then
echo "[init] configuration disk not readable" >/dev/console
fi
FWCFG_ROOT=$CFG_ROOT
fi
fi
# The guest's addresses come from the same netcfg entry as other system VMs.
NETCFG=$FWCFG_ROOT/by_name/opt/spinifex/netcfg/raw
i=0
while [ "$i" -lt 25 ] && [ ! -f "$NETCFG" ]; do
sleep 0.04
i=$((i + 1))
done
if [ -f "$NETCFG" ]; then
# shellcheck source=/dev/null
. "$NETCFG"
for n in 0 1 2 3 4 5; do
eval mac=\$NIC${n}_MAC
[ -z "$mac" ] && continue
iface=""
for _ in 1 2 3 4 5; do
iface=$(ip -o link | awk -v m="$mac" 'tolower($0) ~ tolower(m) {print $2}' | tr -d :)
[ -n "$iface" ] && break
sleep 0.02
done
if [ -z "$iface" ]; then
echo "[init] no interface for MAC $mac (NIC$n)" >/dev/console
continue
fi
cidr="" gw="" default="" rdst="" rvia=""
eval cidr=\$NIC${n}_CIDR
eval gw=\$NIC${n}_GW
eval default=\$NIC${n}_DEFAULT
eval rdst=\$NIC${n}_ROUTE_DST
eval rvia=\$NIC${n}_ROUTE_VIA
ip link set "$iface" up
[ -n "$cidr" ] && ip addr add "$cidr" dev "$iface"
[ "$default" = "1" ] && [ -n "$gw" ] && ip route add default via "$gw" dev "$iface"
[ -n "$rdst" ] && [ -n "$rvia" ] && ip route add "$rdst" via "$rvia" dev "$iface"
done
else
# lambda-agent then fails to reach the host and the guest powers off.
echo "[init] fw_cfg netcfg not available" >/dev/console
fi
exec /sbin/init
-6
View File
@@ -1,6 +0,0 @@
# The power button loads here, not in /init: busybox init has its SIGUSR2
# (poweroff) handler installed before sysinit runs, so a press can never reach
# the pre-exec shell PID 1. A press before this line is lost; the host re-sends.
::sysinit:/sbin/modprobe tiny_power_button power_signal=12
::once:/usr/local/lib/lambda-runtime/run-agent
::ctrlaltdel:/sbin/reboot
-6
View File
@@ -1,6 +0,0 @@
#!/bin/sh
# Runs the Lambda guest agent, then powers the guest off whatever its status.
/usr/local/bin/lambda-agent >/dev/console 2>&1
status=$?
echo "[run-agent] lambda-agent exited with status $status" >/dev/console
poweroff -f
-100
View File
@@ -1,100 +0,0 @@
#!/bin/sh
mount -t proc proc /proc
mount -t sysfs sys /sys
mount -t devtmpfs dev /dev
# virtio_mmio is built-in (CONFIG_VIRTIO_MMIO=y) so virtio-net devices are
# already enumerated by the time PID 1 runs — no modprobe needed.
# qemu_fw_cfg and virtio_net are =m in stock Alpine linux-virt, so load them.
#
# modprobe needs a usable modules.dep; when the image is built on a host whose
# depmod can't produce one, busybox modprobe silently no-ops and the module
# never loads — fw_cfg then never appears and init panics. insmod by explicit
# path needs no modules.dep, so fall back to it. Handles .ko and .ko.gz.
load_mod() {
modprobe "$1" 2>/dev/null && return 0
for ko in $(find /lib/modules -name "$1.ko" -o -name "$1.ko.gz" 2>/dev/null); do
insmod "$ko" 2>/dev/null && return 0
case "$ko" in
*.gz) gunzip -c "$ko" >/tmp/mod.ko 2>/dev/null \
&& insmod /tmp/mod.ko 2>/dev/null && return 0 ;;
esac
done
return 1
}
load_mod qemu_fw_cfg || load_mod fw_cfg_sysfs || true
# virtio_net depends on net_failover, which depends on failover. The load_mod
# insmod fallback loads a single .ko with no dependency resolution, and
# modules.dep is absent when the image is built on a kmod-less host (depmod
# skipped), so modprobe can't resolve the chain either. Load it explicitly
# bottom-up so the NIC comes up regardless of how the image was built —
# without this the agent has no network and the LB hangs in provisioning.
load_mod failover || true
load_mod net_failover || true
load_mod virtio_net || true
# The guest's SQS store listens on 127.0.0.1 and the worker dials it there;
# the kernel leaves lo down, so bring it up before init starts either.
ip link set lo up
NETCFG=/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw
i=0
while [ "$i" -lt 25 ]; do
[ -f "$NETCFG" ] && break
sleep 0.04
i=$((i + 1))
done
if [ ! -f "$NETCFG" ]; then
# Dump host/guest fw_cfg state to the serial console before init exits so a
# "netcfg not available" panic is diagnosable from the captured console log
# (module-not-loaded vs qemu-not-exposing-by_name vs wrong machine type).
echo "[init] FATAL: fw_cfg netcfg not available" >&2
echo "[init] dmi: vendor=$(cat /sys/class/dmi/id/sys_vendor 2>/dev/null) product=$(cat /sys/class/dmi/id/product_name 2>/dev/null) bios=$(cat /sys/class/dmi/id/bios_version 2>/dev/null)" >&2
echo "[init] qemu_fw_cfg dir present: $([ -d /sys/firmware/qemu_fw_cfg ] && echo yes || echo NO)" >&2
echo "[init] modules: $(lsmod 2>/dev/null | awk 'NR>1{print $1}' | tr '\n' ',')" >&2
echo "[init] by_name listing:" >&2
ls -la /sys/firmware/qemu_fw_cfg/by_name/ 2>&1 | sed 's/^/[init] /' >&2
ls -la /sys/firmware/qemu_fw_cfg/by_name/opt/ 2>&1 | sed 's/^/[init] /' >&2
echo "[init] dmesg fw_cfg:" >&2
dmesg 2>/dev/null | grep -i fw_cfg | sed 's/^/[init] /' >&2
exit 1
fi
. "$NETCFG"
for n in 0 1 2 3 4 5; do
eval mac=\$NIC${n}_MAC
[ -z "$mac" ] && continue
iface=""
for attempt in 1 2 3 4 5; do
iface=$(ip -o link | awk -v m="$mac" 'tolower($0) ~ tolower(m) {print $2}' | tr -d :)
[ -n "$iface" ] && break
sleep 0.02
done
if [ -z "$iface" ]; then
echo "[init] WARNING: no interface found for MAC $mac (NIC$n), skipping" >&2
continue
fi
eval cidr=\$NIC${n}_CIDR
eval gw=\$NIC${n}_GW
eval default=\$NIC${n}_DEFAULT
eval rdst=\$NIC${n}_ROUTE_DST
eval rvia=\$NIC${n}_ROUTE_VIA
ip link set "$iface" up
[ -n "$cidr" ] && ip addr add "$cidr" dev "$iface"
[ "$default" = "1" ] && [ -n "$gw" ] && ip route add default via "$gw" dev "$iface"
[ -n "$rdst" ] && [ -n "$rvia" ] && ip route add "$rdst" via "$rvia" dev "$iface"
done
# The image takes no lb-agent environment. The worker reads the cluster CA
# from fw_cfg itself; this copy only serves other tools in the guest.
CA_CERT=/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/ca-cert/raw
mkdir -p /etc/ssl/certs
[ -f "$CA_CERT" ] && cp "$CA_CERT" /etc/ssl/certs/ca-certificates.crt
exec /sbin/init
-9
View File
@@ -1,9 +0,0 @@
# The power button loads here, not in /init: busybox init has its SIGUSR2
# (poweroff) handler installed before sysinit runs, so a press can never reach
# the pre-exec shell PID 1. A press before this line is lost; the host re-sends.
::sysinit:/sbin/modprobe tiny_power_button power_signal=12
::sysinit:/usr/local/lib/sqs-worker/sqs-worker-prepare
::respawn:/usr/local/lib/sqs-worker/sqs-worker-store
::respawn:/usr/local/lib/sqs-worker/sqs-worker-ready
::ctrlaltdel:/sbin/reboot
::shutdown:/usr/local/lib/sqs-worker/sqs-worker-stop
-203
View File
@@ -1,203 +0,0 @@
# sqs-worker-lib.sh — shared by the SQS guest's store, readiness and stop
# steps. Sourced, never executed. Every path and command is overridable so the
# host test can drive the same code against test-owned processes and stub tools.
#
# The guest runs its own SQS nats-server on loopback and the worker beside it.
# The server starts only after boot settled the receipt key; store-ready exists
# only while the running watcher's last check of that server passed.
SQSW_RUN=${SQSW_RUN:-/run/sqs-worker}
SQSW_DATA=${SQSW_DATA:-/data}
SQSW_CONSOLE=${SQSW_CONSOLE:-/dev/console}
SQSW_MOUNTS=${SQSW_MOUNTS:-/proc/mounts}
SQSW_BIN=${SQSW_BIN:-/usr/local/bin/sqs-worker}
SQSW_LIB=${SQSW_LIB:-/usr/local/lib/sqs-worker/sqs-worker-lib.sh}
SQSW_NATS_BIN=${SQSW_NATS_BIN:-/usr/local/bin/nats-server}
SQSW_STORE_PORT=${SQSW_STORE_PORT:-4222}
SQSW_STOP_BUDGET_TENTHS=${SQSW_STOP_BUDGET_TENTHS:-100}
SQSW_RESPAWN_GAP=${SQSW_RESPAWN_GAP:-5}
SQSW_PARK=${SQSW_PARK:-sleep 2147483647}
SQSW_BOOT_RETRY=${SQSW_BOOT_RETRY:-30}
UMOUNT=${UMOUNT:-umount}
sqsw_log() { echo "[sqs-worker] $*" >> "$SQSW_CONSOLE"; }
# sqsw_alive <pid>: running or stopped, not exited (a zombie counts as exited).
sqsw_alive() { [ -r "/proc/$1/status" ] && ! grep -q '^State:[[:space:]]*Z' "/proc/$1/status" 2>/dev/null; }
# sqsw_starttime <pid>: kernel start time (stat field 22). It survives exec, and
# pairing it with the pid reduces PID-reuse risk; it is not an absolute guarantee.
sqsw_starttime() { sed 's/.*) //' "/proc/$1/stat" 2>/dev/null | cut -d' ' -f20; }
# sqsw_same <pid> <starttime>: the recorded process is still alive.
sqsw_same() { [ -n "$2" ] && sqsw_alive "$1" && [ "$(sqsw_starttime "$1")" = "$2" ]; }
# sqsw_write_config <file>: the guest's own server. It listens only on
# loopback, with no monitor, route or cluster listener. Its max_payload admits
# a 1 MiB SQS message with its headers (ADR-0011:S4); the 1MB default does not.
sqsw_write_config() {
cat > "$1" <<CONF
listen: "127.0.0.1:$SQSW_STORE_PORT"
server_name: "sqs-store"
max_payload: 2MiB
jetstream {
store_dir: "$SQSW_DATA/jetstream"
sync_interval: always
}
CONF
}
# sqsw_prepare_store: write and test the server config once boot has settled
# the receipt key. Returns non-zero, leaving the store unstarted, otherwise.
sqsw_prepare_store() {
rm -f "$SQSW_RUN/store.conf"
[ -s "$SQSW_RUN/receipt-ready" ] || { sqsw_log "store: receipt key not settled, store not prepared"; return 1; }
umask 077
sqsw_write_config "$SQSW_RUN/store.conf.new" || return 1
if ! "$SQSW_NATS_BIN" -t -c "$SQSW_RUN/store.conf.new" > "$SQSW_RUN/config-test.out" 2>&1; then
sqsw_log "store: FATAL: server config test failed: $(tr '\n' ' ' < "$SQSW_RUN/config-test.out")"
return 1
fi
mv "$SQSW_RUN/store.conf.new" "$SQSW_RUN/store.conf"
sqsw_log "store: config written for 127.0.0.1:$SQSW_STORE_PORT; server not started yet"
}
# sqsw_park <why>: init respawns a step that exits, so a step that must not run
# logs once and waits for shutdown instead of looping.
sqsw_park() {
sqsw_log "$1"
exec $SQSW_PARK
}
# sqsw_record <name>: record this process's identity before the stopping
# check, so a stop that misses the record has already set stopping.
sqsw_record() { echo "$$ $(sqsw_starttime $$)" > "$SQSW_RUN/$1.pid"; }
# sqsw_boot: settle the disk and receipt key once per guest boot. A transient
# failure (exit 75) waits and exits so init runs it again; boot never remounts
# a disk already mounted. Any other failure is final for this guest boot.
sqsw_boot() {
[ ! -e "$SQSW_RUN/boot-refused" ] || sqsw_park "store: boot refused, not starting"
[ -s "$SQSW_RUN/receipt-ready" ] && return 0
# The boot child records itself before its stopping check, as the other
# steps do, so shutdown either sees its record or it never starts.
SQSW_RUN="$SQSW_RUN" SQSW_BIN="$SQSW_BIN" sh -c '. "$1"; sqsw_record boot; [ ! -e "$SQSW_RUN/stopping" ] || exit 75; exec "$SQSW_BIN" boot' \
sh "$SQSW_LIB" >> "$SQSW_CONSOLE" 2>&1 &
child=$!
rc=0
wait "$child" || rc=$?
rm -f "$SQSW_RUN/boot.pid"
[ ! -e "$SQSW_RUN/stopping" ] || sqsw_park "store: shutdown during boot, not starting"
case "$rc" in
0) return 0 ;;
75) sqsw_log "store: boot not settled, retrying in ${SQSW_BOOT_RETRY}s"
sleep "$SQSW_BOOT_RETRY"
exit 1 ;;
*) : > "$SQSW_RUN/boot-refused"
sqsw_park "store: boot refused (rc=$rc), not starting" ;;
esac
}
# sqsw_store: the respawned server body. It starts only on a prepared config.
sqsw_store() {
mkdir -p "$SQSW_RUN"
sqsw_record store
[ ! -e "$SQSW_RUN/stopping" ] || sqsw_park "store: shutdown in progress, not starting"
sqsw_boot
if [ ! -s "$SQSW_RUN/store.conf" ] && ! sqsw_prepare_store; then
: > "$SQSW_RUN/boot-refused"
sqsw_park "store: not prepared, not starting"
fi
sqsw_log "store: starting pid=$$"
# ADR-0011:S11 bounds the store's Go heap; the guest must be larger.
GOMEMLIMIT=512MiB exec "$SQSW_NATS_BIN" -c "$SQSW_RUN/store.conf"
}
# sqsw_run: the respawned readiness body. A predecessor that was killed or
# crashed could not remove its marker, so every start removes it first.
sqsw_run() {
mkdir -p "$SQSW_RUN"
sqsw_record ready
if [ -e "$SQSW_RUN/store-ready" ]; then
rm -f "$SQSW_RUN/store-ready"
sqsw_log "ready: removed a store-ready marker left by an earlier watcher"
fi
[ ! -e "$SQSW_RUN/stopping" ] || sqsw_park "ready: shutdown in progress, not starting"
[ ! -e "$SQSW_RUN/boot-refused" ] || sqsw_park "ready: boot refused, readiness not started"
# Boot may still be retrying; check again after a gap rather than parking.
if [ ! -s "$SQSW_RUN/store.conf" ]; then
sleep "$SQSW_RESPAWN_GAP"
exit 0
fi
# Init respawns at once; a watcher that keeps exiting restarts at most
# once per gap instead of in a tight loop.
now=$(date +%s)
last=$(cat "$SQSW_RUN/ready.last" 2>/dev/null || echo 0)
wait=$((last + SQSW_RESPAWN_GAP - now))
if [ "$wait" -gt 0 ] && [ "$wait" -le "$SQSW_RESPAWN_GAP" ]; then sleep "$wait"; fi
date +%s > "$SQSW_RUN/ready.last"
sqsw_log "ready: starting watcher pid=$$"
exec "$SQSW_BIN" ready >> "$SQSW_CONSOLE" 2>&1
}
# sqsw_end <name>: SIGTERM the recorded process, SIGKILL only after the
# budget. Returns non-zero while it is still alive.
sqsw_end() {
rec=$(cat "$SQSW_RUN/$1.pid" 2>/dev/null || true)
pid=${rec%% *} st=${rec#* }
[ "$st" != "$rec" ] || st=""
if [ -z "$pid" ] || ! sqsw_same "$pid" "$st"; then
sqsw_log "stop: no running $1"
return 0
fi
kill -TERM "$pid" 2>/dev/null || true
sqsw_log "stop: SIGTERM sent to $1 pid=$pid"
i=0
while sqsw_same "$pid" "$st" && [ "$i" -lt "$SQSW_STOP_BUDGET_TENTHS" ]; do sleep 0.1; i=$((i + 1)); done
if sqsw_same "$pid" "$st"; then
sqsw_log "stop: $1 pid=$pid still running after $((SQSW_STOP_BUDGET_TENTHS / 10))s, SIGKILL"
kill -KILL "$pid" 2>/dev/null || true
j=0
while sqsw_same "$pid" "$st" && [ "$j" -lt 30 ]; do sleep 0.1; j=$((j + 1)); done
fi
if sqsw_same "$pid" "$st"; then
sqsw_log "stop: $1 pid=$pid STILL ALIVE"
return 1
fi
sqsw_log "stop: $1 pid=$pid exited"
rm -f "$SQSW_RUN/$1.pid"
}
# sqsw_stop: block any respawn, drop the marker, stop the watcher, the server
# and any boot in progress, and unmount the disk only once all are confirmed
# gone. A survivor keeps the disk mounted.
sqsw_stop() {
mkdir -p "$SQSW_RUN"
: > "$SQSW_RUN/stopping"
rm -f "$SQSW_RUN/store-ready"
if ! sqsw_end ready || ! sqsw_end store || ! sqsw_end boot; then
rm -f "$SQSW_RUN/store-ready"
sqsw_log "stop: leaving $SQSW_DATA mounted"
return 1
fi
rm -f "$SQSW_RUN/store-ready"
sync
if grep -q " $SQSW_DATA " "$SQSW_MOUNTS" 2>/dev/null; then
rc=0
"$UMOUNT" "$SQSW_DATA" || rc=$?
sqsw_log "stop: umount $SQSW_DATA rc=$rc"
fi
}
# sqsw_shutdown: the only shutdown action. The generic read-only remount runs
# only after both processes stopped; a survivor keeps every mount as it is.
sqsw_shutdown() {
if ! sqsw_stop; then
sync
sqsw_log "shutdown: a guest process survived, skipping umount -a -r"
return 1
fi
rc=0
"$UMOUNT" -a -r || rc=$?
echo "[init] shutdown: umount -a -r rc=$rc" >> "$SQSW_CONSOLE"
}
@@ -1,9 +0,0 @@
#!/bin/sh
# sysinit: load the disk drivers and clear this boot's markers. The respawned
# store step mounts the disk, settles the receipt key and starts the store.
. /usr/local/lib/sqs-worker/sqs-worker-lib.sh
modprobe virtio_blk 2>/dev/null || true
modprobe ext4 2>/dev/null || true
mkdir -p "$SQSW_RUN"
rm -f "$SQSW_RUN/receipt-ready" "$SQSW_RUN/store.conf" "$SQSW_RUN/boot-refused"
exit 0
@@ -1,5 +0,0 @@
#!/bin/sh
# Respawned by init: the worker, reporting ready only while the local store
# answers. A watcher that must not run parks instead of exiting.
. /usr/local/lib/sqs-worker/sqs-worker-lib.sh
sqsw_run
-5
View File
@@ -1,5 +0,0 @@
#!/bin/sh
# The only shutdown action: stop the worker, then the store, and unmount the
# data disk, then run the generic read-only remount only if both are gone.
. /usr/local/lib/sqs-worker/sqs-worker-lib.sh
sqsw_shutdown
@@ -1,5 +0,0 @@
#!/bin/sh
# Respawned by init: the guest's own SQS nats-server, started only on a
# prepared config. A store that must not run parks instead of exiting.
. /usr/local/lib/sqs-worker/sqs-worker-lib.sh
sqsw_store
+4 -6
View File
@@ -1,10 +1,10 @@
# spinifex-unit-version: 6
# spinifex-unit-version: 3
[Unit]
Description=Spinifex Daemon (VM Orchestration)
PartOf=spinifex.target
After=spinifex-nats.service spinifex-viperblock.service ovsdb-server.service ovs-vswitchd.service ovn-controller.service openvswitch-ipsec.service spinifex-hostnet.service
# Wants (not Requires): boot without openvswitch-ipsec, survive OVS restarts via hostnet
Wants=openvswitch-ipsec.service spinifex-hostnet.service
After=spinifex-nats.service spinifex-viperblock.service openvswitch-switch.service ovn-controller.service openvswitch-ipsec.service
# Wants (not Requires): stay bootable when openvswitch-ipsec is absent
Wants=openvswitch-ipsec.service
[Service]
EnvironmentFile=-/etc/spinifex/telemetry.env
@@ -14,8 +14,6 @@ Type=simple
Slice=spinifex-system.slice
User=spinifex-daemon
Group=spinifex
ExecStartPre=/usr/local/bin/spx admin hostnet check
ExecStartPre=-+/usr/local/bin/spx admin node-facts collect
ExecStart=/usr/local/bin/spx service spinifex start
Restart=on-failure
RestartSec=5
+4 -6
View File
@@ -1,10 +1,11 @@
# spinifex-unit-version: 2
# spinifex-unit-version: 1
[Unit]
Description=Spinifex host firewall (nftables input policy)
Documentation=file:/etc/spinifex/firewall/spinifex.nft
Wants=network-pre.target
Before=network-pre.target shutdown.target
DefaultDependencies=no
ConditionPathExists=/etc/spinifex/firewall/peers.nft
[Service]
Type=oneshot
@@ -12,11 +13,8 @@ RemainAfterExit=yes
StandardInput=null
ProtectSystem=full
ProtectHome=true
# Unconditional: connection tracking loads on every node, so a flow that opens
# while the policy is not armed survives the arm. The policy itself still loads
# only where the daemon has written a peer file.
ExecStart=/usr/local/lib/spinifex/spinifex-firewall-apply boot
ExecReload=/usr/local/lib/spinifex/spinifex-firewall-apply boot
ExecStart=/usr/local/lib/spinifex/spinifex-firewall-apply
ExecReload=/usr/local/lib/spinifex/spinifex-firewall-apply
TimeoutStopSec=10s
# No ExecStop: flushing on stop would take vpcd's MASQUERADE and per-EIP FORWARD
-25
View File
@@ -1,25 +0,0 @@
# spinifex-unit-version: 2
[Unit]
Description=Spinifex host network wiring (veth pairs, mgmt address, encap route)
# PartOf/WantedBy ovs-vswitchd: re-applied whenever OVS starts; daemon and vpcd only Want it
Requires=ovs-vswitchd.service
After=ovs-vswitchd.service NetworkManager-wait-online.service
PartOf=ovs-vswitchd.service
# Stopping OVS stops ovn-controller through its Requires= chain and starting OVS
# does not bring it back, so the start that re-applies hostnet restarts it too
Wants=ovn-controller.service
Before=spinifex.target
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/usr/local/bin/spx admin hostnet apply
# Covers the 120s wait behind setup-ovn.sh plus the 60s wait for links
TimeoutStartSec=240
TimeoutStopSec=10
ProtectSystem=full
ProtectHome=yes
PrivateTmp=yes
[Install]
WantedBy=spinifex.target ovs-vswitchd.service
+3 -4
View File
@@ -1,9 +1,9 @@
# spinifex-unit-version: 3
# spinifex-unit-version: 1
[Unit]
Description=Spinifex VPC Daemon (OVN Networking)
PartOf=spinifex.target
After=spinifex-nats.service spinifex-daemon.service systemd-networkd.service ovsdb-server.service ovs-vswitchd.service ovn-controller.service spinifex-hostnet.service
Wants=spinifex-nats.service ovn-controller.service spinifex-hostnet.service
After=spinifex-nats.service spinifex-daemon.service systemd-networkd.service openvswitch-switch.service ovn-controller.service
Wants=spinifex-nats.service ovn-controller.service
[Service]
EnvironmentFile=-/etc/spinifex/telemetry.env
@@ -11,7 +11,6 @@ Type=simple
Slice=spinifex-system.slice
User=spinifex-vpcd
Group=spinifex
ExecStartPre=/usr/local/bin/spx admin hostnet check
ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
ExecStart=/usr/local/bin/spx service vpcd start
Restart=on-failure
+2 -2
View File
@@ -1,9 +1,9 @@
# spinifex-unit-version: 4
# spinifex-unit-version: 1
[Unit]
Description=Spinifex Infrastructure Platform
After=network-online.target
Wants=network-online.target
Wants=spinifex-nats.service spinifex-predastore.service spinifex-northstar.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer ovsdb-server.service ovs-vswitchd.service ovn-controller.service
Wants=spinifex-nats.service spinifex-predastore.service spinifex-northstar.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer openvswitch-switch.service ovn-controller.service
[Install]
WantedBy=multi-user.target
+3 -36
View File
@@ -4,22 +4,10 @@ import (
"flag"
"fmt"
"os"
"slices"
"github.com/mulgadc/predastore/s3api"
"github.com/mulgadc/spinifex/internal/awsmodel"
acmawsapi "github.com/mulgadc/spinifex/spinifex/domains/acm/awsapi"
ec2awsapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi"
awsapi "github.com/mulgadc/spinifex/spinifex/domains/ecr/awsapi"
ecsawsapi "github.com/mulgadc/spinifex/spinifex/domains/ecs/awsapi"
eksawsapi "github.com/mulgadc/spinifex/spinifex/domains/eks/awsapi"
elbv2awsapi "github.com/mulgadc/spinifex/spinifex/domains/elbv2/awsapi"
iamawsapi "github.com/mulgadc/spinifex/spinifex/domains/iam/awsapi"
rdsawsapi "github.com/mulgadc/spinifex/spinifex/domains/rds/awsapi"
route53awsapi "github.com/mulgadc/spinifex/spinifex/domains/route53/awsapi"
sqsawsapi "github.com/mulgadc/spinifex/spinifex/domains/sqs/awsapi"
stsawsapi "github.com/mulgadc/spinifex/spinifex/domains/sts/awsapi"
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
"github.com/mulgadc/spinifex/spinifex/gateway"
_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
)
@@ -55,31 +43,10 @@ func fail(err error) {
}
func compareAll() ([]awsmodel.OperationCoverage, error) {
// Registered services contribute their declared inventory, which needs no
// live dependencies; nothing here serves a request.
inventories := dispatch.AWSOperationInventory(map[string]dispatch.Inventory{
awsapi.ServiceName: awsapi.OperationInventory(),
acmawsapi.ServiceName: acmawsapi.OperationInventory(),
route53awsapi.ServiceName: route53awsapi.OperationInventory(),
elbv2awsapi.InventoryName: elbv2awsapi.OperationInventory(),
rdsawsapi.ServiceName: rdsawsapi.OperationInventory(),
ec2awsapi.ServiceName: ec2awsapi.OperationInventory(),
ecsawsapi.ServiceName: ecsawsapi.OperationInventory(),
eksawsapi.ServiceName: eksawsapi.OperationInventory(),
iamawsapi.ServiceName: iamawsapi.OperationInventory(),
sqsawsapi.ServiceName: sqsawsapi.OperationInventory(),
stsawsapi.ServiceName: stsawsapi.OperationInventory(),
})
// The loop below walks the embedded models, so a served service with no
// model would otherwise drop out of the report without a word.
for name := range inventories {
if !slices.Contains(awsmodel.Services(), awsmodel.Service(name)) {
return nil, fmt.Errorf("no embedded AWS model for served service %s", name)
}
}
dispatch := gateway.AWSOperationInventory()
coverages := make([]awsmodel.OperationCoverage, 0, len(awsmodel.Services()))
for _, service := range awsmodel.Services() {
inventory, ok := inventories[string(service)]
inventory, ok := dispatch[string(service)]
var modelInventory awsmodel.DispatchInventory
switch {
case ok:
+1 -1
View File
@@ -23,7 +23,7 @@ import (
"github.com/insomniacslk/dhcp/dhcpv4"
"github.com/insomniacslk/dhcp/dhcpv4/nclient4"
"github.com/mulgadc/spinifex/spinifex/domains/network/external/dhcp"
"github.com/mulgadc/spinifex/spinifex/network/external/dhcp"
_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
)
+166 -6
View File
@@ -1,15 +1,175 @@
// ecr-credential-provider is the kubelet exec credential provider for Spinifex
// EKS worker nodes; the implementation is in agents/eks/credentialprovider.
// ecr-credential-provider is a kubelet exec credential provider that lets a
// Spinifex EKS worker node pull images from the internal Spinifex ECR registry.
//
// kubelet invokes it per-pull, feeding a CredentialProviderRequest on STDIN and
// reading a CredentialProviderResponse on STDOUT. The provider fetches the
// node IAM-role credentials from IMDS, SigV4-signs GetAuthorizationToken against
// the AWS gateway's ECR endpoint, and returns the minted AWS:<jwt> as Basic auth
// for the requested registry host.
//
// Static node config (gateway URL, CA, region) is read from the cloud-init env
// file /etc/spinifex-eks/agent.env (KEY=value); real env vars override it.
package main
import (
"encoding/json"
"fmt"
"io"
"log/slog"
"net/http"
"os"
"time"
"github.com/mulgadc/spinifex/internal/ecrauth"
"github.com/mulgadc/spinifex/internal/guestenv"
"github.com/mulgadc/spinifex/internal/imdscreds"
_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
"github.com/mulgadc/spinifex/spinifex/agents/eks/credentialprovider"
)
func main() {
os.Exit(credentialprovider.Main(os.Stdin, os.Stdout))
const (
credProviderAPIVersion = "credentialprovider.kubelet.k8s.io/v1"
defaultCacheDuration = "10h"
// expiryMargin trims the token's real TTL so kubelet re-invokes the provider
// before the JWT actually expires mid-run.
expiryMargin = 30 * time.Minute
defaultEnvFile = "/etc/spinifex-eks/agent.env"
defaultGatewayCA = "/etc/spinifex-eks/gateway-ca.pem"
defaultIMDSBase = "http://169.254.169.254/latest"
)
// CredentialProviderRequest is the kubelet exec-provider request on STDIN.
type CredentialProviderRequest struct {
APIVersion string `json:"apiVersion"`
Kind string `json:"kind"`
Image string `json:"image"`
}
// AuthConfig is a single registry credential entry.
type AuthConfig struct {
Username string `json:"username"`
Password string `json:"password"`
}
// CredentialProviderResponse is the kubelet exec-provider response on STDOUT.
type CredentialProviderResponse struct {
APIVersion string `json:"apiVersion"`
Kind string `json:"kind"`
CacheKeyType string `json:"cacheKeyType"`
CacheDuration string `json:"cacheDuration"`
Auth map[string]AuthConfig `json:"auth"`
}
// config holds the static node settings the provider needs.
type config struct {
GatewayURL string
GatewayCA string
Region string
IMDSBase string
}
func main() {
cfg := loadConfig(defaultEnvFile)
httpClient, err := ecrauth.GatewayHTTPClient(cfg.GatewayCA)
if err != nil {
slog.Error("ecr-credential-provider: build gateway client", "err", err)
emitEmpty(os.Stdout)
os.Exit(1)
}
if err := run(os.Stdin, os.Stdout, cfg, httpClient); err != nil {
slog.Error("ecr-credential-provider: run failed", "err", err)
// Still emit a valid empty-auth response so kubelet has a parseable reply.
emitEmpty(os.Stdout)
os.Exit(1)
}
}
// run is the testable core: it reads a CredentialProviderRequest from in,
// resolves credentials via IMDS + the gateway, and writes a
// CredentialProviderResponse to out. The httpClient must trust the gateway CA.
func run(in io.Reader, out io.Writer, cfg config, httpClient *http.Client) error {
var req CredentialProviderRequest
if err := json.NewDecoder(in).Decode(&req); err != nil {
return fmt.Errorf("decode request: %w", err)
}
if req.Image == "" {
return fmt.Errorf("request image is empty")
}
imageHost := ecrauth.HostFromImage(req.Image)
creds, err := imdscreds.Fetch(httpClient, cfg.IMDSBase)
if err != nil {
return fmt.Errorf("fetch IMDS credentials: %w", err)
}
tok, err := ecrauth.GetAuthorizationToken(cfg.Region, cfg.GatewayURL, httpClient,
creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken)
if err != nil {
return fmt.Errorf("get authorization token: %w", err)
}
auth := AuthConfig{Username: tok.Username, Password: tok.Password}
authMap := map[string]AuthConfig{imageHost: auth}
if tok.ProxyHost != "" && tok.ProxyHost != imageHost {
authMap[tok.ProxyHost] = auth
}
resp := CredentialProviderResponse{
APIVersion: credProviderAPIVersion,
Kind: "CredentialProviderResponse",
CacheKeyType: "Registry",
CacheDuration: cacheDurationFrom(tok.ExpiresAt),
Auth: authMap,
}
return json.NewEncoder(out).Encode(&resp)
}
// cacheDurationFrom derives a Go-duration string from the token expiry minus a
// safety margin, falling back to the default when expiry is unknown or too near.
func cacheDurationFrom(expiresAt time.Time) string {
if expiresAt.IsZero() {
return defaultCacheDuration
}
d := time.Until(expiresAt) - expiryMargin
if d <= 0 {
return defaultCacheDuration
}
return d.Round(time.Second).String()
}
// loadConfig reads the cloud-init env file then lets real env vars override. The
// IMDS base is fixed (not in the env file) but overridable for tests.
func loadConfig(envFile string) config {
get := guestenv.Load(envFile).Get
cfg := config{
GatewayURL: get("EKS_GATEWAY_URL"),
GatewayCA: get("EKS_GATEWAY_CA"),
Region: get("EKS_REGION"),
IMDSBase: get("EKS_IMDS_BASE"),
}
if cfg.GatewayCA == "" {
cfg.GatewayCA = defaultGatewayCA
}
if cfg.IMDSBase == "" {
cfg.IMDSBase = defaultIMDSBase
}
return cfg
}
// emitEmpty writes a valid CredentialProviderResponse with no auth entries.
func emitEmpty(out io.Writer) {
resp := CredentialProviderResponse{
APIVersion: credProviderAPIVersion,
Kind: "CredentialProviderResponse",
CacheKeyType: "Registry",
CacheDuration: defaultCacheDuration,
Auth: map[string]AuthConfig{},
}
if err := json.NewEncoder(out).Encode(&resp); err != nil {
slog.Error("ecr-credential-provider: encode empty response", "err", err)
}
}
+126
View File
@@ -0,0 +1,126 @@
package main
import (
"bytes"
"crypto/tls"
"crypto/x509"
"encoding/base64"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
// fakeBackend serves both the IMDS role/credentials endpoints and the gateway
// ECR GetAuthorizationToken on one TLS server, routed by request path.
func fakeBackend(t *testing.T) *httptest.Server {
t.Helper()
mux := http.NewServeMux()
// IMDSv2 token. Real IMDS echoes the requested TTL back as a response
// header; the SDK client parses it from there, not the body.
mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPut {
w.WriteHeader(http.StatusMethodNotAllowed)
return
}
w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
_, _ = w.Write([]byte("imds-v2-token"))
})
// IMDS role name.
mux.HandleFunc("/latest/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/latest/meta-data/iam/security-credentials/" {
if r.Header.Get("X-Aws-Ec2-Metadata-Token") != "imds-v2-token" {
w.WriteHeader(http.StatusUnauthorized)
return
}
_, _ = w.Write([]byte("node-role"))
return
}
// Role credentials.
_ = json.NewEncoder(w).Encode(map[string]string{
"Code": "Success",
"Type": "AWS-HMAC",
"AccessKeyId": "AKIATEST",
"SecretAccessKey": "secrettest",
"Token": "sessiontoken",
"AccountId": "123456789012",
})
})
// Gateway ECR GetAuthorizationToken (X-Amz-Target dispatch).
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
if !strings.Contains(r.Header.Get("X-Amz-Target"), "GetAuthorizationToken") {
w.WriteHeader(http.StatusBadRequest)
return
}
authToken := base64.StdEncoding.EncodeToString([]byte("AWS:testjwt"))
w.Header().Set("Content-Type", "application/x-amz-json-1.1")
_ = json.NewEncoder(w).Encode(map[string]any{
"authorizationData": []map[string]any{{
"authorizationToken": authToken,
"proxyEndpoint": "https://123456789012.dkr.ecr.us-east-1.spinifex.internal",
}},
})
})
srv := httptest.NewTLSServer(mux)
t.Cleanup(srv.Close)
return srv
}
func clientTrusting(srv *httptest.Server) *http.Client {
pool := x509.NewCertPool()
pool.AddCert(srv.Certificate())
return &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}}
}
func TestRun_ReturnsBasicAuth(t *testing.T) {
srv := fakeBackend(t)
client := clientTrusting(srv)
cfg := config{
GatewayURL: srv.URL,
Region: "us-east-1",
IMDSBase: srv.URL + "/latest",
}
const image = "123456789012.dkr.ecr.us-east-1.spinifex.internal/spinifex-demo:latest"
reqBody, err := json.Marshal(CredentialProviderRequest{
APIVersion: credProviderAPIVersion,
Kind: "CredentialProviderRequest",
Image: image,
})
if err != nil {
t.Fatalf("marshal request: %v", err)
}
var out bytes.Buffer
if err := run(bytes.NewReader(reqBody), &out, cfg, client); err != nil {
t.Fatalf("run: %v", err)
}
var resp CredentialProviderResponse
if err := json.Unmarshal(out.Bytes(), &resp); err != nil {
t.Fatalf("unmarshal response: %v\n%s", err, out.String())
}
if resp.Kind != "CredentialProviderResponse" {
t.Errorf("Kind = %q, want CredentialProviderResponse", resp.Kind)
}
if resp.CacheKeyType != "Registry" {
t.Errorf("CacheKeyType = %q, want Registry", resp.CacheKeyType)
}
host := "123456789012.dkr.ecr.us-east-1.spinifex.internal"
auth, ok := resp.Auth[host]
if !ok {
t.Fatalf("auth map missing host %q; got %v", host, resp.Auth)
}
if auth.Username != "AWS" {
t.Errorf("username = %q, want AWS", auth.Username)
}
if auth.Password != "testjwt" {
t.Errorf("password = %q, want testjwt", auth.Password)
}
}
+214
View File
@@ -0,0 +1,214 @@
package main
import (
"bufio"
"context"
"fmt"
"log/slog"
"net/http"
"os"
goruntime "runtime"
"strconv"
"strings"
"time"
"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
)
// version is the agent build version, reported in the register message.
// Overridable via -ldflags "-X main.version=...".
var version = "dev"
// Agent wires the ecs-agent's runtime seams: a gateway control-plane client, a
// container runtime, and an ECR resolver. It registers the host as a container
// instance at boot, heartbeats (re-registers) while alive, polls the gateway for
// task assignments, and runs them through containerd, reporting state back over
// the gateway. It never connects to NATS — the bus stays host-internal.
type Agent struct {
cfg config
id identity
cp controlPlane
resolver ctrruntime.Resolver
// rt holds the container runtime, which is resolved by a loop rather than
// once at boot. Empty until the containerd socket answers, and emptied again
// if it stops answering.
rt *runtimeGate
reg *registrar
hb *heartbeater
// netns builds awsvpc task network namespaces (nil-safe; bridge/host skip it).
netns *taskNetns
// cred serves task IAM role credentials at 169.254.170.2 (nil in unit tests).
cred *credEndpoint
// gpu is the local pinning ledger over the host's discovered GPU UUIDs.
// Always non-nil (newAgent seeds an empty one); New() reseeds it from
// nvidia-smi discovery.
gpu *gpuLedger
// dialRuntime connects to the container runtime. nil in unit tests, which
// drive the gate directly.
dialRuntime func() (ctrruntime.Runtime, error)
closers []func() error
}
// newAgent assembles an Agent from already-built seams. Tests use this directly
// with fakes; New builds the production seams and delegates here. rt may be nil
// when containerd is unavailable; the assign path then reports the task STOPPED
// rather than crashing, and the watch loop keeps dialling for it.
func newAgent(cfg config, id identity, cp controlPlane, rt ctrruntime.Runtime, resolver ctrruntime.Resolver) *Agent {
gate := newRuntimeGate(rt)
a := &Agent{
cfg: cfg,
id: id,
cp: cp,
resolver: resolver,
rt: gate,
netns: newTaskNetns(execNetRunner{}),
gpu: newGPULedger(id.Capacity.GPUIDs),
}
// Registration reports the host's capacity only while a runtime is there to
// use it, so an agent that cannot run a container advertises none and takes
// no placement.
a.reg = newRegistrar(cp, id, gate.available)
a.hb = newHeartbeater(cp, id, cfg.Heartbeat, gate.available)
return a
}
// New builds an Agent from config: it resolves the host identity from IMDS,
// builds the gateway control-plane client, the ECR resolver and (best-effort)
// the containerd runtime. A containerd connect failure is logged, not fatal —
// registration and heartbeat still run so the instance is visible while the
// runtime recovers. The ECR gateway client is built lazily on first image pull
// (not here), so a missing or malformed gateway CA does not stop registration.
func New(cfg config) (*Agent, error) {
imdsClient := &http.Client{Timeout: 5 * time.Second}
meta, err := fetchInstanceMetadata(imdsClient, cfg.IMDSBase)
if err != nil {
return nil, fmt.Errorf("instance metadata: %w", err)
}
host, _ := os.Hostname()
id := identity{
AccountID: meta.AccountID,
ClusterName: cfg.ClusterName,
InstanceID: meta.InstanceID,
AZ: meta.AZ,
InstanceType: meta.InstanceType,
Hostname: host,
Capacity: detectCapacity(discoverNvidiaGPUs(execCommandRunner)),
AgentVersion: version,
}
creds := credentials.NewIMDSProvider(imdsClient, cfg.IMDSBase)
resolver := newLazyECRResolver(creds, cfg.Region, cfg.GatewayURL, cfg.GatewayCA)
// A dial that fails here is not fatal and not final: the node starts the
// container daemon and the agent back to back, so the socket is often not
// accepting yet. Run's watch loop keeps dialling.
rt, perr := ctrruntime.New(cfg.ContainerdSocket)
if perr != nil {
slog.Warn("ecs-agent: containerd unavailable at boot, will keep retrying", "err", perr)
}
cp, err := newGatewayControlPlane(cfg, creds)
if err != nil {
return nil, fmt.Errorf("build gateway control-plane: %w", err)
}
a := newAgent(cfg, id, cp, rt, resolver)
a.cred = newCredEndpoint(creds, cfg.Region, cfg.GatewayURL, cfg.GatewayCA,
cfg.CredEndpointIP, cfg.CredEndpointPort, execNetRunner{})
a.dialRuntime = func() (ctrruntime.Runtime, error) { return ctrruntime.New(cfg.ContainerdSocket) }
return a, nil
}
// Run registers the instance, starts the heartbeat and assignment-poll loops,
// and blocks until ctx is cancelled, then tears down.
func (a *Agent) Run(ctx context.Context) error {
if err := a.reg.Register(); err != nil {
slog.Error("ecs-agent: initial registration failed", "err", err)
} else {
slog.Info("ecs-agent: registered", "cluster", a.id.ClusterName, "instance", a.id.InstanceID)
}
if a.cred != nil {
if err := a.cred.Start(); err != nil {
slog.Error("ecs-agent: credential endpoint start failed", "err", err)
}
}
go a.hb.Run(ctx)
// Keeps the runtime current for the life of the process: dials while there
// is none, and drops one that stops answering.
if a.dialRuntime != nil {
go a.rt.watch(ctx, a.dialRuntime, a.cfg.RuntimeRetry)
}
// Re-adopt containers still running from before this restart, then poll with
// their tasks pre-seeded so re-delivered assignments are acked, not re-run.
go a.pollAssignments(ctx, a.reconcile(ctx))
<-ctx.Done()
return a.Stop()
}
// Stop closes the runtime. Safe to call once.
func (a *Agent) Stop() error {
var firstErr error
if a.cred != nil {
_ = a.cred.Stop()
}
for _, c := range a.closers {
if err := c(); err != nil && firstErr == nil {
firstErr = err
}
}
a.rt.clear()
return firstErr
}
// detectCapacity reports the host's total CPU units (1 vCPU = 1024), memory,
// and GPU count/UUIDs from the caller's nvidia-smi discovery (nil on a
// non-GPU host).
func detectCapacity(gpuUUIDs []string) bus.InstanceCapacity {
return bus.InstanceCapacity{
CPU: goruntime.NumCPU() * 1024,
MemoryMiB: memTotalMiB(),
GPU: len(gpuUUIDs),
GPUIDs: gpuUUIDs,
}
}
// memTotalMiB reads MemTotal from /proc/meminfo, returning 0 if unavailable.
func memTotalMiB() int {
f, err := os.Open("/proc/meminfo")
if err != nil {
return 0
}
defer f.Close()
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := scanner.Text()
if !strings.HasPrefix(line, "MemTotal:") {
continue
}
fields := strings.Fields(line)
if len(fields) < 2 {
return 0
}
kb, err := strconv.Atoi(fields[1])
if err != nil {
return 0
}
return kb / 1024
}
return 0
}
+66
View File
@@ -0,0 +1,66 @@
package main
import (
"context"
"testing"
"time"
ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
)
func TestAgent_RunRegistersThenStopsOnContext(t *testing.T) {
cp := &fakeCP{}
cfg := config{Heartbeat: 5 * time.Millisecond, PollInterval: 5 * time.Millisecond}
puller := &ctrruntime.FakePuller{}
a := newAgent(cfg, testIdentity(), cp, puller, nil)
a.closers = append(a.closers, puller.Close)
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- a.Run(ctx) }()
// Give it time to register + re-register (heartbeat) a few times.
time.Sleep(30 * time.Millisecond)
cancel()
select {
case err := <-done:
if err != nil {
t.Fatalf("Run returned error: %v", err)
}
case <-time.After(time.Second):
t.Fatal("Run did not return after cancel")
}
// One boot register + at least one heartbeat re-register.
if cp.registerCount() < 2 {
t.Errorf("registers = %d, want >= 2 (boot + heartbeat)", cp.registerCount())
}
if !puller.Closed {
t.Error("Stop did not close the puller")
}
}
func TestDetectCapacity_Positive(t *testing.T) {
c := detectCapacity(nil)
if c.CPU <= 0 {
t.Errorf("CPU = %d, want > 0", c.CPU)
}
// MemoryMiB may be 0 on platforms without /proc/meminfo; just assert non-negative.
if c.MemoryMiB < 0 {
t.Errorf("MemoryMiB = %d, want >= 0", c.MemoryMiB)
}
if c.GPU != 0 || c.GPUIDs != nil {
t.Errorf("no GPU UUIDs given: want GPU=0/GPUIDs=nil, got %d/%v", c.GPU, c.GPUIDs)
}
}
func TestDetectCapacity_WithGPUUUIDs(t *testing.T) {
c := detectCapacity([]string{"GPU-aaa", "GPU-bbb"})
if c.GPU != 2 {
t.Errorf("GPU = %d, want 2", c.GPU)
}
if len(c.GPUIDs) != 2 {
t.Errorf("GPUIDs = %v, want 2 entries", c.GPUIDs)
}
}
+96
View File
@@ -0,0 +1,96 @@
package main
import (
"strconv"
"time"
"github.com/mulgadc/spinifex/internal/guestenv"
)
const (
defaultEnvFile = "/etc/spinifex-ecs/agent.env"
defaultGatewayCA = "/etc/spinifex-ecs/gateway-ca.pem"
defaultIMDSBase = "http://169.254.169.254/latest"
defaultContainerdSocket = "/run/containerd/containerd.sock"
defaultHeartbeat = 30 * time.Second
defaultPollInterval = 5 * time.Second
)
// config holds the static settings the agent reads at boot. Cluster identity
// (account ID, instance ID) is discovered at runtime from IMDS, not configured;
// only the cluster *name* the instance was launched into is static. AccessKey /
// SecretKey are legacy seeded creds, optional now that the agent SigV4-signs
// gateway calls with instance-role credentials from IMDS; the agent never holds
// a NATS token.
type config struct {
GatewayURL string
GatewayCA string
Region string
ClusterName string
AccessKey string
SecretKey string
IMDSBase string
ContainerdSocket string
Heartbeat time.Duration
PollInterval time.Duration
RuntimeRetry time.Duration
CredEndpointIP string
CredEndpointPort int
}
// loadConfig reads the cloud-init env file then lets real env vars override.
func loadConfig(envFile string) config {
get := guestenv.Load(envFile).Get
cfg := config{
GatewayURL: get("ECS_GATEWAY_URL"),
GatewayCA: get("ECS_GATEWAY_CA"),
Region: get("ECS_REGION"),
ClusterName: get("ECS_CLUSTER"),
AccessKey: get("ECS_ACCESS_KEY"),
SecretKey: get("ECS_SECRET_KEY"),
IMDSBase: get("ECS_IMDS_BASE"),
ContainerdSocket: get("ECS_CONTAINERD_SOCKET"),
}
if cfg.GatewayCA == "" {
cfg.GatewayCA = defaultGatewayCA
}
if cfg.IMDSBase == "" {
cfg.IMDSBase = defaultIMDSBase
}
if cfg.ContainerdSocket == "" {
cfg.ContainerdSocket = defaultContainerdSocket
}
if cfg.ClusterName == "" {
cfg.ClusterName = "default"
}
cfg.Heartbeat = defaultHeartbeat
if v := get("ECS_HEARTBEAT_INTERVAL"); v != "" {
if d, err := time.ParseDuration(v); err == nil && d > 0 {
cfg.Heartbeat = d
}
}
cfg.PollInterval = defaultPollInterval
if v := get("ECS_POLL_INTERVAL"); v != "" {
if d, err := time.ParseDuration(v); err == nil && d > 0 {
cfg.PollInterval = d
}
}
cfg.RuntimeRetry = defaultRuntimeRetry
if v := get("ECS_RUNTIME_RETRY"); v != "" {
if d, err := time.ParseDuration(v); err == nil && d > 0 {
cfg.RuntimeRetry = d
}
}
cfg.CredEndpointIP = get("ECS_CRED_ENDPOINT_IP")
if cfg.CredEndpointIP == "" {
cfg.CredEndpointIP = defaultCredEndpointIP
}
cfg.CredEndpointPort = defaultCredEndpointPort
if v := get("ECS_CRED_ENDPOINT_PORT"); v != "" {
if p, err := strconv.Atoi(v); err == nil && p > 0 {
cfg.CredEndpointPort = p
}
}
return cfg
}
+73
View File
@@ -0,0 +1,73 @@
package main
import (
"os"
"path/filepath"
"testing"
"time"
)
func TestLoadConfig_Defaults(t *testing.T) {
// Ensure env vars don't leak in from the host.
for _, k := range []string{"ECS_GATEWAY_URL", "ECS_GATEWAY_CA", "ECS_REGION",
"ECS_CLUSTER", "ECS_ACCESS_KEY", "ECS_SECRET_KEY", "ECS_IMDS_BASE",
"ECS_CONTAINERD_SOCKET", "ECS_HEARTBEAT_INTERVAL", "ECS_POLL_INTERVAL"} {
t.Setenv(k, "")
}
cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
if cfg.GatewayCA != defaultGatewayCA {
t.Errorf("GatewayCA = %q, want default", cfg.GatewayCA)
}
if cfg.IMDSBase != defaultIMDSBase {
t.Errorf("IMDSBase = %q, want default", cfg.IMDSBase)
}
if cfg.ContainerdSocket != defaultContainerdSocket {
t.Errorf("ContainerdSocket = %q, want default", cfg.ContainerdSocket)
}
if cfg.ClusterName != "default" {
t.Errorf("ClusterName = %q, want default", cfg.ClusterName)
}
if cfg.Heartbeat != defaultHeartbeat {
t.Errorf("Heartbeat = %v, want %v", cfg.Heartbeat, defaultHeartbeat)
}
if cfg.PollInterval != defaultPollInterval {
t.Errorf("PollInterval = %v, want %v", cfg.PollInterval, defaultPollInterval)
}
}
func TestLoadConfig_FileThenEnvOverride(t *testing.T) {
dir := t.TempDir()
envFile := filepath.Join(dir, "agent.env")
body := "# comment\nECS_GATEWAY_URL=https://gw.file\nECS_REGION=us-west-2\nECS_CLUSTER=prod\n"
if err := os.WriteFile(envFile, []byte(body), 0o600); err != nil {
t.Fatal(err)
}
t.Setenv("ECS_GATEWAY_URL", "https://gw.env")
t.Setenv("ECS_ACCESS_KEY", "AKIAENV")
cfg := loadConfig(envFile)
if cfg.GatewayURL != "https://gw.env" {
t.Errorf("GatewayURL = %q, want env override", cfg.GatewayURL)
}
if cfg.Region != "us-west-2" {
t.Errorf("Region = %q, want from file", cfg.Region)
}
if cfg.ClusterName != "prod" {
t.Errorf("ClusterName = %q, want from file", cfg.ClusterName)
}
if cfg.AccessKey != "AKIAENV" {
t.Errorf("AccessKey = %q, want env", cfg.AccessKey)
}
}
func TestLoadConfig_HeartbeatOverride(t *testing.T) {
t.Setenv("ECS_HEARTBEAT_INTERVAL", "5s")
if cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env")); cfg.Heartbeat != 5*time.Second {
t.Errorf("Heartbeat = %v, want 5s", cfg.Heartbeat)
}
// Garbage value falls back to the default, not zero.
t.Setenv("ECS_HEARTBEAT_INTERVAL", "not-a-duration")
if cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env")); cfg.Heartbeat != defaultHeartbeat {
t.Errorf("Heartbeat = %v, want default on bad value", cfg.Heartbeat)
}
}
@@ -1,18 +1,18 @@
package agent
package main
import (
"context"
"encoding/json"
"fmt"
"time"
awsv2 "github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ecs"
ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
"github.com/mulgadc/spinifex/internal/ecsgw"
"github.com/mulgadc/spinifex/internal/gwsign"
ecsdomain "github.com/mulgadc/spinifex/spinifex/domains/ecs"
handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
)
// controlPlane is the agent's channel to the AWS gateway. It replaces the direct
@@ -21,17 +21,17 @@ import (
type controlPlane interface {
// Register (re-)registers the container instance. Called at boot and on every
// heartbeat tick — the gateway's RegisterContainerInstance is idempotent and
// refreshes LastSeen, folding liveness into registration (no ecsv1.Heartbeat).
// refreshes LastSeen, folding liveness into registration (no bus.Heartbeat).
Register(id identity) error
// SubmitTaskState reports a task's RUNNING/STOPPED transition.
SubmitTaskState(st ecsv1.TaskState) error
SubmitTaskState(st bus.TaskState) error
// PollAssignments drains this instance's assignment + stop inboxes, acking the
// assign taskIDs and stop taskIDs accepted on the previous poll. Returns the
// still-pending assignments and stop directives.
PollAssignments(cluster, instance string, ackAssigns, ackStops []string) ([]ecsv1.Assign, []ecsv1.StopDirective, error)
PollAssignments(cluster, instance string, ackAssigns, ackStops []string) ([]bus.Assign, []bus.StopDirective, error)
// ReportTaskGPU reports the local ledger's pinned device UUIDs for a task's
// containers, so the control plane can populate DescribeTasks gpuIds.
ReportTaskGPU(cluster, task string, containers []ecsdomain.ContainerGPUReport) error
ReportTaskGPU(cluster, task string, containers []handlers_ecs.ContainerGPUReport) error
}
// gatewayControlPlane implements controlPlane over the SigV4 ECS gateway client.
@@ -47,14 +47,26 @@ const pollTimeout = 30 * time.Second
// newGatewayControlPlane builds the SigV4 client signing with the instance-role
// credentials from IMDS (fetched per call so rotation is transparent) + pinned CA.
func newGatewayControlPlane(cfg Config, creds awsv2.CredentialsProvider) (*gatewayControlPlane, error) {
client, err := ecsgw.New(cfg.GatewayURL, cfg.GatewayCA, gwsign.New(creds), cfg.Region, pollTimeout)
func newGatewayControlPlane(cfg config, creds credentials.CredentialsProvider) (*gatewayControlPlane, error) {
client, err := ecsgw.New(cfg.GatewayURL, cfg.GatewayCA, credentialsFunc(creds), cfg.Region, pollTimeout)
if err != nil {
return nil, err
}
return &gatewayControlPlane{client: client}, nil
}
// credentialsFunc adapts the agent's IMDS CredentialsProvider to the ecsgw
// per-call credentials hook.
func credentialsFunc(p credentials.CredentialsProvider) ecsgw.CredentialsFunc {
return func(ctx context.Context) (string, string, string, error) {
c, err := p.Retrieve(ctx)
if err != nil {
return "", "", "", err
}
return c.AccessKeyID, c.SecretAccessKey, c.SessionToken, nil
}
}
func (g *gatewayControlPlane) Register(id identity) error {
in := &ecs.RegisterContainerInstanceInput{
Cluster: aws.String(id.ClusterName),
@@ -84,7 +96,7 @@ func (g *gatewayControlPlane) Register(id identity) error {
return nil
}
func (g *gatewayControlPlane) SubmitTaskState(st ecsv1.TaskState) error {
func (g *gatewayControlPlane) SubmitTaskState(st bus.TaskState) error {
in := &ecs.SubmitTaskStateChangeInput{
Cluster: aws.String(st.ClusterName),
Task: aws.String(st.TaskID),
@@ -112,8 +124,8 @@ func (g *gatewayControlPlane) SubmitTaskState(st ecsv1.TaskState) error {
return nil
}
func (g *gatewayControlPlane) PollAssignments(cluster, instance string, ackAssigns, ackStops []string) ([]ecsv1.Assign, []ecsv1.StopDirective, error) {
in := &ecsdomain.PollAssignmentsInput{
func (g *gatewayControlPlane) PollAssignments(cluster, instance string, ackAssigns, ackStops []string) ([]bus.Assign, []bus.StopDirective, error) {
in := &handlers_ecs.PollAssignmentsInput{
Cluster: cluster,
ContainerInstance: instance,
AckTaskIDs: ackAssigns,
@@ -127,15 +139,15 @@ func (g *gatewayControlPlane) PollAssignments(cluster, instance string, ackAssig
if err != nil {
return nil, nil, fmt.Errorf("poll: %w", err)
}
var out ecsdomain.PollAssignmentsOutput
var out handlers_ecs.PollAssignmentsOutput
if err := json.Unmarshal(resp, &out); err != nil {
return nil, nil, fmt.Errorf("decode poll: %w", err)
}
return out.Assignments, out.Stops, nil
}
func (g *gatewayControlPlane) ReportTaskGPU(cluster, task string, containers []ecsdomain.ContainerGPUReport) error {
in := &ecsdomain.ReportTaskGPUInput{Cluster: cluster, Task: task, Containers: containers}
func (g *gatewayControlPlane) ReportTaskGPU(cluster, task string, containers []handlers_ecs.ContainerGPUReport) error {
in := &handlers_ecs.ReportTaskGPUInput{Cluster: cluster, Task: task, Containers: containers}
body, err := json.Marshal(in)
if err != nil {
return fmt.Errorf("marshal report-task-gpu: %w", err)
@@ -1,4 +1,4 @@
package agent
package main
import (
"context"
@@ -8,22 +8,22 @@ import (
"net/http/httptest"
"testing"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go/service/ecs"
ecsdomain "github.com/mulgadc/spinifex/spinifex/domains/ecs"
"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
)
// fixedCredsProvider is a static aws.CredentialsProvider for tests
// fixedCredsProvider is a static credentials.CredentialsProvider for tests
// that only need a valid SigV4 signature, not real IMDS rotation.
type fixedCredsProvider struct{}
func (fixedCredsProvider) Retrieve(context.Context) (aws.Credentials, error) {
return aws.Credentials{AccessKeyID: "AKIATEST", SecretAccessKey: "secret"}, nil
func (fixedCredsProvider) Retrieve(context.Context) (credentials.Credentials, error) {
return credentials.Credentials{AccessKeyID: "AKIATEST", SecretAccessKey: "secret"}, nil
}
func newTestGatewayControlPlane(t *testing.T, srv *httptest.Server) *gatewayControlPlane {
t.Helper()
cp, err := newGatewayControlPlane(Config{GatewayURL: srv.URL, Region: "us-east-1"}, fixedCredsProvider{})
cp, err := newGatewayControlPlane(config{GatewayURL: srv.URL, Region: "us-east-1"}, fixedCredsProvider{})
if err != nil {
t.Fatalf("newGatewayControlPlane: %v", err)
}
@@ -101,7 +101,7 @@ func TestGatewayControlPlane_Register_NoGPUOmitsResource(t *testing.T) {
}
// ReportTaskGPU posts the per-container device UUIDs under the ReportTaskGPU
// action, decodable as the domains/ecs internal shape.
// action, decodable as the handlers_ecs internal shape.
func TestGatewayControlPlane_ReportTaskGPU(t *testing.T) {
var gotTarget string
var body []byte
@@ -113,7 +113,7 @@ func TestGatewayControlPlane_ReportTaskGPU(t *testing.T) {
defer srv.Close()
cp := newTestGatewayControlPlane(t, srv)
err := cp.ReportTaskGPU("web", "t-1", []ecsdomain.ContainerGPUReport{
err := cp.ReportTaskGPU("web", "t-1", []handlers_ecs.ContainerGPUReport{
{Name: "trainer", GPUIDs: []string{"GPU-aaa"}},
})
if err != nil {
@@ -123,7 +123,7 @@ func TestGatewayControlPlane_ReportTaskGPU(t *testing.T) {
t.Fatal("no X-Amz-Target header sent")
}
var in ecsdomain.ReportTaskGPUInput
var in handlers_ecs.ReportTaskGPUInput
if err := json.Unmarshal(body, &in); err != nil {
t.Fatalf("decode body: %v", err)
}
@@ -1,4 +1,4 @@
package agent
package main
import (
"context"
@@ -13,7 +13,7 @@ import (
"sync"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
"github.com/mulgadc/spinifex/internal/ecrauth"
"github.com/mulgadc/spinifex/internal/stsauth"
)
@@ -47,16 +47,9 @@ type credResponse struct {
// credEntry caches one task's assumed-role credentials alongside the role it was
// assumed from.
// credRole is the task role a credID serves and the task it belongs to; the
// task ID names the STS session, since the credID is a secret.
type credRole struct {
roleARN string
taskID string
}
type credEntry struct {
roleARN string
creds aws.Credentials
creds stsauth.Credentials
}
// credEndpoint is the host-netns HTTP server that hands a task its IAM role
@@ -64,7 +57,7 @@ type credEntry struct {
// mints/caches credentials on demand by assuming the role over the gateway with
// the agent's instance credentials.
type credEndpoint struct {
provider aws.CredentialsProvider
provider credentials.CredentialsProvider
region string
gatewayURL string
caPath string
@@ -75,10 +68,10 @@ type credEndpoint struct {
// assume mints fresh credentials for a role; defaults to AssumeRole over the
// gateway and is overridable in tests.
assume func(ctx context.Context, roleARN, sessionName string) (aws.Credentials, error)
assume func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error)
mu sync.Mutex
roles map[string]credRole // credID -> task role
roles map[string]string // credID -> taskRoleARN
cache map[string]credEntry // credID -> cached creds
httpClient *http.Client
@@ -89,7 +82,7 @@ type credEndpoint struct {
// newCredEndpoint builds the endpoint. An empty ip/port falls back to the
// AWS-reserved 169.254.170.2:80; tests pass loopback + an ephemeral port. run
// adds the link-local address to the dummy interface (real or fake).
func newCredEndpoint(provider aws.CredentialsProvider, region, gatewayURL, caPath, ip string, port int, run netCmdRunner) *credEndpoint {
func newCredEndpoint(provider credentials.CredentialsProvider, region, gatewayURL, caPath, ip string, port int, run netCmdRunner) *credEndpoint {
if ip == "" {
ip = defaultCredEndpointIP
}
@@ -105,7 +98,7 @@ func newCredEndpoint(provider aws.CredentialsProvider, region, gatewayURL, caPat
port: port,
proxyPort: defaultCredProxyPort,
run: run,
roles: map[string]credRole{},
roles: map[string]string{},
cache: map[string]credEntry{},
}
c.assume = c.assumeOverGateway
@@ -114,12 +107,12 @@ func newCredEndpoint(provider aws.CredentialsProvider, region, gatewayURL, caPat
// Register associates a credID with the task role it serves; Deregister removes
// it and drops any cached credentials. Both are no-ops for an empty credID.
func (c *credEndpoint) Register(credID, taskID, roleARN string) {
func (c *credEndpoint) Register(credID, roleARN string) {
if credID == "" || roleARN == "" {
return
}
c.mu.Lock()
c.roles[credID] = credRole{roleARN: roleARN, taskID: taskID}
c.roles[credID] = roleARN
c.mu.Unlock()
}
@@ -149,25 +142,25 @@ func (c *credEndpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) {
http.Error(w, "not found", http.StatusNotFound)
return
}
creds, role, err := c.fetch(r.Context(), credID)
creds, roleARN, err := c.fetch(r.Context(), credID)
if err != nil {
if errors.Is(err, errUnknownCredID) {
http.Error(w, "not found", http.StatusNotFound)
return
}
slog.Error("ecs-agent: assume task role failed", "task", role.taskID, "role", role.roleARN, "err", err)
slog.Error("ecs-agent: assume task role failed", "credId", credID, "err", err)
http.Error(w, "credentials unavailable", http.StatusBadGateway)
return
}
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(credResponse{
RoleArn: role.roleARN,
RoleArn: roleARN,
AccessKeyID: creds.AccessKeyID,
SecretAccessKey: creds.SecretAccessKey,
Token: creds.SessionToken,
Expiration: creds.Expires.UTC().Format(time.RFC3339),
Expiration: creds.Expiration.UTC().Format(time.RFC3339),
}); err != nil {
slog.Warn("ecs-agent: encode credential response", "task", role.taskID, "err", err)
slog.Warn("ecs-agent: encode credential response", "credId", credID, "err", err)
}
}
@@ -176,35 +169,35 @@ var errUnknownCredID = fmt.Errorf("unknown credId")
// fetch returns valid credentials for credID, reusing the cache until within the
// refresh margin and otherwise assuming the task role over the gateway.
func (c *credEndpoint) fetch(ctx context.Context, credID string) (aws.Credentials, credRole, error) {
func (c *credEndpoint) fetch(ctx context.Context, credID string) (stsauth.Credentials, string, error) {
c.mu.Lock()
role, known := c.roles[credID]
roleARN, known := c.roles[credID]
if !known {
c.mu.Unlock()
return aws.Credentials{}, credRole{}, errUnknownCredID
return stsauth.Credentials{}, "", errUnknownCredID
}
if e, ok := c.cache[credID]; ok && e.roleARN == role.roleARN && credValid(e.creds, credRefreshMargin) {
if e, ok := c.cache[credID]; ok && e.roleARN == roleARN && credValid(e.creds, credRefreshMargin) {
c.mu.Unlock()
return e.creds, role, nil
return e.creds, roleARN, nil
}
c.mu.Unlock()
creds, err := c.assume(ctx, role.roleARN, sessionName(role.taskID))
creds, err := c.assume(ctx, roleARN, sessionName(credID))
if err != nil {
return aws.Credentials{}, role, err
return stsauth.Credentials{}, "", err
}
c.mu.Lock()
c.cache[credID] = credEntry{roleARN: role.roleARN, creds: creds}
c.cache[credID] = credEntry{roleARN: roleARN, creds: creds}
c.mu.Unlock()
return creds, role, nil
return creds, roleARN, nil
}
// AssumeProvider returns a CredentialsProvider that mints credentials by assuming
// roleARN over the gateway — the execution-role path for ECR image pulls. It
// caches the assumed set and re-assumes within the refresh margin. session scopes
// the STS session name.
func (c *credEndpoint) AssumeProvider(roleARN, session string) aws.CredentialsProvider {
func (c *credEndpoint) AssumeProvider(roleARN, session string) credentials.CredentialsProvider {
return &assumeProvider{ep: c, roleARN: roleARN, session: session}
}
@@ -216,34 +209,39 @@ type assumeProvider struct {
session string
mu sync.Mutex
cached aws.Credentials
cached stsauth.Credentials
}
var _ aws.CredentialsProvider = (*assumeProvider)(nil)
var _ credentials.CredentialsProvider = (*assumeProvider)(nil)
func (p *assumeProvider) Retrieve(ctx context.Context) (aws.Credentials, error) {
func (p *assumeProvider) Retrieve(ctx context.Context) (credentials.Credentials, error) {
p.mu.Lock()
defer p.mu.Unlock()
if !credValid(p.cached, credRefreshMargin) {
creds, err := p.ep.assume(ctx, p.roleARN, p.session)
if err != nil {
return aws.Credentials{}, err
return credentials.Credentials{}, err
}
p.cached = creds
}
return p.cached, nil
return credentials.Credentials{
AccessKeyID: p.cached.AccessKeyID,
SecretAccessKey: p.cached.SecretAccessKey,
SessionToken: p.cached.SessionToken,
Expiration: p.cached.Expiration,
}, nil
}
// assumeOverGateway mints credentials for roleARN by SigV4-signing AssumeRole
// against the gateway with the agent's instance credentials.
func (c *credEndpoint) assumeOverGateway(ctx context.Context, roleARN, sessionName string) (aws.Credentials, error) {
func (c *credEndpoint) assumeOverGateway(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error) {
instCreds, err := c.provider.Retrieve(ctx)
if err != nil {
return aws.Credentials{}, fmt.Errorf("instance credentials: %w", err)
return stsauth.Credentials{}, fmt.Errorf("instance credentials: %w", err)
}
client, err := c.client()
if err != nil {
return aws.Credentials{}, err
return stsauth.Credentials{}, err
}
return stsauth.AssumeRole(c.region, c.gatewayURL, client,
instCreds.AccessKeyID, instCreds.SecretAccessKey, instCreds.SessionToken,
@@ -382,20 +380,20 @@ func (c *credEndpoint) delEndpointAddr() {
}
// credValid reports whether creds are usable now with margin to spare.
func credValid(c aws.Credentials, margin time.Duration) bool {
func credValid(c stsauth.Credentials, margin time.Duration) bool {
if c.AccessKeyID == "" || c.SecretAccessKey == "" {
return false
}
if c.Expires.IsZero() {
if c.Expiration.IsZero() {
return true
}
return time.Now().Add(margin).Before(c.Expires)
return time.Now().Add(margin).Before(c.Expiration)
}
// sessionName derives an AssumeRole RoleSessionName from the task ID, satisfying the
// sessionName derives an AssumeRole RoleSessionName from credID, satisfying the
// STS [\w+=,.@-]{2,64} constraint.
func sessionName(taskID string) string {
name := "ecs-" + taskID
func sessionName(credID string) string {
name := "ecs-" + credID
if len(name) > 64 {
name = name[:64]
}
+197
View File
@@ -0,0 +1,197 @@
package main
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"time"
"github.com/mulgadc/spinifex/internal/stsauth"
)
// newTestCredEndpoint builds an endpoint whose assume seam returns canned creds,
// counting calls so cache behaviour is observable. run is nil (no iface plumbing).
func newTestCredEndpoint(creds stsauth.Credentials) (*credEndpoint, *int32) {
c := newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
var calls int32
c.assume = func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error) {
atomic.AddInt32(&calls, 1)
return creds, nil
}
return c, &calls
}
func TestCredEndpoint_ServesRegisteredCredID(t *testing.T) {
exp := time.Now().Add(time.Hour).UTC().Truncate(time.Second)
c, _ := newTestCredEndpoint(stsauth.Credentials{
AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "tok", Expiration: exp,
})
c.Register("cred-1", "arn:aws:iam::111122223333:role/task")
rr := httptest.NewRecorder()
c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
if rr.Code != http.StatusOK {
t.Fatalf("status = %d, want 200; body=%s", rr.Code, rr.Body.String())
}
var got credResponse
if err := json.Unmarshal(rr.Body.Bytes(), &got); err != nil {
t.Fatalf("decode: %v", err)
}
if got.AccessKeyID != "AKIA" || got.SecretAccessKey != "secret" || got.Token != "tok" {
t.Errorf("bad creds in response: %+v", got)
}
if got.RoleArn != "arn:aws:iam::111122223333:role/task" {
t.Errorf("RoleArn = %q", got.RoleArn)
}
if got.Expiration != exp.Format(time.RFC3339) {
t.Errorf("Expiration = %q, want %q", got.Expiration, exp.Format(time.RFC3339))
}
}
func TestCredEndpoint_UnknownCredIDIs404(t *testing.T) {
c, calls := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
rr := httptest.NewRecorder()
c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/nope", nil))
if rr.Code != http.StatusNotFound {
t.Fatalf("status = %d, want 404", rr.Code)
}
if *calls != 0 {
t.Errorf("assume called %d times for unknown credID, want 0", *calls)
}
}
func TestCredEndpoint_WrongMethodIs405(t *testing.T) {
c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
c.Register("cred-1", "arn:role")
rr := httptest.NewRecorder()
c.ServeHTTP(rr, httptest.NewRequest(http.MethodPost, "/v2/credentials/cred-1", nil))
if rr.Code != http.StatusMethodNotAllowed {
t.Fatalf("status = %d, want 405", rr.Code)
}
}
func TestCredEndpoint_CachesUntilExpiry(t *testing.T) {
c, calls := newTestCredEndpoint(stsauth.Credentials{
AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Hour),
})
c.Register("cred-1", "arn:role")
for hit := range 3 {
rr := httptest.NewRecorder()
c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
if rr.Code != http.StatusOK {
t.Fatalf("hit %d: status %d", hit, rr.Code)
}
}
if *calls != 1 {
t.Errorf("assume called %d times, want 1 (cached)", *calls)
}
}
func TestCredEndpoint_RefreshesNearExpiry(t *testing.T) {
// Expiry inside the refresh margin -> every hit re-assumes.
c, calls := newTestCredEndpoint(stsauth.Credentials{
AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Minute),
})
c.Register("cred-1", "arn:role")
for range 2 {
rr := httptest.NewRecorder()
c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
}
if *calls != 2 {
t.Errorf("assume called %d times, want 2 (expiry within margin)", *calls)
}
}
func TestCredEndpoint_DeregisterDropsCredID(t *testing.T) {
c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
c.Register("cred-1", "arn:role")
c.Deregister("cred-1")
rr := httptest.NewRecorder()
c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
if rr.Code != http.StatusNotFound {
t.Fatalf("status = %d, want 404 after deregister", rr.Code)
}
}
func TestCredEndpoint_AssumeFailureIs502(t *testing.T) {
c, _ := newTestCredEndpoint(stsauth.Credentials{})
c.assume = func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error) {
return stsauth.Credentials{}, context.DeadlineExceeded
}
c.Register("cred-1", "arn:role")
rr := httptest.NewRecorder()
c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
if rr.Code != http.StatusBadGateway {
t.Fatalf("status = %d, want 502", rr.Code)
}
}
func TestCredEndpoint_StartServesOverLoopback(t *testing.T) {
c, _ := newTestCredEndpoint(stsauth.Credentials{
AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Hour),
})
c.Register("cred-1", "arn:role")
c.port = 0 // bind an ephemeral port instead of privileged 80
if err := c.Start(); err != nil {
t.Fatalf("start: %v", err)
}
defer c.Stop()
url := "http://" + c.ln.Addr().String() + credRelativeURI("cred-1")
resp, err := http.Get(url)
if err != nil {
t.Fatalf("get: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("status = %d, want 200", resp.StatusCode)
}
}
func TestCredEndpoint_RedirectInstallsAndRemovesDNAT(t *testing.T) {
// failOn the -C existence check so addRedirect falls through to -A.
f := &fakeNetRunner{failOn: "-t nat -C"}
c := newCredEndpoint(nil, "us-east-1", "https://gw", "", "169.254.170.2", 80, f)
if c.listenPort() != defaultCredProxyPort {
t.Fatalf("listenPort = %d, want %d (proxy)", c.listenPort(), defaultCredProxyPort)
}
if err := c.addRedirect(); err != nil {
t.Fatalf("addRedirect: %v", err)
}
target := "-d 169.254.170.2 -p tcp --dport 80 -j DNAT --to-destination 169.254.170.2:51679"
for _, chain := range []string{"-A OUTPUT", "-A PREROUTING"} {
if !f.sawAny("iptables -t nat " + chain + " " + target) {
t.Errorf("missing add rule for %s; calls=%v", chain, f.joined())
}
}
c.delRedirect()
for _, chain := range []string{"-D OUTPUT", "-D PREROUTING"} {
if !f.sawAny("iptables -t nat " + chain + " " + target) {
t.Errorf("missing delete rule for %s; calls=%v", chain, f.joined())
}
}
}
func TestCredEndpoint_RedirectSkippedOnLoopback(t *testing.T) {
f := &fakeNetRunner{}
c := newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, f)
if err := c.addRedirect(); err != nil {
t.Fatalf("addRedirect: %v", err)
}
c.delRedirect()
if f.sawAny("iptables") {
t.Errorf("loopback mode must not touch iptables; calls=%v", f.joined())
}
if c.listenPort() != c.port {
t.Errorf("loopback listenPort = %d, want caller port %d", c.listenPort(), c.port)
}
}
func TestCredRelativeURI(t *testing.T) {
if got := credRelativeURI("abc"); got != "/v2/credentials/abc" {
t.Errorf("credRelativeURI = %q", got)
}
}
+41
View File
@@ -0,0 +1,41 @@
package credentials
import (
"context"
"net/http"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/mulgadc/spinifex/internal/imdscreds"
)
// IMDSProvider fetches instance-role credentials from IMDS via the AWS SDK's
// EC2 IMDS client and ec2rolecreds provider, cached with aws.CredentialsCache,
// which refetches once the real expiry passes. A failed refresh is an error
// here, not an extended expiry. Safe for concurrent use.
type IMDSProvider struct {
cache *aws.CredentialsCache
}
var _ CredentialsProvider = (*IMDSProvider)(nil)
// NewIMDSProvider builds a provider against the given IMDS base URL (e.g.
// "http://169.254.169.254/latest"). A nil client uses the SDK IMDS client's
// default HTTP client.
func NewIMDSProvider(client *http.Client, base string) *IMDSProvider {
provider := imdscreds.NewProvider(client, base)
return &IMDSProvider{cache: aws.NewCredentialsCache(provider)}
}
// Retrieve returns cached credentials when still valid, else fetches fresh ones.
func (p *IMDSProvider) Retrieve(ctx context.Context) (Credentials, error) {
creds, err := p.cache.Retrieve(ctx)
if err != nil {
return Credentials{}, err
}
return Credentials{
AccessKeyID: creds.AccessKeyID,
SecretAccessKey: creds.SecretAccessKey,
SessionToken: creds.SessionToken,
Expiration: creds.Expires,
}, nil
}
+164
View File
@@ -0,0 +1,164 @@
package credentials
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"time"
)
// imdsStub serves the IMDSv2 token + role + credentials endpoints under the
// real /latest prefix the SDK IMDS client always requests.
func imdsStub(t *testing.T, hits *int32, creds map[string]string) *httptest.Server {
t.Helper()
mux := http.NewServeMux()
mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
// Real IMDS echoes the requested TTL back as a response header; the SDK
// client parses it from there, not the body.
w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
_, _ = w.Write([]byte("v2-token"))
})
mux.HandleFunc("/latest/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/latest/meta-data/iam/security-credentials/" {
_, _ = w.Write([]byte("node-role"))
return
}
if hits != nil {
atomic.AddInt32(hits, 1)
}
_ = json.NewEncoder(w).Encode(creds)
})
srv := httptest.NewServer(mux)
t.Cleanup(srv.Close)
return srv
}
func TestRetrieve_FetchesAndCaches(t *testing.T) {
var hits int32
// Kept under an hour: ec2rolecreds caps Expires to now+1h.
exp := time.Now().Add(30 * time.Minute).UTC().Round(time.Second)
srv := imdsStub(t, &hits, map[string]string{
"Code": "Success",
"AccessKeyId": "AKIA",
"SecretAccessKey": "secret",
"Token": "session",
"Expiration": exp.Format(time.RFC3339),
})
p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
got, err := p.Retrieve(context.Background())
if err != nil {
t.Fatalf("Retrieve: %v", err)
}
if got.AccessKeyID != "AKIA" || !got.Expiration.Equal(exp) {
t.Errorf("creds mismatch: %+v", got)
}
// Second call well within validity must not re-hit IMDS.
if _, err := p.Retrieve(context.Background()); err != nil {
t.Fatalf("Retrieve 2: %v", err)
}
if hits != 1 {
t.Errorf("IMDS credential hits = %d, want 1 (cached)", hits)
}
}
func TestRetrieve_RefetchesWhenExpired(t *testing.T) {
var hits int32
// Already expired: aws.CredentialsCache refetches once the stored
// credentials' real expiry has passed, regardless of ExpiryWindow.
srv := imdsStub(t, &hits, map[string]string{
"Code": "Success",
"AccessKeyId": "AKIA",
"SecretAccessKey": "secret",
"Expiration": time.Now().Add(-time.Minute).UTC().Format(time.RFC3339),
})
p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
if _, err := p.Retrieve(context.Background()); err != nil {
t.Fatal(err)
}
if _, err := p.Retrieve(context.Background()); err != nil {
t.Fatal(err)
}
if hits != 2 {
t.Errorf("hits = %d, want 2 (expired refetch)", hits)
}
}
// imdsStubFailAfterFirst serves creds on the first credential request and 500s
// on every one after it, so a test can drive a refresh that fails against a
// credential the cache has already stored.
func imdsStubFailAfterFirst(t *testing.T, creds map[string]string) *httptest.Server {
t.Helper()
var hits atomic.Int32
mux := http.NewServeMux()
mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
_, _ = w.Write([]byte("v2-token"))
})
mux.HandleFunc("/latest/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/latest/meta-data/iam/security-credentials/" {
_, _ = w.Write([]byte("node-role"))
return
}
if hits.Add(1) > 1 {
w.WriteHeader(http.StatusInternalServerError)
return
}
_ = json.NewEncoder(w).Encode(creds)
})
srv := httptest.NewServer(mux)
t.Cleanup(srv.Close)
return srv
}
// A refresh that fails must surface as an error. Left to itself the SDK's
// ec2rolecreds implements HandleFailToRefresh, which hands back the same dead
// key with Expires pushed 5-15 minutes out, so the agent would sign requests
// with a credential the control plane has already rejected — indefinitely,
// since every subsequent refresh fails the same way.
func TestRetrieve_FailedRefreshDoesNotExtendExpiry(t *testing.T) {
srv := imdsStubFailAfterFirst(t, map[string]string{
"Code": "Success",
"AccessKeyId": "AKIA",
"SecretAccessKey": "secret",
"Token": "session",
// Already expired, so the next Retrieve must go back to IMDS.
"Expiration": time.Now().Add(-time.Minute).UTC().Format(time.RFC3339),
})
p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
first, err := p.Retrieve(context.Background())
if err != nil {
t.Fatalf("first Retrieve: %v", err)
}
got, err := p.Retrieve(context.Background())
if err == nil {
t.Fatalf("refresh failure was reported as success: %+v", got)
}
if got.Expiration.After(first.Expiration) {
t.Errorf("expiry extended on refresh failure: %v -> %v", first.Expiration, got.Expiration)
}
}
func TestRetrieve_CancelledContext(t *testing.T) {
srv := imdsStub(t, nil, map[string]string{"Code": "Success", "AccessKeyId": "A", "SecretAccessKey": "B"})
p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := p.Retrieve(ctx); err == nil {
t.Fatal("expected error from cancelled context")
}
}
func TestRetrieve_CodeNotSuccessErrors(t *testing.T) {
srv := imdsStub(t, nil, map[string]string{"Code": "Failure", "AccessKeyId": "A", "SecretAccessKey": "B"})
p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
if _, err := p.Retrieve(context.Background()); err == nil {
t.Fatal("expected error when Code != Success")
}
}
+24
View File
@@ -0,0 +1,24 @@
// Package credentials supplies AWS SigV4 credentials to the ecs-agent. The agent
// signs gateway calls (ECR GetAuthorizationToken, instance register/heartbeat)
// with the host's instance-role credentials, fetched from IMDS and cached until
// near expiry.
package credentials
import (
"context"
"time"
)
// Credentials is a SigV4 credential set with an expiry.
type Credentials struct {
AccessKeyID string
SecretAccessKey string
SessionToken string
Expiration time.Time
}
// CredentialsProvider yields current credentials, refreshing as needed. Retrieve
// must be safe for concurrent use.
type CredentialsProvider interface {
Retrieve(ctx context.Context) (Credentials, error)
}
+55
View File
@@ -0,0 +1,55 @@
package main
import (
"testing"
"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
)
func TestTaskCredID(t *testing.T) {
cases := []struct {
name string
as bus.Assign
want string
}{
{"no role", bus.Assign{TaskID: "t1"}, ""},
{"role, no credID -> taskID", bus.Assign{TaskID: "t1", TaskRoleARN: "arn:role"}, "t1"},
{"role + credID", bus.Assign{TaskID: "t1", CredID: "c9", TaskRoleARN: "arn:role"}, "c9"},
}
for _, tc := range cases {
if got := taskCredID(&tc.as); got != tc.want {
t.Errorf("%s: taskCredID = %q, want %q", tc.name, got, tc.want)
}
}
}
func TestWithCredEnv(t *testing.T) {
base := map[string]string{"FOO": "bar"}
// No credID -> env returned unchanged (same map).
if got := withCredEnv(base, ""); got["FOO"] != "bar" || got["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"] != "" {
t.Errorf("blank credID mutated env: %v", got)
}
got := withCredEnv(base, "cred-1")
if got["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"] != "/v2/credentials/cred-1" {
t.Errorf("missing relative-uri: %v", got)
}
if got["FOO"] != "bar" {
t.Errorf("dropped existing env: %v", got)
}
// Original map must be untouched (copy semantics).
if _, ok := base["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"]; ok {
t.Errorf("source env mutated: %v", base)
}
}
func TestSessionName(t *testing.T) {
if got := sessionName("abc"); got != "ecs-abc" {
t.Errorf("sessionName = %q", got)
}
long := sessionName(string(make([]byte, 100)))
if len(long) != 64 {
t.Errorf("sessionName not truncated to 64: len=%d", len(long))
}
}
@@ -1,4 +1,4 @@
package agent
package main
import (
"context"
@@ -7,9 +7,9 @@ import (
"strings"
"sync"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
"github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
"github.com/mulgadc/spinifex/internal/ecrauth"
"github.com/mulgadc/spinifex/spinifex/agents/ecs/runtime"
)
// ecrResolver implements runtime.Resolver by minting an ECR authorization token
@@ -22,7 +22,7 @@ import (
// pulled, so the agent can still register + heartbeat on a host with absent or
// wrong ECR config.
type ecrResolver struct {
creds aws.CredentialsProvider
creds credentials.CredentialsProvider
region string
gatewayURL string
caPath string
@@ -35,13 +35,13 @@ var _ runtime.Resolver = (*ecrResolver)(nil)
// newECRResolver builds a resolver with an injected HTTP client (tests). The
// client is used as-is and never rebuilt from caPath.
func newECRResolver(creds aws.CredentialsProvider, region, gatewayURL string, httpClient *http.Client) *ecrResolver {
func newECRResolver(creds credentials.CredentialsProvider, region, gatewayURL string, httpClient *http.Client) *ecrResolver {
return &ecrResolver{creds: creds, region: region, gatewayURL: gatewayURL, httpClient: httpClient}
}
// newLazyECRResolver builds a resolver that constructs its gateway HTTP client
// from caPath on first use (production path).
func newLazyECRResolver(creds aws.CredentialsProvider, region, gatewayURL, caPath string) *ecrResolver {
func newLazyECRResolver(creds credentials.CredentialsProvider, region, gatewayURL, caPath string) *ecrResolver {
return &ecrResolver{creds: creds, region: region, gatewayURL: gatewayURL, caPath: caPath}
}
@@ -1,4 +1,4 @@
package agent
package main
import (
"context"
@@ -14,16 +14,16 @@ import (
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
)
// stubCreds is a aws.CredentialsProvider returning fixed creds or error.
// stubCreds is a credentials.CredentialsProvider returning fixed creds or error.
type stubCreds struct {
c aws.Credentials
c credentials.Credentials
err error
}
func (s stubCreds) Retrieve(context.Context) (aws.Credentials, error) {
func (s stubCreds) Retrieve(context.Context) (credentials.Credentials, error) {
return s.c, s.err
}
@@ -58,9 +58,9 @@ func trusting(srv *httptest.Server) *http.Client {
func TestECRResolver_Authorize(t *testing.T) {
srv := gatewayStub(t, "AWS:jwt", "https://123456789012.dkr.ecr.us-east-1.spinifex.internal", http.StatusOK)
creds := stubCreds{c: aws.Credentials{
creds := stubCreds{c: credentials.Credentials{
AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "sess",
Expires: time.Now().Add(time.Hour),
Expiration: time.Now().Add(time.Hour),
}}
r := newECRResolver(creds, "us-east-1", srv.URL, trusting(srv))
@@ -85,7 +85,7 @@ func TestECRResolver_CredsError(t *testing.T) {
func TestECRResolver_GatewayError(t *testing.T) {
srv := gatewayStub(t, "", "", http.StatusInternalServerError)
creds := stubCreds{c: aws.Credentials{AccessKeyID: "A", SecretAccessKey: "B"}}
creds := stubCreds{c: credentials.Credentials{AccessKeyID: "A", SecretAccessKey: "B"}}
r := newECRResolver(creds, "us-east-1", srv.URL, trusting(srv))
if _, _, _, err := r.Authorize(context.Background(), "123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1"); err == nil {
t.Fatal("expected gateway error")
@@ -93,7 +93,7 @@ func TestECRResolver_GatewayError(t *testing.T) {
}
func TestECRResolver_LazyClient_BadCADeferredToAuthorize(t *testing.T) {
creds := stubCreds{c: aws.Credentials{AccessKeyID: "A", SecretAccessKey: "B"}}
creds := stubCreds{c: credentials.Credentials{AccessKeyID: "A", SecretAccessKey: "B"}}
// Construction with a bogus CA path must not fail — only Authorize does.
r := newLazyECRResolver(creds, "us-east-1", "https://gw", filepath.Join(t.TempDir(), "absent.pem"))
if _, _, _, err := r.Authorize(context.Background(), "123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1"); err == nil {
@@ -141,9 +141,9 @@ func TestIsECRRef(t *testing.T) {
// registry, even though it lacks a ".dkr.ecr." name.
func TestECRResolver_Authorize_BareHostGateway(t *testing.T) {
srv := gatewayStub(t, "AWS:jwt", "https://192.168.1.13:9999", http.StatusOK)
creds := stubCreds{c: aws.Credentials{
creds := stubCreds{c: credentials.Credentials{
AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "sess",
Expires: time.Now().Add(time.Hour),
Expiration: time.Now().Add(time.Hour),
}}
r := newECRResolver(creds, "us-east-1", srv.URL, trusting(srv))
@@ -1,4 +1,4 @@
package agent
package main
import (
"os/exec"
@@ -1,4 +1,4 @@
package agent
package main
import (
"errors"
@@ -1,4 +1,4 @@
package agent
package main
import (
"fmt"
@@ -1,4 +1,4 @@
package agent
package main
import (
"sync"
+51
View File
@@ -0,0 +1,51 @@
package main
import (
"context"
"log/slog"
"time"
)
// heartbeater keeps the container instance alive by periodically re-registering
// it through the gateway. Real ECS folds liveness into ACS; v1 folds it into an
// idempotent RegisterContainerInstance, which refreshes the instance's LastSeen
// so the scheduler can tell live instances from dead ones (no bus.Heartbeat).
type heartbeater struct {
cp controlPlane
id identity
interval time.Duration
// runtimeReady reports whether a container runtime is resolved, so a beat
// advertises capacity only while there is something to run it with. nil
// means always ready.
runtimeReady func() bool
}
func newHeartbeater(cp controlPlane, id identity, interval time.Duration, runtimeReady func() bool) *heartbeater {
if interval <= 0 {
interval = defaultHeartbeat
}
return &heartbeater{cp: cp, id: id, interval: interval, runtimeReady: runtimeReady}
}
// beat re-registers the instance once, refreshing its LastSeen.
func (h *heartbeater) beat() error {
ready := h.runtimeReady == nil || h.runtimeReady()
return h.cp.Register(advertisedIdentity(h.id, ready))
}
// Run beats every interval until ctx is cancelled. A failed beat is logged and
// retried on the next tick rather than killing the loop.
func (h *heartbeater) Run(ctx context.Context) {
ticker := time.NewTicker(h.interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if err := h.beat(); err != nil {
slog.Warn("ecs-agent: heartbeat (re-register) failed", "err", err)
}
}
}
}
+12 -12
View File
@@ -1,6 +1,13 @@
// Command ecs-agent runs inside a Spinifex ECS container instance; the
// implementation is in agents/ecs/agent. This binary owns process lifecycle:
// signal handling, startup and run failure logging, and the exit code.
// ecs-agent runs inside a Spinifex ECS container instance (a guest VM booted from
// the ECS-AMI, with containerd baked in). It registers the host with the ECS
// scheduler through the AWS gateway over TLS+SigV4 (never NATS), heartbeats by
// re-registering, polls the gateway for task assignments, runs them through
// containerd, and reports state back over the gateway.
//
// Static config (gateway URL, CA, region, cluster, seeded IAM creds, containerd
// socket) is read from the cloud-init env file /etc/spinifex-ecs/agent.env
// (KEY=value); real env vars override it. Host identity (account, instance, AZ)
// comes from IMDS at boot.
package main
import (
@@ -11,19 +18,12 @@ import (
"syscall"
_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
ecsagent "github.com/mulgadc/spinifex/spinifex/agents/ecs/agent"
)
// version is the agent build version, reported in the register message.
// Overridable via -ldflags "-X main.version=...".
var version = "dev"
func main() {
cfg := ecsagent.LoadConfig(ecsagent.DefaultEnvFile)
cfg.AgentVersion = version
cfg := loadConfig(defaultEnvFile)
agent, err := ecsagent.New(cfg)
agent, err := New(cfg)
if err != nil {
slog.Error("ecs-agent: startup failed", "err", err)
os.Exit(1)
+66
View File
@@ -0,0 +1,66 @@
package main
import (
"context"
"fmt"
"io"
"net/http"
"strings"
"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
"github.com/mulgadc/spinifex/internal/imdscreds"
)
// instanceMetadata is the IMDS-derived identity of the host the agent runs on.
type instanceMetadata struct {
AccountID string
InstanceID string
AZ string
InstanceType string
}
// fetchInstanceMetadata reads the instance-id, AZ and account-id from IMDSv2
// via the SDK IMDS client. base is the IMDS root, e.g.
// "http://169.254.169.254/latest".
func fetchInstanceMetadata(client *http.Client, base string) (instanceMetadata, error) {
imdsClient := imdscreds.NewClient(client, base)
ctx := context.Background()
instanceID, err := getMetadata(ctx, imdsClient, "instance-id")
if err != nil {
return instanceMetadata{}, fmt.Errorf("instance-id: %w", err)
}
az, err := getMetadata(ctx, imdsClient, "placement/availability-zone")
if err != nil {
return instanceMetadata{}, fmt.Errorf("availability-zone: %w", err)
}
doc, err := imdsClient.GetInstanceIdentityDocument(ctx, &imds.GetInstanceIdentityDocumentInput{})
if err != nil {
return instanceMetadata{}, fmt.Errorf("identity document: %w", err)
}
// The instance type comes off the identity document already in hand rather
// than a further meta-data read; the scheduler sizes the host's awsvpc ENI
// capacity from it.
return instanceMetadata{
AccountID: doc.AccountID,
InstanceID: strings.TrimSpace(instanceID),
AZ: strings.TrimSpace(az),
InstanceType: strings.TrimSpace(doc.InstanceType),
}, nil
}
// getMetadata reads a single meta-data path and returns its body as a string.
func getMetadata(ctx context.Context, imdsClient *imds.Client, path string) (string, error) {
out, err := imdsClient.GetMetadata(ctx, &imds.GetMetadataInput{Path: path})
if err != nil {
return "", err
}
defer out.Content.Close()
body, err := io.ReadAll(out.Content)
if err != nil {
return "", err
}
return string(body), nil
}

Some files were not shown because too many files have changed in this diff Show More