release: update go.mod for v1.22.0

Service limits are per tenancy. vm-single-principal now builds in its own, so
the shape the other topologies share can have zero quota there, and the apply
then fails before anything about the instance principal is exercised.

Measured in ours: standard-e6-core-count is 0 available, against 13 for e5 and
10 for standard3. The first principal run died on 400-LimitExceeded for Core
Instance, which reads as a capacity problem and is a shape with no quota.

OCI_PRINCIPAL_SHAPE overrides it for that topology alone and defaults to the
shape every other topology uses, so nothing changes where quota already exists.

.github/assets/diagrams/exoscale-single-node.svg

@@ -0,0 +1,79 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="1020" height="836" viewBox="0 0 1020 836" role="img" aria-label="Spinifex on a single Exoscale instance: the instance sits on the zone&#x27;s public network with one interface, eth0, holding its own address and the default route. Manual Elastic IPs are delivered to eth0 and routed by the host into OVN through the spx-nat transit veth, and an Exoscale block storage volume backs /var/lib/spinifex.">
+<rect x="0" y="0" width="1020" height="836" rx="20" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="28" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2.5">SPINIFEX ON EXOSCALE — SINGLE NODE</text>
+<text x="992" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="end">one instance · routed NAT · Elastic IPs</text>
+<text x="28" y="72" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="23" fill="#F5F3EE" font-weight="800" text-anchor="start">One Exoscale instance, single Spinifex node</text>
+<rect x="28" y="92" width="964" height="76" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="48" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">INTERNET</text>
+<text x="972" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">AWS SDKs, SSH and guest egress all arrive here</text>
+<rect x="46" y="126" width="260.0" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="176.0" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">aws-cli / Terraform / boto3 → :9999</text>
+<rect x="315.0" y="126" width="171.6" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="400.8" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">ssh → guest Elastic IP</text>
+<rect x="495.6" y="126" width="205.6" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="598.4" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">guest egress ← same address</text>
+<line x1="510" y1="170" x2="510" y2="186" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="28" y="188" width="964" height="582" rx="16" fill="#EE4B12" fill-opacity="0.022" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.45" stroke-dasharray="6 5"/>
+<text x="50" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2">EXOSCALE ZONE DE-FRA-1  ·  PUBLIC NETWORK 91.92.116.0/23</text>
+<text x="970" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">gateway 91.92.116.1 · security group open to all</text>
+<rect x="48" y="232" width="924" height="520" rx="14" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
+<text x="68" y="266" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="16" fill="#F5F3EE" font-weight="800" text-anchor="start">standard.huge</text>
+<text x="212" y="266" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="400" text-anchor="start">spinifex-node-01  ·  8 vCPU / 32 GiB  ·  Ubuntu 26.04</text>
+<text x="952" y="266" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">one host = one Spinifex region</text>
+<line x1="68" y1="282" x2="952" y2="282" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
+<rect x="68" y="298" width="434" height="108" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="88" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">eth0 — PUBLIC INTERFACE</text>
+<text x="482" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">the only NIC</text>
+<text x="88" y="356" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">91.92.116.250/23  ·  holds the default route</text>
+<text x="88" y="378" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">advertise IP · API :9999 · SSH · host masquerade</text>
+<rect x="518" y="298" width="434" height="108" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="538" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">ELASTIC IPs — MANUAL</text>
+<text x="932" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">attached to the instance</text>
+<text x="538" y="356" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">89.145.163.1/32  →  delivered to eth0</text>
+<text x="538" y="378" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">the host routes each /32 into OVN, never owns it</text>
+<line x1="735" y1="406" x2="735" y2="422" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="68" y="424" width="884" height="40" rx="11" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
+<text x="88" y="449" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#EE4B12" font-weight="600" text-anchor="start">spx-nat-host ⇄ spx-nat-ovs</text>
+<text x="296" y="449" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#9AA1A8" font-weight="400" text-anchor="start">transit 100.127.0.0/24  ·  EIP/32 routed in  ·  masquerade out eth0 for the rest</text>
+<line x1="510" y1="464" x2="510" y2="480" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="68" y="482" width="884" height="100" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="88" y="512" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">SPINIFEX CONTROL PLANE</text>
+<text x="932" y="512" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">every service is one binary: spx service &lt;name&gt;</text>
+<rect x="86" y="530" width="93.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="132.5" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">awsgw :9999</text>
+<rect x="188.0" y="530" width="58.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="217.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">daemon</text>
+<rect x="255.0" y="530" width="44.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="277.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">vpcd</text>
+<rect x="308.0" y="530" width="44.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="330.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">NATS</text>
+<rect x="361.0" y="530" width="86.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="404.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">predastore</text>
+<rect x="456.0" y="530" width="86.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="499.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">viperblock</text>
+<rect x="551.0" y="530" width="79.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="590.5" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">northstar</text>
+<rect x="639.0" y="530" width="72.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="675.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">ui :3000</text>
+<text x="86" y="568" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">SigV4 in, NATS in the middle, QEMU/KVM (nested) · OVN/OVS · Predastore · Viperblock underneath</text>
+<line x1="248" y1="582" x2="248" y2="598" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
+<line x1="772" y1="582" x2="772" y2="598" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
+<rect x="68" y="600" width="532" height="130" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="88" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">TENANT VPCs — OVN br-int</text>
+<text x="580" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">geneve-local</text>
+<rect x="86" y="644" width="496" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="100" y="666" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">vpc-a  172.31.0.0/16</text>
+<text x="300" y="666" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-…  172.31.0.4  →  89.145.163.1</text>
+<rect x="86" y="686" width="496" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="100" y="708" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">vpc-b  10.0.0.0/16</text>
+<text x="300" y="708" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-…  10.0.1.9  ·  private, egress only</text>
+<rect x="616" y="600" width="336" height="130" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="636" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">BLOCK STORAGE VOLUME</text>
+<text x="932" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">256 GiB+</text>
+<text x="636" y="658" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">virtio-blk  →  /dev/vdb</text>
+<text x="636" y="680" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">/var/lib/spinifex</text>
+<text x="636" y="700" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11.5" fill="#5B636C" font-weight="400" text-anchor="start">found by-id, defaults,nofail</text>
+<text x="636" y="718" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11.5" fill="#5B636C" font-weight="400" text-anchor="start">viperblock · predastore · JetStream</text>
+<text x="28" y="796" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">A guest&#x27;s public address is a manual Exoscale Elastic IP, delivered to eth0 addressed to itself — no private/public pair to translate.</text>
+<text x="28" y="814" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">The host routes it into OVN, so inbound and outbound are the same address and the guest sees its private IP only.</text>
+</svg>

.github/assets/diagrams/exoscale-three-node.svg

@@ -0,0 +1,123 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="1020" height="796" viewBox="0 0 1020 796" role="img" aria-label="Spinifex across three Exoscale instances in one zone, spread by an anti-affinity group. Each has a public eth0 carrying its own Elastic IPs, a private-network eth1 for the cluster mesh, its own block storage volume and its share of the guests. Geneve, NATS JetStream, predastore erasure coding and the OVN raft join them east-west over the private network. This topology is not yet tested.">
+<rect x="0" y="0" width="1020" height="796" rx="20" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="28" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2.5">SPINIFEX ON EXOSCALE — THREE NODES</text>
+<text x="992" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="end">anti-affinity group · one zone · not yet tested</text>
+<text x="28" y="72" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="23" fill="#F5F3EE" font-weight="800" text-anchor="start">Three Exoscale instances, redundant multi-node install</text>
+<rect x="28" y="92" width="964" height="76" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="48" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">INTERNET</text>
+<text x="972" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">one endpoint, whichever node answers</text>
+<rect x="46" y="126" width="260.0" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="176.0" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">aws-cli / Terraform / boto3 → :9999</text>
+<rect x="315.0" y="126" width="171.6" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="400.8" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">ssh → guest Elastic IP</text>
+<rect x="495.6" y="126" width="205.6" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="598.4" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">guest egress ← same address</text>
+<line x1="510" y1="170" x2="510" y2="186" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="28" y="188" width="964" height="440" rx="16" fill="#EE4B12" fill-opacity="0.022" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.45" stroke-dasharray="6 5"/>
+<text x="50" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2">EXOSCALE ZONE DE-FRA-1  ·  PUBLIC NETWORK + PRIVATE NETWORK</text>
+<text x="970" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">one security group · one anti-affinity group</text>
+<rect x="48" y="232" width="300" height="380" rx="14" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
+<text x="64" y="260" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-01</text>
+<text x="332" y="260" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">standard.huge</text>
+<line x1="64" y1="272" x2="332" y2="272" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
+<rect x="64" y="284" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="78" y="305" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth0  91.92.116.250</text>
+<text x="78" y="323" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">public · default route · API</text>
+<rect x="64" y="342" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="78" y="363" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth1  10.10.0.11</text>
+<text x="78" y="381" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">private network · mesh · advertise · encap</text>
+<rect x="64" y="400" width="268" height="50" rx="10" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
+<text x="78" y="421" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start">EIP  89.145.163.1</text>
+<text x="78" y="439" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">→ eth0 · /32 via spx-nat-host</text>
+<rect x="64" y="462" width="51.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="89.5" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">awsgw</text>
+<rect x="124.0" y="462" width="57.6" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="152.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">daemon</text>
+<rect x="190.6" y="462" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="212.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">vpcd</text>
+<rect x="64" y="492" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="86.2" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">nats</text>
+<rect x="117.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="159.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">predastore</text>
+<rect x="210.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="252.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">viperblock</text>
+<rect x="64" y="526" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="76" y="548" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-23d2…  172.31.0.4 → 89.145.163.1</text>
+<rect x="64" y="568" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="76" y="590" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">volume /dev/vdb → /var/lib/spinifex</text>
+<rect x="360" y="232" width="300" height="380" rx="14" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
+<text x="376" y="260" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-02</text>
+<text x="644" y="260" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">standard.huge</text>
+<line x1="376" y1="272" x2="644" y2="272" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
+<rect x="376" y="284" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="390" y="305" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth0  91.92.117.14</text>
+<text x="390" y="323" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">public · default route · API</text>
+<rect x="376" y="342" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="390" y="363" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth1  10.10.0.12</text>
+<text x="390" y="381" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">private network · mesh · advertise · encap</text>
+<rect x="376" y="400" width="268" height="50" rx="10" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
+<text x="390" y="421" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start">EIP  89.145.160.107</text>
+<text x="390" y="439" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">→ eth0 · /32 via spx-nat-host</text>
+<rect x="376" y="462" width="51.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="401.5" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">awsgw</text>
+<rect x="436.0" y="462" width="57.6" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="464.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">daemon</text>
+<rect x="502.6" y="462" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="524.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">vpcd</text>
+<rect x="376" y="492" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="398.2" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">nats</text>
+<rect x="429.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="471.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">predastore</text>
+<rect x="522.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="564.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">viperblock</text>
+<rect x="376" y="526" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="388" y="548" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-6b91…  172.31.0.9 → 89.145.160.107</text>
+<rect x="376" y="568" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="388" y="590" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">volume /dev/vdb → /var/lib/spinifex</text>
+<rect x="672" y="232" width="300" height="380" rx="14" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
+<text x="688" y="260" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-03</text>
+<text x="956" y="260" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">standard.huge</text>
+<line x1="688" y1="272" x2="956" y2="272" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
+<rect x="688" y="284" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="702" y="305" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth0  91.92.116.88</text>
+<text x="702" y="323" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">public · default route · API</text>
+<rect x="688" y="342" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="702" y="363" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth1  10.10.0.13</text>
+<text x="702" y="381" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">private network · mesh · advertise · encap</text>
+<rect x="688" y="400" width="268" height="50" rx="10" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
+<text x="702" y="421" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start">EIP  89.145.162.131</text>
+<text x="702" y="439" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">→ eth0 · /32 via spx-nat-host</text>
+<rect x="688" y="462" width="51.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="713.5" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">awsgw</text>
+<rect x="748.0" y="462" width="57.6" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="776.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">daemon</text>
+<rect x="814.6" y="462" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="836.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">vpcd</text>
+<rect x="688" y="492" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="710.2" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">nats</text>
+<rect x="741.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="783.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">predastore</text>
+<rect x="834.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
+<text x="876.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">viperblock</text>
+<rect x="688" y="526" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="700" y="548" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-0ae4…  172.31.1.6 → 89.145.162.131</text>
+<rect x="688" y="568" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="700" y="590" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">volume /dev/vdb → /var/lib/spinifex</text>
+<line x1="510" y1="628" x2="510" y2="644" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
+<rect x="28" y="646" width="964" height="70" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="48" y="676" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">EAST–WEST, OVER THE EXOSCALE PRIVATE NETWORK</text>
+<text x="972" y="676" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">no Elastic IP is involved · MTU 1500</text>
+<rect x="46" y="684" width="124.0" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="108.0" y="698.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">Geneve UDP 6081</text>
+<rect x="179.0" y="684" width="137.6" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="247.8" y="698.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">NATS JetStream R3</text>
+<rect x="325.6" y="684" width="144.4" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="397.8" y="698.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">predastore RS(2,1)</text>
+<rect x="479.0" y="684" width="151.2" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="554.6" y="698.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">OVN Northbound raft</text>
+<rect x="639.2" y="684" width="56.0" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
+<text x="667.2" y="698.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">OVSDB</text>
+<text x="28" y="744" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Guest-to-guest traffic rides Geneve between the nodes&#x27; private-network interfaces — it never touches eth0 or an Elastic IP.</text>
+<text x="28" y="762" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Each node carries the Elastic IPs of its own guests. Moving one between nodes is an Exoscale API call, not yet automated.</text>
+<text x="28" y="780" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Three nodes is the smallest cluster that tolerates losing one; the anti-affinity group keeps them on different hypervisors.</text>
+</svg>

.github/workflows/dev-release.yml

@@ -0,0 +1,188 @@
+name: Dev Prerelease
+
+# Publishes the dev channel that install.mulgadc.com already serves. setup.sh has
+# understood INSTALL_SPINIFEX_CHANNEL=dev since it was written and the install
+# Worker resolves that channel to the newest GitHub prerelease, but nothing has
+# ever produced one -- so /download/dev/amd64 returned 404. This is the producer.
+#
+# A separate workflow rather than conditionals in release.yml, because a dev build
+# is a different artifact and not a release with flags off: amd64 only, no ISO, no
+# AMI refresh, no bare-metal gate, and nothing that touches a self-hosted runner.
+on:
+  push:
+    branches: [dev]
+    # Nothing here changes the binary, and a prerelease per docs commit would
+    # churn the channel our contacts are installing from.
+    paths-ignore:
+      - "docs/**"
+      - "**/*.md"
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+# Supersede rather than queue: on a run of merges to dev only the newest commit is
+# worth publishing, and the older build would land after it and become the channel.
+concurrency:
+  group: dev-release
+  cancel-in-progress: true
+
+jobs:
+  prerelease:
+    runs-on: ubuntu-26.04
+    steps:
+      # Full history and tags, because the tag name is git describe and a shallow
+      # clone has no tag to describe against.
+      - name: Checkout Spinifex
+        uses: actions/checkout@v7
+        with:
+          path: spinifex
+          fetch-depth: 0
+
+      # dev, not each repository's default branch, which is what release.yml takes.
+      # A dev build of spinifex against main of the others is a pairing nobody runs.
+      - name: Checkout Predastore
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/predastore
+          ref: dev
+          path: predastore
+
+      - name: Checkout Viperblock
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/viperblock
+          ref: dev
+          path: viperblock
+
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: dev
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: dev
+          path: bluebottle
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: spinifex/go.mod
+
+      # The tag carries the describe string so the commit is recoverable from the
+      # version alone, and the -dev suffix is what release.yml's own convention
+      # already treats as a prerelease. VERSION must equal the tag: the install
+      # Worker derives the asset name from the tag it resolves.
+      - name: Resolve the version
+        id: version
+        working-directory: spinifex
+        run: |
+          set -euo pipefail
+          # Describing against a dev tag would nest it, growing the name every release.
+          TAG="$(git describe --tags --always --exclude '*-dev')-dev"
+          case "$TAG" in
+              v*) ;;
+              *) echo "::error::git describe gave '$TAG', which is not a v-prefixed version"; exit 1 ;;
+          esac
+          if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
+              echo "::notice::$TAG already published; nothing to do"
+              echo "exists=true" >> "$GITHUB_OUTPUT"
+          fi
+          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
+          echo "Publishing $TAG"
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Build microVM artifacts (lb-agent + kernel + initramfs)
+        if: steps.version.outputs.exists != 'true'
+        run: cd spinifex && make build-microvm-image
+
+      - name: Build amd64 Tarball
+        if: steps.version.outputs.exists != 'true'
+        env:
+          VERSION: ${{ steps.version.outputs.tag }}
+        run: cd spinifex && make distro-amd64 VERSION="$VERSION"
+
+      # --prerelease on create, never a later promote step. release.yml sets it in
+      # a job that runs after its verification gates, which leaves a window where
+      # the release is not flagged; GitHub's releases/latest excludes prereleases
+      # only, so in that window a dev build is what every customer curl installs.
+      - name: Publish the prerelease
+        if: steps.version.outputs.exists != 'true'
+        working-directory: spinifex
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TAG: ${{ steps.version.outputs.tag }}
+        run: |
+          set -euo pipefail
+          # A file, not an inline string: the notes are Markdown, and leading
+          # whitespace from indenting a shell literal turns them into code blocks.
+          cat > "$RUNNER_TEMP/notes.md" <<EOF
+          Automated dev-channel build of \`$GITHUB_SHA\`.
+
+          **Not a release.** amd64 only, no ISO and no system images, and it has not been through the release verification gates. Install it with:
+
+          \`\`\`
+          curl -sfL https://install.mulgadc.com | sudo bash -s -- --channel dev
+          \`\`\`
+          EOF
+          sed -i 's/^          //' "$RUNNER_TEMP/notes.md"
+          gh release create "$TAG" \
+              --repo "$GITHUB_REPOSITORY" \
+              --target "$GITHUB_SHA" \
+              --prerelease \
+              --title "$TAG" \
+              --notes-file "$RUNNER_TEMP/notes.md" \
+              "dist/spinifex-$TAG-linux-amd64.tar.gz" \
+              "dist/spinifex-$TAG-linux-amd64.tar.gz.sha256" \
+              "scripts/setup.sh"
+
+      # The assertion that matters. A prerelease must never become the latest
+      # release, because latest is what an unqualified curl | bash installs -- so
+      # this checks the published state rather than trusting the flag we passed.
+      - name: Assert the stable channel is untouched
+        if: steps.version.outputs.exists != 'true'
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TAG: ${{ steps.version.outputs.tag }}
+        run: |
+          set -euo pipefail
+          latest="$(gh api "repos/$GITHUB_REPOSITORY/releases/latest" --jq .tag_name)"
+          if [ "$latest" = "$TAG" ]; then
+              echo "::error::$TAG became the latest release: every customer curl | bash would install it"
+              gh release edit "$TAG" --repo "$GITHUB_REPOSITORY" --prerelease
+              exit 1
+          fi
+          echo "latest is still $latest"
+          dev="$(gh api "repos/$GITHUB_REPOSITORY/releases?per_page=20" --jq 'map(select(.prerelease))[0].tag_name')"
+          [ "$dev" = "$TAG" ] || { echo "::error::the dev channel resolves to $dev, not $TAG"; exit 1; }
+
+      # Kept rather than unbounded: the Worker reads the first page of releases to
+      # resolve the channel, so enough prereleases would eventually push the real
+      # ones off it, and each build carries two assets.
+      #
+      # Twenty, not five. A pinned dev tag is how an external evaluator installs a
+      # specific build, and five is about a week of them -- short enough that the tag
+      # someone was given stops resolving while they are still using it.
+      - name: Prune old dev prereleases
+        if: steps.version.outputs.exists != 'true'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+          gh api "repos/$GITHUB_REPOSITORY/releases?per_page=100" \
+              --jq 'map(select(.prerelease and (.tag_name | endswith("-dev")))) | .[20:] | .[].tag_name' \
+              > stale.txt
+          while read -r tag; do
+              [ -n "$tag" ] || continue
+              echo "deleting $tag"
+              gh release delete "$tag" --repo "$GITHUB_REPOSITORY" --yes --cleanup-tag
+          done < stale.txt

.github/workflows/e2e-baremetal.yml

@@ -123,7 +123,7 @@ jobs:
             INSTALLER_BIN="${RUNNER_TEMP}/bm-bins/spinifex-installer" \
             SPINIFEX_SRC="${GITHUB_WORKSPACE}" \
             VIPERBLOCK_SRC="${RUNNER_TEMP}/bm-bins/viperblock" \
-            VERSION="$(cd "$GITHUB_WORKSPACE" && git describe --tags --always 2>/dev/null || echo dev)"
+            VERSION="$(cd "$GITHUB_WORKSPACE" && git describe --tags --always --exclude '*-dev' 2>/dev/null || echo dev)"
 
       - name: Write ePDU credentials
         run: |

.github/workflows/e2e-cloudvendor-nightly.yml

@@ -0,0 +1,287 @@
+name: E2E Cloud Vendor Nightly
+
+# Dispatched by the same Cloudflare Worker that drives e2e-nightly.yml, once per
+# ref, rather than by a `schedule:` here. GitHub registers a schedule only from
+# the default branch, so a cron in this file could never run `dev` and a feature
+# branch as separate nightlies, and it would always execute the default branch's
+# copy of this file -- leaving a fix to the workflow itself untestable until it
+# had already shipped. A dispatch takes both the file and the source from its ref.
+on:
+  workflow_dispatch:
+    inputs:
+      topologies:
+        description: "Space-separated topologies (blank = vm-single vm-multi bm vm-single-principal). One name runs one topology."
+        required: false
+      workbooks:
+        description: "Space-separated workbooks (blank = the driver's default list)"
+        required: false
+      source:
+        description: "tree (build this ref) or release (the published installer)"
+        required: false
+        default: tree
+      channel:
+        description: "With source=release: latest, or dev for the newest prerelease"
+        required: false
+        default: latest
+      keep_on_fail:
+        description: "Leave a failed topology up to inspect. It keeps billing."
+        required: false
+        type: boolean
+        default: false
+      auth:
+        description: "How nodes reach the OCI API: instance_principal (needs the tenancy dynamic group) or api_key (needs the credential hook). The vm-single-principal topology always uses instance_principal."
+        required: false
+        default: instance_principal
+        type: choice
+        options: [instance_principal, api_key]
+      no_external_pool:
+        description: "Form without the OCI allocator. Skips every workbook needing a public address. Not applied to vm-single-principal."
+        required: false
+        type: boolean
+        default: false
+      runner:
+        description: "Runner label (blank = the OCI_E2E_RUNNER variable, else ubuntu-26.04)"
+        required: false
+
+permissions:
+  contents: read
+
+# Every topology is built in one OCI compartment under one set of resource names,
+# so two runs at once would each see the other's instances and the sweep in
+# e2e-oci.sh could destroy a cluster the other run was still testing. Queue the
+# second rather than cancelling it: a dispatch is usually a verification run.
+concurrency:
+  group: e2e-oci
+  cancel-in-progress: false
+
+jobs:
+  oci:
+    # Hosted by default: this job only drives Terraform and SSH, and the work
+    # happens on OCI, so nothing here needs a hypervisor. Overridable because the
+    # argument for a per-provider self-hosted runner is control, not capacity --
+    # a dedicated label keeps a two-hour cloud wait off the hypervisor labels and
+    # lifts the six-hour cap a hosted job carries.
+    runs-on: ${{ inputs.runner || vars.OCI_E2E_RUNNER || 'ubuntu-26.04' }}
+    # Three topologies including a bare-metal build, each with its own install,
+    # formation and workbook pass. The sweep still runs if this fires.
+    timeout-minutes: 300
+    steps:
+      # Set here rather than in a job-level env block: the runner context is not
+      # available at job level, only from a step onwards, and a workflow that
+      # references it there is rejected as invalid before any job starts.
+      #
+      # Outside github.workspace on purpose. The checkout wipes untracked files, so
+      # on a persistent runner it would delete the Terraform state of a killed run
+      # before the next sweep could read it, leaving instances nothing can find.
+      - name: Put the Terraform state outside the checkout
+        run: echo "OCI_STATE_ROOT=$RUNNER_WORKSPACE/oci-state" >> "$GITHUB_ENV"
+
+      - uses: actions/checkout@v7
+        with:
+          path: spinifex
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          # mulga alone. It is the only private repository of the six, and the
+          # five that build-install-artifacts.sh clones are public, so the builder
+          # needs no credential and this token is not widened to reach them.
+          repositories: mulga
+
+      # build-install-artifacts.sh lives in mulga. The workbooks and their driver
+      # are in spinifex, so only the one directory is needed here.
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/mulga
+          path: mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: scripts/tofu-cluster
+
+      - uses: hashicorp/setup-terraform@v4
+        with:
+          terraform_wrapper: false
+
+      # Written to the runner's own filesystem and nowhere else. The OCI API
+      # credential is not here at all: it reaches Terraform as OCI_PRIVATE_KEY in
+      # the environment, so no key file is ever created.
+      - name: Install the throwaway SSH key
+        env:
+          OCI_E2E_SSH_PRIVATE_KEY: ${{ secrets.OCI_E2E_SSH_PRIVATE_KEY }}
+          OCI_E2E_SSH_PUBLIC_KEY: ${{ secrets.OCI_E2E_SSH_PUBLIC_KEY }}
+        run: |
+          set -euo pipefail
+          mkdir -p ~/.ssh
+          printf '%s\n' "$OCI_E2E_SSH_PRIVATE_KEY" > ~/.ssh/oci-spx
+          printf '%s\n' "$OCI_E2E_SSH_PUBLIC_KEY" > ~/.ssh/oci-spx.pub
+          chmod 600 ~/.ssh/oci-spx
+          ssh-keygen -y -f ~/.ssh/oci-spx > /dev/null
+
+      - name: Run the OCI topologies
+        working-directory: spinifex/scripts/terraform/oci-spx
+        env:
+          OCI_TENANCY_OCID: ${{ secrets.OCI_TENANCY_OCID }}
+          OCI_USER_OCID: ${{ secrets.OCI_USER_OCID }}
+          OCI_FINGERPRINT: ${{ secrets.OCI_FINGERPRINT }}
+          OCI_PRIVATE_KEY: ${{ secrets.OCI_PRIVATE_KEY }}
+          OCI_REGION: ${{ vars.OCI_REGION || 'ap-sydney-1' }}
+          TF_VAR_compartment_ocid: ${{ secrets.OCI_COMPARTMENT_OCID }}
+          MULGA_ROOT: ${{ github.workspace }}/mulga
+          SPX_REF: ${{ github.ref_name }}
+          OCI_TOPOLOGIES: ${{ inputs.topologies }}
+          OCI_SOURCE: ${{ inputs.source }}
+          OCI_CHANNEL: ${{ inputs.channel }}
+          # Independent of the pool, deliberately. These were one switch, which made
+          # the combination we actually need -- an API key with the allocator running
+          # -- impossible to ask for, so public-IP allocation went untested.
+          OCI_INSTANCE_PRINCIPAL: ${{ inputs.auth == 'api_key' && '0' || '1' }}
+          # Skipping the pool records the allocator and every public-address workbook
+          # as SKIPPED, so a green run with this set is not a claim about guest
+          # networking. It exists for a formation-only check, not as a way past a
+          # missing credential.
+          OCI_NO_EXTERNAL_POOL: ${{ inputs.no_external_pool && '1' || '0' }}
+          # Only an API-key run needs one: an instance principal is the node's own
+          # certificate, so there is nothing to hand over. Unset is not an error, and
+          # the allocator gate then fails rather than passing a node that cannot
+          # authenticate.
+          OCI_CREDENTIAL_HOOK: ${{ inputs.auth == 'api_key' && vars.OCI_CREDENTIAL_HOOK || '' }}
+          # vm-single-principal builds in its own tenancy. The dynamic group it needs
+          # is a tenancy-root resource, and the shared test tenancy grants us only
+          # compartment rights, so that topology can never pass there however correct
+          # the code is. Unset skips it rather than spending a provision to fail.
+          OCI_PRINCIPAL_TENANCY_OCID: ${{ secrets.OCI_PRINCIPAL_TENANCY_OCID }}
+          OCI_PRINCIPAL_USER_OCID: ${{ secrets.OCI_PRINCIPAL_USER_OCID }}
+          OCI_PRINCIPAL_FINGERPRINT: ${{ secrets.OCI_PRINCIPAL_FINGERPRINT }}
+          OCI_PRINCIPAL_PRIVATE_KEY: ${{ secrets.OCI_PRINCIPAL_PRIVATE_KEY }}
+          OCI_PRINCIPAL_COMPARTMENT_OCID: ${{ secrets.OCI_PRINCIPAL_COMPARTMENT_OCID }}
+          OCI_PRINCIPAL_REGION: ${{ vars.OCI_PRINCIPAL_REGION }}
+          # Service limits are per tenancy, so the shape the other topologies use
+          # can have zero quota in this one. Ours has no E6 cores at all, and the
+          # apply then fails before anything about the principal is exercised.
+          OCI_PRINCIPAL_SHAPE: ${{ vars.OCI_PRINCIPAL_SHAPE }}
+          OCI_ARTIFACT_DIR: ${{ runner.temp }}/e2e-oci
+          OCI_KEEP_ON_FAIL: ${{ inputs.keep_on_fail && '1' || '0' }}
+          # Through the environment, never interpolated into the script below: a
+          # dispatch input is attacker-controlled text as far as the shell is
+          # concerned, and a quote in it would otherwise end the string.
+          WORKBOOKS_INPUT: ${{ inputs.workbooks }}
+        run: |
+          set -uo pipefail
+          # WORKBOOKS is exported only when the dispatch named some: the harness
+          # distinguishes unset (the driver's default list) from empty (run none).
+          if [ -n "$WORKBOOKS_INPUT" ]; then export WORKBOOKS="$WORKBOOKS_INPUT"; fi
+          ./e2e-oci.sh 2>&1 | tee "$RUNNER_TEMP/e2e-oci.log"
+          exit "${PIPESTATUS[0]}"
+
+      # Before the sweep, which destroys the state the tables are read from, and
+      # always(): a run whose verdict is a failure is exactly the one somebody
+      # wants a table for. The successful path writes the same tables itself, so
+      # this step is for the run that did not get there.
+      #
+      # The markdown lives in the script rather than here. A heredoc in a YAML
+      # string is the one place in this repository nothing can lint or run.
+      - name: Publish the result tables
+        if: always()
+        working-directory: spinifex/scripts/terraform/oci-spx
+        env:
+          OCI_REGION: ${{ vars.OCI_REGION || 'ap-sydney-1' }}
+          OCI_ARTIFACT_DIR: ${{ runner.temp }}/e2e-oci
+          OCI_TOPOLOGIES: ${{ inputs.topologies }}
+          OCI_SOURCE: ${{ inputs.source }}
+          OCI_CHANNEL: ${{ inputs.channel }}
+          SPX_REF: ${{ github.ref_name }}
+          # The same two the run step passes. The header states what was and was not
+          # tested, so a summary rendered without them would claim more than the run.
+          OCI_INSTANCE_PRINCIPAL: ${{ inputs.auth == 'api_key' && '0' || '1' }}
+          OCI_NO_EXTERNAL_POOL: ${{ inputs.no_external_pool && '1' || '0' }}
+        run: ./e2e-oci.sh --summary-only
+
+      # The sweep below deletes the per-topology state directories, so anything
+      # still only in one of them at that point is unrecoverable -- and a failing
+      # workbook writes its assertion output to exactly there. Copying into the
+      # artifact directory first is what makes a failure diagnosable by whoever
+      # reads the run page, rather than only on the runner that produced it.
+      - name: Collect the per-topology logs
+        if: always()
+        run: |
+          set -uo pipefail
+          dest="$RUNNER_TEMP/e2e-oci"
+          mkdir -p "$dest"
+          # The topologies this run drove, taken from the per-topology log the
+          # harness writes rather than from the dispatch input: a run that names
+          # none gets the harness's own default, and reading that back here would
+          # mean a second copy of the list to disagree with the first. A state
+          # directory left by an earlier run is then not collected, so the artifact
+          # cannot show a topology this run never attempted.
+          topologies=""
+          for log in "$dest"/*.log; do
+              [ -f "$log" ] || continue
+              name="${log##*/}"; name="${name%.log}"
+              [ -d "$OCI_STATE_ROOT/.validate-$name" ] && topologies="$topologies $name"
+          done
+          echo "collecting:${topologies:- nothing}"
+          for topology in $topologies; do
+              for file in "$OCI_STATE_ROOT/.validate-$topology"/*.log \
+                          "$OCI_STATE_ROOT/.validate-$topology"/*.txt \
+                          "$OCI_STATE_ROOT/.validate-$topology"/*.tsv; do
+                  [ -f "$file" ] || continue
+                  # The hook handles a private key and the one thing known about an
+                  # operator's executable is that it is not ours to publish.
+                  case "${file##*/}" in credential-hook.log) continue ;; esac
+                  cp -- "$file" "$dest/$topology-${file##*/}"
+              done
+          done
+          ls -l "$dest"
+
+      # always(), and before the artifact upload: an OCI bare-metal host left
+      # running by a cancelled or timed-out job is the most expensive way this
+      # workflow can fail. The sweep works from the Terraform state files, so it
+      # can only destroy what this run created.
+      - name: Sweep anything left behind
+        if: always() && !inputs.keep_on_fail
+        working-directory: spinifex/scripts/terraform/oci-spx
+        env:
+          OCI_TENANCY_OCID: ${{ secrets.OCI_TENANCY_OCID }}
+          OCI_USER_OCID: ${{ secrets.OCI_USER_OCID }}
+          OCI_FINGERPRINT: ${{ secrets.OCI_FINGERPRINT }}
+          OCI_PRIVATE_KEY: ${{ secrets.OCI_PRIVATE_KEY }}
+          OCI_REGION: ${{ vars.OCI_REGION || 'ap-sydney-1' }}
+          TF_VAR_compartment_ocid: ${{ secrets.OCI_COMPARTMENT_OCID }}
+          # The sweep destroys by state file, so it needs whichever tenancy each
+          # topology built in. Without these a leftover principal deployment is
+          # swept against a tenancy that never held it and reported clean.
+          OCI_PRINCIPAL_TENANCY_OCID: ${{ secrets.OCI_PRINCIPAL_TENANCY_OCID }}
+          OCI_PRINCIPAL_USER_OCID: ${{ secrets.OCI_PRINCIPAL_USER_OCID }}
+          OCI_PRINCIPAL_FINGERPRINT: ${{ secrets.OCI_PRINCIPAL_FINGERPRINT }}
+          OCI_PRINCIPAL_PRIVATE_KEY: ${{ secrets.OCI_PRINCIPAL_PRIVATE_KEY }}
+          OCI_PRINCIPAL_COMPARTMENT_OCID: ${{ secrets.OCI_PRINCIPAL_COMPARTMENT_OCID }}
+          OCI_PRINCIPAL_REGION: ${{ vars.OCI_PRINCIPAL_REGION }}
+          OCI_ARTIFACT_DIR: ${{ runner.temp }}/e2e-oci-sweep
+        run: ./e2e-oci.sh --sweep-only
+
+      # Explicit, not left to the runner's own cleanup: a hosted runner is
+      # discarded after the job, but a self-hosted one keeps $HOME, so the key
+      # would outlive the run that needed it and sit there between nightlies.
+      - name: Remove the throwaway SSH key
+        if: always()
+        run: rm -f ~/.ssh/oci-spx ~/.ssh/oci-spx.pub
+
+      # Named extensions rather than a directory glob: the artifact directory also
+      # holds the distro tarball and the test archive, which are hundreds of
+      # megabytes and reproducible from the ref. Everything here is a file the
+      # collect step above chose, so terraform.tfstate cannot reach an artifact.
+      - name: Upload the run
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: e2e-oci-${{ github.run_id }}
+          path: |
+            ${{ runner.temp }}/e2e-oci.log
+            ${{ runner.temp }}/e2e-oci/*.log
+            ${{ runner.temp }}/e2e-oci/verdict.txt
+            ${{ runner.temp }}/e2e-oci/summary.md
+            ${{ runner.temp }}/e2e-oci/*.txt
+            ${{ runner.temp }}/e2e-oci/*.tsv
+            ${{ runner.temp }}/e2e-oci-sweep/*.log
+          retention-days: 14

.github/workflows/release.yml

@@ -46,7 +46,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
-          go-version: "1.27.0"
+          go-version-file: spinifex/go.mod
 
       - name: Build microVM artifacts (lb-agent + kernel + initramfs)
         run: cd spinifex && make build-microvm-image

.golangci.yml

@@ -1,5 +1,8 @@
 version: "2"
 
+run:
+  allow-serial-runners: true
+
 issues:
   max-same-issues: 10
 
@@ -142,14 +145,10 @@ linters:
     staticcheck:
       checks:
         - all
-        - -ST1000 # package comment
         - -ST1003 # naming conventions (Id→ID etc.) — too noisy to retrofit
         - -ST1012 # error var naming — errname covers this + error type suffixes
         - -ST1013 # HTTP status constants — usestdlibvars is broader
         - -ST1016 # receiver name consistency (too noisy to retrofit)
-        - -ST1020 # godoc on exported types — not enforcing yet
-        - -ST1021 # godoc on exported type groups — not enforcing yet
-        - -ST1022 # godoc on exported vars/consts — not enforcing yet
         - -SA4006 # value overwritten — ineffassign catches strictly more of these
         - -SA9005 # marshaling structs with no exported fields — intended {} for empty API-response types
 

Makefile

@@ -57,7 +57,7 @@ build-ui:
 	cd spinifex/services/spinifexui/frontend && pnpm build
 
 # GO commands
-VERSION ?= $(shell git describe --tags --always --dirty)
+VERSION ?= $(shell git describe --tags --always --dirty --exclude '*-dev')
 COMMIT  ?= $(shell git rev-parse --short HEAD)
 LDFLAGS := -s -w -X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Version=$(VERSION) -X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Commit=$(COMMIT)
 
@@ -153,8 +153,9 @@ install-microvm: $(MICROVM_ARTIFACTS) ## Install microVM artifacts to /usr/share
 # -j2 overlaps the only two heavy gates, lint and test-cover: alone each leaves
 # most of the machine idle, so pairing them cuts a cold run by about a quarter.
 # -Otarget keeps each gate's output in one block instead of interleaving them.
+# manifest-check manifest-lint test-package-check removed, can add back later
 preflight:
-	@$(MAKE) --no-print-directory -j2 -Otarget QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-package-check test-harness test-build-scripts
+	@$(MAKE) --no-print-directory -j2 -Otarget QUIET=1 lint govulncheck test-cover diff-coverage test-harness test-build-scripts
 	@echo -e "\n ✅ Preflight passed — safe to commit."
 
 # Shell suites + shellcheck for build/scripts/, the systemd-unit helpers that
@@ -166,7 +167,7 @@ preflight:
 # reason: between them they install the software, wire OVN, erase the state and
 # remove the install, so a shellcheck warning in one is an outage or a wrong
 # deletion, not a style note.
-NODE_SCRIPTS := scripts/setup.sh scripts/setup-ovn.sh scripts/node-reset.sh scripts/uninstall-spx.sh
+NODE_SCRIPTS := scripts/setup.sh scripts/setup-ovn.sh scripts/install-node.sh scripts/smoke-test.sh scripts/node-reset.sh scripts/uninstall-spx.sh
 
 test-build-scripts:
 	@echo -e "\n....Running build/scripts/**/*_test.sh...."
@@ -261,6 +262,17 @@ test-cover:
 	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -timeout 180s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... ./cmd/... ./internal/... $(_COVQ)
 	@scripts/check-coverage.sh $(COVERPROFILE) $(QUIET)
 
+# Refresh the README coverage badge. Measured as awesome-go documents
+# (-coverpkg=./... over ./...), with cmd/ entrypoints left out of the total.
+coverage-badge:
+	@echo -e "\n....Measuring whole-module coverage for the README badge...."
+	LOG_IGNORE=1 go test $(GOTESTFLAGS) -timeout 600s -covermode=atomic -coverpkg=./... -coverprofile=coverage-badge.out ./...
+	@grep -v '/cmd/' coverage-badge.out > coverage-badge-nocmd.out
+	@pct=$$(go tool cover -func=coverage-badge-nocmd.out | tail -1 | awk '{print $$NF}' | tr -d '%'); \
+	color=$$(awk -v p="$$pct" 'BEGIN { print (p >= 80) ? "brightgreen" : (p >= 70) ? "yellow" : "red" }'); \
+	sed -i -E "s|badge/coverage-[0-9.]+%25-[a-z]+|badge/coverage-$${pct}%25-$${color}|" README.md; \
+	echo "Coverage $${pct}% ($${color}); README badge updated"
+
 # Run unit tests with race detector
 test-race:
 	@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
@@ -317,6 +329,17 @@ bench:
 	@echo -e "\n....Running benchmarks for $(GO_PROJECT_NAME)...."
 	LOG_IGNORE=1 go test -benchmem -run=. -bench=. ./...
 
+# Fuzz every Fuzz* target for FUZZTIME each; -fuzz takes one target per run.
+# Plain `go test` already replays the seeds and testdata/fuzz on preflight.
+FUZZTIME ?= 30s
+fuzz:
+	@set -e; grep -rlE '^func Fuzz\w+\(' --include='*_test.go' spinifex cmd internal | while read -r file; do \
+		for name in $$(grep -oE '^func Fuzz\w+' "$$file" | cut -d' ' -f2); do \
+			echo -e "\n....Fuzzing $$name ($$(dirname $$file)) for $(FUZZTIME)...."; \
+			LOG_IGNORE=1 go test -run='^$$' -fuzz="^$$name$$" -fuzztime=$(FUZZTIME) ./$$(dirname $$file); \
+		done; \
+	done
+
 # Fast iteration: build + install binary + restart all services.
 # Microvm artifacts are reinstalled when they already exist on disk — the rule's
 # input timestamps drive a rebuild only if anything actually changed. On a fresh
@@ -452,7 +475,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: test-package-check build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
+.PHONY: test-package-check build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image install-microvm go_build preflight test test-cover coverage-badge test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck nilaway \

README.md

@@ -4,6 +4,8 @@
 
 <p align="center">
   <a href="https://go.dev"><img src="https://img.shields.io/badge/Go-1.27+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
+  <a href="https://pkg.go.dev/github.com/mulgadc/spinifex"><img src="https://img.shields.io/badge/pkg.go.dev-reference-007D9C?style=flat-square&logo=go&logoColor=white" alt="Go Reference"></a>
+  <a href="https://github.com/mulgadc/spinifex/blob/main/Makefile"><img src="https://img.shields.io/badge/coverage-82.9%25-brightgreen?style=flat-square" alt="Coverage"></a>
   <a href="LICENSE"><img src="https://img.shields.io/badge/License-AGPL--3.0-3fb950?style=flat-square" alt="License"></a>
   <a href="https://mulgadc.com"><img src="https://img.shields.io/badge/home-mulga-orange?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTE2LjcxOCA4Ljg5MWMtMS4yODUgMS4zNy0zLjE3OCAyLjMxNy00LjY5NSAzLjQ0LS44NTQuNjMtMy4wODkgMi4yNy0xLjYxIDMuMzQ0IDEuNzg4IDEuMjk4IDYuMjQzLjE2NCA3Ljc4NS0xLjI2Ljg0LS43NzUuODE0LTEuODIyLS4zMjgtMi4yNTktMS4yMzUtLjQ3Mi0yLjY2LS4xMTEtMy45MTMuMDg3LS4wNDIuMDA3LS4xMzEuMDU1LS4xMjMtLjAyLjUyMy0uMzM2Ljk5My0uNzUgMS41MDMtMS4xMDIuNDkyLS4zNCAxLjA5Ny0uODI3IDEuNy0uODM4IDEuODcxLS4wMzQgMy43OTkuODkgNC4yODcgMi44MTUuODExIDMuMjAzLTMuMDA2IDUuNzE1LTUuNzg0IDUuOTQybDEuNjE0LS43NDdjLjYwNS0uMzYgMS4yMTctLjczNCAxLjc1Mi0xLjE5Ni4xMzMtLjExNS4yMy0uMjYzLjM0Mi0uMzczLjAyNy0uMDI2LjMwMi0uMTQ0LjE0NC0uMTUtMS40NDEuOTQ1LTMuMTI3IDEuNTcyLTQuODEzIDEuOTMyLS41OC4xMjMtMS4xOTUuMjQyLTEuNzg1LjI1Ny4wMTUuMDguMDg3LjA2NC4xNC4wOC40NzYuMTU0IDEuMDIuMjQ1IDEuNTE2LjMyLjA2OC4wNTItLjAwNi4wNzQtLjA2LjA4MS0uNDQ4LjA1OC0uOTIzLjE0My0xLjM3LjE2Ni0xLjI3LjA2NC0yLjU1LS4wNjgtMy43NzctLjM4bC0uNjktMS45NTdjLS4wNi4wMDYtLjA2My4wNi0uMDc5LjEwMy0uMDYuMTctLjMwNSAxLjQxNy0uMzg3IDEuNDM1LS42Ni0uMi0xLjI1MS0uNjQ3LTEuNzM2LTEuMTMybDEuMTUtMi4wNi0xLjgzNSAxLjAxMWMtLjI1OC0uNTgzLS4zMDUtMS4yNDMtLjIxOS0xLjg3bDIuMzc1LTEuMThjLS43MzktLjA0OS0xLjQ4Ni4wNjgtMi4yMi4xNDUuMTI1LS4zNi4yNzctLjcxOC40NjItMS4wNTEuMDY3LS4xMjIuNDI2LS43MjEuNTItLjcyNC44Ni4xNjQgMS43ODYuMjQxIDIuNjQ4LjA3NGwtMS44Ny0uNzcxLS4wNjktLjA5M2MuMzgyLS4zNi43Ny0uNzE3IDEuMTk5LTEuMDI0LjgwNC4yMzQgMS42My40MjIgMi40NzMuNDMzTDkuMzUgOS4zNDJhNyA3IDAgMCAxIC41NjctLjQwMWMuMTYtLjEwMy42MDktLjQwMi43NzItLjM4LjY1LjIyMSAxLjMyNC4zNzggMi4wMS40MzMtLjMxMy0uMzI1LS45MDktLjU2NC0xLjIxLS44NjgtLjAzMy0uMDM0LS4wNTgtLjAzNi0uMDQzLS4wOTguNzg3LS40NTIgMS42NjItLjkwMSAyLjM0LTEuNTE3LjY5LS42MjkgMS40MjEtMS42NTYuOTI0LTIuNjA1LjU3My4xODIgMS4wNjcuOTA2IDEuMDU0IDEuNTEyLS4wMzQgMS42NzYtMS44MjIgMy4xNy0yLjk0MyA0LjIyMiAxLjczMi0uNzI4IDMuNzE0LTIuMjMgMy43MS00LjMwNS0uMDAzLTEuNjg4LTEuNTQtMi4zNjUtMi45OTMtMi41MThhMy4yIDMuMiAwIDAgMS0uMzg1IDEuMDg3Yy0uNDI4LjcxOS0xLjMwMiAxLjE2OC0xLjc1OCAxLjkxNS0uMzExLjUxLS4zNyAxLjE5NS0xLjAzMSAxLjM5LS4yNy4wOC0uNjE3LjA5My0uODk3LjA5NGwuNjE1LS4zMjVjLjY4OS0uNTA2LjY3Ny0xLjQyIDEuMDk5LTIuMS0uMDUtLjA3LS41NDUtLjItLjY2LS4yMjktLjUxLS4xMjUtMS4zNzUtLjI4NS0xLjg4Ni0uMjUtLjE1Ny4wMS0uODY5LjEyMS0uOTMuMjQyLS4wODguMTc3LjM0MS45My40NjggMS4xMDEuMDI1LjAzNS4wNzcuMDI2LjA4LjAzLjAyNC4wMzEuMDA3LjEwNS0uMDYuMDhhNSA1IDAgMCAxLS40MDQtLjI4M2MtLjUwMy0uMzg4LTEuMDc4LS45NzQtMS40OTktMS40NDctLjA2OC0uMTY2LS4xNi0uMzUuMDAyLS40ODcuODg4LS41NSAxLjc2OC0xLjEyNiAyLjY3LTEuNjUxIDIuNjcxLTEuNTU4IDUuNzExLTMuMzE4IDguMjY3LS40NjUgMi4xMTkgMi4zNjYgMS41MTQgNS4yMTItLjUxMiA3LjM3MnptLTguMjI0LTUuMzhjLjY5OS0uMTIyIDIuMDE4LjU3MiAyLjIzNS0uNDU2LjAyMS0uMS0uMDM2LS4xNTcuMDItLjI1NS4wNDMtLjA3My4yODYtLjI1LjM3LS4zMTguMjctLjIxNy41NzctLjM4Ny44NDItLjYxMi0xLjAyOS4xNjYtMi4wNjUuNzU2LTIuOTY0IDEuMjc3LS4xNDkuMDg2LS4zMS4xNzYtLjQ1MS4yNzMtLjAzNy4wMjUtLjA3NS0uMDA2LS4wNTMuMDltLTQuODMgMTAuODQ0Yy0xLjcxNSAxLjg4MS0xLjMyNSA0LjU3LjQ5NCA2LjIxNyAxLjgxMSAxLjY0MSA0LjY2IDIuMjIzIDcuMDQ4IDIuMTA4IDIuMTUyLS4xMDMgNC4zMzctLjgxMiA2LjQ2LS4wOTEuODI1LjI4IDEuNTQ2LjgwNSAyLjE2IDEuNDExLS4wODMtLjQtLjMyNC0uODE5LS41NTgtMS4xNTktMi4xMy0zLjA5NS02LjI3LTEuOTM1LTkuNDI2LTIuNTUzLTEuNzExLS4zMzUtMy40OTEtMS4xMTgtNC41MzMtMi41NjctLjkwMS0xLjI1My0xLjA0Mi0yLjczLS41OTUtNC4xOTQtLjA5MS0uMDktLjk1Mi43MjEtMS4wNS44MjhtNi4zNjYtOC42NjdjLS4zODIuMzM5LS43ODcuNjYtMS4yMTIuOTQ4bC0xLjIwNy41OWMxLjAyMi4wMzkgMi4wOC0uNTQ2IDIuNDItMS41MzciLz48L3N2Zz4=" alt="mulgadc.com"></a>
   <a href="https://docs.mulgadc.com"><img src="https://img.shields.io/badge/docs-spinifex-0969da?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTYuNSAySDIwYTEgMSAwIDAgMSAxIDF2MTQuNWExIDEgMCAwIDEtMSAxSDYuNWExLjUgMS41IDAgMCAwIDAgM0gyMWExIDEgMCAwIDEgMCAySDYuNUEzLjUgMy41IDAgMCAxIDMgMjBWNS41QTMuNSAzLjUgMCAwIDEgNi41IDJ6bTEgNGExIDEgMCAwIDAgMCAyaDlhMSAxIDAgMCAwIDAtMmgtOXptMCA0YTEgMSAwIDAgMCAwIDJoOWExIDEgMCAwIDAgMC0yaC05eiIvPjwvc3ZnPg==" alt="docs.mulgadc.com"></a>

cmd/installer/install/install.go

@@ -870,7 +870,7 @@ func fireInstallCallback() {
 		return
 	}
 	client := &http.Client{Timeout: 10 * time.Second}
-	resp, err := client.Get(url) //nolint:noctx // installer has no context; best-effort fire-and-forget
+	resp, err := client.Get(url)
 	if err != nil {
 		slog.Warn("fireInstallCallback: request failed", "url", url, "err", err)
 		return

cmd/rds-agent/agent.go

@@ -220,7 +220,8 @@ func newAgent(cfg config, cp controlPlane, probe *engineProbe) (*Agent, error) {
 	return a, nil
 }
 
-// Does not wait for IMDS: the register loop rides out a datapath still coming up.
+// New builds the agent from cfg, failing if no gateway URL is set. It does not
+// wait for IMDS: the register loop rides out a datapath still coming up.
 func New(cfg config) (*Agent, error) {
 	if cfg.GatewayURL == "" {
 		return nil, fmt.Errorf("no gateway URL configured (RDS_GATEWAY_URL)")
@@ -365,16 +366,16 @@ func unescapeMountField(field string) string {
 
 // Boot-retry bounds. Tight to start, since the usual cause is a control plane
 // still creating this instance's record, and capped so an outage does not turn
-// a booting fleet into a retry storm.
-const (
+// a booting fleet into a retry storm. Vars so tests can shorten the wait.
+var (
 	retryMin = 1 * time.Second
 	retryMax = 30 * time.Second
-
-	// With the production backoff, attempt five is about 15 seconds into the
-	// failure and still precedes the first 30-second heartbeat.
-	retryErrorAttempt = 5
 )
 
+// With the production backoff, attempt five is about 15 seconds into the
+// failure and still precedes the first 30-second heartbeat.
+const retryErrorAttempt = 5
+
 // Wrapping an error in this stops retryObserved. For the failures the control
 // plane decides from its own record, no retry can change the answer, and looping
 // on one only keeps the caller blocked.

cmd/rds-agent/engine_mariadb_test.go

@@ -948,8 +948,8 @@ func TestMariaDBProbe_LiveButUnreachableEngineResetsTheRollbackDeadline(t *testi
 	recovery := &fakeRecovery{restored: true}
 	guard := newTestGuard(recovery, probe)
 	// Bounded so the deadline-resetting loop cannot run forever if it stops
-	// resetting; the assertion is that the rollback was never reached.
-	ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond)
+	// resetting; four rollback budgets, so a missed reset would roll back.
+	ctx, cancel := context.WithTimeout(t.Context(), 80*time.Millisecond)
 	defer cancel()
 	guard.Run(ctx)
 

cmd/rds-agent/main_test.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"os"
+	"testing"
+	"time"
+)
+
+// TestMain shortens the boot-retry backoff for the whole package: the tests
+// assert which calls are retried, not how long the agent waits between them.
+func TestMain(m *testing.M) {
+	retryMin, retryMax = time.Millisecond, time.Millisecond
+	os.Exit(m.Run())
+}

cmd/rds-agent/paramrecovery_test.go

@@ -460,8 +460,8 @@ func TestParamGuard_LeavesARecoveringEngineAlone(t *testing.T) {
 	engine := &fakeRecovery{restored: true}
 	guard := newTestGuard(engine, stubProbe(t, 1, nil))
 	// Bounded so the deadline-resetting loop cannot run forever if it stops
-	// resetting; the assertion is that the rollback was never reached.
-	ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond)
+	// resetting; four rollback budgets, so a missed reset would roll back.
+	ctx, cancel := context.WithTimeout(t.Context(), 80*time.Millisecond)
 	defer cancel()
 	guard.Run(ctx)
 

cmd/spinifex/cmd/admin.go

@@ -425,7 +425,7 @@ func init() {
 	adminJoinCmd.MarkFlagRequired("host")
 	adminJoinCmd.MarkFlagRequired("token")
 
-	imagesImportCmd.Flags().String("tmp-dir", os.TempDir(), "Temporary directory for image import processing")
+	imagesImportCmd.Flags().String("tmp-dir", "", "Scratch directory for extracting the image (default: <spinifex-dir>/images); must be disk-backed, not tmpfs")
 
 	imagesImportCmd.Flags().String("name", "", "Import specified image by name")
 	imagesImportCmd.Flags().String("ami-name", "", "Override the registered AMI name (DescribeImages name). Defaults to ami-{distro}-{version}-{arch}. Use for locally-built appliances (e.g. --ami-name spinifex-eks-node).")
@@ -585,7 +585,6 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	var image utils.Images
 
 	var imageFile string
-	var imageStat os.FileInfo
 	var err error
 
 	forceCmd, _ := cmd.Flags().GetBool("force")
@@ -753,25 +752,59 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	// Next, validate if the image is raw, tar, gz, xv, etc. We need to upload the raw image
-	tmpDir, err := os.MkdirTemp(ostmpDir, "spinifex-image-tmp-*")
+	// An empty --tmp-dir scratches beside the catalog: disk-backed, unlike a
+	// tmpfs /tmp, and already root:spinifex 0770.
+	scratchRoot := ostmpDir
+	if scratchRoot == "" {
+		scratchRoot = imageDir
+	}
 
+	extractedImagePath, cleanupScratch, err := extractToScratch(imageFile, scratchRoot)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Could not create temp dir: %v\n", err)
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
 		os.Exit(1)
 	}
 
-	extractedImagePath, err := utils.ExtractDiskImageFromFile(imageFile, imagePath)
+	// os.Exit skips deferred calls, so scratch is removed before any exit.
+	volumeId, err := registerExtractedImage(image, amiNameOverride, imagePath, extractedImagePath, sourceDigest)
+	cleanupScratch()
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Could not extract image: %v\n", err)
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
 		os.Exit(1)
 	}
 
-	imageStat, err = os.Stat(extractedImagePath)
+	fmt.Printf("✅ Image import complete. Image-ID (AMI): %s\n", volumeId)
+	fmt.Printf("✅ Source digest: %s:%s (%s)\n", sourceDigest.Algorithm, sourceDigest.Value, sourceDigest.Verification)
+}
 
+// extractToScratch extracts imageFile into a fresh hidden directory under
+// scratchRoot and returns the extracted path with a cleanup that removes that
+// directory. On error the directory is already removed.
+func extractToScratch(imageFile, scratchRoot string) (string, func(), error) {
+	scratchDir, err := os.MkdirTemp(scratchRoot, ".import-*")
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Could not stat image: %v\n", err)
-		os.Exit(1)
+		return "", nil, fmt.Errorf("create scratch dir under %s: %w", scratchRoot, err)
+	}
+	cleanup := func() {
+		if err := os.RemoveAll(scratchDir); err != nil {
+			fmt.Fprintf(os.Stderr, "Warning: could not remove scratch dir %s: %v\n", scratchDir, err)
+		}
+	}
+
+	extracted, err := utils.ExtractDiskImageFromFile(imageFile, scratchDir)
+	if err != nil {
+		cleanup()
+		return "", nil, fmt.Errorf("extract image: %w", err)
+	}
+	return extracted, cleanup, nil
+}
+
+// registerExtractedImage writes the AMI manifest into imagePath, imports
+// extractedImagePath into storage and registers the AMI, returning its ID.
+func registerExtractedImage(image utils.Images, amiNameOverride, imagePath, extractedImagePath string, sourceDigest ebsmetadata.ImageDigest) (string, error) {
+	imageStat, err := os.Stat(extractedImagePath)
+	if err != nil {
+		return "", fmt.Errorf("stat image: %w", err)
 	}
 
 	// Describe the image/AMI as the control-plane document it will become.
@@ -810,24 +843,19 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	// Save as JSON
 	jsonData, err := json.Marshal(ami)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Could not marshal manifest: %v\n", err)
-		os.Exit(1)
+		return "", fmt.Errorf("marshal manifest: %w", err)
 	}
 
 	manifestFilename := fmt.Sprintf("%s/%s.json", imagePath, ami.Name)
 	// Write to file
 	err = os.WriteFile(manifestFilename, jsonData, 0600)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Could not write manifest: %v\n", err)
-		os.Exit(1)
+		return "", fmt.Errorf("write manifest: %w", err)
 	}
 
-	defer os.RemoveAll(tmpDir)
-
 	appConfig, nc, err := loadConfigAndConnect()
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Could not connect to the cluster: %v\n", err)
-		os.Exit(1)
+		return "", fmt.Errorf("connect to the cluster: %w", err)
 	}
 	defer nc.Close()
 	node := appConfig.Nodes[appConfig.Node]
@@ -844,8 +872,7 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 		Progress:         os.Stdout,
 	})
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Could not import image: %v\n", err)
-		os.Exit(1)
+		return "", fmt.Errorf("import image: %w", err)
 	}
 
 	// admin.ImportImage only wrote the provider's half of the snapshot (its
@@ -853,8 +880,7 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	// DescribeSnapshots and GetAMISourceVolumeID cannot resolve the import.
 	metaStore := objectstore.NewS3ObjectStoreFromConfig(node.Predastore.Host, node.Predastore.Region, node.Predastore.AccessKey, node.Predastore.SecretKey)
 	if err := registerImportedAMISnapshot(metaStore, node.Predastore.Bucket, ami, node.AZ, node.Viperblock.EncryptionKeyFile != ""); err != nil {
-		fmt.Fprintf(os.Stderr, "Imported %s but could not register its snapshot: %v\n", volumeId, err)
-		os.Exit(1)
+		return "", fmt.Errorf("imported %s but could not register its snapshot: %w", volumeId, err)
 	}
 
 	// The document is what DescribeImages enumerates, so it is written last:
@@ -862,12 +888,10 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	// partway leaves no half-registered image behind.
 	ami.State = "available"
 	if err := ebsmetadata.NewStore(metaStore, node.Predastore.Bucket).PutAMI(context.Background(), ami); err != nil {
-		fmt.Fprintf(os.Stderr, "Imported %s but could not register it: %v\n", volumeId, err)
-		os.Exit(1)
+		return "", fmt.Errorf("imported %s but could not register it: %w", volumeId, err)
 	}
 
-	fmt.Printf("✅ Image import complete. Image-ID (AMI): %s\n", volumeId)
-	fmt.Printf("✅ Source digest: %s:%s (%s)\n", sourceDigest.Algorithm, sourceDigest.Value, sourceDigest.Verification)
+	return volumeId, nil
 }
 
 // registerImportedAMISnapshot writes the EC2 control plane's snapshot document
@@ -2970,7 +2994,6 @@ func runAccountCreate(cmd *cobra.Command, args []string) {
 			endpointHost = h
 		}
 	}
-	endpointURL := "https://" + net.JoinHostPort(endpointHost, "9999")
 
 	credPath := filepath.Join(homeDir, ".aws", "credentials")
 	configPath := filepath.Join(homeDir, ".aws", "config")
@@ -2985,12 +3008,7 @@ func runAccountCreate(cmd *cobra.Command, args []string) {
 	if region == "" {
 		region = "ap-southeast-2"
 	}
-	if err := admin.UpdateAWSINIFile(configPath, "profile "+profileName, map[string]string{
-		"region":       region,
-		"endpoint_url": endpointURL,
-		"ca_bundle":    certPath,
-		"output":       "json",
-	}); err != nil {
+	if err := admin.WriteAWSProfileConfig(configPath, profileName, region, certPath, endpointHost); err != nil {
 		fmt.Fprintf(os.Stderr, "Warning: could not update AWS config: %v\n", err)
 	}
 
@@ -3499,11 +3517,11 @@ func resolvePublicPoolFlags(source, poolRange, bindBridge, gateway, defaultBindB
 		if gateway == "" {
 			return "", "", "", "", fmt.Errorf("--external-gateway is required with --external-source=static")
 		}
-		parts := strings.SplitN(poolRange, "-", 2)
-		if len(parts) != 2 || net.ParseIP(parts[0]) == nil || net.ParseIP(parts[1]) == nil {
+		var ok bool
+		start, end, ok = strings.Cut(poolRange, "-")
+		if !ok || net.ParseIP(start) == nil || net.ParseIP(end) == nil {
 			return "", "", "", "", fmt.Errorf("--external-pool must be start-end IPs (e.g., 192.168.1.150-192.168.1.250), got: %s", poolRange)
 		}
-		start, end = parts[0], parts[1]
 	default:
 		return "", "", "", "", fmt.Errorf("--external-source must be 'static' or 'dhcp', got: %s", source)
 	}

cmd/spinifex/cmd/admin_account_quota.go

@@ -3,7 +3,8 @@ package cmd
 import (
 	"context"
 	"fmt"
-	"sort"
+	"maps"
+	"slices"
 	"strconv"
 
 	"github.com/mulgadc/spinifex/spinifex/gateway"
@@ -166,11 +167,7 @@ func printAccountQuota(resp *gateway.AccountQuotaResponse) {
 	fmt.Printf("%-16s %10s  %s\n", "DIMENSION", "LIMIT", "SOURCE")
 	fmt.Printf("%-16s %10s  %s\n", "---------", "-----", "------")
 
-	names := make([]string, 0, len(resp.Limits))
-	for name := range resp.Limits {
-		names = append(names, name)
-	}
-	sort.Strings(names)
+	names := slices.Sorted(maps.Keys(resp.Limits))
 	for _, name := range names {
 		limit := strconv.Itoa(resp.Limits[name])
 		if resp.Limits[name] == handlers_quota.Unlimited {

cmd/spinifex/cmd/admin_images_import_test.go

@@ -0,0 +1,117 @@
+package cmd
+
+import (
+	"bytes"
+	"compress/gzip"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func readRawDiskFixture(t *testing.T) []byte {
+	t.Helper()
+	data, err := os.ReadFile(filepath.Join("..", "..", "..", "tests", "unit-test-disk-image.raw"))
+	require.NoError(t, err)
+	return data
+}
+
+func gzipData(t *testing.T, data []byte) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	gw := gzip.NewWriter(&buf)
+	_, err := gw.Write(data)
+	require.NoError(t, err)
+	require.NoError(t, gw.Close())
+	return buf.Bytes()
+}
+
+func qcow2Data(t *testing.T, raw []byte) []byte {
+	t.Helper()
+	if _, err := exec.LookPath("qemu-img"); err != nil {
+		t.Skip("qemu-img not found")
+	}
+	dir := t.TempDir()
+	rawPath := filepath.Join(dir, "src.raw")
+	qcowPath := filepath.Join(dir, "src.qcow2")
+	require.NoError(t, os.WriteFile(rawPath, raw, 0o600))
+	out, err := exec.Command("qemu-img", "convert", "-f", "raw", "-O", "qcow2", rawPath, qcowPath).CombinedOutput()
+	require.NoError(t, err, string(out))
+	data, err := os.ReadFile(qcowPath)
+	require.NoError(t, err)
+	return data
+}
+
+func dirNames(t *testing.T, dir string) []string {
+	t.Helper()
+	entries, err := os.ReadDir(dir)
+	require.NoError(t, err)
+	names := make([]string, 0, len(entries))
+	for _, e := range entries {
+		names = append(names, e.Name())
+	}
+	sort.Strings(names)
+	return names
+}
+
+// The catalog dir doubles as the download cache, so the source lives there
+// as it does for a --name import, with the images dir as the scratch root.
+func TestExtractToScratch_LeavesNoIntermediateInCatalog(t *testing.T) {
+	raw := readRawDiskFixture(t)
+
+	cases := []struct {
+		name    string
+		source  string
+		payload func(t *testing.T) []byte
+		wantErr string
+	}{
+		{
+			name:    "gzip raw is decompressed into scratch",
+			source:  "disk.raw.gz",
+			payload: func(t *testing.T) []byte { return gzipData(t, raw) },
+		},
+		{
+			name:    "qcow2 is converted into scratch",
+			source:  "disk.qcow2",
+			payload: func(t *testing.T) []byte { return qcow2Data(t, raw) },
+		},
+		{
+			name:    "intermediate that fails the disk sniff is removed",
+			source:  "disk.raw.gz",
+			payload: func(t *testing.T) []byte { return gzipData(t, []byte("not a disk image")) },
+			wantErr: "extract image",
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			imagesDir := t.TempDir()
+			catalogDir := filepath.Join(imagesDir, "debian", "13", "x86_64")
+			require.NoError(t, os.MkdirAll(catalogDir, 0o770))
+			source := filepath.Join(catalogDir, tc.source)
+			require.NoError(t, os.WriteFile(source, tc.payload(t), 0o600))
+
+			extracted, cleanup, err := extractToScratch(source, imagesDir)
+
+			if tc.wantErr != "" {
+				require.ErrorContains(t, err, tc.wantErr)
+				assert.Nil(t, cleanup)
+			} else {
+				require.NoError(t, err)
+				scratchDir := filepath.Dir(extracted)
+				assert.Equal(t, imagesDir, filepath.Dir(scratchDir))
+				assert.True(t, strings.HasPrefix(filepath.Base(scratchDir), ".import-"), "extracted into %s", extracted)
+				assert.FileExists(t, extracted)
+				cleanup()
+			}
+
+			assert.Equal(t, []string{"debian"}, dirNames(t, imagesDir), "scratch dir must be removed")
+			assert.Equal(t, []string{tc.source}, dirNames(t, catalogDir), "catalog must hold only the source")
+		})
+	}
+}

cmd/spinifex/cmd/admin_jsprobe_test.go

@@ -61,6 +61,7 @@ func TestCanaryRoundTrip_WritesADistinctValueEachRun(t *testing.T) {
 // A deleted bucket stands in for the unreachable-store case: the probe must
 // report an error rather than treating a failed write as success.
 func TestCanaryRoundTrip_FailsWhenBucketIsGone(t *testing.T) {
+	t.Parallel()
 	kv, js := newCanaryBucket(t)
 
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)

cmd/spinifex/cmd/admin_kv.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
+	"maps"
 	"os"
 	"path/filepath"
 	"slices"
@@ -420,11 +421,7 @@ func hashStoredMsg(m *jetstream.RawStreamMsg) string {
 	field([]byte(m.Subject))
 	binary.BigEndian.PutUint64(n[:], uint64(m.Time.UnixNano()))
 	h.Write(n[:])
-	keys := make([]string, 0, len(m.Header))
-	for k := range m.Header {
-		keys = append(keys, k)
-	}
-	sort.Strings(keys)
+	keys := slices.Sorted(maps.Keys(m.Header))
 	for _, k := range keys {
 		field([]byte(k))
 		for _, v := range m.Header[k] {

cmd/spinifex/cmd/admin_kv_compare.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"maps"
 	"os"
 	"slices"
 	"sort"
@@ -153,11 +154,7 @@ func compareNodeDigests(nodes []nodeDigest) (kvCompareResult, error) {
 			byName[d.Name] = append(byName[d.Name], v)
 		}
 	}
-	names := make([]string, 0, len(byName))
-	for name := range byName {
-		names = append(names, name)
-	}
-	sort.Strings(names)
+	names := slices.Sorted(maps.Keys(byName))
 
 	var res kvCompareResult
 	for _, name := range names {

cmd/spinifex/cmd/admin_kv_test.go

@@ -215,11 +215,15 @@ func TestKVDigestClusteredReplicas(t *testing.T) {
 	js, err := jetstream.New(nc)
 	require.NoError(t, err)
 
+	// Each attempt gets its own deadline: before a meta leader is elected a
+	// create blocks for the client's default timeout instead of failing fast.
 	var kv jetstream.KeyValue
 	require.Eventually(t, func() bool {
-		kv, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "rep", History: 3, Replicas: n, TTL: time.Hour})
+		attemptCtx, cancel := context.WithTimeout(ctx, 250*time.Millisecond)
+		defer cancel()
+		kv, err = js.CreateKeyValue(attemptCtx, jetstream.KeyValueConfig{Bucket: "rep", History: 3, Replicas: n, TTL: time.Hour})
 		return err == nil
-	}, 30*time.Second, 250*time.Millisecond, "clustered KV create: %v", err)
+	}, 30*time.Second, 20*time.Millisecond, "clustered KV create: %v", err)
 	for i := range 5 {
 		_, err := kv.PutString(ctx, fmt.Sprintf("k%d", i%2), fmt.Sprintf("v%d", i))
 		require.NoError(t, err)
@@ -279,10 +283,12 @@ func TestKVDigestIgnoresExpiry(t *testing.T) {
 	require.NoError(t, err)
 	js, err := jetstream.New(nc)
 	require.NoError(t, err)
-	kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "ttl", TTL: time.Second})
+	const ttl = 200 * time.Millisecond
+	kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "ttl", TTL: ttl})
 	require.NoError(t, err)
 	_, err = kv.PutString(ctx, "k", "v")
 	require.NoError(t, err)
+	putAt := time.Now()
 
 	work := t.TempDir()
 	_, metas, err := copyStreamTree(filepath.Join(parent, "jetstream"), filepath.Join(work, "jetstream"))
@@ -291,7 +297,9 @@ func TestKVDigestIgnoresExpiry(t *testing.T) {
 	ns.Shutdown()
 	ns.WaitForShutdown()
 
-	time.Sleep(1500 * time.Millisecond)
+	// The server stamped the message before the put returned, so it is now
+	// older than the TTL.
+	time.Sleep(time.Until(putAt.Add(ttl + 50*time.Millisecond)))
 	d, err := digestStore(ctx, work, nil, false, metas)
 	require.NoError(t, err)
 	require.Len(t, d, 1)

cmd/spinifex/cmd/admin_ochre.go

@@ -438,14 +438,10 @@ func parseWeightsS3URI(uri string) (bucket, prefix string, err error) {
 	if trimmed == uri {
 		return "", "", fmt.Errorf("invalid --s3-uri %q: expected s3://bucket/prefix", uri)
 	}
-	parts := strings.SplitN(trimmed, "/", 2)
-	bucket = parts[0]
+	bucket, prefix, _ = strings.Cut(trimmed, "/")
 	if bucket == "" {
 		return "", "", fmt.Errorf("invalid --s3-uri %q: missing bucket", uri)
 	}
-	if len(parts) == 2 {
-		prefix = parts[1]
-	}
 	if prefix != "" && !strings.HasSuffix(prefix, "/") {
 		prefix += "/"
 	}

cmd/spinifex/cmd/admin_ochre_vector.go

@@ -283,11 +283,9 @@ func truncateForTable(s string, maxRunes int) string {
 }
 
 // formatQueryResults renders 'ochre vector query' output: a truncated table
-// by default, or the full response as indented JSON when asJSON is set
-// (D10's "full text via source ref" -- here, via --json).
+// by default, or the full response as indented JSON when asJSON is set.
 func formatQueryResults(results []handlers_ochrevector.QueryResult, asJSON bool) (string, error) {
 	if asJSON {
-		//nolint:musttag // QueryResult is an internal, non-wire Go type (predates this CLI); it round-trips fine on its default field names.
 		data, err := json.MarshalIndent(results, "", "  ")
 		if err != nil {
 			return "", fmt.Errorf("encode results as JSON: %w", err)
@@ -317,7 +315,6 @@ func parseFilterFlag(raw string) (*handlers_ochrevector.Filter, error) {
 		return nil, nil
 	}
 	var f handlers_ochrevector.Filter
-	//nolint:musttag // Filter is the existing filter-AST type (D9); it round-trips fine on its default field names.
 	if err := json.Unmarshal([]byte(raw), &f); err != nil {
 		return nil, fmt.Errorf("decode --filter: %w", err)
 	}

cmd/spinifex/cmd/admin_principal.go

@@ -188,13 +188,14 @@ func describePrincipal(svc handlers_iam.IAMService, accountID, userName string)
 }
 
 // principalPolicyActions returns the actions an inline policy allows, shortened
-// to the method name. An undecodable document reports nothing rather than
-// guessing, and the listing shows the user with no grants.
+// to the method name. A NotAction grant is one entry naming what it excludes.
+// An undecodable document reports nothing rather than guessing.
 func principalPolicyActions(document string) []string {
 	var doc struct {
 		Statement []struct {
-			Effect string                   `json:"Effect"`
-			Action handlers_iam.StringOrArr `json:"Action"`
+			Effect    string                   `json:"Effect"`
+			Action    handlers_iam.StringOrArr `json:"Action"`
+			NotAction handlers_iam.StringOrArr `json:"NotAction"`
 		} `json:"Statement"`
 	}
 	if err := json.Unmarshal([]byte(document), &doc); err != nil {
@@ -209,6 +210,13 @@ func principalPolicyActions(document string) []string {
 		for _, action := range stmt.Action {
 			actions = append(actions, strings.TrimPrefix(action, adminPrincipalService+":"))
 		}
+		if len(stmt.NotAction) > 0 {
+			excluded := make([]string, 0, len(stmt.NotAction))
+			for _, action := range stmt.NotAction {
+				excluded = append(excluded, strings.TrimPrefix(action, adminPrincipalService+":"))
+			}
+			actions = append(actions, "NotAction("+strings.Join(excluded, ", ")+")")
+		}
 	}
 	sort.Strings(actions)
 	return actions

cmd/spinifex/cmd/admin_principal_test.go

@@ -102,6 +102,18 @@ func TestPrincipalPolicyActionsRoundTrip(t *testing.T) {
 		cmd.PrincipalPolicyActions(cmd.PrincipalPolicyDocument(grants)))
 }
 
+// A NotAction grant covers everything but its list, so rendering only Action
+// would show the principal as holding nothing.
+func TestPrincipalPolicyActionsRenderNotAction(t *testing.T) {
+	document := `{"Version":"2012-10-17","Statement":[
+		{"Effect":"Allow","NotAction":["spinifex:DeleteAccount","spinifex:CreateAccount"],"Resource":"*"},
+		{"Effect":"Allow","Action":"spinifex:ListAccounts","Resource":"*"},
+		{"Effect":"Deny","NotAction":"spinifex:ListAccounts","Resource":"*"}]}`
+
+	assert.Equal(t, []string{"ListAccounts", "NotAction(DeleteAccount, CreateAccount)"},
+		cmd.PrincipalPolicyActions(document))
+}
+
 // A document nobody can parse is reported as no grants rather than as a guess.
 func TestPrincipalPolicyActionsIgnoreAnUndecodableDocument(t *testing.T) {
 	assert.Empty(t, cmd.PrincipalPolicyActions("{not json"))

cmd/spinifex/cmd/cluster_test.go

@@ -35,26 +35,29 @@ func fakeDrainResponder(t *testing.T, nc *nats.Conn, released int, errMsg string
 }
 
 func TestDrainDHCPLeasesSumsReleased(t *testing.T) {
+	t.Parallel()
 	_, nc, _ := testutil.StartTestJetStream(t)
 	fakeDrainResponder(t, nc, 7, "")
 
-	released, responders := drainDHCPLeases(nc, 2*time.Second)
+	released, responders := drainDHCPLeases(nc, 250*time.Millisecond)
 	assert.Equal(t, 7, released)
 	assert.Equal(t, 1, responders)
 }
 
 func TestDrainDHCPLeasesReportsResponderError(t *testing.T) {
+	t.Parallel()
 	_, nc, _ := testutil.StartTestJetStream(t)
 	fakeDrainResponder(t, nc, 0, "list leases for drain: boom")
 
 	// An erroring responder still counts as a responder but contributes
 	// zero released leases.
-	released, responders := drainDHCPLeases(nc, 2*time.Second)
+	released, responders := drainDHCPLeases(nc, 250*time.Millisecond)
 	assert.Equal(t, 0, released)
 	assert.Equal(t, 1, responders)
 }
 
 func TestDrainDHCPLeasesNoResponders(t *testing.T) {
+	t.Parallel()
 	_, nc, _ := testutil.StartTestJetStream(t)
 
 	// No vpcd subscribed: the collection window elapses with zero replies.
@@ -281,7 +284,7 @@ func captureSlog(t *testing.T, fn func()) string {
 // at WARN level, and names only the running instance, not the stopped one.
 func TestWarnIfGuestsLeftRunningNamesRunningGuests(t *testing.T) {
 	origTimeout := guestEnumerationTimeout
-	guestEnumerationTimeout = 200 * time.Millisecond
+	guestEnumerationTimeout = 50 * time.Millisecond
 	t.Cleanup(func() { guestEnumerationTimeout = origTimeout })
 
 	_, nc, _ := testutil.StartTestJetStream(t)
@@ -312,7 +315,7 @@ func TestWarnIfGuestsLeftRunningNamesRunningGuests(t *testing.T) {
 // running guests produces no warning.
 func TestWarnIfGuestsLeftRunningNoGuestsIsSilent(t *testing.T) {
 	origTimeout := guestEnumerationTimeout
-	guestEnumerationTimeout = 200 * time.Millisecond
+	guestEnumerationTimeout = 50 * time.Millisecond
 	t.Cleanup(func() { guestEnumerationTimeout = origTimeout })
 
 	_, nc, _ := testutil.StartTestJetStream(t)
@@ -510,7 +513,7 @@ func TestCollectLocalShutdownACKExhaustedReturnsWithinBudget(t *testing.T) {
 // must log at ERROR, not WARN, and still name only the running instance.
 func TestReportGuestsLeftRunningSevereLogsErrorAndNamesGuests(t *testing.T) {
 	origTimeout := guestEnumerationTimeout
-	guestEnumerationTimeout = 200 * time.Millisecond
+	guestEnumerationTimeout = 50 * time.Millisecond
 	t.Cleanup(func() { guestEnumerationTimeout = origTimeout })
 
 	_, nc, _ := testutil.StartTestJetStream(t)

cmd/spinifex/cmd/export_test.go

@@ -29,6 +29,13 @@ func SetFirewallApplyHelper(path string) func() {
 // InitCLILogger exposes initCLILogger for testing.
 var InitCLILogger = initCLILogger
 
+// OutputStylingEnabled, IsTerminal and SetOutputStyling expose the spx colour guard.
+var (
+	OutputStylingEnabled = outputStylingEnabled
+	IsTerminal           = isTerminal
+	SetOutputStyling     = setOutputStyling
+)
+
 // CLILogLevel exposes the CLI logger's level var for testing.
 var CLILogLevel = cliLogLevel
 

cmd/spinifex/cmd/root.go

@@ -1,13 +1,18 @@
+// Package cmd defines the spx command-line interface: the cobra commands that
+// install, configure, start and administer a Spinifex node and its cluster.
 package cmd
 
 import (
 	"fmt"
+	"io"
 	"log/slog"
 	"os"
 
 	"github.com/mulgadc/spinifex/spinifex/config"
+	"github.com/pterm/pterm"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+	"golang.org/x/term"
 )
 
 var (
@@ -50,6 +55,8 @@ func Execute() {
 	cliLogLevel.Set(slog.LevelError)
 	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: cliLogLevel})))
 
+	setOutputStyling(outputStylingEnabled(os.Getenv("NO_COLOR"), os.Getenv("TERM"), isTerminal(os.Stdout)))
+
 	err := rootCmd.Execute()
 	if err != nil {
 		fmt.Fprintln(os.Stderr, err)
@@ -57,6 +64,28 @@ func Execute() {
 	}
 }
 
+// outputStylingEnabled reports whether spx may emit ANSI colour and text styling:
+// only on a terminal that is not TERM=dumb, with NO_COLOR unset or empty.
+func outputStylingEnabled(noColor, termName string, stdoutIsTerminal bool) bool {
+	return noColor == "" && termName != "dumb" && stdoutIsTerminal
+}
+
+// isTerminal reports whether w is a file attached to a terminal.
+func isTerminal(w io.Writer) bool {
+	f, ok := w.(*os.File)
+	return ok && term.IsTerminal(int(f.Fd()))
+}
+
+// setOutputStyling switches pterm's colour and bold/underline styling for every
+// printer at once. pterm forces colour on at init regardless of the terminal.
+func setOutputStyling(enabled bool) {
+	if enabled {
+		pterm.EnableStyling()
+	} else {
+		pterm.DisableStyling()
+	}
+}
+
 // initCLILogger retunes the CLI logger from SPX_LOG_LEVEL, then --verbose so an
 // explicit flag wins over the environment. There is no root --debug flag: four
 // service subcommands already define one with a service-specific meaning, and a

cmd/spinifex/cmd/root_output_test.go

@@ -0,0 +1,149 @@
+package cmd_test
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/spinifex/cmd/spinifex/cmd"
+	"github.com/pterm/pterm"
+)
+
+func TestOutputStylingEnabled(t *testing.T) {
+	tests := []struct {
+		name     string
+		noColor  string
+		term     string
+		terminal bool
+		want     bool
+	}{
+		{name: "terminal keeps colour", term: "xterm-256color", terminal: true, want: true},
+		{name: "unset TERM on a terminal keeps colour", terminal: true, want: true},
+		{name: "TERM=dumb drops colour on a terminal", term: "dumb", terminal: true, want: false},
+		{name: "pipe drops colour", terminal: false, want: false},
+		{name: "NO_COLOR drops colour on a terminal", noColor: "1", terminal: true, want: false},
+		{name: "any non-empty NO_COLOR counts, even 0", noColor: "0", terminal: true, want: false},
+		{name: "NO_COLOR on a pipe", noColor: "1", terminal: false, want: false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := cmd.OutputStylingEnabled(tt.noColor, tt.term, tt.terminal); got != tt.want {
+				t.Fatalf("OutputStylingEnabled(%q, %q, %v) = %v, want %v", tt.noColor, tt.term, tt.terminal, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestIsTerminalFalseForNonTTYWriters(t *testing.T) {
+	r, w, err := os.Pipe()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer r.Close()
+	defer w.Close()
+
+	f, err := os.CreateTemp(t.TempDir(), "out")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+
+	for name, wr := range map[string]io.Writer{"pipe": w, "regular file": f, "buffer": &bytes.Buffer{}} {
+		if cmd.IsTerminal(wr) {
+			t.Errorf("IsTerminal(%s) = true, want false", name)
+		}
+	}
+}
+
+// renderNodesTable renders a table shaped like `spx get nodes` through the same
+// pterm.DefaultTable the CLI uses, after applying the styling decision.
+func renderNodesTable(t *testing.T, noColor, termName string, terminal bool) string {
+	t.Helper()
+	t.Cleanup(func() { cmd.SetOutputStyling(true) })
+	cmd.SetOutputStyling(cmd.OutputStylingEnabled(noColor, termName, terminal))
+
+	var buf bytes.Buffer
+	data := pterm.TableData{
+		{"NAME", "STATUS", "IP"},
+		{"node1", "Ready", "10.0.0.1"},
+		{"node-long-name", "NotReady", "10.0.0.22"},
+	}
+	if err := pterm.DefaultTable.WithHasHeader().WithLeftAlignment().WithData(data).WithWriter(&buf).Render(); err != nil {
+		t.Fatalf("render: %v", err)
+	}
+	return buf.String()
+}
+
+func TestTableOutputPlainWhenNotTerminal(t *testing.T) {
+	r, w, err := os.Pipe()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer r.Close()
+	defer w.Close()
+
+	out := renderNodesTable(t, "", "xterm", cmd.IsTerminal(w))
+	assertPlainAlignedTable(t, out)
+}
+
+func TestTableOutputPlainWithNoColorOnTerminal(t *testing.T) {
+	out := renderNodesTable(t, "1", "xterm", true)
+	assertPlainAlignedTable(t, out)
+}
+
+func TestTableOutputPlainOnDumbTerminal(t *testing.T) {
+	out := renderNodesTable(t, "", "dumb", true)
+	assertPlainAlignedTable(t, out)
+}
+
+func TestTableOutputStyledOnTerminal(t *testing.T) {
+	out := renderNodesTable(t, "", "xterm", true)
+	if !strings.Contains(out, "\x1b[") {
+		t.Fatalf("table on a terminal carries no ANSI styling, so the plain-output tests prove nothing:\n%q", out)
+	}
+}
+
+func assertPlainAlignedTable(t *testing.T, out string) {
+	t.Helper()
+	if strings.ContainsRune(out, '\x1b') {
+		t.Fatalf("output contains an escape byte:\n%q", out)
+	}
+
+	lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
+	var rows []string
+	for _, l := range lines {
+		if strings.Contains(l, "node") || strings.HasPrefix(l, "NAME") {
+			rows = append(rows, l)
+		}
+	}
+	if len(rows) != 3 {
+		t.Fatalf("want header and two node rows, got %d:\n%s", len(rows), out)
+	}
+
+	// Columns line up only if the width calculation saw the same bytes it printed.
+	statusCol := strings.Index(rows[0], "STATUS")
+	ipCol := strings.Index(rows[0], "IP")
+	for _, row := range rows[1:] {
+		fields := strings.Fields(row)
+		var status string
+		for _, f := range fields {
+			if strings.Contains(f, "Ready") {
+				status = f
+			}
+		}
+		if got := strings.Index(row, status); got != statusCol {
+			t.Errorf("STATUS column at %d in %q, header at %d", got, row, statusCol)
+		}
+		if got := strings.Index(row, "10.0.0."); got != ipCol {
+			t.Errorf("IP column at %d in %q, header at %d", got, row, ipCol)
+		}
+	}
+
+	// grep -w Ready must match: the byte before the word is not a word character.
+	ready := strings.Index(rows[1], "Ready")
+	if ready <= 0 || rows[1][ready-1] != ' ' {
+		t.Errorf("Ready is not preceded by a space in %q", rows[1])
+	}
+}

cmd/spinifex/cmd/service.go

@@ -225,42 +225,6 @@ var predastoreStartCmd = &cobra.Command{
 	},
 }
 
-var predastoreStopCmd = &cobra.Command{
-	Use:           "stop",
-	Short:         "Stop the predastore service",
-	SilenceErrors: true,
-	SilenceUsage:  true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		fmt.Println("Stopping predastore service...")
-
-		// Only the pid directory matters here: Stop finds the running process
-		// through it, and start wrote the file there.
-		service, err := service.New("predastore", &predastore.Config{
-			BasePath: viper.GetString("predastore-base-path"),
-		})
-
-		if err != nil {
-			return fmt.Errorf("create predastore service: %w", err)
-		}
-
-		if err = service.Stop(); err != nil {
-			return fmt.Errorf("stop predastore service: %w", err)
-		}
-
-		fmt.Println("Predastore service stopped")
-		return nil
-	},
-}
-
-var predastoreStatusCmd = &cobra.Command{
-	Use:   "status",
-	Short: "Get status of the predastore service",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Add your status logic here
-		fmt.Println("Predastore service status: ...")
-	},
-}
-
 // Repeat for viperblock.
 var viperblockStartCmd = &cobra.Command{
 	Use:           "start",
@@ -413,37 +377,6 @@ var viperblockStartCmd = &cobra.Command{
 	},
 }
 
-var viperblockStopCmd = &cobra.Command{
-	Use:           "stop",
-	Short:         "Stop the viperblock service",
-	SilenceErrors: true,
-	SilenceUsage:  true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		fmt.Println("Stopping viperblock service...")
-
-		service, err := service.New("viperblock", &viperblockd.Config{})
-
-		if err != nil {
-			return fmt.Errorf("create viperblock service: %w", err)
-		}
-
-		if err = service.Stop(); err != nil {
-			return fmt.Errorf("stop viperblock service: %w", err)
-		}
-
-		fmt.Println("Viperblock service stopped")
-		return nil
-	},
-}
-
-var viperblockStatusCmd = &cobra.Command{
-	Use:   "status",
-	Short: "Get status of the viperblock service",
-	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("Viperblock service status: ...")
-	},
-}
-
 var qemunbdCmd = &cobra.Command{
 	Use:   "qemunbd",
 	Short: "Manage the qemunbd service",
@@ -518,37 +451,6 @@ var qemunbdStartCmd = &cobra.Command{
 	},
 }
 
-var qemunbdStopCmd = &cobra.Command{
-	Use:           "stop",
-	Short:         "Stop the qemunbd service",
-	SilenceErrors: true,
-	SilenceUsage:  true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		fmt.Println("Stopping qemunbd service...")
-
-		service, err := service.New("qemunbd", &qemunbdd.Config{})
-
-		if err != nil {
-			return fmt.Errorf("create qemunbd service: %w", err)
-		}
-
-		if err = service.Stop(); err != nil {
-			return fmt.Errorf("stop qemunbd service: %w", err)
-		}
-
-		fmt.Println("qemunbd service stopped")
-		return nil
-	},
-}
-
-var qemunbdStatusCmd = &cobra.Command{
-	Use:   "status",
-	Short: "Get status of the qemunbd service",
-	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("qemunbd service status: ...")
-	},
-}
-
 // Repeat for nats.
 var natsStartCmd = &cobra.Command{
 	Use:           "start",
@@ -589,37 +491,6 @@ var natsStartCmd = &cobra.Command{
 	},
 }
 
-var natsStopCmd = &cobra.Command{
-	Use:           "stop",
-	Short:         "Stop the nats service",
-	SilenceErrors: true,
-	SilenceUsage:  true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		fmt.Println("Stopping nats service...")
-
-		service, err := service.New("nats", &nats.Config{})
-
-		if err != nil {
-			return fmt.Errorf("create nats service: %w", err)
-		}
-
-		if err = service.Stop(); err != nil {
-			return fmt.Errorf("stop nats service: %w", err)
-		}
-
-		fmt.Println("Nats service stopped")
-		return nil
-	},
-}
-
-var natsStatusCmd = &cobra.Command{
-	Use:   "status",
-	Short: "Get status of the nats service",
-	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("Nats service status: ...")
-	},
-}
-
 // Repeat for spinifex.
 var spinifexStartCmd = &cobra.Command{
 	Use:           "start",
@@ -682,37 +553,6 @@ var spinifexStartCmd = &cobra.Command{
 	},
 }
 
-var spinifexStopCmd = &cobra.Command{
-	Use:           "stop",
-	Short:         "Stop the spinifex service",
-	SilenceErrors: true,
-	SilenceUsage:  true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		fmt.Println("Stopping spinifex service...")
-
-		service, err := service.New("spinifex", &config.ClusterConfig{})
-
-		if err != nil {
-			return fmt.Errorf("create spinifex service: %w", err)
-		}
-
-		if err = service.Stop(); err != nil {
-			return fmt.Errorf("stop spinifex service: %w", err)
-		}
-
-		fmt.Println("Spinifex service stopped")
-		return nil
-	},
-}
-
-var spinifexStatusCmd = &cobra.Command{
-	Use:   "status",
-	Short: "Get status of the spinifex service",
-	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("Spinifex service status: ...")
-	},
-}
-
 // AWS GW
 
 var awsgwStartCmd = &cobra.Command{
@@ -785,37 +625,6 @@ var awsgwStartCmd = &cobra.Command{
 	},
 }
 
-var awsgwStopCmd = &cobra.Command{
-	Use:           "stop",
-	Short:         "Stop the awsgw service",
-	SilenceErrors: true,
-	SilenceUsage:  true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		fmt.Println("Stopping awsgw service...")
-
-		service, err := service.New("awsgw", &config.ClusterConfig{})
-
-		if err != nil {
-			return fmt.Errorf("create awsgw service: %w", err)
-		}
-
-		if err = service.Stop(); err != nil {
-			return fmt.Errorf("stop awsgw service: %w", err)
-		}
-
-		fmt.Println("AWSGW service stopped")
-		return nil
-	},
-}
-
-var awsgwStatusCmd = &cobra.Command{
-	Use:   "status",
-	Short: "Get status of the awsgw service",
-	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("AWSGW service status: ...")
-	},
-}
-
 // consoleRegion is the region the console signs with. awsgw pins signature
 // verification to this same node field, so reading it from anywhere else —
 // including the cluster-wide [aws] region — would let the two drift and fail
@@ -876,50 +685,6 @@ var spinifexUIStartCmd = &cobra.Command{
 	},
 }
 
-var spinifexUIStopCmd = &cobra.Command{
-	Use:           "stop",
-	Short:         "Stop the spinifex-ui service",
-	SilenceErrors: true,
-	SilenceUsage:  true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		fmt.Println("Stopping spinifex-ui service...")
-
-		svc, err := service.New("spinifex-ui", &spinifexui.Config{})
-
-		if err != nil {
-			return fmt.Errorf("create spinifex-ui service: %w", err)
-		}
-
-		if err = svc.Stop(); err != nil {
-			return fmt.Errorf("stop spinifex-ui service: %w", err)
-		}
-		fmt.Println("spinifex-ui service stopped")
-		return nil
-	},
-}
-
-var spinifexUIStatusCmd = &cobra.Command{
-	Use:           "status",
-	Short:         "Get status of the spinifex-ui service",
-	SilenceErrors: true,
-	SilenceUsage:  true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		svc, err := service.New("spinifex-ui", &spinifexui.Config{})
-
-		if err != nil {
-			return fmt.Errorf("create spinifex-ui service: %w", err)
-		}
-
-		status, err := svc.Status()
-		if err != nil {
-			return fmt.Errorf("get spinifex-ui service status: %w", err)
-		}
-
-		fmt.Println("spinifex-ui service status:", status)
-		return nil
-	},
-}
-
 // checkLegacyWanBridgeKey fails vpcd startup if the deprecated `wan_bridge`
 // TOML key or `SPINIFEX_VPCD_WAN_BRIDGE` env-var is present. There is no silent
 // alias or auto-rewrite; the operator must remove the key before vpcd will start.
@@ -1047,35 +812,6 @@ var vpcdStartCmd = &cobra.Command{
 	},
 }
 
-var vpcdStopCmd = &cobra.Command{
-	Use:           "stop",
-	Short:         "Stop the vpcd service",
-	SilenceErrors: true,
-	SilenceUsage:  true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		fmt.Println("Stopping vpcd service...")
-
-		svc, err := service.New("vpcd", &vpcd.Config{})
-		if err != nil {
-			return fmt.Errorf("create vpcd service: %w", err)
-		}
-
-		if err = svc.Stop(); err != nil {
-			return fmt.Errorf("stop vpcd service: %w", err)
-		}
-		fmt.Println("vpcd service stopped")
-		return nil
-	},
-}
-
-var vpcdStatusCmd = &cobra.Command{
-	Use:   "status",
-	Short: "Get status of the vpcd service",
-	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("vpcd service status: ...")
-	},
-}
-
 var northstarCmd = &cobra.Command{
 	Use:   "northstar",
 	Short: "Manage the northstar (DNS) service",
@@ -1198,35 +934,6 @@ func resolveNorthstarConfigPath(nodePath, override string) string {
 	return nodePath
 }
 
-var northstarStopCmd = &cobra.Command{
-	Use:           "stop",
-	Short:         "Stop the northstar service",
-	SilenceErrors: true,
-	SilenceUsage:  true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		fmt.Println("Stopping northstar service...")
-
-		svc, err := service.New("northstar", &northstar.Config{})
-		if err != nil {
-			return fmt.Errorf("create northstar service: %w", err)
-		}
-
-		if err = svc.Stop(); err != nil {
-			return fmt.Errorf("stop northstar service: %w", err)
-		}
-		fmt.Println("northstar service stopped")
-		return nil
-	},
-}
-
-var northstarStatusCmd = &cobra.Command{
-	Use:   "status",
-	Short: "Get status of the northstar service",
-	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("northstar service status: ...")
-	},
-}
-
 var qmpCollectorCmd = &cobra.Command{
 	Use:   "qmp-collector",
 	Short: "Manage the qmp-collector (guest metrics) service",
@@ -1270,34 +977,6 @@ var qmpCollectorStartCmd = &cobra.Command{
 	},
 }
 
-var qmpCollectorStopCmd = &cobra.Command{
-	Use:           "stop",
-	Short:         "Stop the qmp-collector service",
-	SilenceErrors: true,
-	SilenceUsage:  true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		fmt.Println("Stopping qmp-collector service...")
-
-		svc, err := service.New("qmp-collector", &qmpcollector.Config{})
-		if err != nil {
-			return fmt.Errorf("create qmp-collector service: %w", err)
-		}
-		if err = svc.Stop(); err != nil {
-			return fmt.Errorf("stop qmp-collector service: %w", err)
-		}
-		fmt.Println("qmp-collector service stopped")
-		return nil
-	},
-}
-
-var qmpCollectorStatusCmd = &cobra.Command{
-	Use:   "status",
-	Short: "Get status of the qmp-collector service",
-	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("qmp-collector service status: ...")
-	},
-}
-
 // registerPredastoreNamespacedFlags defines predastore's host, base-path and
 // config-path flags and binds them to viper. It must only run once (from
 // init()); pflag panics on a redefined flag.
@@ -1427,8 +1106,6 @@ func init() {
 	bindPredastoreCollisionEnv()
 
 	predastoreCmd.AddCommand(predastoreStartCmd)
-	predastoreCmd.AddCommand(predastoreStopCmd)
-	predastoreCmd.AddCommand(predastoreStatusCmd)
 
 	serviceCmd.AddCommand(viperblockCmd)
 
@@ -1454,8 +1131,6 @@ func init() {
 	viper.BindPFlag("viperblock-debug", viperblockCmd.PersistentFlags().Lookup("debug"))
 
 	viperblockCmd.AddCommand(viperblockStartCmd)
-	viperblockCmd.AddCommand(viperblockStopCmd)
-	viperblockCmd.AddCommand(viperblockStatusCmd)
 
 	serviceCmd.AddCommand(qemunbdCmd)
 
@@ -1464,15 +1139,11 @@ func init() {
 	viper.BindPFlag("qemunbd-debug", qemunbdCmd.PersistentFlags().Lookup("debug"))
 
 	qemunbdCmd.AddCommand(qemunbdStartCmd)
-	qemunbdCmd.AddCommand(qemunbdStopCmd)
-	qemunbdCmd.AddCommand(qemunbdStatusCmd)
 
 	// Nats
 	serviceCmd.AddCommand(natsCmd)
 
 	natsCmd.AddCommand(natsStartCmd)
-	natsCmd.AddCommand(natsStopCmd)
-	natsCmd.AddCommand(natsStatusCmd)
 
 	// Add NATS flags
 	natsCmd.PersistentFlags().Int("port", 4222, "NATS server port")
@@ -1494,8 +1165,6 @@ func init() {
 	serviceCmd.AddCommand(spinifexCmd)
 
 	spinifexCmd.AddCommand(spinifexStartCmd)
-	spinifexCmd.AddCommand(spinifexStopCmd)
-	spinifexCmd.AddCommand(spinifexStatusCmd)
 
 	spinifexCmd.PersistentFlags().String("wal-dir", "", "Write-ahead log (WAL) directory. Place on high-speed NVMe disk, or tmpfs for development.")
 	viper.BindEnv("wal-dir", "SPINIFEX_WAL_DIR")
@@ -1518,8 +1187,6 @@ func init() {
 	bindAwsgwCollisionEnv()
 
 	awsgwCmd.AddCommand(awsgwStartCmd)
-	awsgwCmd.AddCommand(awsgwStopCmd)
-	awsgwCmd.AddCommand(awsgwStatusCmd)
 
 	// spinifex-ui
 	serviceCmd.AddCommand(spinifexUICmd)
@@ -1549,15 +1216,11 @@ func init() {
 	viper.BindPFlag("spinifex-ui-ochre-enabled", spinifexUICmd.PersistentFlags().Lookup("ochre-enabled"))
 
 	spinifexUICmd.AddCommand(spinifexUIStartCmd)
-	spinifexUICmd.AddCommand(spinifexUIStopCmd)
-	spinifexUICmd.AddCommand(spinifexUIStatusCmd)
 
 	// vpcd
 	serviceCmd.AddCommand(vpcdCmd)
 
 	vpcdCmd.AddCommand(vpcdStartCmd)
-	vpcdCmd.AddCommand(vpcdStopCmd)
-	vpcdCmd.AddCommand(vpcdStatusCmd)
 
 	// northstar
 	serviceCmd.AddCommand(northstarCmd)
@@ -1567,13 +1230,9 @@ func init() {
 	viper.BindPFlag("northstar-config", northstarCmd.PersistentFlags().Lookup("northstar-config"))
 
 	northstarCmd.AddCommand(northstarStartCmd)
-	northstarCmd.AddCommand(northstarStopCmd)
-	northstarCmd.AddCommand(northstarStatusCmd)
 
 	// qmp-collector
 	serviceCmd.AddCommand(qmpCollectorCmd)
 
 	qmpCollectorCmd.AddCommand(qmpCollectorStartCmd)
-	qmpCollectorCmd.AddCommand(qmpCollectorStopCmd)
-	qmpCollectorCmd.AddCommand(qmpCollectorStatusCmd)
 }

cmd/spinifex/cmd/service_exit_test.go

@@ -62,16 +62,8 @@ func writeMalformedToml(t *testing.T) string {
 	return writeSpinifexToml(t, "[section\nkey = value without quotes\n")
 }
 
-// isolateRuntimeDir redirects the pid-file fallback (utils.pidPath, via
-// XDG_RUNTIME_DIR) to an empty temp dir, so a stop command with no explicit
-// directory override can't resolve against a real, shared, ambient pid dir.
-func isolateRuntimeDir(t *testing.T) {
-	t.Helper()
-	t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
-}
-
 // TestServiceStartCmdsReturnErrorOnMissingConfig exercises each service
-// start/stop command's RunE directly and asserts a fatal precondition
+// start command's RunE directly and asserts a fatal precondition
 // failure surfaces as a returned error, not a silent zero-exit return.
 // Cases with a setup func drive further into RunE toward a specific branch.
 func TestServiceStartCmdsReturnErrorOnMissingConfig(t *testing.T) {
@@ -339,99 +331,6 @@ func TestServiceStartCmdsReturnErrorOnMissingConfig(t *testing.T) {
 			},
 			wantErr: "start qmp-collector service",
 		},
-		// The stop commands below resolve their pid directory to an isolated,
-		// always-empty temp dir, so Stop() safely fails to find a pid file
-		// rather than risking a real running process via an ambient path.
-		{
-			name: "predastore stop (no running process)",
-			cmd:  predastoreStopCmd,
-			setup: func(t *testing.T) {
-				resetGlobalViper(t)
-				viper.Set("predastore-base-path", t.TempDir())
-			},
-			wantErr: "stop predastore service",
-		},
-		{
-			name: "viperblock stop (no running process)",
-			cmd:  viperblockStopCmd,
-			setup: func(t *testing.T) {
-				resetGlobalViper(t)
-				isolateRuntimeDir(t)
-			},
-			wantErr: "stop viperblock service",
-		},
-		{
-			name: "qemunbd stop (no running process)",
-			cmd:  qemunbdStopCmd,
-			setup: func(t *testing.T) {
-				resetGlobalViper(t)
-				isolateRuntimeDir(t)
-			},
-			wantErr: "stop qemunbd service",
-		},
-		{
-			name: "nats stop (no running process)",
-			cmd:  natsStopCmd,
-			setup: func(t *testing.T) {
-				resetGlobalViper(t)
-				isolateRuntimeDir(t)
-			},
-			wantErr: "stop nats service",
-		},
-		{
-			name: "spinifex stop (no running process)",
-			cmd:  spinifexStopCmd,
-			setup: func(t *testing.T) {
-				resetGlobalViper(t)
-				isolateRuntimeDir(t)
-			},
-			wantErr: "stop spinifex service",
-		},
-		{
-			name: "awsgw stop (no running process)",
-			cmd:  awsgwStopCmd,
-			setup: func(t *testing.T) {
-				resetGlobalViper(t)
-				isolateRuntimeDir(t)
-			},
-			wantErr: "stop awsgw service",
-		},
-		{
-			name: "spinifex-ui stop (no running process)",
-			cmd:  spinifexUIStopCmd,
-			setup: func(t *testing.T) {
-				resetGlobalViper(t)
-				isolateRuntimeDir(t)
-			},
-			wantErr: "stop spinifex-ui service",
-		},
-		{
-			name: "vpcd stop (no running process)",
-			cmd:  vpcdStopCmd,
-			setup: func(t *testing.T) {
-				resetGlobalViper(t)
-				isolateRuntimeDir(t)
-			},
-			wantErr: "stop vpcd service",
-		},
-		{
-			name: "northstar stop (no running process)",
-			cmd:  northstarStopCmd,
-			setup: func(t *testing.T) {
-				resetGlobalViper(t)
-				isolateRuntimeDir(t)
-			},
-			wantErr: "stop northstar service",
-		},
-		{
-			name: "qmp-collector stop (no running process)",
-			cmd:  qmpCollectorStopCmd,
-			setup: func(t *testing.T) {
-				resetGlobalViper(t)
-				isolateRuntimeDir(t)
-			},
-			wantErr: "stop qmp-collector service",
-		},
 	}
 
 	for _, tt := range tests {
@@ -451,36 +350,6 @@ func TestServiceStartCmdsReturnErrorOnMissingConfig(t *testing.T) {
 	}
 }
 
-// TestSpinifexUIStatusCmdReportsStoppedWithNoRunningProcess drives the
-// status command's success path: with no pid file at an isolated
-// XDG_RUNTIME_DIR, Status reports "stopped" and RunE must return nil.
-func TestSpinifexUIStatusCmdReportsStoppedWithNoRunningProcess(t *testing.T) {
-	isolateRuntimeDir(t)
-
-	require.NotNil(t, spinifexUIStatusCmd.RunE)
-	err := spinifexUIStatusCmd.RunE(spinifexUIStatusCmd, nil)
-	require.NoError(t, err)
-}
-
-// TestSpinifexUIStatusCmdReturnsErrorOnUnreadablePidDir drives the status
-// command's other branch: a genuine read failure (permission denied, not
-// ENOENT) must surface as an error rather than the benign "stopped" case.
-func TestSpinifexUIStatusCmdReturnsErrorOnUnreadablePidDir(t *testing.T) {
-	if os.Geteuid() == 0 {
-		t.Skip("root bypasses directory permission checks")
-	}
-
-	dir := t.TempDir()
-	require.NoError(t, os.WriteFile(filepath.Join(dir, "spinifex-ui.pid"), []byte("123"), 0o600))
-	require.NoError(t, os.Chmod(dir, 0o000))
-	t.Cleanup(func() { _ = os.Chmod(dir, 0o755) })
-	t.Setenv("XDG_RUNTIME_DIR", dir)
-
-	require.NotNil(t, spinifexUIStatusCmd.RunE)
-	err := spinifexUIStatusCmd.RunE(spinifexUIStatusCmd, nil)
-	require.ErrorContains(t, err, "get spinifex-ui service status")
-}
-
 // TestServiceStartExitsNonZeroOnFailure proves a fatal service-start error
 // terminates the process non-zero (systemd's Restart=on-failure depends on
 // it). os.Exit can't be observed in-process, so this re-execs the test

docs/COMMANDS.md

@@ -166,7 +166,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 
 | Command | Flags | Description |
 |---------|-------|-------------|
-| `spx admin images import` | `--name`, `--file`, `--distro`, `--version`, `--arch`, `--platform`, `--boot-mode` (bios/uefi/uefi-preferred), `--tag`, `--force`, `--skip-verify`, `--checksum`, `--ami-name` (override the registered `DescribeImages` name, which otherwise defaults to `ami-{distro}-{version}-{arch}`; used for locally-built appliances such as `--ami-name spinifex-eks-node`), `--tmp-dir` (default: `/tmp`) | Catalog imports (`--name`) download the image, fetch the catalog `Checksum` URL, verify the SHA-256/SHA-512 digest, and inherit `BootMode` from the catalog entry. `--boot-mode` overrides the catalog value when set. Mismatch fails closed; the cached file is left on disk and `--force` re-downloads. `--file` imports require an explicit `--boot-mode` because there is no catalog metadata to inherit from, and are verified only when `--checksum <sums-file>` is passed. `--checksum` takes the sums file the image was published with (`SHA256SUMS`, `SHA512SUMS`, `<image>.sha256`/`.sha512`, coreutils, BSD or bare-digest format), looks up the entry for the image's basename, and infers sha256 or sha512 from the digest length. A mismatch exits 1 before anything is written to storage and leaves the file in place. `--checksum` proves only that the image matches the sums file supplied with it, not that either is authentic; verify the sums file's vendor signature separately. `--checksum` is refused without `--file` or with `--skip-verify`. A `--file` import without `--checksum` prints `Importing <file> without checksum validation` to stderr and proceeds; `--name --file` does not fall back to the catalog digest. Every import records the source artifact's digest on the AMI with who verified it (`operator`, `catalog` or `unverified`, the unverified cases hashed as sha256), prints it on success, and shows it in `spx admin images describe`. The digest is of the file as supplied, not of the stored volume. `--skip-verify` bypasses verification for catalog imports and prints a warning to stderr; use only for debugging or when upstream mirrors are confirmed-broken. Import never modifies the image: the bytes written to storage are the bytes that were verified. A guest that needs to trust the deployment CA for in-guest SDK-over-TLS calls fetches it at runtime from `http://169.254.169.254/spinifex/ca.pem` (see the IMDS section), so a CA rotation reaches guests without re-importing anything. |
+| `spx admin images import` | `--name`, `--file`, `--distro`, `--version`, `--arch`, `--platform`, `--boot-mode` (bios/uefi/uefi-preferred), `--tag`, `--force`, `--skip-verify`, `--checksum`, `--ami-name` (override the registered `DescribeImages` name, which otherwise defaults to `ami-{distro}-{version}-{arch}`; used for locally-built appliances such as `--ami-name spinifex-eks-node`), `--tmp-dir` (default: `<spinifex-dir>/images`) | Catalog imports (`--name`) download the image, fetch the catalog `Checksum` URL, verify the SHA-256/SHA-512 digest, and inherit `BootMode` from the catalog entry. `--boot-mode` overrides the catalog value when set. Mismatch fails closed; the cached file is left on disk and `--force` re-downloads. `--file` imports require an explicit `--boot-mode` because there is no catalog metadata to inherit from, and are verified only when `--checksum <sums-file>` is passed. `--checksum` takes the sums file the image was published with (`SHA256SUMS`, `SHA512SUMS`, `<image>.sha256`/`.sha512`, coreutils, BSD or bare-digest format), looks up the entry for the image's basename, and infers sha256 or sha512 from the digest length. A mismatch exits 1 before anything is written to storage and leaves the file in place. `--checksum` proves only that the image matches the sums file supplied with it, not that either is authentic; verify the sums file's vendor signature separately. `--checksum` is refused without `--file` or with `--skip-verify`. A `--file` import without `--checksum` prints `Importing <file> without checksum validation` to stderr and proceeds; `--name --file` does not fall back to the catalog digest. Every import records the source artifact's digest on the AMI with who verified it (`operator`, `catalog` or `unverified`, the unverified cases hashed as sha256), prints it on success, and shows it in `spx admin images describe`. The digest is of the file as supplied, not of the stored volume. `--skip-verify` bypasses verification for catalog imports and prints a warning to stderr; use only for debugging or when upstream mirrors are confirmed-broken. Import never modifies the image: the bytes written to storage are the bytes that were verified. Compressed and qcow2 images are extracted into a hidden `.import-*` scratch directory under `--tmp-dir`, which is removed when the import succeeds or fails, so the catalog directory keeps only the downloaded source (for `--name`) and the AMI manifest. Point `--tmp-dir` at a disk-backed filesystem with room for the full raw image, never at a tmpfs such as `/tmp`. A guest that needs to trust the deployment CA for in-guest SDK-over-TLS calls fetches it at runtime from `http://169.254.169.254/spinifex/ca.pem` (see the IMDS section), so a CA rotation reaches guests without re-importing anything. |
 | `spx admin images list` | — | Lists available OS images that can be imported or downloaded |
 | `spx admin images describe` | `--image-id` (required) | Read-only. Prints the AMI's name, owner, creation date, boot mode and architecture, then its source digest: algorithm and value, verification (`operator` with the sums file name, `catalog` with the sums URL, or `unverified`), and the source file name. AMIs imported before digest recording print `Source digest:  not recorded`. The digest is not exposed through `DescribeImages`. |
 | `spx admin images promote` | `--image-id` (required), `--yes` | Reads `ami-<id>/config.json`, validates the AMI is account-owned, then rewrites `ImageOwnerAlias` to `"system"` in-place. No block data is copied. The change takes effect immediately — the AMI becomes visible to all accounts via `DescribeImages`. Prompts for confirmation (skipped with `--yes`). Already-system AMIs are refused. |

docs/admin/setting-up-your-cluster/README.md

@@ -71,6 +71,12 @@ ubuntu-26.04-x86_64     | ubuntu | 26.04   | x86_64 | uefi
 spx admin images import --name ubuntu-26.04-x86_64
 ```
 
+Verify the import and note the AMI ID:
+
+```bash
+AMI_ID=$(aws ec2 describe-images --query 'Images[0].ImageId' --output text)
+```
+
 ### Option B: Import a local image file
 
 ```bash
@@ -95,12 +101,12 @@ for img in spinifex-rds-postgres spinifex-rds-mariadb spinifex-ecs-node spinifex
 done
 ```
 
-| Image | Serves | Contents |
-| --- | --- | --- |
-| `spinifex-rds-postgres` | RDS, `--engine postgres` | Alpine 3.24.1 + PostgreSQL 18 + `rds-init` |
-| `spinifex-rds-mariadb` | RDS, `--engine mariadb` | Alpine 3.24.1 + MariaDB 11.8 + `rds-init` |
-| `spinifex-ecs-node` | ECS container instances | Alpine 3.21.7 + containerd + ecs-agent |
-| `spinifex-eks-node` | EKS control plane and nodegroups | Alpine 3.21.7 + K3s v1.32.5 + eks-token-webhook |
+| Image                   | Serves                           | Contents                                        |
+| ----------------------- | -------------------------------- | ----------------------------------------------- |
+| `spinifex-rds-postgres` | RDS, `--engine postgres`         | Alpine 3.24.1 + PostgreSQL 18 + `rds-init`      |
+| `spinifex-rds-mariadb`  | RDS, `--engine mariadb`          | Alpine 3.24.1 + MariaDB 11.8 + `rds-init`       |
+| `spinifex-ecs-node`     | ECS container instances          | Alpine 3.21.7 + containerd + ecs-agent          |
+| `spinifex-eks-node`     | EKS control plane and nodegroups | Alpine 3.21.7 + K3s v1.32.5 + eks-token-webhook |
 
 All four boot **BIOS**, unlike the distro images, which boot UEFI. That is a property of how they are built and nothing you need to pass.
 
@@ -168,7 +174,7 @@ aws ec2 attach-internet-gateway \
 
 ### Create a Subnet
 
-Create the subnet your instances will launch into. The routing and public-IP steps below are what make it a *public* subnet.
+Create the subnet your instances will launch into. The routing and public-IP steps below are what make it a _public_ subnet.
 
 ```bash
 SUBNET_ID=$(aws ec2 create-subnet \
@@ -349,17 +355,17 @@ https://YOUR_NODE_IP:3000
 
 On first load, your browser will show a TLS warning — Spinifex generates a self-signed certificate at install time. This is expected.
 
-1. Accept the warning to reach the login page (exact wording varies by browser — e.g. Chrome: *Advanced → Proceed to ...*, Firefox: *Advanced → Accept the Risk and Continue*).
+1. Accept the warning to reach the login page (exact wording varies by browser — e.g. Chrome: _Advanced → Proceed to ..._, Firefox: _Advanced → Accept the Risk and Continue_).
 2. On the login page, click **Download Certificate** and save `spinifex-ca.pem` to your machine.
 3. Install the certificate as a **trusted root** on your workstation, following the steps for your platform below.
 4. Restart your browser and reload `https://YOUR_NODE_IP:3000`. The padlock should now show a valid certificate.
 
 Step 3, by platform:
 
-- **macOS:** open `spinifex-ca.pem` in Keychain Access → *System* keychain → set *Trust* to **Always Trust**.
+- **macOS:** open `spinifex-ca.pem` in Keychain Access → _System_ keychain → set _Trust_ to **Always Trust**.
 - **Linux:** `sudo cp spinifex-ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt && sudo update-ca-certificates`
-- **Windows:** double-click the file → *Install Certificate* → *Local Machine* → *Trusted Root Certification Authorities*.
-- **Browser-only (Firefox):** *Settings → Privacy & Security → Certificates → View Certificates → Authorities → Import* and tick *Trust this CA to identify websites*.
+- **Windows:** double-click the file → _Install Certificate_ → _Local Machine_ → _Trusted Root Certification Authorities_.
+- **Browser-only (Firefox):** _Settings → Privacy & Security → Certificates → View Certificates → Authorities → Import_ and tick _Trust this CA to identify websites_.
 
 > **Why this is required:** the UI logs in by reading your AWS credentials through a trusted TLS channel. Browsers refuse to send credentials over an untrusted connection, so the certificate must be installed as trusted — temporary "Proceed anyway" exceptions won't work for login.
 

docs/containers/eks/README.md

@@ -55,7 +55,7 @@ Authentication uses **EKS access entries** (the API authentication mode) — IAM
 
 **Spinifex specifics**
 
-- **Auth mode is API-only.** `authentication_mode` must be `API`; the legacy `CONFIG_MAP` and `API_AND_CONFIG_MAP` modes are rejected. Grant access with access entries, not an `aws-auth` ConfigMap.
+- **Access entries are the only auth path.** `authentication_mode` accepts `API` (the default when unset) and `API_AND_CONFIG_MAP`; bare `CONFIG_MAP` is rejected. Spinifex has no `aws-auth` ConfigMap path, so `API_AND_CONFIG_MAP` behaves as `API`: grant access with access entries.
 - **Security groups are auto-managed.** Spinifex creates the cluster and node-group security groups deterministically (e.g. `eks-cluster-<name>-nodegroup-sg`); `vpc_config.security_group_ids` is ignored. To expose a workload you add an ingress rule to the auto-managed node-group SG.
 - **The worker image is fixed.** Node groups always boot Spinifex's `eks-node` image. `ami_type` is recorded but does not select the image, and the image must be registered on the cluster before you create a node group.
 - **Default Kubernetes version is `1.32`.**
@@ -262,7 +262,7 @@ The workbooks form a ladder. **eks-quickstart** is a NodePort demo; **eks-https-
 
 **Workload unreachable from outside the cluster.** The node-group security group only allows intra-cluster traffic by default. Add an ingress rule for your NodePort to `eks-cluster-<name>-nodegroup-sg`, as shown in the CLI tab.
 
-**`create-cluster` rejected with an authentication-mode error.** `authentication_mode` must be `API`. Remove any `CONFIG_MAP` / `API_AND_CONFIG_MAP` setting and grant access with access entries instead.
+**`create-cluster` rejected with an authentication-mode error.** `CONFIG_MAP` is the one mode Spinifex rejects, because it has no `aws-auth` ConfigMap path. Set `authentication_mode` to `API` or `API_AND_CONFIG_MAP`, or leave it unset, and grant access with access entries.
 
 **Addon install fails.** Only addons bundled into the registered `eks-node` image install successfully. If you pin an addon version in Terraform, leave `addon_version` unset so Spinifex selects the catalog default.
 

docs/coverage/README.md

@@ -17,18 +17,18 @@ tags:
 
 ## Overview
 
-The platform serves **397 operations** across the AWS APIs below. Every page names the operations its service implements from the pinned model, generated from the dispatch tables on each build rather than written by hand.
+The platform serves **392 operations** across the AWS APIs below. Every page names the operations its service implements from the pinned model, generated from the dispatch tables on each build rather than written by hand.
 
 | Service | Operations |
 |---|---:|
 | [ACM](/coverage/acm) | 9 |
 | [EC2](/coverage/ec2) | 127 |
-| [ECR](/coverage/ecr) | 22 |
+| [ECR](/coverage/ecr) | 23 |
 | [ECS](/coverage/ecs) | 31 |
-| [EKS](/coverage/eks) | 36 |
+| [EKS](/coverage/eks) | 27 |
 | [ELBv2](/coverage/elbv2) | 36 |
-| [IAM](/coverage/iam) | 79 |
-| [RDS](/coverage/rds) | 26 |
+| [IAM](/coverage/iam) | 81 |
+| [RDS](/coverage/rds) | 27 |
 | [S3](/coverage/s3) | 26 |
 | [STS](/coverage/sts) | 5 |
-| **Total** | **397** |
+| **Total** | **392** |

docs/coverage/coverage.json

@@ -1983,6 +1983,7 @@
           "DeleteRepository",
           "DeleteRepositoryPolicy",
           "DescribeImages",
+          "DescribeRegistry",
           "DescribeRepositories",
           "GetAuthorizationToken",
           "GetLifecyclePolicy",
@@ -2002,7 +2003,6 @@
         "stubbed": [
           "BatchCheckLayerAvailability",
           "CompleteLayerUpload",
-          "DescribeRegistry",
           "GetDownloadUrlForLayer",
           "GetRegistryPolicy",
           "InitiateLayerUpload",
@@ -2348,7 +2348,6 @@
         ],
         "registered": [
           "AssociateAccessPolicy",
-          "AssociateIdentityProviderConfig",
           "CreateAccessEntry",
           "CreateAddon",
           "CreateCluster",
@@ -2361,36 +2360,27 @@
           "DescribeAddon",
           "DescribeAddonVersions",
           "DescribeCluster",
-          "DescribeIdentityProviderConfig",
           "DescribeNodegroup",
-          "DescribeUpdate",
           "DisassociateAccessPolicy",
-          "DisassociateIdentityProviderConfig",
           "GetRecoveryDirective",
           "ListAccessEntries",
           "ListAccessPolicies",
           "ListAddons",
           "ListAssociatedAccessPolicies",
           "ListClusters",
-          "ListIdentityProviderConfigs",
           "ListInternalAddons",
           "ListNodegroups",
           "ListTagsForResource",
-          "ListUpdates",
           "PublishInternal",
           "TagResource",
           "UntagResource",
           "UpdateAccessEntry",
           "UpdateAddon",
-          "UpdateClusterConfig",
-          "UpdateClusterVersion",
           "UpdateNodegroupConfig",
-          "UpdateNodegroupVersion",
           "WebhookTokenReview"
         ],
         "implemented": [
           "AssociateAccessPolicy",
-          "AssociateIdentityProviderConfig",
           "CreateAccessEntry",
           "CreateAddon",
           "CreateCluster",
@@ -2403,34 +2393,27 @@
           "DescribeAddon",
           "DescribeAddonVersions",
           "DescribeCluster",
-          "DescribeIdentityProviderConfig",
           "DescribeNodegroup",
-          "DescribeUpdate",
           "DisassociateAccessPolicy",
-          "DisassociateIdentityProviderConfig",
           "ListAccessEntries",
           "ListAccessPolicies",
           "ListAddons",
           "ListAssociatedAccessPolicies",
           "ListClusters",
-          "ListIdentityProviderConfigs",
           "ListNodegroups",
           "ListTagsForResource",
-          "ListUpdates",
           "TagResource",
           "UntagResource",
           "UpdateAccessEntry",
           "UpdateAddon",
-          "UpdateClusterConfig",
-          "UpdateClusterVersion",
-          "UpdateNodegroupConfig",
-          "UpdateNodegroupVersion"
+          "UpdateNodegroupConfig"
         ],
         "stubbed": [],
         "unsupported": [],
         "missing": [
           "ActivateCertificateAuthority",
           "AssociateEncryptionConfig",
+          "AssociateIdentityProviderConfig",
           "CancelUpdate",
           "CreateCapability",
           "CreateCertificateAuthority",
@@ -2449,19 +2432,27 @@
           "DescribeClusterVersions",
           "DescribeEksAnywhereSubscription",
           "DescribeFargateProfile",
+          "DescribeIdentityProviderConfig",
           "DescribeInsight",
           "DescribeInsightsRefresh",
           "DescribePodIdentityAssociation",
+          "DescribeUpdate",
+          "DisassociateIdentityProviderConfig",
           "ListCapabilities",
           "ListCertificateAuthorities",
           "ListEksAnywhereSubscriptions",
           "ListFargateProfiles",
+          "ListIdentityProviderConfigs",
           "ListInsights",
           "ListPodIdentityAssociations",
+          "ListUpdates",
           "RegisterCluster",
           "StartInsightsRefresh",
           "UpdateCapability",
+          "UpdateClusterConfig",
+          "UpdateClusterVersion",
           "UpdateEksAnywhereSubscription",
+          "UpdateNodegroupVersion",
           "UpdatePodIdentityAssociation"
         ],
         "extra": [
@@ -2862,6 +2853,7 @@
           "GetUser",
           "GetUserPolicy",
           "ListAccessKeys",
+          "ListAccountAliases",
           "ListAttachedGroupPolicies",
           "ListAttachedRolePolicies",
           "ListAttachedUserPolicies",
@@ -2901,7 +2893,8 @@
           "UntagUser",
           "UpdateAccessKey",
           "UpdateAssumeRolePolicy",
-          "UpdateRole"
+          "UpdateRole",
+          "UpdateRoleDescription"
         ],
         "implemented": [
           "AddRoleToInstanceProfile",
@@ -2943,6 +2936,7 @@
           "GetUser",
           "GetUserPolicy",
           "ListAccessKeys",
+          "ListAccountAliases",
           "ListAttachedGroupPolicies",
           "ListAttachedRolePolicies",
           "ListAttachedUserPolicies",
@@ -2982,7 +2976,8 @@
           "UntagUser",
           "UpdateAccessKey",
           "UpdateAssumeRolePolicy",
-          "UpdateRole"
+          "UpdateRole",
+          "UpdateRoleDescription"
         ],
         "stubbed": [],
         "unsupported": [],
@@ -3042,7 +3037,6 @@
           "GetServiceLastAccessedDetails",
           "GetServiceLastAccessedDetailsWithEntities",
           "GetServiceLinkedRoleDeletionStatus",
-          "ListAccountAliases",
           "ListDelegationRequests",
           "ListMFADeviceTags",
           "ListMFADevices",
@@ -3078,7 +3072,6 @@
           "UpdateGroup",
           "UpdateLoginProfile",
           "UpdateOpenIDConnectProviderThumbprint",
-          "UpdateRoleDescription",
           "UpdateSAMLProvider",
           "UpdateSSHPublicKey",
           "UpdateServerCertificate",
@@ -3297,6 +3290,7 @@
           "ModifyDBCluster",
           "ModifyDBInstance",
           "ModifyDBParameterGroup",
+          "ModifyDBSubnetGroup",
           "ModifyOptionGroup",
           "PollDBCommands",
           "PromoteReadReplica",
@@ -3331,6 +3325,7 @@
           "ListTagsForResource",
           "ModifyDBInstance",
           "ModifyDBParameterGroup",
+          "ModifyDBSubnetGroup",
           "RebootDBInstance",
           "RemoveTagsFromResource",
           "RestoreDBInstanceFromDBSnapshot",
@@ -3448,7 +3443,6 @@
           "ModifyDBShardGroup",
           "ModifyDBSnapshot",
           "ModifyDBSnapshotAttribute",
-          "ModifyDBSubnetGroup",
           "ModifyEventSubscription",
           "ModifyGlobalCluster",
           "ModifyIntegration",

docs/coverage/ec2/README.md

@@ -24,6 +24,10 @@ Spinifex implements **127 operations** in the EC2 `2016-11-15` API model.
 
 Spot Instance Requests are a mock over the on-demand `RunInstances` path. A request synchronously launches real VMs on the operator's own compute and is then reported `active` and `fulfilled`. There is no spot market: no bidding, no price rejection, no interruption and no reclamation, and instances are never taken back.
 
+### AMI Launch Permissions Are Refused
+
+An AMI cannot be shared with another account or made public. `ModifyImageAttribute` refuses `LaunchPermission`, `UserIds`, `UserGroups`, `OrganizationArns`, `OrganizationalUnitArns` and `OperationType`, and `ResetImageAttribute` resets only `description`, not `launchPermission`. Each refusal is `InvalidParameterValue`. Terraform's `aws_ami_launch_permission` therefore cannot be created. `description` is the only image attribute that can be modified or reset.
+
 ### Operations
 
 | Operation |

docs/coverage/ec2/intro.md

@@ -1,3 +1,7 @@
 ### Spot Instances Are a Mock
 
 Spot Instance Requests are a mock over the on-demand `RunInstances` path. A request synchronously launches real VMs on the operator's own compute and is then reported `active` and `fulfilled`. There is no spot market: no bidding, no price rejection, no interruption and no reclamation, and instances are never taken back.
+
+### AMI Launch Permissions Are Refused
+
+An AMI cannot be shared with another account or made public. `ModifyImageAttribute` refuses `LaunchPermission`, `UserIds`, `UserGroups`, `OrganizationArns`, `OrganizationalUnitArns` and `OperationType`, and `ResetImageAttribute` resets only `description`, not `launchPermission`. Each refusal is `InvalidParameterValue`. Terraform's `aws_ami_launch_permission` therefore cannot be created. `description` is the only image attribute that can be modified or reset.

docs/coverage/ecr/README.md

@@ -18,7 +18,7 @@ tags:
 
 ## Overview
 
-Spinifex implements **22 operations** in the ECR `2015-09-21` API model.
+Spinifex implements **23 operations** in the ECR `2015-09-21` API model.
 
 ### Two endpoints, one registry
 
@@ -26,7 +26,7 @@ Repository metadata is served over the AWS API on the gateway endpoint. Image da
 
 The split explains the stubs below. The layer-transfer operations — `BatchCheckLayerAvailability`, `InitiateLayerUpload`, `UploadLayerPart`, `CompleteLayerUpload` and `GetDownloadUrlForLayer` — are registered stubs because the `/v2/` endpoint carries that traffic instead. A client using `docker` or any OCI-compatible tool never calls them.
 
-Registry replication is a stub for the same kind of reason: a deployment is a single registry, with no cross-region peer to replicate to.
+Registry replication is a stub for the same kind of reason: a deployment is a single registry, with no cross-region peer to replicate to. `DescribeRegistry` reports this as a replication configuration with no rules.
 
 ### Operations
 
@@ -39,6 +39,7 @@ Registry replication is a stub for the same kind of reason: a deployment is a si
 | `DeleteRepository` |
 | `DeleteRepositoryPolicy` |
 | `DescribeImages` |
+| `DescribeRegistry` |
 | `DescribeRepositories` |
 | `GetAuthorizationToken` |
 | `GetLifecyclePolicy` |

docs/coverage/ecr/intro.md

@@ -4,4 +4,4 @@ Repository metadata is served over the AWS API on the gateway endpoint. Image da
 
 The split explains the stubs below. The layer-transfer operations — `BatchCheckLayerAvailability`, `InitiateLayerUpload`, `UploadLayerPart`, `CompleteLayerUpload` and `GetDownloadUrlForLayer` — are registered stubs because the `/v2/` endpoint carries that traffic instead. A client using `docker` or any OCI-compatible tool never calls them.
 
-Registry replication is a stub for the same kind of reason: a deployment is a single registry, with no cross-region peer to replicate to.
+Registry replication is a stub for the same kind of reason: a deployment is a single registry, with no cross-region peer to replicate to. `DescribeRegistry` reports this as a replication configuration with no rules.

docs/coverage/eks/README.md

@@ -18,14 +18,13 @@ tags:
 
 ## Overview
 
-Spinifex implements **36 operations** in the EKS `2017-11-01` API model.
+Spinifex implements **27 operations** in the EKS `2017-11-01` API model.
 
 ### Operations
 
 | Operation |
 |---|
 | `AssociateAccessPolicy` |
-| `AssociateIdentityProviderConfig` |
 | `CreateAccessEntry` |
 | `CreateAddon` |
 | `CreateCluster` |
@@ -38,25 +37,17 @@ Spinifex implements **36 operations** in the EKS `2017-11-01` API model.
 | `DescribeAddon` |
 | `DescribeAddonVersions` |
 | `DescribeCluster` |
-| `DescribeIdentityProviderConfig` |
 | `DescribeNodegroup` |
-| `DescribeUpdate` |
 | `DisassociateAccessPolicy` |
-| `DisassociateIdentityProviderConfig` |
 | `ListAccessEntries` |
 | `ListAccessPolicies` |
 | `ListAddons` |
 | `ListAssociatedAccessPolicies` |
 | `ListClusters` |
-| `ListIdentityProviderConfigs` |
 | `ListNodegroups` |
 | `ListTagsForResource` |
-| `ListUpdates` |
 | `TagResource` |
 | `UntagResource` |
 | `UpdateAccessEntry` |
 | `UpdateAddon` |
-| `UpdateClusterConfig` |
-| `UpdateClusterVersion` |
 | `UpdateNodegroupConfig` |
-| `UpdateNodegroupVersion` |

docs/coverage/iam/README.md

@@ -18,7 +18,7 @@ tags:
 
 ## Overview
 
-Spinifex implements **79 operations** in the IAM `2010-05-08` API model.
+Spinifex implements **81 operations** in the IAM `2010-05-08` API model.
 
 ### Scope
 
@@ -67,6 +67,7 @@ All IAM operations are account-scoped. The root user of an account bypasses poli
 | `GetUser` |
 | `GetUserPolicy` |
 | `ListAccessKeys` |
+| `ListAccountAliases` |
 | `ListAttachedGroupPolicies` |
 | `ListAttachedRolePolicies` |
 | `ListAttachedUserPolicies` |
@@ -107,3 +108,4 @@ All IAM operations are account-scoped. The root user of an account bypasses poli
 | `UpdateAccessKey` |
 | `UpdateAssumeRolePolicy` |
 | `UpdateRole` |
+| `UpdateRoleDescription` |

docs/coverage/internal-report.md

@@ -762,7 +762,7 @@ spot-price: On owned hardware there is no spot-to-on-demand price differential,
 
 ## ecr
 
-`2015-09-21` — 22 of 58 modelled operations implemented (37.9%), 11 stubbed, 0 not applicable, 3 registered outside the pinned model.
+`2015-09-21` — 23 of 58 modelled operations implemented (39.7%), 10 stubbed, 0 not applicable, 3 registered outside the pinned model.
 
 ### Not implemented
 
@@ -793,7 +793,6 @@ spot-price: On owned hardware there is no spot-to-on-demand price differential,
 
 - `BatchCheckLayerAvailability`
 - `CompleteLayerUpload`
-- `DescribeRegistry`
 - `GetDownloadUrlForLayer`
 - `GetRegistryPolicy`
 - `InitiateLayerUpload`
@@ -882,12 +881,13 @@ spot-price: On owned hardware there is no spot-to-on-demand price differential,
 
 ## eks
 
-`2017-11-01` — 36 of 70 modelled operations implemented (51.4%), 0 stubbed, 0 not applicable, 4 registered outside the pinned model.
+`2017-11-01` — 27 of 70 modelled operations implemented (38.6%), 0 stubbed, 0 not applicable, 4 registered outside the pinned model.
 
 ### Not implemented
 
 - `ActivateCertificateAuthority`
 - `AssociateEncryptionConfig`
+- `AssociateIdentityProviderConfig`
 - `CancelUpdate`
 - `CreateCapability`
 - `CreateCertificateAuthority`
@@ -906,19 +906,27 @@ spot-price: On owned hardware there is no spot-to-on-demand price differential,
 - `DescribeClusterVersions`
 - `DescribeEksAnywhereSubscription`
 - `DescribeFargateProfile`
+- `DescribeIdentityProviderConfig`
 - `DescribeInsight`
 - `DescribeInsightsRefresh`
 - `DescribePodIdentityAssociation`
+- `DescribeUpdate`
+- `DisassociateIdentityProviderConfig`
 - `ListCapabilities`
 - `ListCertificateAuthorities`
 - `ListEksAnywhereSubscriptions`
 - `ListFargateProfiles`
+- `ListIdentityProviderConfigs`
 - `ListInsights`
 - `ListPodIdentityAssociations`
+- `ListUpdates`
 - `RegisterCluster`
 - `StartInsightsRefresh`
 - `UpdateCapability`
+- `UpdateClusterConfig`
+- `UpdateClusterVersion`
 - `UpdateEksAnywhereSubscription`
+- `UpdateNodegroupVersion`
 - `UpdatePodIdentityAssociation`
 
 ### Registered outside the pinned model
@@ -957,7 +965,7 @@ spot-price: On owned hardware there is no spot-to-on-demand price differential,
 
 ## iam
 
-`2010-05-08` — 79 of 180 modelled operations implemented (43.9%), 0 stubbed, 19 not applicable, 0 registered outside the pinned model.
+`2010-05-08` — 81 of 180 modelled operations implemented (45.0%), 0 stubbed, 19 not applicable, 0 registered outside the pinned model.
 
 ### Not implemented
 
@@ -1006,7 +1014,6 @@ spot-price: On owned hardware there is no spot-to-on-demand price differential,
 - `GetServerCertificate`
 - `GetServiceLastAccessedDetails`
 - `GetServiceLastAccessedDetailsWithEntities`
-- `ListAccountAliases`
 - `ListDelegationRequests`
 - `ListMFADeviceTags`
 - `ListMFADevices`
@@ -1038,7 +1045,6 @@ spot-price: On owned hardware there is no spot-to-on-demand price differential,
 - `UpdateGroup`
 - `UpdateLoginProfile`
 - `UpdateOpenIDConnectProviderThumbprint`
-- `UpdateRoleDescription`
 - `UpdateSAMLProvider`
 - `UpdateServerCertificate`
 - `UpdateUser`
@@ -1079,7 +1085,7 @@ signing-certificates: X.509 signing certificates are an EC2-Classic SOAP credent
 
 ## rds
 
-`2014-10-31` — 26 of 164 modelled operations implemented (15.9%), 0 stubbed, 4 not applicable, 5 registered outside the pinned model.
+`2014-10-31` — 27 of 164 modelled operations implemented (16.5%), 0 stubbed, 4 not applicable, 5 registered outside the pinned model.
 
 ### Not implemented
 
@@ -1178,7 +1184,6 @@ signing-certificates: X.509 signing certificates are an EC2-Classic SOAP credent
 - `ModifyDBShardGroup`
 - `ModifyDBSnapshot`
 - `ModifyDBSnapshotAttribute`
-- `ModifyDBSubnetGroup`
 - `ModifyEventSubscription`
 - `ModifyGlobalCluster`
 - `ModifyIntegration`

docs/coverage/rds/README.md

@@ -19,7 +19,7 @@ tags:
 
 ## Overview
 
-Spinifex implements **26 operations** in the RDS `2014-10-31` API model.
+Spinifex implements **27 operations** in the RDS `2014-10-31` API model.
 
 ### Engines
 
@@ -54,7 +54,7 @@ A parameter whose omission would create a false safety, security or availability
 | `DBSecurityGroups` | EC2-Classic security groups — use `VpcSecurityGroupIds` |
 | `DBClusterIdentifier`, `DBClusterSnapshotIdentifier` | Clustered engines are not offered |
 | `EnableCloudwatchLogsExports` | Log export is not implemented |
-| `EngineVersion` other than the engine's pin, `Engine` on modify | No in-place engine or version change |
+| `EngineVersion` other than the engine's pin; on modify, any `EngineVersion` and `Engine` | No in-place engine or version change |
 | `Engine=mysql` (and Aurora engines) | Oracle MySQL is not offered; `mariadb` is a distinct engine, not an alias for it |
 | `NewDBInstanceIdentifier` | The identifier is the DNS label and the KV key |
 | `DBPortNumber`, `DBSubnetGroupName` on modify | Both would move the endpoint |
@@ -67,6 +67,8 @@ A parameter whose omission would create a false safety, security or availability
 | `EnableCustomerOwnedIp` | An Outposts feature |
 | `ForceFailover` (reboot) | No standby to fail over to |
 | `DBSnapshotIdentifier` (stop) | Snapshot-on-stop is not implemented |
+| `DBInstanceAutomatedBackupsArn` (DescribeDBInstanceAutomatedBackups) | Cross-region automated backup replication is not offered, so no replicated backup has an ARN |
+| `dbi-resource-id` filter (DescribeDBSnapshots, DescribeDBInstanceAutomatedBackups) | Snapshots and automated backups do not record their source instance's resource ID |
 
 ### Operations
 
@@ -93,6 +95,7 @@ A parameter whose omission would create a false safety, security or availability
 | `ListTagsForResource` |
 | `ModifyDBInstance` |
 | `ModifyDBParameterGroup` |
+| `ModifyDBSubnetGroup` |
 | `RebootDBInstance` |
 | `RemoveTagsFromResource` |
 | `RestoreDBInstanceFromDBSnapshot` |

docs/coverage/rds/intro.md

@@ -31,7 +31,7 @@ A parameter whose omission would create a false safety, security or availability
 | `DBSecurityGroups` | EC2-Classic security groups — use `VpcSecurityGroupIds` |
 | `DBClusterIdentifier`, `DBClusterSnapshotIdentifier` | Clustered engines are not offered |
 | `EnableCloudwatchLogsExports` | Log export is not implemented |
-| `EngineVersion` other than the engine's pin, `Engine` on modify | No in-place engine or version change |
+| `EngineVersion` other than the engine's pin; on modify, any `EngineVersion` and `Engine` | No in-place engine or version change |
 | `Engine=mysql` (and Aurora engines) | Oracle MySQL is not offered; `mariadb` is a distinct engine, not an alias for it |
 | `NewDBInstanceIdentifier` | The identifier is the DNS label and the KV key |
 | `DBPortNumber`, `DBSubnetGroupName` on modify | Both would move the endpoint |
@@ -44,3 +44,5 @@ A parameter whose omission would create a false safety, security or availability
 | `EnableCustomerOwnedIp` | An Outposts feature |
 | `ForceFailover` (reboot) | No standby to fail over to |
 | `DBSnapshotIdentifier` (stop) | Snapshot-on-stop is not implemented |
+| `DBInstanceAutomatedBackupsArn` (DescribeDBInstanceAutomatedBackups) | Cross-region automated backup replication is not offered, so no replicated backup has an ARN |
+| `dbi-resource-id` filter (DescribeDBSnapshots, DescribeDBInstanceAutomatedBackups) | Snapshots and automated backups do not record their source instance's resource ID |

docs/exoscale-integration/README.md

@@ -0,0 +1,258 @@
+---
+title: "Spinifex on Exoscale"
+seoTitle: "Run Spinifex on Exoscale with Elastic IPs — Spinifex Docs"
+description: "Install Spinifex on one Exoscale instance and run EC2, load balancers and EKS on real Exoscale Elastic IPs."
+category: "Install"
+tags:
+  - install
+  - exoscale
+  - cloud
+  - elastic ip
+resources:
+  - title: "Single-Node Install"
+    url: "/docs/install"
+  - title: "Spinifex on Oracle Cloud"
+    url: "/docs/oci-integration"
+  - title: "VPC Networking"
+    url: "/docs/vpc-networking"
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+---
+
+# Running Spinifex on Exoscale
+
+> [!WARNING]
+> **Work in progress.** This page is not yet published on docs.mulgadc.com. Do not add it to `scripts/docs-config.json` in `mulga-docs` until the [Limits](#limits) section is settled.
+
+> One Exoscale instance, the AWS APIs on its address, and guests reachable on real Exoscale Elastic IPs that Spinifex creates and deletes for you.
+
+## Table of Contents
+
+- [What you get](#what-you-get)
+- [Before you start](#before-you-start)
+- [Install](#install)
+- [Use it from your laptop](#use-it-from-your-laptop)
+- [Troubleshooting](#troubleshooting)
+- [Limits](#limits)
+
+---
+
+## What you get
+
+A single Exoscale instance runs the whole Spinifex stack: EC2, EBS, S3, VPC, load balancers and EKS, behind an endpoint the ordinary AWS CLI, SDKs and Terraform provider talk to unmodified.
+
+Whenever something needs a public address, such as an instance in a public subnet, an Elastic IP, an internet-facing load balancer or a NAT gateway, Spinifex creates an Exoscale Elastic IP, attaches it to your instance, and deletes it again when the resource goes away. You never manage the addresses by hand.
+
+<p align="center">
+  <img src="../../.github/assets/diagrams/exoscale-single-node.svg" alt="Single node on Exoscale — the instance on the zone's public network with one interface, Elastic IPs routed into OVN through the routed-NAT transit veth, the control plane, tenant VPCs and the block storage volume" width="900">
+</p>
+
+---
+
+## Before you start
+
+You need:
+
+- An Exoscale organisation and an **API key** that can create, attach, detach and delete Elastic IPs.
+- An SSH key pair.
+- The AWS CLI on your laptop.
+
+Then create these in one zone, from the Exoscale console or the `exo` CLI. This guide uses `de-fra-1`.
+
+| Resource | Setting |
+| --- | --- |
+| Compute instance | `standard.huge` (8 vCPU / 32 GiB) or larger, template `Linux Ubuntu 26.04 LTS 64-bit`, 100 GiB disk, your SSH key |
+| Block storage volume | 256 GiB or more, attached to the instance |
+| Security group | Allow all inbound TCP, UDP and ICMP from `0.0.0.0/0`, attached to the instance |
+
+Note the instance's **ID** (a UUID, shown on its page in the console) and the volume's **ID**. You do not need to create any Elastic IPs.
+
+> [!NOTE]
+> The security group is open on purpose. Guests are filtered by their own Spinifex security groups, and the node by the Spinifex host firewall you turn on in step 5.
+
+---
+
+## Install
+
+Everything below runs on the instance, over SSH as `ubuntu`.
+
+### 1. Check that the instance can run VMs
+
+```bash
+ls -l /dev/kvm && cat /sys/module/kvm_intel/parameters/nested
+```
+
+`/dev/kvm` must exist and the second line must print `Y`. If either fails, pick a different instance type before going further.
+
+### 2. Put the data volume at `/var/lib/spinifex`
+
+Every guest disk and S3 object lives here, so it belongs on the volume and not on the 100 GiB root disk.
+
+```bash
+VOL_ID=<volume ID>
+DEV=/dev/disk/by-id/virtio-${VOL_ID:0:20}
+
+sudo mkfs.ext4 -L spinifex-data "$DEV"          # only on a new, empty volume
+sudo mkdir -p /var/lib/spinifex
+echo "UUID=$(sudo blkid -o value -s UUID "$DEV") /var/lib/spinifex ext4 defaults,nofail 0 2" | sudo tee -a /etc/fstab
+sudo mount /var/lib/spinifex
+findmnt /var/lib/spinifex                       # must print a line
+```
+
+### 3. Install the `exo` CLI and give it your key
+
+Spinifex manages Elastic IPs by running `exo`, so it needs the CLI and a key file.
+
+```bash
+curl -fsSLO https://github.com/exoscale/cli/releases/download/v1.101.0/exoscale-cli_1.101.0_linux_amd64.deb
+sudo dpkg -i exoscale-cli_1.101.0_linux_amd64.deb
+
+sudo mkdir -p /etc/spinifex/exoscale
+sudo tee /etc/spinifex/exoscale/exoscale.toml >/dev/null <<'EOF'
+defaultaccount = "spinifex"
+
+[[accounts]]
+  name        = "spinifex"
+  key         = "EXO..."
+  secret      = "..."
+  defaultZone = "de-fra-1"
+EOF
+sudo chmod 700 /etc/spinifex/exoscale && sudo chmod 600 /etc/spinifex/exoscale/exoscale.toml
+```
+
+Check the key works: `sudo exo --config /etc/spinifex/exoscale/exoscale.toml compute elastic-ip list -z de-fra-1`.
+
+### 4. Install Spinifex
+
+```bash
+curl -sfL https://install.mulgadc.com | sudo bash
+sudo /usr/local/share/spinifex/setup-ovn.sh --management --nat-uplink
+
+sudo spx admin init --node node1 --nodes 1 --region eu-central-1 --az eu-central-1a \
+  --ipsec=false --external-mode=nat
+sudo chown -R spinifex-daemon: /etc/spinifex/exoscale
+```
+
+Now tell Spinifex where to get public addresses. Add this to the end of `/etc/spinifex/spinifex.toml`, just above the `[bootstrap]` section, using your instance ID:
+
+```toml
+[[network.external_pools]]
+name                 = "exo"
+source               = "exoscale"
+exoscale_zone        = "de-fra-1"
+exoscale_instance_id = "<instance ID>"
+```
+
+Leave the `nat-transit` pool that `init` wrote where it is.
+
+> [!NOTE]
+> `eu-central-1` is the region name your AWS tools will see. It is your choice and does not have to match the Exoscale zone.
+
+### 5. Turn on the firewall, then start
+
+```bash
+sudo env SETUP_STAGES=firewall /usr/local/share/spinifex/setup.sh --firewall on
+sudo systemctl start spinifex.target
+systemctl list-units 'spinifex-*' --no-pager
+```
+
+Every `spinifex-*` unit should be `active`. The daemon log confirms the Elastic IP integration is working:
+
+```bash
+sudo journalctl -u spinifex-daemon | grep -E 'exo CLI|Exoscale allocator ready'
+```
+
+### 6. Import an image
+
+```bash
+sudo spx admin images import --name ubuntu-26.04-x86_64 --config /etc/spinifex/spinifex.toml
+```
+
+For EKS, also import `spinifex-eks-node`. `sudo spx admin images list --config /etc/spinifex/spinifex.toml` shows the full catalogue.
+
+---
+
+## Use it from your laptop
+
+Copy the cluster's CA and the admin credentials from the instance:
+
+```bash
+NODE=<instance public IP>
+mkdir -p ~/.spinifex
+ssh ubuntu@$NODE 'sudo cat /etc/spinifex/ca.pem' > ~/.spinifex/exo-ca.pem
+```
+
+Copy the `[spinifex]` section of `~/.aws/credentials` on the instance into your own `~/.aws/credentials` as `[spinifex-exo]`, then:
+
+```bash
+aws configure set --profile spinifex-exo region eu-central-1
+export AWS_PROFILE=spinifex-exo AWS_CA_BUNDLE=~/.spinifex/exo-ca.pem
+export AWS_ENDPOINT_URL=https://$NODE:9999
+
+aws ec2 describe-regions        # proves the connection
+```
+
+### Launch an instance
+
+```bash
+aws ec2 import-key-pair --key-name demo --public-key-material fileb://~/.ssh/id_ed25519.pub
+SG=$(aws ec2 describe-security-groups --filters Name=group-name,Values=default --query 'SecurityGroups[0].GroupId' --output text)
+aws ec2 authorize-security-group-ingress --group-id "$SG" --protocol tcp --port 22 --cidr 0.0.0.0/0
+
+AMI=$(aws ec2 describe-images --filters Name=name,Values='*ubuntu-26.04*' --query 'Images[0].ImageId' --output text)
+aws ec2 run-instances --image-id "$AMI" --instance-type t3.small --key-name demo \
+  --query 'Instances[0].InstanceId' --output text
+```
+
+After about a minute, `describe-instances` shows a public IP. That is a new Exoscale Elastic IP, and it appears in the Exoscale console with a description beginning `spinifex:`.
+
+```bash
+ssh ubuntu@<public IP> 'curl -s ifconfig.me'    # prints the same address
+```
+
+Terminating the instance deletes the Elastic IP at Exoscale.
+
+### Load balancers and EKS
+
+The [Terraform workbooks](https://github.com/mulgadc/spinifex/tree/main/docs/terraform-workbooks) run unchanged. Pass the region and endpoint:
+
+```bash
+cd nginx-alb
+terraform init
+terraform apply -var region=eu-central-1 -var spinifex_endpoint=https://$NODE:9999
+```
+
+`nginx-alb` builds two web servers behind an internet-facing load balancer. `eks-quickstart` builds a Kubernetes cluster with one worker. Run `terraform destroy` before starting the next one, because of the quota below.
+
+> [!IMPORTANT]
+> **Exoscale allows five Elastic IPs per organisation by default**, and every public address Spinifex hands out uses one. An instance with a public IP uses one, an internet-facing load balancer one, a NAT gateway one, and `eks-quickstart` three. Past the quota, `allocate-address` returns `AddressLimitExceeded` and a launch returns `InsufficientAddressCapacity`. Ask Exoscale support to raise it.
+>
+> **Exoscale keeps counting a deleted Elastic IP for a while.** After a `terraform destroy`, `exo limits` still shows the old usage even though `exo compute elastic-ip list` no longer shows the addresses, so an `apply` straight afterwards can hit the quota. Wait until `exo limits` agrees with the list before starting the next workbook.
+
+---
+
+## Troubleshooting
+
+**A guest's public IP does not answer from the instance itself.** That is expected: traffic from the host to a guest's Elastic IP is dropped. Test from your laptop.
+
+**SSH or HTTP is refused straight after launch.** Guests take 30 to 60 seconds to boot. Wait and retry.
+
+**An EKS cluster goes to `FAILED` within seconds, or a load balancer never becomes active.** Run `exo limits`. If Elastic IP usage is at the maximum, the cluster could not get its public endpoint; destroy it, wait for usage to drop, and apply again.
+
+**The daemon logs an `exo` error at start.** Run the check at the end of step 3. A wrong key, secret or zone is the usual cause.
+
+**An Elastic IP you created yourself.** Spinifex only ever touches Elastic IPs whose description starts with `spinifex:<instance ID>:`. Anything else in the organisation is left alone, but it still counts against the quota.
+
+---
+
+## Limits
+
+Proven on one `standard.huge` instance in `de-fra-1` on 2026-09-30: EC2 with a public IP, Elastic IP allocate and release, the `nginx-alb` workbook and the `eks-quickstart` workbook with one worker. Each was reached from the internet and left no Elastic IP behind at Exoscale.
+
+Not yet tested:
+
+- **More than one node.** Each node would carry its own guests' Elastic IPs, with cluster traffic over an Exoscale private network.
+- **Moving an Elastic IP between nodes** with its guest.
+- **A narrowly scoped API key.** The proof used an unrestricted key.
+- **Other zones, bare metal and GPU instances.**
+- **Guest disk performance** on Exoscale block storage.

docs/install/install-multi-node/README.md

@@ -213,6 +213,13 @@ Choose one method and apply it to **every** server in the cluster.
 curl -fsSL https://install.mulgadc.com | bash
 ```
 
+This installs the latest release. To try the newest development build instead, set `INSTALL_SPINIFEX_CHANNEL=dev` on **every** node — a cluster must not mix versions. It carries unreleased work and has not been through release testing.
+
+```bash
+curl -fsSL https://install.mulgadc.com -o /tmp/spx-install.sh
+sudo INSTALL_SPINIFEX_CHANNEL=dev bash /tmp/spx-install.sh
+```
+
 **Option B — bare metal, from the ISO.** Boot each server from the Spinifex ISO and follow [Bootable USB Install](/docs/install-usb). This installs the operating system, partitions the disks, and configures the hostname and network interfaces alongside Spinifex. The ISO installer does not form a cluster — that is what the remaining steps do.
 
 Complete this step on all three servers before continuing. Step 4 requires every node to be installed, reachable, and available at the same time.

docs/install/install/README.md

@@ -75,6 +75,13 @@ curl -fsSL https://install.mulgadc.com | bash
 
 The installer downloads the Spinifex binary and bootstraps all dependencies (QEMU, OVN/OVS, AWS CLI).
 
+This installs the latest release. To try the newest development build instead, set the channel — it carries unreleased work and has not been through release testing, so use it to evaluate a fix or a new feature rather than for anything you depend on.
+
+```bash
+curl -fsSL https://install.mulgadc.com -o /tmp/spx-install.sh
+sudo INSTALL_SPINIFEX_CHANNEL=dev bash /tmp/spx-install.sh
+```
+
 ## Step 2. Setup OVN Networking
 
 If your WAN interface is already a bridge (e.g. `br-wan`), setup-ovn.sh auto-detects it:

docs/oci-integration/oci-architecture/README.md

@@ -0,0 +1,483 @@
+---
+title: "OCI Architecture"
+seoTitle: "Spinifex on OCI: Architecture and Operations — Spinifex Docs"
+description: "How Spinifex is wired on OCI: both VNICs, the iSCSI data volume and the public-address allocator, with the IAM policy, the quotas and the troubleshooting."
+category: "Cloud Install"
+sections:
+  - overview
+  - troubleshooting
+tags:
+  - install
+  - oci
+  - oracle cloud
+  - terraform
+  - architecture
+  - troubleshooting
+resources:
+  - title: "Spinifex on Oracle Cloud"
+    url: "/docs/oci-integration"
+  - title: "Host Firewall"
+    url: "/docs/host-firewall"
+  - title: "VPC Networking"
+    url: "/docs/vpc-networking"
+  - title: "Multi-Node Install"
+    url: "/docs/install-multi-node"
+---
+
+# Spinifex on OCI: Architecture and Operations
+
+> The reference half of the OCI integration. [Spinifex on Oracle Cloud](/docs/oci-integration) is the step-by-step deployment; everything here is what sits underneath it, plus the troubleshooting.
+
+## Table of Contents
+
+- [Why run Spinifex on OCI](#why-run-spinifex-on-oci)
+- [Sizing](#sizing)
+- [How it fits together](#how-it-fits-together)
+- [Credentials: an API key, or an instance principal](#credentials-an-api-key-or-an-instance-principal)
+- [The IAM policy](#the-iam-policy)
+- [Quotas](#quotas)
+- [Firewalls, in all three layers](#firewalls-in-all-three-layers)
+- [What Terraform builds, per node](#what-terraform-builds-per-node)
+- [Every Terraform variable](#every-terraform-variable)
+- [The `oci` CLI](#the-oci-cli)
+- [Deploying without Terraform](#deploying-without-terraform)
+- [Troubleshooting](#troubleshooting)
+
+---
+
+## Overview
+
+## Why run Spinifex on OCI
+
+**Lower cost for the same shape of workload.** OCI prices compute and block storage well below the large US clouds for equivalent OCPU and IOPS, and Spinifex turns one large instance into a whole EC2/EBS/S3 surface. A tenant's twenty small instances become twenty QEMU guests on hardware you are billed for once, instead of twenty billable cloud instances.
+
+**Egress stops being the line item that decides the architecture.** Oracle publishes a large monthly outbound-transfer allowance per tenancy, then charges per GB beyond it at roughly an order of magnitude below the majors. Check the current pricing page rather than a number in a document, but that gap is the reason this integration exists. Traffic _between_ guests never leaves the host, or on a cluster never leaves the VCN: it is Geneve over the private plane, which is not billed as internet egress at all.
+
+**The workload stops being AWS-shaped and becomes yours.** Your tenants code against Spinifex's API, so one set of Terraform, SDK calls and AMIs runs on OCI today and on bare metal in your own rack whenever you decide to move it. Leaving AWS becomes a deployment decision rather than a rewrite, and the second move costs nothing extra.
+
+**You keep the parts of OCI worth keeping.** Bare-metal shapes, fault domains, block volumes with a chosen performance tier, and real public addresses out of OCI's own pool. Spinifex registers each guest's address with OCI rather than hiding the cloud underneath it, so the guest is genuinely reachable on the internet instead of sitting behind a shared NAT.
+
+## Sizing
+
+**Bare metal is the recommendation.** `BM.Standard.E2.64` or similar gives you the hardware directly, with no nested virtualisation, full NIC performance, and nothing between your guests and the wire. Where bare metal is out of scope, a single `VM.Standard.E6.Flex` suits testing and development, or three of them production.
+
+|                 | Minimum               | Recommended                                                       |
+| --------------- | --------------------- | ----------------------------------------------------------------- |
+| **Shape**       | `VM.Standard.E6.Flex` | Bare metal, e.g. `BM.Standard.E2.64`                              |
+| **OCPU**        | 8 (16 vCPU)           | 32+                                                               |
+| **RAM**         | 32 GB                 | 128 GB+                                                           |
+| **Data volume** | 256 GiB block volume  | 1 TB+ NVMe-backed                                                 |
+| **VNICs**       | 2                     | 2                                                                 |
+| **Nodes**       | 1                     | 3, see [Cluster sizing](/docs/install-multi-node#cluster-sizing)  |
+
+## How it fits together
+
+Everything in [Single-Node Install](/docs/install) and [Multi-Node Install](/docs/install-multi-node) still applies: formation, storage and OVN behave exactly as they do on hardware you own. The layer underneath is what changes. An OCI VCN is not an Ethernet segment, so the external datapath is wired differently, and public addresses come from OCI's API rather than from a range you write in a config file.
+
+### Single node: the whole AWS surface inside one OCI instance
+
+One instance runs the control plane and every tenant guest. Customers reach the AWS APIs on the node's own address, while their instances reach the internet through addresses registered on the node's second VNIC.
+
+<p align="center">
+  <img src="../../../.github/assets/diagrams/oci-single-node.svg" alt="Single node on OCI: the instance inside the VCN public subnet, both VNICs, br-wan, the routed-NAT transit veth, the control plane, tenant VPCs and the iSCSI data volume" width="900">
+</p>
+
+Two things in that picture are the whole integration. The guest's private address (`10.0.1.4`) never leaves the host, because OVN and the host route both hold it. The address OCI delivers to is a **secondary private IP registered on VNIC 1**, appearing on `br-wan`, and the customer's public address (`150.230.13.131`) is a reserved public IP that OCI 1:1-NATs onto it upstream. `describe-instances` reports the public one while every datapath object holds the private one, which is why the two never appear together on the host.
+
+The third is storage. Every guest disk, every S3 object and the JetStream state all live under `/var/lib/spinifex`, which is an **OCI block volume attached over iSCSI**, not the boot disk. Size it for the guests you intend to run. A boot volume that quietly ends up carrying the stack is the most common way an install on OCI fails later rather than immediately.
+
+### Three nodes: the same surface, distributed
+
+Each node registers the addresses for **its own** guests on **its own** second VNIC, and the allocator runs per node, so nothing has to know another node's OCIDs. When a guest moves, whether through a stop/start that lands elsewhere or a host failure, the node it lands on claims the address's private half from the old VNIC with a single OCI call. The OCID and the public half are untouched, so the customer's address never changes.
+
+<p align="center">
+  <img src="../../../.github/assets/diagrams/oci-three-node.svg" alt="Three nodes on OCI: three instances inside the VCN public subnet, one per fault domain, each with its own VNICs, addresses and iSCSI volume, joined east-west by Geneve, NATS, predastore and the OVN raft" width="900">
+</p>
+
+The overlay, the object shards and the OVN databases all cross **VNIC 0**, inside the VCN. Size that plane rather than the external one: guest-to-guest traffic between nodes is Geneve over the VCN, and it is the link every distributed layer shares.
+
+> [!NOTE]
+> **The Spinifex region is your own naming and has nothing to do with the OCI region.** `ap-southeast-2` in `spinifex.toml` beside `ap-sydney-1` in OCI is correct and expected. Do not read one as evidence about the other.
+
+## Credentials: an API key, or an instance principal
+
+The allocator needs OCI credentials at runtime. **A node with none forms, passes every health check, and then refuses every launch that wants a public address** with `InsufficientAddressCapacity`, naming no cause outside its own journal.
+
+### An API key, the documented path
+
+An API key needs nothing from a tenancy admin, which also makes it the only option when the compartment belongs to someone else's tenancy. The cost is key material on disk, with a rotation obligation, on every node:
+
+```bash
+# On your workstation, not the node.
+openssl genrsa -out ~/.oci/oci_api_key.pem 2048
+chmod 600 ~/.oci/oci_api_key.pem
+openssl rsa -pubout -in ~/.oci/oci_api_key.pem -out ~/.oci/oci_api_key_public.pem
+openssl rsa -pubout -outform DER -in ~/.oci/oci_api_key.pem \
+  | openssl md5 -c        # this is the fingerprint
+```
+
+Upload the public key to the user under **Identity → Users → API Keys**, and record the user OCID, the fingerprint and the tenancy OCID.
+
+Each node then needs **two** files, and a node holding only the key is the silent failure described above.
+
+| Path                                | Mode                 | Contents                                                                 |
+| ----------------------------------- | -------------------- | ------------------------------------------------------------------------ |
+| `/etc/spinifex/oci/oci_api_key.pem` | `0640 root:spinifex` | The private key                                                          |
+| `/etc/spinifex/oci/config`          | `0640 root:spinifex` | An ordinary OCI SDK config, profile `[spinifex]`, naming the key by path |
+
+The profile name must be `spinifex` to match `oci_config_profile` in the pool:
+
+```ini
+[spinifex]
+user=ocid1.user.oc1..aaaa...
+fingerprint=12:34:56:78:90:ab:cd:ef:12:34:56:78:90:ab:cd:ef
+tenancy=ocid1.tenancy.oc1..aaaa...
+region=ap-sydney-1
+key_file=/etc/spinifex/oci/oci_api_key.pem
+```
+
+**`spx-oci-config.sh` writes both files on every node, and the deploy calls it for you.** It runs after formation and before the pool is configured, which is the only window where a node has `/etc/spinifex` but has not started the allocator, and it reads the credential from your `[spinifex]` profile, a dedicated key, a set of `OCI_SPX_*` environment variables, or failing all three the credential Terraform itself used — resolved through the same `scripts/oci_env.py` as the apply, so the two cannot disagree.
+
+**The key reaches the nodes over SSH and goes nowhere else.** It is never rendered into user-data or Terraform state, where it would be readable from instance metadata for the life of the instance, and it is carried in the remote shell's stdin rather than its arguments, where `ps` would expose it to every other tenant on the host. `--credential-hook PATH` still takes any executable called as `hook <ssh-key> <host>...`, so a tenancy that keeps credentials in a vault can substitute its own.
+
+### An instance principal: no key material, but it needs a tenancy admin
+
+The instance certificate is served at `/opc/v2/identity/cert.pem` and the Go SDK authenticates as the instance with no key on disk. It must also be _authorised_, by a dynamic group and a policy, and **both are tenancy-root resources**: `oci_identity_dynamic_group` is created with `compartment_id` set to the tenancy OCID. A compartment-scoped user cannot create them, and the attempt fails with `404-NotAuthorizedOrNotFound` on `CreateDynamicGroup`, which reads like a misconfiguration and is not one.
+
+```bash
+cd scripts/terraform/oci-spx
+./setup-identity.sh --dry-run     # always first
+./setup-identity.sh               # needs a tenancy-admin OCI profile
+```
+
+Then set `instance_principal = "adopt"` in your tfvars and pass `--instance-principal` to the deploy instead of `--credential-hook`. The dynamic group matches on `instance.compartment.id`, so it covers every node you ever build in that compartment and nothing needs updating when nodes are replaced.
+
+## The IAM policy
+
+Spinifex calls exactly ten operations, all of them in `spinifex/cloud/oci/client.go`. Grant no more than these:
+
+| Operation                                  | Why                                                                              |
+| ------------------------------------------ | -------------------------------------------------------------------------------- |
+| `CreatePrivateIp`                          | Register the secondary private IP the datapath carries                           |
+| `DeletePrivateIp`                          | Release it                                                                       |
+| `GetPrivateIp`                             | Poll for `AVAILABLE` after an asynchronous assign                                |
+| `ListPrivateIps`                           | The startup reconcile, and the affinity pass that moves an address between nodes |
+| `UpdatePrivateIp`                          | Reassign a private IP to this node's VNIC when a guest moves here                |
+| `CreatePublicIp`                           | Create the RESERVED public IP                                                    |
+| `DeletePublicIp`                           | Release it                                                                       |
+| `GetPublicIp` / `GetPublicIpByPrivateIpId` | Poll lifecycle state, and reconcile a private IP back to its public half         |
+| `UpdatePublicIp`                           | Attach and detach the public IP from a private IP                                |
+
+The matching policy, scoped to the one compartment Spinifex allocates in:
+
+```text
+Allow group SpinifexOperators to use    vnics       in compartment <compartment-name>
+Allow group SpinifexOperators to manage private-ips in compartment <compartment-name>
+Allow group SpinifexOperators to manage public-ips  in compartment <compartment-name>
+Allow group SpinifexOperators to use    subnets     in compartment <compartment-name>
+```
+
+`instance-principal.tf` grants the dynamic group these same four statements, so an instance principal can allocate a public address exactly as an IAM user can. The two credential routes differ in who is authorised, never in what.
+
+> [!IMPORTANT]
+> **Allocating one address needs permissions on three resource types, and a 404 names none of them.** `CreatePrivateIp` is checked against `PRIVATE_IP_CREATE`, `PRIVATE_IP_ASSIGN`, `VNIC_ASSIGN` and `SUBNET_ATTACH`, and `DeletePrivateIp` against `SUBNET_DETACH`. `VNIC_ASSIGN` lives in the vnics `use` verb, so `read vnics` is not enough; `SUBNET_ATTACH` and `SUBNET_DETACH` live in the subnets `use` verb, so a policy naming only vnics and IPs authorises nothing. Either omission fails every allocation with `NotAuthorizedOrNotFound` on a node that looks entirely healthy, and OCI uses that one error for "you may not" and "it is not there" alike, so it reads as a wrong compartment OCID rather than a missing verb.
+
+To tell the two apart, make one call per permission class and see which is the first to fail. `GetVnic` tests `VNIC_READ`, `ListPrivateIps` tests `PRIVATE_IP_READ`, and `GetSubnet` tests `SUBNET_READ`. If the first two succeed and the third does not, the credential is authentic and in the right compartment, and the policy is missing its subnet statement.
+
+**Scope it to a compartment, not the tenancy.** `manage public-ips` at tenancy level lets a compromised node consume the whole regional reserved-public-IP quota.
+
+**Do not grant `manage vnics` or `manage instances`.** Spinifex never creates, attaches or deletes a VNIC, and never touches an instance. It only adds addresses to a VNIC that already exists, which `use` permits. `manage` also permits VNIC lifecycle, which is strictly more than the integration can use.
+
+## Quotas
+
+Two limits bound how many public addresses a deployment can hand out, and they do not scale the same way:
+
+| Quota                              | Default | Scope                     | Notes                                                                                          |
+| ---------------------------------- | ------- | ------------------------- | ---------------------------------------------------------------------------------------------- |
+| **Reserved public IPs**            | 50      | Per region, whole tenancy | Shared by every node, so it does **not** grow with node count. This is the wall you hit first. |
+| **Secondary private IPs per VNIC** | 64      | Per VNIC, so per node     | Not raisable. Three nodes gives you 192, so this is the limit that does scale.                 |
+
+**On a new tenancy, check the compute limits too.** A fresh account commonly carries a service limit of zero for bare-metal shapes and a low one for flex VMs, which can leave `BM.Standard.E2.64` unavailable until you request it. That refusal arrives at apply time, not at plan time.
+
+A raise is a support request, not a setting: Console → Governance & Administration → Limits, Quotas and Usage → "Request a service limit increase". Cite the exact limit name from `oci limits definition list`, not a name from documentation.
+
+```bash
+oci limits definition list --service-name vcn --compartment-id <tenancy-ocid>
+oci limits value list       --service-name vcn --compartment-id <tenancy-ocid>
+oci limits resource-availability get --service-name vcn \
+    --limit-name <name-from-above> --compartment-id <tenancy-ocid>
+```
+
+**These calls need a grant the node does not have, and must not be given.** Limits and usage are tenancy-level, which cannot be compartment-scoped the way the policy above insists the node's is. Create a **second, read-only principal** for an operator or a reporting job, and keep it off every node:
+
+```text
+Allow group SpinifexReporting to read limits        in tenancy
+Allow group SpinifexReporting to read usage-reports in tenancy
+```
+
+The second verb is what Cost Analysis and `oci usage-api usage-summary request-summarized-usages` read. Adding either to `SpinifexOperators` would hand every node tenancy-wide read, which is exactly what the node policy refuses.
+
+## Firewalls, in all three layers
+
+Three independent things filter traffic to an OCI deployment, and they are easy to mistake for one. Nearly every "my port is closed" report on OCI turns out to be the wrong layer being adjusted.
+
+| Layer | What it guards | Who configures it |
+| --- | --- | --- |
+| **OCI security list** | The whole public subnet, including the addresses guests are reached on | `node_client_cidr_allow_list` |
+| **The host firewall** | The node's own listeners: console, S3 gate, AWS gateway, SSH, the cluster plane | `node_service_ports`, then the Spinifex nft policy |
+| **OVN security groups** | Each guest, per AWS security-group rules | Your tenants, through the EC2 API |
+
+### The OCI security list is deliberately open, and should stay that way
+
+```hcl
+# network.tf, security_list.public
+ingress_security_rules { protocol = "all", source = var.vcn_cidr }      # intra-VCN
+dynamic "ingress_security_rules" { ... source = each allow-list CIDR }  # protocol = "all"
+```
+
+A guest's public address is a **secondary private IP on VNIC 1, in the public subnet**, so inbound traffic to a guest crosses this security list. Narrowing `node_client_cidr_allow_list` to your own office `/32` therefore cuts internet ingress to **every guest**, not just to the node. A tenant who then opens 443 on their own security group finds it does nothing, with no error anywhere naming the cause.
+
+That is also why the rule is `protocol = "all"` rather than the three service ports. Enumerating ports here would mean a Terraform change for every customer port.
+
+**So leave it at `0.0.0.0/0` for any deployment that serves guests**, and do the restricting in the host firewall, which is the layer that can tell the node's own listeners apart from a guest's. Narrow the allow list only for a private lab with no public guest services.
+
+The separate intra-VCN rule exists so that narrowing the allow list can never cut Geneve, OVN, NATS or predastore between nodes.
+
+### The host firewall is what you harden
+
+Terraform's cloud-init runs `open-service-ports.sh`, which opens all protocols from the peer CIDR plus `node_service_ports` (`3000 8443 9999`) to any source, in the Ubuntu image's **iptables**. SSH is already open to the world from the image's own INPUT rule, and nothing in that script narrows it.
+
+The Spinifex nft policy is installed but **not armed** on this path, because the installer arms it only when asked. Arming it is what gives you the proper policy: public plane, peer-scoped cluster plane, and an SSH scope you own.
+
+```bash
+# On each node. Arms the policy the ISO installer arms by default.
+sudo env SETUP_STAGES=firewall /usr/local/share/spinifex/setup.sh --firewall on
+```
+
+Then narrow SSH in the one file `setup.sh` creates and never overwrites:
+
+```bash
+# /etc/spinifex/firewall/custom.nft is yours, and upgrades leave it alone.
+define trusted_ssh_peers = { 203.0.113.10/32 }
+```
+
+```bash
+sudo nft -c -f /etc/spinifex/firewall/spinifex.nft    # check before applying
+sudo /usr/local/lib/spinifex/spinifex-firewall-apply
+```
+
+What the armed policy allows, and why each port is where it is, is in [Host Firewall](/docs/host-firewall), along with [Narrowing SSH access](/docs/host-firewall#narrowing-ssh-access) and the rule that **forming or expanding a cluster starts by turning the policy off**, because nodes mid-formation are not yet each other's peers.
+
+Two OCI-specific notes on top of that guide:
+
+- **Northstar's DNS on 53 is in the armed policy's public plane, but `node_service_ports` does not include it.** Until the nft policy is armed, a node serving a public zone does not answer on 53 from the internet.
+- **Never flush iptables on an OCI node.** The image's `InstanceServices` chain in OUTPUT is what permits iSCSI to `169.254.2.0/24:3260` and the metadata service, so clearing it takes the data volume and IMDS with it.
+
+## What Terraform builds, per node
+
+| Resource                     | Detail                                                                                                                                             |
+| ---------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `oci_core_instance`          | One per `node_count`, round-robined across **fault domains** so three nodes land on three sets of racks rather than wherever placement puts them   |
+| Primary VNIC                 | Public IP, `hostname_label`, the host plane: SSH, the advertise address, the Geneve endpoint                                                       |
+| `oci_core_vnic_attachment`   | A **second VNIC** in the same subnet. Every Spinifex external address becomes a secondary private IP here at runtime                               |
+| `oci_core_volume`            | A block volume per node, `data_volume_size_in_gbs` at `data_volume_vpus_per_gb`                                                                    |
+| `oci_core_volume_attachment` | **`attachment_type = "iscsi"`**, with the Oracle Cloud Agent's Block Volume Management plugin enabled so the node logs the iSCSI session in itself |
+| cloud-init                   | Partitions, formats, mounts and `fstab`s the volume; builds `br-wan` over the second VNIC; opens the service ports                                 |
+
+The resource-by-resource architecture is in [`scripts/terraform/oci-spx`](https://github.com/mulgadc/spinifex/tree/dev/scripts/terraform/oci-spx).
+
+**The data volume is iSCSI, and that is the detail that bites.** OCI presents the volume as an iSCSI target reachable at `169.254.2.2:3260`, so attaching it in the API does not put a block device on the host. `is_agent_auto_iscsi_login_enabled` makes the Oracle Cloud Agent do the login, and it does that _asynchronously_, so the device is not there when cloud-init first runs. The mount script waits up to ten minutes for it, formats it **only if `blkid` reports no filesystem** so that a re-run cannot erase a populated volume, and writes an fstab entry by UUID:
+
+```text
+UUID=<uuid>  /var/lib/spinifex  ext4  defaults,_netdev,nofail  0 2
+```
+
+`_netdev` orders the mount after the network, and therefore after `iscsid` has a session. `nofail` keeps a missing volume from wedging the boot. Mounting `/var/lib/spinifex` _itself_ rather than somewhere else and symlinking is deliberate: viperblock, predastore and JetStream all land on the volume with no configuration pointing anywhere unusual.
+
+## Every Terraform variable
+
+All of these have defaults that build a working single node, except the first:
+
+| Variable                      | Default               | What it decides                                                                                                                                                                                                                                      |
+| ----------------------------- | --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `compartment_ocid`            | none                  | **Required.** An OCID, not a name: this deploys into a compartment that already exists, because creating one needs tenancy-root rights the API user must not have                                                                                    |
+| `deployment_name`             | `spinifex`            | Prefixes every resource and becomes the VCN `dns_label`, so 1–15 lowercase alphanumerics starting with a letter                                                                                                                                      |
+| `node_count`                  | `1`                   | Nodes, 1–25. `1` for a single node, `3` for a cluster. Each gets its own volume, its own second VNIC and its own fault domain                                                                                                                        |
+| `compute_shape`               | `VM.Standard.E6.Flex` | **`BM.*` for bare metal, `VM.*.Flex` for a VM.** A fixed bare-metal shape carries its own size, so `shape_config` is emitted only when the name contains `Flex`. Set a BM shape and the two sizing variables below are ignored rather than rejected  |
+| `compute_ocpus`               | `8`                   | OCPUs on a flex shape. Validated at ≥ 8 (16 vCPU), the minimum recommended for a node                                                                                                                                                                |
+| `compute_memory_in_gbs`       | `32`                  | GiB on a flex shape. Validated at ≥ 32                                                                                                                                                                                                               |
+| `data_volume_size_in_gbs`     | `256`                 | Size of each node's data volume. Guest disks, S3 objects and JetStream all live here, so size it for the guests rather than for the control plane                                                                                                    |
+| `data_volume_vpus_per_gb`     | `120`                 | Performance tier, 0–120 VPUs/GB. 120 is Ultra High Performance; 10 is Balanced. Higher is more IOPS and more cost                                                                                                                                    |
+| `data_mountpoint`             | `/var/lib/spinifex`   | Where cloud-init mounts it. Change only if you know why                                                                                                                                                                                              |
+| `vcn_cidr`                    | `10.200.0.0/22`       | VCN address space. A `/22` leaves room for the secondary private IPs Spinifex registers per node                                                                                                                                                     |
+| `public_subnet_cidr`          | `10.200.0.0/23`       | The subnet every node sits in, **on both VNICs**. Nodes serve the UI, the S3 gate and the AWS gateway directly, so they cannot live in a private subnet                                                                                              |
+| `node_client_cidr_allow_list` | `["0.0.0.0/0"]`       | OCI security-list ingress, **all protocols**, for the whole public subnet. Guest public addresses cross it too, so read [Firewalls](#firewalls-in-all-three-layers) before narrowing it                                                              |
+| `node_service_ports`          | `[3000, 8443, 9999]`  | Console, predastore gate, AWS gateway, opened in the host's iptables by cloud-init. SSH comes from the image's own rule                                                                                                                              |
+| `instance_principal`          | unset                 | `"adopt"` to authenticate the allocator as the instance rather than with a key file. Needs the tenancy dynamic group and policy to exist                                                                                                            |
+| `ubuntu_version`              | `26.04`               | Canonical platform image version. The newest image for the shape and region is resolved at plan time rather than pinned to a regional OCID                                                                                                           |
+| `wan_bridge_name`             | `br-wan`              | The Linux bridge built over the second VNIC                                                                                                                                                                                                          |
+| `wan_bridge_mtu`              | `9000`                | The OCI VCN carries 9000                                                                                                                                                                                                                             |
+
+**Six more exist and are not yours to set**: `tenancy_ocid`, `user_ocid`, `fingerprint`, `private_key`, `private_key_path` and `region`. `scripts/oci_env.py` fills them as `TF_VAR_*` from the profile in `~/.oci/config` on every `terraform` call the driver makes, so **the region you deploy into is the profile's region** and the credential never reaches a `.tfvars` file. Setting them by hand works, and is how a run with no config file at all would have to work, but then two sources disagree about which tenancy this is.
+
+The profile is resolved as `--oci-profile`, then `$OCI_CLI_PROFILE`, then the reference tenancy, then `DEFAULT`. A name passed explicitly and absent from the config is an error that lists the profiles present; only an unasked-for name falls through. CI takes a different route entirely, reading `OCI_TENANCY_OCID`, `OCI_USER_OCID`, `OCI_FINGERPRINT`, `OCI_PRIVATE_KEY` and `OCI_REGION` from the environment so a runner writes no key to disk. A partial set there is an error rather than a fallback, because half a credential means a secret failed to reach the job.
+
+## The `oci` CLI
+
+**The `oci` CLI is not on the Ubuntu image**, and `apt` has no `python3-oci-cli` package. Nothing in the deployment needs it; install it when you want to inspect allocations by hand:
+
+```bash
+sudo apt-get update && sudo apt-get install -y python3-venv
+python3 -m venv ~/.oci-cli-venv
+~/.oci-cli-venv/bin/pip install --upgrade pip oci-cli
+sudo ln -sf ~/.oci-cli-venv/bin/oci /usr/local/bin/oci
+oci --version
+```
+
+**What the CLI is and is not for.** Spinifex's daemon uses the OCI **Go SDK** for every allocation, so the CLI is not on that path at all. You need it for bootstrap and for operational inspection. Do not build automation that shells out to it during an allocation: allocations run inside a request budget that a forked Python process will not respect.
+
+---
+
+## Deploying without Terraform
+
+**Terraform automates every step of a normal install, and that is the point of it.** The `oci-spx` configuration builds the VCN, the subnets, the gateways, the block volumes and both VNICs per node, then renders the cloud-init that installs Spinifex and sets up OVN. The sequence below happens without anyone typing it, in the same order every time, on one node or three.
+
+If you do need to install by hand, whether for an existing OCI tenancy you cannot run Terraform against, an unsupported shape, or a stage you are debugging, the install itself is not OCI-specific. Follow the standard guides:
+
+- [Installing Spinifex](../../install/install/README.md) for a single node.
+- [Multi-Node Installation](../../install/install-multi-node/README.md) for a cluster, which covers `spx admin init` on the leader and `spx admin join` on the rest.
+
+Four things about OCI are not in those guides, and each has its own section above:
+
+- **The external pool** is `source = "oci"`, which needs the API key or instance principal from [Credentials](#credentials-an-api-key-or-an-instance-principal) and the [IAM policy](#the-iam-policy). Omit it and you get the silent failure that section opens with: a healthy node that cannot hand out an address.
+- **`br-wan` stays a Linux bridge** owned by netplan, linked to OVS `br-ext` by a veth pair. Pass `setup-ovn.sh --wan-bridge=br-wan` and it detects this itself. The NIC never becomes an OVS port, so the VNIC keeps its MAC identity. See [How it fits together](#how-it-fits-together).
+- **The block volume must be mounted with `_netdev`**, because it arrives over iSCSI and the mount is otherwise attempted before a session exists.
+- **IMDS needs the 169.254 remap** described under [Guests cannot reach instance metadata](#guests-cannot-reach-instance-metadata), since the guest's metadata address collides with OCI's own.
+
+---
+
+## Troubleshooting
+
+### If a deploy stage fails
+
+The driver stops at the first failure, names the log, and leaves the rest alone. Logs are all under `.validate-<topology>/`:
+
+| Stage                                           | Log                                      |
+| ----------------------------------------------- | ---------------------------------------- |
+| Building the infrastructure                     | `apply.log`                              |
+| Installing Spinifex                             | `install-<host>.log`                     |
+| Forming the cluster                             | `form.log`                               |
+| Installing the credential, configuring the pool | `credential-hook.log`, `pool-<host>.log` |
+| Verifying public addressing                     | `allocator-<host>.log`                   |
+| The Terraform workbooks                         | `workbooks.log`, `workbooks.tsv`         |
+
+The allocator log is the one worth knowing about, because public addressing is the part with the most moving pieces. If the allocator never comes up, the driver captures the daemon journal, the pool as configured, the bridge and OCI MAC addresses, and whether each credential file exists and is readable, never the credentials themselves, before anything is torn down.
+
+Add `--keep-on-fail` to leave a failed deployment running so you can log in and look. **It keeps billing** until you run `--destroy-only`.
+
+### Addresses and quotas
+
+**Spinifex enforces no limit of its own.** It attempts the allocation and reports what OCI says, so the ceiling you hit is always your tenancy's and never a number baked into the software. A quota is raised with Oracle, not reconfigured here, and there is nothing to restart afterwards.
+
+**`InsufficientAddressCapacity` on `allocate-address` is what exhaustion looks like.** It is the AWS error code for "out of addresses", and Spinifex returns it for either quota, mapped from OCI's own refusal. On a cluster the regional one is the likelier of the two.
+
+**A detached reserved public IP still bills.** `disassociate-address` returns the address to `AVAILABLE` without deleting it, exactly as an unassociated AWS Elastic IP behaves, and it keeps consuming your regional quota as well as your bill. `release-address` frees both.
+
+**A stopped instance costs nothing for its auto-assigned address.** Stopping an instance returns its auto-assigned address to the pool and deletes the reserved public IP object behind it, as AWS does, and the start that follows creates a new one. An **Elastic IP** is yours and survives the stop untouched. That is the whole distinction between the two, and on OCI it is also the difference between a stopped fleet that bills for addresses and one that does not.
+
+### A guest's public address is unreachable
+
+**Check where you are probing from first.** A node is a poor vantage point for a public address, for two independent reasons: the security list may not admit the node's own public address as a source, and a VCN may or may not reflect its public addresses back into itself. Either one fails the probe on a perfectly healthy cluster. Test from outside the VCN, which is where a customer is.
+
+Then, in order:
+
+1. `sudo ovn-nbctl lr-nat-list <router>` for a `dnat_and_snat` row, holding the _private_ address.
+2. `ip route get <private-addr>` to confirm it routes via the transit veth.
+3. `sudo iptables -S FORWARD | grep spinifex-eip-ingress` for both directions.
+4. `oci network private-ip list --vnic-id <vnic>` to confirm the private address is registered with OCI, **and on this node's VNIC**. If it is not, nothing else matters.
+
+**Egress works from one address and not another on the same NIC.** OCI enforces the source address per VNIC, so that means the address is not a registered private-IP object on it. It is never a routing problem on the host.
+
+**An address went dark after a guest moved node.** The affinity pass should have claimed it within 15 seconds. Check the new node's daemon journal for `ocinet affinity`, then use step 4 above to confirm which VNIC OCI thinks holds the private half.
+
+### A node that was fine and then was not
+
+Terraform's cloud-init does the host jobs on each node: the iSCSI volume, `br-wan` and its source routing, the firewall. When one of them did not take, the symptom arrives much later and rarely looks like a networking or storage problem. This one command answers all of it:
+
+```bash
+ssh -i ~/.ssh/oci-spx ubuntu@<node-public-ip> '
+  cloud-init status                  # done
+  sudo iscsiadm -m session           # a session to 169.254.2.2:3260
+  findmnt /var/lib/spinifex          # mounted by UUID, with _netdev
+  ip -br addr show br-wan            # UP, second VNIC address as a /32
+  ip -br addr show enp1s0            # the bridge MEMBER holds NO address
+  ip route show default              # still on the PRIMARY interface
+  ls -l /dev/kvm                     # present
+'
+```
+
+Three answers are worth knowing in advance, because each looks wrong and is not:
+
+- **`br-wan` holds a `/32` and the default route is on the other interface.** That is how two VNICs share one subnet without the second stealing the first one's traffic, and it is what satisfies OCI's per-VNIC source check. `br-wan`'s address appears in `ip rule` instead of in the default route.
+- **`br-wan`'s address is not one you chose.** It is the second VNIC's own private IP, assigned by OCI, and not predictable before the apply.
+- **`enp1s0` carrying an address is the one real fault here.** The kernel derives an on-link route from it that beats the primary VNIC's, so every packet to another node leaves the wrong VNIC and OCI drops it. The node still forms a cluster and still reports `Ready`, because formation uses outbound connections, and it surfaces later as `predastore … the stripe is short one holder` on the first AMI import. `sudo /usr/local/sbin/spinifex-setup-wan-bridge` repairs it in place.
+
+### The allocator
+
+**Confirm it came up**, which `spx get nodes` cannot tell you:
+
+```bash
+sudo journalctl -u spinifex-vpcd --since -5m | grep -i ocinet
+```
+
+```text
+"msg":"ocinet resolved the external VNIC from its interface","pool":"oci-public","iface":"br-wan",
+  "vnic_id":"ocid1.vnic.oc1.ap-sydney-1.abzxsljrjwlf...kimuq","private_ip":"10.200.1.31","subnet":"10.200.0.0/23"
+"msg":"ocinet credential authorised to allocate","pool":"oci-public",
+  "vnic_id":"ocid1.vnic.oc1.ap-sydney-1.abzxsljrjwlf...kimuq","compartment":"ocid1.compartment.oc1..aaaa"
+"msg":"OCI allocator ready","pool":"oci-public","collected":0,"stale_bindings":0,"skipped":1
+```
+
+**`credential authorised to allocate` is the line that matters, and `resolved the external VNIC` is not.** The authorisation line is logged after the node has made one real read in each permission class an allocation needs, so it proves the credential is accepted, the compartment is right and every verb in the policy is held. The resolution line above it comes from the instance metadata service and needs no IAM policy at all, so a node with a policy missing a verb logs it and still refuses every allocation.
+
+The refusal has its own line, which names the verb:
+
+```text
+"msg":"ocinet credential cannot allocate an external address","pool":"oci-public",
+  "error":"reading subnet ocid1.subnet... failed, which needs `use subnets` in compartment ocid1...;
+    AssignPrivateIP requires SUBNET_ATTACH from that same verb: ... NotAuthorizedOrNotFound"
+```
+
+On a cold cluster start you may instead see `OCI allocator reconcile failed … nats: no responders available for request` on some nodes. That is vpcd racing JetStream's KV at boot, where the startup reconcile is skipped and not retried. It is harmless on a new cluster where nothing has been allocated, and it is tracked; restart `spinifex-vpcd` on that node to run it.
+
+### Host
+
+**`RunInstances` returns `ServerInternal`.** Check the daemon journal for the real error with `journalctl -u spinifex-daemon --since -10m`. An OCI allocation failure surfaces this way.
+
+**The stack ends up on the boot disk and the node fills.** The data volume was mounted without `_netdev` and lost the race with `iscsid`. `findmnt /var/lib/spinifex` is the check, and `ls` cannot tell the difference.
+
+### Guests cannot reach instance metadata
+
+**Guests boot but cloud-init hangs on metadata**, or the host stops answering its own. OCI uses `169.254.169.254` for its instance metadata, and so does every AWS-compatible guest, so the host and the guests want the same address.
+
+Spinifex resolves this by moving the _host_ side of the per-guest metadata link to a different link-local pair, leaving guests to ask `169.254.169.254` exactly as they do on AWS. The deploy sets `imds_host_meta_ip` and `imds_host_dns_ip` for you. **Set both or neither**, because a half-configured pair is ignored and the node behaves as if the remap were off, which is what this symptom looks like.
+
+Check it after your first guest launches:
+
+```bash
+ip route get 169.254.169.254                 # your real NIC, not lo
+curl -sH 'Authorization: Bearer Oracle' -o /dev/null -w '%{http_code}\n' \
+     http://169.254.169.254/opc/v2/instance/ # 200
+ip -br addr show | grep ime-                 # holds 169.254.42.x, not .169.x
+```
+
+`ip route get 169.254.169.254` returning `dev lo` is the signature of a missing remap. The same signature can appear after a guest terminates, from a leftover `ime-` endpoint still holding the address; with the remap configured that is harmless, and removing the stale OVS port clears it.
+
+### An OVS problem, when `ovs-vsctl` says nothing is wrong
+
+`ovs-vsctl` exits 0 even when a bridge could not be instantiated. Read the error column and the log, never the exit status:
+
+```bash
+sudo ovs-vsctl --columns=name,error list Interface
+sudo journalctl -u ovs-vswitchd --since -10m
+```

docs/terraform-workbooks/eks-gitops-argocd/README.md

@@ -256,7 +256,7 @@ kubectl -n kube-system logs deploy/aws-load-balancer-controller --tail=50
 
 Confirm the cluster carries `spinifex.io/managed-ingress = "false"` so the LBC owns ingress.
 
-If the controller logs show `FailedBuildModel ... DescribeAvailabilityZones ... 403 ... AccessDenied`, the node role is missing the LBC permissions. The controller runs with the node instance-profile credentials, so the `${var.cluster_name}-node-lbc` policy (`aws_iam_policy.node_lbc`) must be attached to the node role — re-run `tofu apply` on the parent module if it was provisioned before that policy existed.
+If the controller logs show `FailedBuildModel ... DescribeAvailabilityZones ... 403 ... UnauthorizedOperation`, the node role is missing the LBC permissions. The controller runs with the node instance-profile credentials, so the `${var.cluster_name}-node-lbc` policy (`aws_iam_policy.node_lbc`) must be attached to the node role — re-run `tofu apply` on the parent module if it was provisioned before that policy existed.
 
 ### Provider Connection Refused
 

docs/terraform-workbooks/eks-quickstart/README.md

@@ -66,7 +66,7 @@ Under the hood it keeps things minimal: a VPC with two public subnets, an IAM ro
 
 - The cluster's security groups are **auto-managed** — `vpc_config.security_group_ids` is ignored. To reach a NodePort, this workbook looks the worker SG up by its deterministic name (`eks-cluster-<name>-nodegroup-sg`) and adds a single ingress rule.
 - The worker AMI is always Spinifex's `eks-node` image; `ami_type` is recorded but does **not** select the image.
-- `authentication_mode` must be `"API"` (the `API_AND_CONFIG_MAP` mode is rejected).
+- `authentication_mode` can be `"API"` or `"API_AND_CONFIG_MAP"`, and bare `"CONFIG_MAP"` is rejected. Access is granted through access entries, as there is no `aws-auth` ConfigMap path.
 - The workers need outbound internet (here via the IGW) to pull the demo container image.
 
 **Prerequisites:**

go.mod

@@ -4,33 +4,33 @@ go 1.27.0
 
 require (
 	github.com/aws/aws-sdk-go v1.55.8
-	github.com/aws/aws-sdk-go-v2 v1.47.0
+	github.com/aws/aws-sdk-go-v2 v1.47.1
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20
-	github.com/aws/aws-sdk-go-v2/config v1.33.5
-	github.com/aws/aws-sdk-go-v2/credentials v1.20.5
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0
+	github.com/aws/aws-sdk-go-v2/config v1.33.6
+	github.com/aws/aws-sdk-go-v2/credentials v1.20.6
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1
 	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/charmbracelet/bubbles v1.0.0
 	github.com/charmbracelet/bubbletea v1.3.10
 	github.com/charmbracelet/lipgloss v1.1.0
-	github.com/containerd/containerd/v2 v2.4.0
+	github.com/containerd/containerd/v2 v2.4.1
 	github.com/distribution/reference v0.6.0
 	github.com/go-chi/chi/v5 v5.3.2
 	github.com/golang-jwt/jwt/v5 v5.3.1
 	github.com/insomniacslk/dhcp v0.0.0-20260719225207-c76316d4aa82
 	github.com/jackc/pgx/v5 v5.11.0
-	github.com/klauspost/compress v1.20.0
+	github.com/klauspost/compress v1.20.1
 	github.com/klauspost/cpuid/v2 v2.4.0
 	github.com/miekg/dns v1.1.73
 	github.com/minio/highwayhash v1.0.4
-	github.com/mulgadc/bluebottle v1.21.0
-	github.com/mulgadc/northstar v1.21.0
-	github.com/mulgadc/predastore v1.21.0
-	github.com/mulgadc/viperblock v1.21.0
+	github.com/mulgadc/bluebottle v1.22.0
+	github.com/mulgadc/northstar v1.22.0
+	github.com/mulgadc/predastore v1.22.0
+	github.com/mulgadc/viperblock v1.22.0
 	github.com/nats-io/nats-server/v2 v2.15.0
 	github.com/nats-io/nats.go v1.54.0
 	github.com/opencontainers/runtime-spec v1.3.0
-	github.com/oracle/oci-go-sdk/v65 v65.126.0
+	github.com/oracle/oci-go-sdk/v65 v65.126.1
 	github.com/ovn-kubernetes/libovsdb v0.8.1
 	github.com/pelletier/go-toml/v2 v2.4.3
 	github.com/pterm/pterm v0.12.83
@@ -48,6 +48,7 @@ require (
 	golang.org/x/crypto v0.57.0
 	golang.org/x/sync v0.23.0
 	golang.org/x/sys v0.48.0
+	golang.org/x/term v0.46.0
 	golang.org/x/time v0.16.0
 	golang.org/x/tools v0.50.0
 	gopkg.in/ini.v1 v1.67.3
@@ -61,18 +62,18 @@ require (
 	github.com/Microsoft/hcsshim v0.15.0-rc.4 // indirect
 	github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect
 	github.com/atotto/clipboard v0.1.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.3 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.3 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.113.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/signin v1.10.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.38.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.51.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 // indirect
 	github.com/aws/smithy-go v1.28.2 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -177,7 +178,7 @@ require (
 	github.com/spf13/cast v1.10.0 // indirect
 	github.com/spf13/pflag v1.0.10 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
-	github.com/tidwall/btree v1.8.1 // indirect
+	github.com/tidwall/btree v1.8.2 // indirect
 	github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
 	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
@@ -197,7 +198,6 @@ require (
 	golang.org/x/mod v0.41.0 // indirect
 	golang.org/x/net v0.59.0 // indirect
 	golang.org/x/telemetry v0.0.0-20260908163034-4bcc4b2ee518 // indirect
-	golang.org/x/term v0.46.0 // indirect
 	golang.org/x/text v0.42.0 // indirect
 	golang.org/x/vuln v1.6.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20260825221802-da73d73af1c5 // indirect

go.sum

@@ -30,40 +30,40 @@ github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z
 github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
-github.com/aws/aws-sdk-go-v2 v1.47.0 h1:0jsHallhJCeaU0Ko48c/3FK1ctOQ7NpzggxriJOQ8MQ=
-github.com/aws/aws-sdk-go-v2 v1.47.0/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU=
+github.com/aws/aws-sdk-go-v2 v1.47.1 h1:uOIZnp4PK3ZhKI0dNrJrhTEsLxbpXHTAJlwoS1pvAtw=
+github.com/aws/aws-sdk-go-v2 v1.47.1/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20/go.mod h1:g7PNzKcsOKWb4fkSRBA7BZVAS6Y8IcxzN+nRohhQ1Q8=
-github.com/aws/aws-sdk-go-v2/config v1.33.5 h1:UA1dmokBFOLFoOyVBhO6HjM6edy0MIk5AZSkJVcksQw=
-github.com/aws/aws-sdk-go-v2/config v1.33.5/go.mod h1:Dop8axzz0xx38GExIYWXdeyc8QQ7Cr+nPsxpD/LYy4U=
-github.com/aws/aws-sdk-go-v2/credentials v1.20.5 h1:wklUVvHMc9xTQ3rcp49/ISpiMnhbCicJcA6n6S8m7J8=
-github.com/aws/aws-sdk-go-v2/credentials v1.20.5/go.mod h1:fyEdrn6ccLFOkoK84j5bQyGTxp9zPt5l2XMhxf4DVZs=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0 h1:AM4hHjww+PSFtt6E+UrBrPlZkWsePCLEt9AjkfQX+yM=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0/go.mod h1:3x/yXezeQjpOvBb4jEMxrS8SXvpdvJ5abv6l5c1gWM8=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.3 h1:Hp/VgjP0BysR3OgLlR057Vz2LcbbVnoWeJ+3qWiS/fY=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.3/go.mod h1:nwGV5qw7F1IZPgxCvA/ph8N2TAuz+BkRG/bXn808qMA=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.3 h1:MUaM4f+kj1ZIBPZfUS8cxP1GKXXZtHJjAthy93AN7SM=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.3/go.mod h1:6YmVmEVRI5ZZzRjCSsb9SryKH0hAlMRdgA7kG9aDvBU=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3 h1:fuSCw4Z2qfRCztMPO3GXJNSiEp6Wee+WOLwrHHUMy9c=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3/go.mod h1:6SxcHheD1pPR5+kWm1wGvjlL/YqUsh267sAfEmN4K7A=
+github.com/aws/aws-sdk-go-v2/config v1.33.6 h1:MBjkSTLczek/UgiK+EYPIoRTqE7gP8vtW3OFbFo7Nug=
+github.com/aws/aws-sdk-go-v2/config v1.33.6/go.mod h1:grRAFzdAZJrwcbasJRg2MPvIrVjtlfXllHssN6+E1JE=
+github.com/aws/aws-sdk-go-v2/credentials v1.20.6 h1:NpAFXCU7NzXNkdGK3zQTtsRJ+3v9tZQV0xcdRw8uBdw=
+github.com/aws/aws-sdk-go-v2/credentials v1.20.6/go.mod h1:mcZCoiPnyMvP8VMNbygNX5lLqSlkYJIMPODylQMurOk=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 h1:8gALAAmacnIXh+z6VkdDanv4/IkG5APdg4DZLDTmLog=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1/go.mod h1:Z7IJhJU+poOdJjUR2wpyY21ossQ1XS/R3Lk9Msq5kM4=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 h1:CLq4+8UHCI+ZZYl/EuJxXovaIVN2xeeT8JV+dsApQ5E=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4/go.mod h1:Wv4q5sAM04xAMkoOedxLx2inVf6K5FdxYp+A61L+q/0=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 h1:dD4MR81I7YkpEBRk6UP9rocC2QnT3qVuXwzlYTtfGEs=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4/go.mod h1:EcXV1kAFd5XwSkDHlj94gnF3q5CkJyYiIJfH8N0VmrE=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 h1:7Wo47d/xn/7KttCSBd8EGYeZ7ULRFRkUHr6vkZPBzVQ=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4/go.mod h1:tDB2IVC1xC3vX8o+6uRlzhTxP3g1b77CZXFX/oD2FnQ=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 h1:bAdDl/HkGCcGPoe25ToSHEw23VIxt6CT5fLcg111BKg=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19/go.mod h1:KaUzbLxv4CeSxh6ZCl9B4m7CuFenS8kUEaDs+f/DQr4=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.4 h1:7TXEkbDzy4BkYYTfTVjKcADTfkDnvHxDwcF1pJG1yF0=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.4/go.mod h1:GqWeeKfYfezihA2KfFL9l7ohEdZWe1tuFWh3GfyNSnE=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3 h1:bON1rJf67TSTDCKg816AAIE4xSTtoo9tl0XRkO72R+I=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3/go.mod h1:c5BBpjJcQXpfeq9iASyVKA3T6vX6B6LEXY4mL/gklDY=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.3 h1:L8vIOxylma91TcR96NFTEC07G3JDwSl+CvK2b+IODms=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.3/go.mod h1:fmPIZQzTExYuBNWFyi1P7IoDjvskgphXqK1yObMzusM=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.113.3 h1:IDIchqG5C/o/JdprtYn1NirCi7iUx7XQ8+WwZ6odFX8=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.113.3/go.mod h1:iJ69H4lkK6a9zQ+L5i9pDERMok5Jvts0iZaMjWEi/78=
-github.com/aws/aws-sdk-go-v2/service/signin v1.10.0 h1:ZD5qFpWcaOKdTuhBi431pIDkCgrMkMlMT6jlpSPoIRI=
-github.com/aws/aws-sdk-go-v2/service/signin v1.10.0/go.mod h1:8Nuuf+tR346PjJ3MvZPh9pekbLiLQFWJhzMXfwy7alA=
-github.com/aws/aws-sdk-go-v2/service/sso v1.38.0 h1:JGeeBcMlhg1xtOXYpeCaTQBZObtXMPQCUqBcmr65NRA=
-github.com/aws/aws-sdk-go-v2/service/sso v1.38.0/go.mod h1:XwteswG9EOMRFm73UT0t+MbTwyLxMrEXkU6e+v92Lzo=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0 h1:obhahQXDEdVEv8y5bTKXR30LVaxYe1kyYM0L7l2Iq+k=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0/go.mod h1:6twZZ/aXHNy1vXUO8koUbp++MYzMASkOgEBdkbJYmO0=
-github.com/aws/aws-sdk-go-v2/service/sts v1.51.0 h1:Zpnqa6XtrNzXZnwbdCqHOXpXhMsa01ql/pcRQ1sb4hk=
-github.com/aws/aws-sdk-go-v2/service/sts v1.51.0/go.mod h1:/8JRcdTt//hG0Q4BTmGbuOplT7ABe+5rdtqUHqXvYIM=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 h1:/TYsZXdA8UTa+WCtCYSAJIr1vwl0+eho6TUgJGwFFO8=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5/go.mod h1:qPqp1Uwd/BqdhPufv6oem9j5J7HNsgc2V22dUiDPn+s=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 h1:29SvnfGhXjTl8ONxFwbj2rs6lbhiFXD2CgFQmbT/bXY=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4/go.mod h1:wm04I5DMuNVvZHFe/dHnUxincvNbbK7AiNBbYsQivek=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 h1:pPiWfgeNxqluKEph7hvU88kuGKBPOWzO+Dk9t2zqqNs=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4/go.mod h1:YlwGoIUDG/3kBQbdNOVs/xKZ9J01G8e/6D1mRBj9uTk=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 h1:VMAdYqr4Jn/8ATs9BHC5riwrs0d6m1Z2ohFriSwZwm0=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0/go.mod h1:9APRWGLFITKD+xzWSIyT9V7QV4bNlEuIieWlzXgGFlI=
+github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 h1:DzCCWLzcIRQ77F3DEUljud7bEjTgFOIKXP52NmVRyhU=
+github.com/aws/aws-sdk-go-v2/service/signin v1.10.1/go.mod h1:xpo/geVldu8payT375WekctUzopG/hBU7miiqItMUlw=
+github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 h1:Umtl/0YZhng4xndfW3lKJrYYP7NLEjI6bGXVomwLcs0=
+github.com/aws/aws-sdk-go-v2/service/sso v1.38.1/go.mod h1:rRD/dnm7q0HYE/I5TMaPgkWyyUGLcwuxHLABsLnQ3e0=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 h1:orIWdNiLgzrhu/11RcPPKO/SBzUUymbUQuZbSPImghg=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1/go.mod h1:skwM/xsbR/1ReUTesv9BhpJp1VjajR7DWQnuVLwiXsQ=
+github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 h1:0HOqZXRvMytH6bFHVIc0oJX07sZjfhz0zXtjs6gdE8s=
+github.com/aws/aws-sdk-go-v2/service/sts v1.51.1/go.mod h1:26zA0GhDrLo+yiLI2yXWxqB1PdsShfLikoI7GOEgugM=
 github.com/aws/smithy-go v1.28.2 h1:myhcykQcatTul2B/zITjDk203G7t0awUAs1hVry5Bvg=
 github.com/aws/smithy-go v1.28.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
@@ -111,8 +111,8 @@ github.com/containerd/console v1.0.5 h1:R0ymNeydRqH2DmakFNdmjR2k0t7UPuiOV/N/27/q
 github.com/containerd/console v1.0.5/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk=
 github.com/containerd/containerd/api v1.12.0 h1:kuQm82SbDrCuO4n7hf2L8zsBtZLuympyq5X/VotfX2A=
 github.com/containerd/containerd/api v1.12.0/go.mod h1:EBcSzoi9Vl18cdODaXUCskf3D2NT8lsSXeZJnU5jIUc=
-github.com/containerd/containerd/v2 v2.4.0 h1:mZLYWkPAgF4tfTjinClrRfvUxmxHxg1PQL7cjrrpTtk=
-github.com/containerd/containerd/v2 v2.4.0/go.mod h1:gHZz+v5y8mGt9grF3ynuaa3r6bXLmghBZWTAftHAUtg=
+github.com/containerd/containerd/v2 v2.4.1 h1:DUx/ZJN7cEu0WuzHClDB+68H/bqMEH5pWoEjf0ae4hc=
+github.com/containerd/containerd/v2 v2.4.1/go.mod h1:vgLdtvl3prFk1d3ZVqQcsDD5Q9IKM+vAIdU54U85w+I=
 github.com/containerd/continuity v0.5.0 h1:7a85HZpCSs+1Zps0Ee3DPSuAWY+0SJM1JNM51nlEVDg=
 github.com/containerd/continuity v0.5.0/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE=
 github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
@@ -251,8 +251,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC
 github.com/josharian/native v1.0.1-0.20221213033349-c1e37c09b531/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
 github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA=
 github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
-github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
-github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
+github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
+github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
 github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
@@ -312,14 +312,14 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU
 github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
 github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
 github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
-github.com/mulgadc/bluebottle v1.21.0 h1:TH2/B2fVlqZ2SFgr2lebbjkCH9R/4vIVWjXDznNu72A=
-github.com/mulgadc/bluebottle v1.21.0/go.mod h1:LH0MuQcFxet0dDEISlZsuULIYj592eCN2Fv39A6EH5o=
-github.com/mulgadc/northstar v1.21.0 h1:LE/5kiT3kBelCg02cXa3b7awhaYkGHsOXeJ4nHRLNaQ=
-github.com/mulgadc/northstar v1.21.0/go.mod h1:O446cApNMB6qebqieBhDZfCIGVWX++v3QdOz+wIGhUk=
-github.com/mulgadc/predastore v1.21.0 h1:pAm4fFWf0vnlwRgnoL2oJ+KSCw8mtjT2rQoBHFmnjzk=
-github.com/mulgadc/predastore v1.21.0/go.mod h1:icCfKGnyqDaJ/Jlbp2aYeo2qwrKHZfxmPHDYK1MM7wc=
-github.com/mulgadc/viperblock v1.21.0 h1:RbQ/GpHCg/uoPffYioSKqzqyoz22Uc7wpQ7XY7etLg0=
-github.com/mulgadc/viperblock v1.21.0/go.mod h1:n0wCOGV58Lt2IVPXEvpbHDkTITlXtFw+2CMrQXgbJ7s=
+github.com/mulgadc/bluebottle v1.22.0 h1:pGZ5/Ly3CYem62pPu2zKBRpvMpAG0LHMIMw9Yt/lkIg=
+github.com/mulgadc/bluebottle v1.22.0/go.mod h1:DwT91bJyUsrpg1gCJmcCOTG+tgw5J124LloHYvGAQu0=
+github.com/mulgadc/northstar v1.22.0 h1:fTndlk47hK76fcNr2MqsyeKhBNVEDt8+BC6Z30ohokA=
+github.com/mulgadc/northstar v1.22.0/go.mod h1:yLH8waqfHZx177Ae6dGQGTDt/02rPE9alnUKyxDNhVs=
+github.com/mulgadc/predastore v1.22.0 h1:mUQJMTxMiRDVvemMP+34BP/+r1U0iCptc9B++u4NJ8o=
+github.com/mulgadc/predastore v1.22.0/go.mod h1:/cMIoKtnQG/DWyeX/0matEQrG2/abY60tNHWcBKS9Kc=
+github.com/mulgadc/viperblock v1.22.0 h1:8vS2AtpIDkIhGNOazQPicKXV2y/8PAy8BNrVWIOCP6I=
+github.com/mulgadc/viperblock v1.22.0/go.mod h1:Hq8yMagAX8c4wZ68oUkwZPyxTNMwvRwB685eHFewisc=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc=
@@ -338,8 +338,8 @@ github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJw
 github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
 github.com/opencontainers/runtime-spec v1.3.0 h1:YZupQUdctfhpZy3TM39nN9Ika5CBWT5diQ8ibYCRkxg=
 github.com/opencontainers/runtime-spec v1.3.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/oracle/oci-go-sdk/v65 v65.126.0 h1:RuV0MEcLOOgNOBadYbbkUQriCK4Gm5348F/GdWvYPcI=
-github.com/oracle/oci-go-sdk/v65 v65.126.0/go.mod h1:Pzy+BpgkDesvGZXEHgslwhIYobHCPHg6wRta1mWnlqQ=
+github.com/oracle/oci-go-sdk/v65 v65.126.1 h1:WmQ2Igq7/L/cHlR2jZnJkBC2UI51Ams4DJZW3CZGVBo=
+github.com/oracle/oci-go-sdk/v65 v65.126.1/go.mod h1:YmvgbsnUfrsiJ410XAGQ1BRsrUYJJ7W3O5wUikmeI8w=
 github.com/ovn-kubernetes/libovsdb v0.8.1 h1:M2J8bcJt5mXCom0HqzfEtuHkT80CTSQRcYG7acT8gf4=
 github.com/ovn-kubernetes/libovsdb v0.8.1/go.mod h1:ZlnHLzagmLOSvyd9qfxBIZp6wOSOw0IsRsc+6lNUGbU=
 github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY=
@@ -417,8 +417,8 @@ github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWD
 github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
-github.com/tidwall/btree v1.8.1 h1:27ehoXvm5AG/g+1VxLS1SD3vRhp/H7LuEfwNvddEdmA=
-github.com/tidwall/btree v1.8.1/go.mod h1:jBbTdUWhSZClZWoDg54VnvV7/54modSOzDN7VXftj1A=
+github.com/tidwall/btree v1.8.2 h1:mTY5iM4UTh2e80TG7RHDQQmnAY3xGNQA/DbU0olkhkA=
+github.com/tidwall/btree v1.8.2/go.mod h1:jBbTdUWhSZClZWoDg54VnvV7/54modSOzDN7VXftj1A=
 github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923 h1:tHNk7XK9GkmKUR6Gh8gVBKXc2MVSZ4G/NnWLtzw4gNA=
 github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923/go.mod h1:eLL9Nub3yfAho7qB0MzZizFhTU2QkLeoVsWdHtDW264=
 github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs=

internal/awsmodel/ec2error_test.go

@@ -77,8 +77,8 @@ func TestEC2ErrorCatalogMetadataAndClasses(t *testing.T) {
 	if catalog.metadata.VerifiedOn == "" {
 		t.Fatal("verifiedOn is empty")
 	}
-	if len(catalog.codes) != 112 {
-		t.Fatalf("catalog code count = %d, want 112", len(catalog.codes))
+	if len(catalog.codes) != 114 {
+		t.Fatalf("catalog code count = %d, want 114", len(catalog.codes))
 	}
 	if catalog.codes["InvalidParameterValue"] != ec2ErrorClassClient {
 		t.Fatal("InvalidParameterValue is not classified as client")

internal/awsmodel/error.go

@@ -48,7 +48,14 @@ func ValidateErrorResponse(service Service, operationName string, body []byte) (
 // they are not treated as proof that an operation-specific code conforms.
 func isUnmodelledCommonError(service Service, code string) bool {
 	switch service {
-	case IAM, ElasticLoadBalancingV2:
+	case IAM:
+		return isUnmodelledQueryError(code)
+	case ElasticLoadBalancingV2:
+		// ELBv2's common errors list the parameter validation codes too.
+		switch code {
+		case "InvalidParameterCombination", "InvalidParameterValue", "MissingParameter", "ValidationError":
+			return true
+		}
 		return isUnmodelledQueryError(code)
 	case STS:
 		// STS also uses these two runtime input errors even though the model
@@ -107,9 +114,7 @@ func (m *Model) decodeErrorCode(body []byte) (string, error) {
 		if separator := strings.LastIndex(code, "#"); separator >= 0 {
 			code = code[separator+1:]
 		}
-		if separator := strings.Index(code, ":"); separator >= 0 {
-			code = code[:separator]
-		}
+		code, _, _ = strings.Cut(code, ":")
 		if code == "" {
 			return "", fmt.Errorf("JSON error envelope has no code")
 		}

internal/awsmodel/error_test.go

@@ -97,6 +97,12 @@ func TestValidateErrorResponseClassifiesCommonErrorsAsUnmodelled(t *testing.T) {
 			operation: "AssumeRole",
 			body:      `<ErrorResponse><Error><Code>ValidationError</Code></Error></ErrorResponse>`,
 		},
+		{
+			name:      "ELBv2 parameter validation",
+			service:   ElasticLoadBalancingV2,
+			operation: "AddTags",
+			body:      `<ErrorResponse><Error><Code>InvalidParameterValue</Code></Error></ErrorResponse>`,
+		},
 		{
 			name:      "JSON authorization",
 			service:   ECS,

internal/awsmodel/model.go

@@ -131,12 +131,7 @@ var modelCache sync.Map
 
 // Services returns the supported service identifiers in stable order.
 func Services() []Service {
-	services := make([]Service, 0, len(modelFiles))
-	for service := range modelFiles {
-		services = append(services, service)
-	}
-	slices.Sort(services)
-	return services
+	return slices.Sorted(maps.Keys(modelFiles))
 }
 
 // Load parses and indexes the embedded model for service. Each service is

internal/awsmodel/models/ec2/error-codes.json

@@ -68,6 +68,7 @@
     "InvalidLaunchTemplateName.AlreadyExistsException",
     "InvalidLaunchTemplateName.MalformedException",
     "InvalidLaunchTemplateName.NotFoundException",
+    "InvalidMaxResults",
     "InvalidNatGatewayID.NotFound",
     "InvalidNetworkInterface.InUse",
     "InvalidNetworkInterface.NotFound",
@@ -77,6 +78,7 @@
     "InvalidPlacementGroup.Duplicate",
     "InvalidPlacementGroup.InUse",
     "InvalidPlacementGroup.Unknown",
+    "InvalidRequest",
     "InvalidRoute.NotFound",
     "InvalidRouteTableID.NotFound",
     "InvalidSecurityGroupRuleId.Malformed",

internal/awsmodel/page.go

@@ -4,8 +4,9 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"maps"
 	"regexp"
-	"sort"
+	"slices"
 	"strconv"
 	"strings"
 	"unicode/utf8"
@@ -253,12 +254,7 @@ func validateFrontmatter(owner string, page PageMetadata) error {
 // key rather than by first use, so adding an operation cannot renumber a note
 // that is already published.
 func noteOrder(notes map[string]string) []string {
-	keys := make([]string, 0, len(notes))
-	for key := range notes {
-		keys = append(keys, key)
-	}
-	sort.Strings(keys)
-	return keys
+	return slices.Sorted(maps.Keys(notes))
 }
 
 // checkNotes rejects a note nothing references and a reference to a note that

internal/awsmodel/pattern.go

@@ -51,8 +51,24 @@ func newPatternSampler(pattern string) (*patternSampler, error) {
 	return sampler, err
 }
 
+// re2Equivalents rewrite model patterns that use lookahead, which Go regexp
+// lacks, as patterns matching the same strings.
+var re2Equivalents = map[string]string{
+	// ACM DomainNameString: labels start and end alphanumeric, the last has two
+	// or more characters, with an optional leading wildcard label.
+	`^(\*\.)?(((?!-)[A-Za-z0-9-]{0,62}[A-Za-z0-9])\.)+((?!-)[A-Za-z0-9-]{1,62}[A-Za-z0-9])$`: `^(\*\.)?([A-Za-z0-9]([A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.)+[A-Za-z0-9][A-Za-z0-9-]{0,61}[A-Za-z0-9]$`,
+}
+
+// re2Pattern spells a model pattern in Go regexp syntax.
+func re2Pattern(pattern string) string {
+	if equivalent, ok := re2Equivalents[pattern]; ok {
+		return equivalent
+	}
+	return javaEscape.ReplaceAllString(pattern, `\x{$1}`)
+}
+
 func compilePatternSampler(pattern string) (*patternSampler, error) {
-	pattern = javaEscape.ReplaceAllString(pattern, `\x{$1}`)
+	pattern = re2Pattern(pattern)
 	re, err := regexp.Compile(pattern)
 	if err != nil {
 		return nil, err

internal/awsmodel/request.go

@@ -65,22 +65,33 @@ func GenerateRequests(service Service, operationName string, options RequestOpti
 	if !ok {
 		return RequestPlan{}, fmt.Errorf("awsmodel: %s operation %q is not modelled", service, operationName)
 	}
+	if service == ACM {
+		if _, err := importMaterial(); err != nil {
+			return RequestPlan{}, err
+		}
+	}
 	declared := model.operationErrorCodes(operation)
 	if operation.Input == nil {
 		return RequestPlan{Cases: []RequestCase{{Operation: operationName, Input: map[string]any{}}}, DeclaredErrors: declared}, nil
 	}
 
-	first := model.newRequestGenerator(0, options)
+	// A create or delete names a resource of its own, so it neither collides
+	// with a fixture nor removes one.
+	if strings.HasPrefix(operationName, "Create") || strings.HasPrefix(operationName, "Delete") {
+		options.Fixtures = nil
+	}
+	first := model.newRequestGenerator(operationName, 0, options)
 	full, ok := first.structure(operation.Input.Shape, nil, 0)
 	plan := RequestPlan{Skipped: first.skipped, Unbroken: first.unbroken, DeclaredErrors: declared}
 	if !ok {
 		plan.Skipped = append(plan.Skipped, Skip{Path: "$", Reason: "a required member cannot be generated"})
 		return plan, nil
 	}
-	input := model.shapes[operation.Input.Shape]
+	// Conditionally required members join every request but are never omitted.
+	required := append(slices.Clone(model.shapes[operation.Input.Shape].Required), conditionallyRequired[service][operationName]...)
 	optional := make([]string, 0, len(full))
 	for _, member := range slices.Sorted(maps.Keys(full)) {
-		if !slices.Contains(input.Required, member) {
+		if !slices.Contains(required, member) {
 			optional = append(optional, member)
 		}
 	}
@@ -90,7 +101,7 @@ func GenerateRequests(service Service, operationName string, options RequestOpti
 	// what an earlier case created.
 	salt := 0
 	generate := func() (map[string]any, []constraintSite, error) {
-		generator := model.newRequestGenerator(salt, options)
+		generator := model.newRequestGenerator(operationName, salt, options)
 		salt++
 		value, _ := generator.structure(operation.Input.Shape, nil, 0)
 		if len(generator.sites) != len(first.sites) {
@@ -111,7 +122,7 @@ func GenerateRequests(service Service, operationName string, options RequestOpti
 			Operation: operationName,
 			Member:    member,
 			Path:      memberPathOf(member),
-			Input:     keepMembers(value, input.Required, member),
+			Input:     keepMembers(value, required, member),
 		})
 	}
 	for index := range first.sites {
@@ -121,10 +132,10 @@ func GenerateRequests(service Service, operationName string, options RequestOpti
 		}
 		site := sites[index]
 		member, _ := site.path[0].(string)
-		if slices.Contains(input.Required, member) {
+		if slices.Contains(required, member) {
 			member = ""
 		}
-		broken, err := site.apply(keepMembers(value, input.Required, member))
+		broken, err := site.apply(keepMembers(value, required, member))
 		if err != nil {
 			return RequestPlan{}, err
 		}
@@ -269,6 +280,7 @@ var generatedTimestamp = time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC)
 
 type requestGenerator struct {
 	model      *Model
+	operation  string
 	options    RequestOptions
 	salt       string
 	saltNumber int
@@ -278,8 +290,8 @@ type requestGenerator struct {
 	visiting   map[string]bool
 }
 
-func (m *Model) newRequestGenerator(salt int, options RequestOptions) *requestGenerator {
-	return &requestGenerator{model: m, options: options, salt: saltLetters(salt), saltNumber: salt, visiting: map[string]bool{}}
+func (m *Model) newRequestGenerator(operation string, salt int, options RequestOptions) *requestGenerator {
+	return &requestGenerator{model: m, operation: operation, options: options, salt: saltLetters(salt), saltNumber: salt, visiting: map[string]bool{}}
 }
 
 // saltLetters spells n in base 26 with lower-case letters, which fit more
@@ -462,6 +474,9 @@ func (g *requestGenerator) mapValue(shape *Shape, path []pathStep, depth int) (a
 func (g *requestGenerator) stringValue(shape *Shape, ref ShapeRef, path []pathStep) (any, bool) {
 	if len(shape.Enum) > 0 {
 		g.site(path, constraintSite{constraint: ConstraintEnum, detail: "value not in enum", replacement: invalidEnumValue(shape.Enum)})
+		if value, ok := g.enumHint(path); ok && slices.Contains(shape.Enum, value) {
+			return value, true
+		}
 		return shape.Enum[0], true
 	}
 
@@ -583,7 +598,10 @@ func invalidEnumValue(values []string) string {
 }
 
 func (g *requestGenerator) integer(shape *Shape, path []pathStep) int64 {
-	value := int64(1)
+	value, ok := g.integerHint(path)
+	if !ok {
+		value = 1
+	}
 	if shape.Min != nil {
 		value = max(value, int64(math.Ceil(*shape.Min)))
 	}
@@ -630,5 +648,8 @@ func (g *requestGenerator) blob(shape *Shape, path []pathStep) []byte {
 			g.leaveUnbroken(path, fmt.Sprintf("max length %g bytes is above the generated limit", *shape.Max))
 		}
 	}
+	if value, ok := g.blobHint(path); ok && len(value) >= minLength && len(value) <= maxLength {
+		return value
+	}
 	return []byte(fitLength("spx"+g.salt, minLength, maxLength))
 }

internal/awsmodel/request_hints.go

@@ -1,17 +1,29 @@
 package awsmodel
 
 import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
 	"fmt"
+	"math/big"
 	"regexp"
 	"slices"
 	"strings"
+	"sync"
+	"time"
 )
 
 // RequestOptions places generated ARNs in the account and region the
-// requests are sent as.
+// requests are sent as. AccessKeyID is the key the requests are signed with.
+// Fixtures maps a member to an existing resource it can name.
 type RequestOptions struct {
-	AccountID string
-	Region    string
+	AccountID   string
+	Region      string
+	AccessKeyID string
+	Fixtures    map[string]string
 }
 
 // Models constrain most identifiers only by length, so "spxa" is a model-valid
@@ -56,11 +68,71 @@ var ec2IDPrefixes = map[string]string{
 
 var ec2IDMember = regexp.MustCompile(`^(.*?)Ids?$`)
 
+// conditionallyRequired names members the model marks optional that the
+// operation still needs in its base request. The services refuse their absence
+// with a code the judge cannot tell from a real finding.
+var conditionallyRequired = map[Service]map[string][]string{
+	EC2: {"CreateCapacityReservation": {"AvailabilityZone"}},
+	ECS: {
+		"CreateCapacityProvider": {"autoScalingGroupProvider"},
+		"CreateService":          {"taskDefinition"},
+	},
+	RDS: {
+		"CreateDBInstance":                {"AllocatedStorage", "MasterUsername", "MasterUserPassword"},
+		"RestoreDBInstanceFromDBSnapshot": {"DBSnapshotIdentifier"},
+	},
+}
+
 const (
+	// imageManifest is well formed but names blobs no repository holds.
+	imageManifest    = `{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json","config":{"mediaType":"application/vnd.docker.container.image.v1+json","size":2,"digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000"},"layers":[]}`
+	lifecyclePolicy  = `{"rules":[{"rulePriority":1,"description":"expire untagged","selection":{"tagStatus":"untagged","countType":"sinceImagePushed","countUnit":"days","countNumber":14},"action":{"type":"expire"}}]}`
+	repositoryPolicy = `{"Version":"2012-10-17","Statement":[{"Sid":"pull","Effect":"Allow","Principal":{"AWS":"arn:aws:iam::%s:root"},"Action":"ecr:BatchGetImage"}]}`
+
 	policyDocument      = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"*"}]}`
 	trustPolicyDocument = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"ec2.amazonaws.com"},"Action":"sts:AssumeRole"}]}`
 )
 
+// ec2TaggedTypes is the resource type each EC2 create tags through its
+// TagSpecifications. AWS refuses any other type, so the enum's first value
+// would make an otherwise valid request invalid.
+var ec2TaggedTypes = map[string]string{
+	"AllocateAddress":                 "elastic-ip",
+	"AuthorizeSecurityGroupEgress":    "security-group-rule",
+	"AuthorizeSecurityGroupIngress":   "security-group-rule",
+	"CopyImage":                       "image",
+	"CopySnapshot":                    "snapshot",
+	"CreateCapacityReservation":       "capacity-reservation",
+	"CreateEgressOnlyInternetGateway": "egress-only-internet-gateway",
+	"CreateImage":                     "image",
+	"CreateInternetGateway":           "internet-gateway",
+	"CreateKeyPair":                   "key-pair",
+	"CreateLaunchTemplate":            "launch-template",
+	"CreateNatGateway":                "natgateway",
+	"CreateNetworkInterface":          "network-interface",
+	"CreatePlacementGroup":            "placement-group",
+	"CreateRouteTable":                "route-table",
+	"CreateSecurityGroup":             "security-group",
+	"CreateSnapshot":                  "snapshot",
+	"CreateSubnet":                    "subnet",
+	"CreateVolume":                    "volume",
+	"CreateVpc":                       "vpc",
+	"ImportKeyPair":                   "key-pair",
+	"RegisterImage":                   "image",
+	"RequestSpotInstances":            "spot-instances-request",
+	"RunInstances":                    "instance",
+}
+
+// enumHint returns the enum value a valid request needs at path, where the
+// first value would not do.
+func (g *requestGenerator) enumHint(path []pathStep) (string, bool) {
+	if g.model.service != EC2 || len(path) == 0 || path[0] != "TagSpecifications" || memberName(path) != "ResourceType" {
+		return "", false
+	}
+	value, ok := ec2TaggedTypes[g.operation]
+	return value, ok
+}
+
 // hint returns a value of the right form for the member at path, if one is
 // known. salt keeps names distinct between request cases.
 func (g *requestGenerator) hint(path []pathStep) (string, bool) {
@@ -69,11 +141,20 @@ func (g *requestGenerator) hint(path []pathStep) (string, bool) {
 		return "", false
 	}
 	options := g.options
+	if fixture, ok := options.Fixtures[member]; ok {
+		return fixture, true
+	}
 	suffix := fmt.Sprintf("%017x", g.saltNumber)
 	name := "spx" + g.salt
 
 	switch g.model.service {
 	case EC2:
+		switch member {
+		case "AvailabilityZone":
+			return options.Region + "a", true
+		case "SourceRegion":
+			return options.Region, true
+		}
 		if match := ec2IDMember.FindStringSubmatch(capitalize(member)); match != nil {
 			if prefix, ok := ec2IDPrefixes[match[1]]; ok {
 				return prefix + "-" + suffix, true
@@ -90,6 +171,10 @@ func (g *requestGenerator) hint(path []pathStep) (string, bool) {
 			return base + "listener/app/" + name + "/" + suffix[1:] + "/" + suffix[1:], true
 		case "RuleArn", "RuleArns":
 			return base + "listener-rule/app/" + name + "/" + suffix[1:] + "/" + suffix[1:] + "/" + suffix[1:], true
+		case "ProtocolVersion":
+			return "HTTP1", true
+		case "HttpCode":
+			return "200", true
 		}
 	case RDS:
 		switch member {
@@ -101,6 +186,8 @@ func (g *requestGenerator) hint(path []pathStep) (string, bool) {
 			return "db.t3.medium", true
 		case "DBParameterGroupFamily":
 			return "postgres18", true
+		case "MasterUserPassword":
+			return "spx-conformance-" + g.salt, true
 		case "SnapshotType":
 			return "manual", true
 		case "Source":
@@ -113,13 +200,40 @@ func (g *requestGenerator) hint(path []pathStep) (string, bool) {
 		case "taskDefinition":
 			return name + ":1", true
 		}
+	case ACM:
+		// The integration tenant CA permits example.com.
+		switch member {
+		case "DomainName", "SubjectAlternativeNames", "ValidationDomain":
+			return name + ".example.com", true
+		}
 	case ECR:
-		if member == "resourceArn" {
+		switch member {
+		case "resourceArn":
 			return fmt.Sprintf("arn:aws:ecr:%s:%s:repository/%s", options.Region, options.AccountID, name), true
+		case "imageManifest":
+			return imageManifest, true
+		case "lifecyclePolicyText":
+			return lifecyclePolicy, true
+		case "policyText":
+			return fmt.Sprintf(repositoryPolicy, options.AccountID), true
 		}
 	case EKS:
-		if member == "resourceArn" {
+		switch member {
+		case "resourceArn":
 			return fmt.Sprintf("arn:aws:eks:%s:%s:cluster/%s", options.Region, options.AccountID, name), true
+		case "policyArn":
+			return "arn:aws:eks::aws:cluster-access-policy/AmazonEKSViewPolicy", true
+		case "addonName":
+			return "aws-ebs-csi-driver", true
+		}
+		// An access entry's type; a nested type names an identity provider's.
+		if member == "type" && len(path) == 1 {
+			return "STANDARD", true
+		}
+	case STS:
+		// GetAccessKeyInfo answers only for a key that exists.
+		if member == "AccessKeyId" && options.AccessKeyID != "" {
+			return options.AccessKeyID, true
 		}
 	}
 
@@ -142,6 +256,103 @@ func (g *requestGenerator) hint(path []pathStep) (string, bool) {
 	return "", false
 }
 
+// integerHint returns a value AWS accepts for an integer member whose range
+// the model leaves out, if one is known.
+func (g *requestGenerator) integerHint(path []pathStep) (int64, bool) {
+	if g.model.service == RDS && memberName(path) == "AllocatedStorage" {
+		// The smallest gp2/gp3 allocation PostgreSQL, the Engine hint, accepts.
+		return 20, true
+	}
+	return 0, false
+}
+
+// blobHint returns a value AWS accepts for a blob member, if one is known.
+func (g *requestGenerator) blobHint(path []pathStep) ([]byte, bool) {
+	if g.model.service != ACM {
+		return nil, false
+	}
+	// GenerateRequests has already surfaced a generation error.
+	material, err := importMaterial()
+	if err != nil {
+		return nil, false
+	}
+	switch memberName(path) {
+	case "Certificate":
+		return material.certificate, true
+	case "PrivateKey":
+		return material.privateKey, true
+	case "CertificateChain":
+		return material.chain, true
+	}
+	return nil, false
+}
+
+// certificateMaterial is a PEM leaf certificate, its key and the CA that
+// signed it, as ImportCertificate takes them.
+type certificateMaterial struct {
+	certificate, privateKey, chain []byte
+}
+
+// importMaterial is generated once per run; minting a key per request would
+// dominate the ACM cases.
+var importMaterial = sync.OnceValues(func() (certificateMaterial, error) {
+	now := time.Now().UTC()
+	caKey, caDER, err := newCertificate(&x509.Certificate{
+		SerialNumber:          big.NewInt(1),
+		Subject:               pkix.Name{CommonName: "Spinifex Conformance CA"},
+		NotBefore:             now.Add(-time.Hour),
+		NotAfter:              now.AddDate(1, 0, 0),
+		KeyUsage:              x509.KeyUsageCertSign,
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+	}, nil, nil)
+	if err != nil {
+		return certificateMaterial{}, err
+	}
+	ca, err := x509.ParseCertificate(caDER)
+	if err != nil {
+		return certificateMaterial{}, fmt.Errorf("awsmodel: parse conformance CA: %w", err)
+	}
+	leafKey, leafDER, err := newCertificate(&x509.Certificate{
+		SerialNumber: big.NewInt(2),
+		Subject:      pkix.Name{CommonName: "spx.example.com"},
+		DNSNames:     []string{"spx.example.com"},
+		NotBefore:    now.Add(-time.Hour),
+		NotAfter:     now.AddDate(1, 0, 0),
+		KeyUsage:     x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+	}, ca, caKey)
+	if err != nil {
+		return certificateMaterial{}, err
+	}
+	keyDER, err := x509.MarshalPKCS8PrivateKey(leafKey)
+	if err != nil {
+		return certificateMaterial{}, fmt.Errorf("awsmodel: marshal conformance key: %w", err)
+	}
+	return certificateMaterial{
+		certificate: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: leafDER}),
+		privateKey:  pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}),
+		chain:       pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER}),
+	}, nil
+})
+
+// newCertificate creates a P-256 key and a certificate for it from template,
+// signed by parent's key, or self-signed when parent is nil.
+func newCertificate(template, parent *x509.Certificate, parentKey *ecdsa.PrivateKey) (*ecdsa.PrivateKey, []byte, error) {
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		return nil, nil, fmt.Errorf("awsmodel: generate conformance key: %w", err)
+	}
+	if parent == nil {
+		parent, parentKey = template, key
+	}
+	der, err := x509.CreateCertificate(rand.Reader, template, parent, &key.PublicKey, parentKey)
+	if err != nil {
+		return nil, nil, fmt.Errorf("awsmodel: create conformance certificate: %w", err)
+	}
+	return key, der, nil
+}
+
 // memberName is the structure member a value at path belongs to, looking
 // through list indices.
 func memberName(path []pathStep) string {

internal/awsmodel/request_test.go

@@ -4,6 +4,8 @@ package awsmodel
 //shapes, and the unexported pattern sampler and salt helpers.
 
 import (
+	"crypto/tls"
+	"crypto/x509"
 	"fmt"
 	"maps"
 	"regexp"
@@ -17,7 +19,7 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-var testRequestOptions = RequestOptions{AccountID: "123456789012", Region: "ap-southeast-2"}
+var testRequestOptions = RequestOptions{AccountID: "123456789012", Region: "ap-southeast-2", AccessKeyID: "AKIAIOSFODNN7EXAMPLE"}
 
 // constraintViolation is one model constraint an input breaks, found by
 // walking the input against the model independently of the generator.
@@ -99,7 +101,7 @@ func testPattern(pattern string) *regexp.Regexp {
 	if re, ok := testPatterns.Load(pattern); ok {
 		return re.(*regexp.Regexp)
 	}
-	re := regexp.MustCompile(javaEscape.ReplaceAllString(pattern, `\x{$1}`))
+	re := regexp.MustCompile(re2Pattern(pattern))
 	testPatterns.Store(pattern, re)
 	return re
 }
@@ -136,7 +138,7 @@ func TestGeneratedRequestsBreakOnlyTheirConstraint(t *testing.T) {
 					var violations []constraintViolation
 					model.inputViolations(operation.Input.Shape, "$", request.Input, &violations)
 					for member := range request.Input {
-						if !slices.Contains(input.Required, member) {
+						if !slices.Contains(input.Required, member) && !slices.Contains(conditionallyRequired[service][operationName], member) {
 							require.Equal(t, request.Member, member, "%s %s %s carries an unrelated optional member", operationName, request.Constraint, request.Path)
 						}
 					}
@@ -180,6 +182,65 @@ func TestGenerateRequestsIsolatesEachOptionalMember(t *testing.T) {
 	}
 }
 
+func TestGenerateRequestsSeedsConditionallyRequiredMembers(t *testing.T) {
+	for service, operations := range conditionallyRequired {
+		model, err := Load(service)
+		require.NoError(t, err)
+		for operationName, members := range operations {
+			operation, ok := model.Operation(operationName)
+			require.True(t, ok, "%s %s is not modelled", service, operationName)
+			input := model.shapes[operation.Input.Shape]
+			for _, member := range members {
+				require.Contains(t, input.Members, member, "%s %s", service, operationName)
+				require.NotContains(t, input.Required, member, "%s %s now models %s as required", service, operationName, member)
+			}
+
+			plan, err := GenerateRequests(service, operationName, testRequestOptions)
+			require.NoError(t, err)
+			for _, request := range plan.Cases {
+				for _, member := range members {
+					require.NotEqual(t, member, request.Member, "%s %s", service, operationName)
+					require.Contains(t, request.Input, member, "%s %s %s %s", service, operationName, request.Constraint, request.Path)
+				}
+			}
+		}
+	}
+
+	plan, err := GenerateRequests(RDS, "CreateDBInstance", testRequestOptions)
+	require.NoError(t, err)
+	require.Equal(t, int64(20), plan.Cases[0].Input["AllocatedStorage"])
+	plan, err = GenerateRequests(EC2, "CreateCapacityReservation", testRequestOptions)
+	require.NoError(t, err)
+	require.Equal(t, "ap-southeast-2a", plan.Cases[0].Input["AvailabilityZone"])
+
+	// A constraint inside a seeded member is judged against the base request.
+	plan, err = GenerateRequests(ECS, "CreateCapacityProvider", testRequestOptions)
+	require.NoError(t, err)
+	nested := 0
+	for _, request := range plan.Cases {
+		if strings.HasPrefix(request.Path, "$.autoScalingGroupProvider.") {
+			nested++
+			require.Empty(t, request.Member, request.Path)
+		}
+	}
+	require.NotZero(t, nested)
+}
+
+func TestGenerateRequestsTargetsFixturesOutsideCreateAndDelete(t *testing.T) {
+	options := testRequestOptions
+	options.Fixtures = map[string]string{"RoleName": "fixture"}
+	for operation, wantFixture := range map[string]bool{"TagRole": true, "CreateRole": false, "DeleteRole": false} {
+		plan, err := GenerateRequests(IAM, operation, options)
+		require.NoError(t, err)
+		for _, request := range plan.Cases {
+			if request.Path == "$.RoleName" {
+				continue
+			}
+			require.Equal(t, wantFixture, request.Input["RoleName"] == "fixture", "%s %s %s", operation, request.Constraint, request.Path)
+		}
+	}
+}
+
 func TestGenerateRequestsAttributesRejections(t *testing.T) {
 	plan, err := GenerateRequests(IAM, "CreateRole", testRequestOptions)
 	require.NoError(t, err)
@@ -278,6 +339,13 @@ func TestGenerateRequestsUsesResourceHints(t *testing.T) {
 		{RDS, "CreateDBInstance", "Engine", regexp.MustCompile(`^postgres$`)},
 		{ECR, "TagResource", "resourceArn", regexp.MustCompile(`^arn:aws:ecr:ap-southeast-2:123456789012:repository/`)},
 		{EKS, "TagResource", "resourceArn", regexp.MustCompile(`^arn:aws:eks:ap-southeast-2:123456789012:cluster/`)},
+		{EKS, "AssociateAccessPolicy", "policyArn", regexp.MustCompile(`^arn:aws:eks::aws:cluster-access-policy/AmazonEKSViewPolicy$`)},
+		{EKS, "CreateAddon", "addonName", regexp.MustCompile(`^aws-ebs-csi-driver$`)},
+		{EKS, "CreateAccessEntry", "type", regexp.MustCompile(`^STANDARD$`)},
+		{ElasticLoadBalancingV2, "CreateTargetGroup", "ProtocolVersion", regexp.MustCompile(`^HTTP1$`)},
+		{EC2, "CopyImage", "SourceRegion", regexp.MustCompile(`^ap-southeast-2$`)},
+		{STS, "GetAccessKeyInfo", "AccessKeyId", regexp.MustCompile(`^AKIAIOSFODNN7EXAMPLE$`)},
+		{IAM, "DeleteAccessKey", "AccessKeyId", regexp.MustCompile(`^AKIA[0-9A-F]{16}$`)},
 	}
 	for _, test := range tests {
 		t.Run(test.operation+"."+test.member, func(t *testing.T) {
@@ -302,6 +370,80 @@ func TestGenerateRequestsUsesResourceHints(t *testing.T) {
 	}
 }
 
+func TestGenerateRequestsHintsNestedMembers(t *testing.T) {
+	plan, err := GenerateRequests(ElasticLoadBalancingV2, "CreateTargetGroup", testRequestOptions)
+	require.NoError(t, err)
+	for _, request := range plan.Cases {
+		if request.Acceptance() && request.Member == "Matcher" {
+			require.Equal(t, "200", request.Input["Matcher"].(map[string]any)["HttpCode"])
+			return
+		}
+	}
+	t.Fatal("no acceptance request sets Matcher")
+}
+
+// AWS refuses a tag resource type the operation does not create, so a valid
+// request tags the operation's own type rather than the enum's first value.
+func TestGenerateRequestsTagsTheOperationsOwnResourceType(t *testing.T) {
+	for operation, want := range map[string]string{"CreateVpc": "vpc", "CreateLaunchTemplate": "launch-template"} {
+		t.Run(operation, func(t *testing.T) {
+			plan, err := GenerateRequests(EC2, operation, testRequestOptions)
+			require.NoError(t, err)
+			for _, request := range plan.Cases {
+				if request.Acceptance() && request.Member == "TagSpecifications" {
+					spec := request.Input["TagSpecifications"].([]any)[0].(map[string]any)
+					require.Equal(t, want, spec["ResourceType"])
+					return
+				}
+			}
+			t.Fatal("no acceptance request sets TagSpecifications")
+		})
+	}
+}
+
+func TestGenerateRequestsImportsAVerifiableCertificate(t *testing.T) {
+	plan, err := GenerateRequests(ACM, "ImportCertificate", testRequestOptions)
+	require.NoError(t, err)
+	var input map[string]any
+	for _, request := range plan.Cases {
+		if request.Acceptance() && request.Member == "CertificateChain" {
+			input = request.Input
+		}
+	}
+	require.NotNil(t, input, "no acceptance request sets CertificateChain")
+
+	pair, err := tls.X509KeyPair(input["Certificate"].([]byte), input["PrivateKey"].([]byte))
+	require.NoError(t, err)
+	leaf, err := x509.ParseCertificate(pair.Certificate[0])
+	require.NoError(t, err)
+	roots := x509.NewCertPool()
+	require.True(t, roots.AppendCertsFromPEM(input["CertificateChain"].([]byte)))
+	_, err = leaf.Verify(x509.VerifyOptions{Roots: roots, DNSName: "spx.example.com"})
+	require.NoError(t, err)
+}
+
+func TestRE2EquivalentsMatchTheModelledSemantics(t *testing.T) {
+	modelled := map[string]bool{}
+	for _, service := range Services() {
+		model, err := Load(service)
+		require.NoError(t, err)
+		for _, shape := range model.shapes {
+			modelled[shape.Pattern] = true
+		}
+	}
+	for pattern := range re2Equivalents {
+		require.True(t, modelled[pattern], "no model uses %s", pattern)
+	}
+
+	domain := testPattern(`^(\*\.)?(((?!-)[A-Za-z0-9-]{0,62}[A-Za-z0-9])\.)+((?!-)[A-Za-z0-9-]{1,62}[A-Za-z0-9])$`)
+	for _, name := range []string{"example.com", "*.example.com", "a.b.co", "x-y.example.com", "a." + strings.Repeat("b", 63)} {
+		require.True(t, domain.MatchString(name), name)
+	}
+	for _, name := range []string{"example", "-a.com", "a-.com", "a.c", "a.-com", "a.com-", "a..com", "*.com.*", strings.Repeat("a", 64) + ".com"} {
+		require.False(t, domain.MatchString(name), name)
+	}
+}
+
 func TestPatternSamplerMatchesWholeValue(t *testing.T) {
 	tests := []struct {
 		pattern  string

internal/awsmodel/verdict.go

@@ -135,9 +135,7 @@ func DecodeRequestResult(service Service, status int, header http.Header, body [
 }
 
 func trimErrorCode(code string) string {
-	if separator := strings.Index(code, ":"); separator >= 0 {
-		code = code[:separator]
-	}
+	code, _, _ = strings.Cut(code, ":")
 	if separator := strings.LastIndex(code, "#"); separator >= 0 {
 		code = code[separator+1:]
 	}

internal/ecrauth/ecrauth.go

@@ -109,11 +109,9 @@ func GatewayHTTPClient(caPath string) (*http.Client, error) {
 // HostFromImage returns the host[:port] portion of an image ref or URL. It strips
 // any scheme and trims at the first path separator.
 func HostFromImage(image string) string {
-	if i := strings.Index(image, "://"); i >= 0 {
-		image = image[i+3:]
+	if _, rest, ok := strings.Cut(image, "://"); ok {
+		image = rest
 	}
-	if i := strings.IndexByte(image, '/'); i >= 0 {
-		image = image[:i]
-	}
-	return image
+	host, _, _ := strings.Cut(image, "/")
+	return host
 }

internal/guestenv/guestenv.go

@@ -10,11 +10,13 @@ import (
 	"strings"
 )
 
+// Loader holds the KEY=value settings read from an agent.env file, keyed by
+// variable name. Read values through Get so the process environment applies.
 type Loader map[string]string
 
-// A missing or unreadable file yields an empty Loader rather than an error: the
-// settings are all optional, and a guest whose file never arrived must still
-// start far enough to report why.
+// Load parses the KEY=value file at path, skipping blanks and # comments. A
+// missing or unreadable file yields an empty Loader rather than an error: the
+// settings are optional, and a guest without its file must still report why.
 func Load(path string) Loader {
 	out := Loader{}
 	f, err := os.Open(path)
@@ -38,8 +40,9 @@ func Load(path string) Loader {
 	return out
 }
 
-// A real environment variable wins, so an operator or test can override what
-// cloud-init wrote. An empty value is treated as unset rather than as a mask.
+// Get returns the setting for key. A real environment variable wins, so an
+// operator or test can override what cloud-init wrote. An empty value is
+// treated as unset rather than as a mask.
 func (l Loader) Get(key string) string {
 	if v := os.Getenv(key); v != "" {
 		return v

internal/rdsgw/client.go

@@ -37,7 +37,8 @@ const (
 	defaultTimeout = 40 * time.Second
 )
 
-// One client is reused for register/heartbeat/bootstrap/poll.
+// Client sends SigV4-signed RDS Query calls to the gateway. One client is
+// reused for register/heartbeat/bootstrap/poll.
 type Client struct {
 	baseURL    string
 	signer     *gwsign.Signer
@@ -45,8 +46,9 @@ type Client struct {
 	httpClient *http.Client
 }
 
-// caPath optionally pins the gateway TLS CA; empty relies on the system trust
-// store. region defaults to us-east-1, since SigV4 requires a non-empty one.
+// New builds a TLS 1.3 client for baseURL. caPath optionally pins the gateway
+// TLS CA; empty relies on the system trust store. region defaults to us-east-1,
+// since SigV4 requires a non-empty one.
 func New(baseURL, caPath string, signer *gwsign.Signer, region string, timeout time.Duration) (*Client, error) {
 	if baseURL == "" {
 		return nil, fmt.Errorf("rdsgw: baseURL is required")
@@ -88,8 +90,8 @@ func New(baseURL, caPath string, signer *gwsign.Signer, region string, timeout t
 	}, nil
 }
 
-// Code is the AWS error code, so a caller branches on the failure class rather
-// than matching message text.
+// APIError is a non-2xx gateway response. Code is the AWS error code, so a
+// caller branches on the failure class rather than matching message text.
 type APIError struct {
 	Action     string
 	StatusCode int
@@ -113,8 +115,9 @@ type errorResponse struct {
 	} `xml:"Error"`
 }
 
+// Call POSTs action and decodes its <ActionResult> into out, which may be nil.
 // params carries the action's own arguments; Action and Version are set here.
-// out may be nil. A non-2xx yields an *APIError. No retry; callers wrap.
+// A non-2xx yields an *APIError. No retry; callers wrap.
 func (c *Client) Call(ctx context.Context, action string, params url.Values, out any) error {
 	form := make(url.Values, len(params)+2)
 	maps.Copy(form, params)

scripts/setup.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 # Spinifex binary installer
 # Usage: curl -sfL https://install.mulgadc.com | bash
+#        curl -sfL https://install.mulgadc.com | sudo bash -s -- --channel dev
 #
 # Environment variables:
 #   INSTALL_SPINIFEX_CHANNEL   Release channel: latest (default), dev
@@ -897,6 +898,17 @@ install_files() {
         info "  /usr/local/share/spinifex/setup.sh"
     fi
 
+    # Formation. Shipping these means a multi-node cluster can be formed from a
+    # `curl | bash` install with no checkout, which is what a cloud operator has.
+    # smoke-test.sh comes too: install-node.sh scp's it from its own directory for
+    # the post-formation verification, so without it formation fails at the last step.
+    for formation in install-node.sh smoke-test.sh; do
+        if [ -f "$EXTRACT_DIR/$formation" ]; then
+            $SUDO install -m 0755 "$EXTRACT_DIR/$formation" "/usr/local/share/spinifex/$formation"
+            info "  /usr/local/share/spinifex/$formation"
+        fi
+    done
+
     # Teardown scripts. An install delivered by `curl | bash` with no way to
     # undo it is a gap in the product, not just in our tooling, and node-reset.sh
     # ships alongside because uninstall-spx.sh delegates its state teardown to it
@@ -1390,11 +1402,22 @@ main() {
         case "$1" in
             --firewall=*) INSTALL_SPINIFEX_FIREWALL="${1#*=}" ;;
             --firewall)   INSTALL_SPINIFEX_FIREWALL="${2:-}"; shift ;;
+            --channel=*)  INSTALL_SPINIFEX_CHANNEL="${1#*=}" ;;
+            --channel)    INSTALL_SPINIFEX_CHANNEL="${2:-}"; shift ;;
+            --version=*)  INSTALL_SPINIFEX_VERSION="${1#*=}" ;;
+            --version)    INSTALL_SPINIFEX_VERSION="${2:-}"; shift ;;
             *) fatal "unknown option: $1" ;;
         esac
         shift
     done
 
+    case "$INSTALL_SPINIFEX_CHANNEL" in
+        latest|dev) ;;
+        # Anything else is passed to the install endpoint as a literal tag, so a
+        # typo would 404 on the download rather than here. Name the two channels.
+        *) fatal "--channel must be 'latest' or 'dev', got: ${INSTALL_SPINIFEX_CHANNEL} (use --version for a specific tag)" ;;
+    esac
+
     case "${INSTALL_SPINIFEX_FIREWALL}" in
         on|off) ;;
         # Unset resolves by install path: the ISO arms, everything else does not.

scripts/terraform/oci-spx/.gitignore

@@ -8,3 +8,8 @@ terraform.tfstate.*
 crash.log
 terraform.tfvars
 *.pem
+*.log
+.validate-*/
+# Per-run artifacts and the identity state, which is not a topology's to destroy.
+.e2e-oci-*/
+.identity/

scripts/terraform/oci-spx/README.md

@@ -31,7 +31,7 @@ Every resource is prefixed with `deployment_name` (default `spinifex`), so nothi
 
 ## Quick Start
 
-Prerequisites: Terraform, Python 3, OCI credentials in `~/.oci/config`, and the public and private halves of the SSH key. The Python helper creates and uses this repository's isolated `.venv` and installs [requirements.txt](requirements.txt).
+Prerequisites: Terraform, Python 3, OCI credentials in `~/.oci/config` or the environment, and the public and private halves of the SSH key. The Python helper is standard library only, so there is nothing to install: it once used the OCI SDK, which meant a virtualenv and therefore `python3-venv` on every host that runs it.
 
 ```bash
 cd scripts/terraform/oci-spx
@@ -53,13 +53,11 @@ python3 scripts/oci_env.py --ssh-public-key-path <path_to_public_ssh_key> -- ter
 
 | Item | Purpose |
 | --- | --- |
-| `scripts/oci_env.py` | Loads OCI profile inputs, discovers the tenancy home region, exports Terraform variables, and runs Terraform in the repository `.venv`. |
-| `requirements.txt` | Python dependency set for the environment helper. |
+| `scripts/oci_env.py` | Loads the credential from a profile or the environment, checks its shape, exports Terraform variables, and runs Terraform. |
 | `terraform.tfvars.example` | Example non-secret Terraform inputs. Copy it to untracked `terraform.auto.tfvars` for local overrides. |
-| `.venv/` | Repository-local virtual environment; generated locally and not committed. |
 | `terraform.tfstate` | Terraform state; generated locally unless a remote backend is configured. Do not commit it. |
 
-The helper defaults to OCI profile `apacanzset03child03`. If it is absent and `apacanzset03child3` is locally configured, the helper explicitly falls back to that profile. Override either setting when needed:
+The helper reads the `DEFAULT` profile, which is what `oci setup config` writes. Set `OCI_CLI_PROFILE` or pass `--profile` to read a different one; a named profile the config does not hold is an error rather than a fallback, so a deployment cannot land in a tenancy nobody chose.
 
 ```bash
 python3 scripts/oci_env.py --profile my-profile --region ap-sydney-1 -- terraform plan
@@ -125,7 +123,6 @@ flowchart TB
 ├── outputs.tf           # OCIDs and policy outputs
 ├── versions.tf          # Terraform and provider version constraints
 ├── terraform.tfvars.example
-├── requirements.txt
 └── scripts/oci_env.py   # Python environment and Terraform command helper
 ```
 
@@ -208,9 +205,77 @@ The private subnet is **unused today** and kept only so it exists: Spinifex gues
 node_client_cidr_allow_list = ["203.0.113.10/32"]
 ```
 
-## The API user this seeds
+## How each node authenticates to OCI
 
-The OCI provider integration needs credentials on each node so Spinifex can allocate addresses at runtime. **Grant that user only the operations Spinifex performs on addresses** — create/get/list/delete private IPs, create/get/update/delete public IPs — and nothing else. It is not a tenancy admin. Broader rights widen the blast radius of a node compromise for no benefit.
+The allocator needs OCI credentials at runtime, and there are two ways to give it them. A node with neither forms, passes every health check, and then refuses every launch that wants a public address with `InsufficientAddressCapacity` — the cause appears only in the node's journal, so this is worth getting right before first start.
+
+**Instance principal is the better one.** Each node authenticates with the certificate its own metadata service serves, so no key material exists on any node, there is nothing to rotate, and nothing sensitive reaches Terraform state. The cost is a dynamic group and a policy, which are tenancy-root resources — so creating them needs a tenancy-admin principal, which is why the rest of this configuration deliberately creates nothing at tenancy root.
+
+`instance_principal` has three values, because using an instance principal and being allowed to create one are different rights:
+
+| Value | Pool auth | Creates the dynamic group and policy | Credential needed |
+| --- | --- | --- | --- |
+| `off` (default) | API key file | No | Compartment-scoped |
+| `adopt` | `instance_principal` | No — assumes they exist | Compartment-scoped |
+| `create` | `instance_principal` | Yes | **Tenancy admin** |
+
+Create them once per tenancy, then every deployment and rebuild afterwards uses `adopt`:
+
+```bash
+./setup-identity.sh --dry-run    # always first
+./setup-identity.sh
+```
+
+`setup-identity.sh` targets only those two resources and keeps them in `.identity/terraform.tfstate`, separate from every topology's state. That separation is load-bearing: `validate-topology.sh` destroys its own state at the end of each run, so holding tenancy resources there would let a nightly teardown delete the tenancy's policy.
+
+**`adopt` references the dynamic group by nothing at all.** Its matching rule is `instance.compartment.id`, so it covers every instance in the compartment and names no OCID — which is why adopting needs no read on an identity resource and no tenancy rights. The cost is that a missing policy is invisible at apply time: the node forms, passes every health check, and then refuses every launch wanting a public address. The allocator gate in `validate-topology.sh` is what catches that, by requiring `ocinet credential authorised to allocate` in each node's `spinifex-daemon` journal.
+
+The policy grants four verbs in one compartment: `use vnics`, `manage private-ips`, `manage public-ips` and `use subnets`.
+That is exactly what allocating an external address does and nothing more.
+`use subnets` looks unrelated and is not: `CreatePrivateIp` is checked against `SUBNET_ATTACH` and `CreatePrivateIp` is how an address is registered, so a policy without it authorises nothing and every allocation returns a 404.
+
+**An API key is the fallback**, and the default because it needs nothing from a tenancy admin. Grant that user only the operations Spinifex performs on addresses and nothing else; it is not a tenancy admin, and broader rights widen the blast radius of a node compromise for no benefit.
+
+`spx-oci-config.sh` installs it on **every** node, as `/etc/spinifex/oci/{oci_api_key.pem,config}` — not a home directory, because the daemon's unit sets `ProtectHome=yes`. Both files are needed: `oci_config_file` names an ordinary OCI SDK config, so a node holding only the PEM forms and then fails every allocation. The profile it writes is `spinifex`, matching `oci_config_profile`:
+
+```ini
+[spinifex]
+user=ocid1.user.oc1..<yours>
+fingerprint=<yours>
+tenancy=ocid1.tenancy.oc1..<yours>
+region=ap-sydney-1
+key_file=/etc/spinifex/oci/oci_api_key.pem
+```
+
+The script takes `validate-topology.sh`'s hook contract, `hook <ssh-key> <host>...`, so it is both the default thing to pass to `--credential-hook` and runnable on its own against an existing cluster. `--dry-run` resolves and validates the credential without touching a host.
+
+It resolves the credential in four steps and takes the first complete one: the `OCI_SPX_*` environment variables, the `[spinifex]` profile in `~/.oci/config`, `~/.oci/oci_api_key_spx.pem` with the Terraform profile's identifiers, then the Terraform credential itself with a warning that it is wider than needed. Steps 2 to 4 go through `scripts/oci_env.py`, the same resolver the apply uses, so the node credential and the one that built the infrastructure cannot come from different profiles. The fingerprint is checked against the key before anything is written, because a mismatched pair is otherwise an OCI 401 at the first allocation.
+
+Credential material reaches the nodes over SSH, in the remote shell's stdin rather than its arguments, and goes into neither user-data nor Terraform state. `--credential-hook` still accepts any executable with that argument shape, so a tenancy holding credentials in a vault can substitute its own. Instance principal needs no hook at all, which is the reason to prefer it wherever a dynamic group can be created.
+
+Either way, Terraform stages the matching pool block at `/etc/spinifex/oci/external-pool.toml` on each node, with `oci_auth` set to match. Append it to `/etc/spinifex/spinifex.toml` after `spx admin init` and restart `spinifex.target`. Under instance principal that file holds no secret at all, which is the point of it.
+
+## Validating a topology end to end
+
+`validate-topology.sh` builds one topology from nothing, installs the published Spinifex release, forms the cluster, runs a Terraform workbook against it, proves the workbook serves traffic, and destroys everything. Three topologies, because each breaks differently — bare metal presents VNICs unlike a VM, a single node has no Geneve underlay to get wrong, and only a cluster exercises RAFT, the gateway chassis and cross-node allocation.
+
+```bash
+./validate-topology.sh --topology bm        --dry-run
+./validate-topology.sh --topology vm-single --instance-principal
+./validate-topology.sh --topology vm-multi  --instance-principal
+```
+
+`--topology` has no default on purpose: a command aimed at the wrong one is the easiest expensive mistake here. Each topology keeps its own state under `.validate-<topology>/`, so two can be built from one checkout without either destroying the other's instances, and every log from the run lands there.
+
+**A topology's shape and node count are defaults, not settings.** They go to Terraform as `TF_VAR_compute_shape` and `TF_VAR_node_count`, which is the weakest source Terraform reads, so a `terraform.auto.tfvars` in the checkout outranks them and a deployment sizes itself in that file with no flag to pass. CI has no such file — `*.auto.tfvars` is gitignored — so a named topology stays the same every run. `instance_principal` is the exception and remains a `-var`, because `--instance-principal` is a choice about the run rather than about the infrastructure's size, and a stale tfvars must not quietly contradict it.
+
+The count that later gates read is the number of addresses the `hosts_file` output names, not the number asked for, and the run logs the shape and count it built. A tfvars that changes either is reported rather than silently diverging from the topology's name.
+
+**The teardown decides the verdict.** A topology or workbook that cannot be destroyed is half proved, and has been a real defect before, so `destroy` runs from an `EXIT` trap even on failure and a teardown failure fails the run. `--keep` leaves everything up and records no verdict. Other flags: `--skip-workload` (form and verify, launch no guests), `--workbook NAME`, `--ssh-public-key` / `--ssh-private-key`.
+
+**A clean teardown can still leave public IPs behind, and they count against the tenancy.** The addresses Spinifex allocates for guests are created by the running node, so they are in no Terraform state and `destroy` neither sees nor removes them. The node's own reconcile collects detached ones every ten minutes, which is exactly the sweep a destroy takes away. Twenty accumulated in `spxbm` over four runs and exhausted the tenancy's 50 reserved-public-IP limit, at which point every later launch in every compartment failed. [Check for leftover public IPs afterwards](../../../docs/oci-integration/README.md#check-for-leftover-public-ips-afterwards) has the query and the rule for reading it.
+
+The workbook runs **on the node** against `127.0.0.1`, because the node certificate carries no SAN for its public address — a workbook driven from outside the VCN is still blocked.
 
 ## Common Commands
 

scripts/terraform/oci-spx/cloud-init/mount-data-volume.sh

@@ -18,18 +18,39 @@ die() {
     exit 1
 }
 
+MULTIPATHD_UNIT=/usr/lib/systemd/system/multipathd.service
+MULTIPATH_ALIAS=/etc/systemd/system/multipath-tools.service
+
+# OCI makes a bare-metal attachment multipath, and the agent refuses to log one in
+# unless multipathd runs. It probes "multipath-tools", the name Debian retired, so
+# on Ubuntu 26.04 it gets exit 5 forever while multipathd is active all along. A
+# symlink in /etc/systemd/system is a systemd alias, which is all the agent needs.
+#
+# Asserted on every poll rather than written once: on the first bare-metal run the
+# symlink was created and then vanished, cause unknown, and the volume only logged
+# in once it was back. Converging on each pass beats trusting a single write.
+assert_multipath_alias() {
+    [ -e "$MULTIPATHD_UNIT" ] || return 0
+    [ "$(readlink -f "$MULTIPATH_ALIAS" 2>/dev/null)" = "$MULTIPATHD_UNIT" ] && return 0
+
+    ln -sfn "$MULTIPATHD_UNIT" "$MULTIPATH_ALIAS"
+    systemctl daemon-reload
+    log "aliased multipath-tools.service to multipathd.service for the Oracle agent"
+}
+
 # The Oracle Cloud Agent logs the iSCSI session in asynchronously, so the device
 # is not present when cloud-init first runs. Wait for it instead of racing it.
 wait_for_device() {
     local i
     for i in $(seq 1 120); do
+        assert_multipath_alias
         if [ -b "$DEVICE" ]; then
             log "$DEVICE appeared after $((i * 5))s"
             return 0
         fi
         sleep 5
     done
-    die "$DEVICE did not appear within 600s; check the iSCSI session with 'iscsiadm -m session'"
+    die "$DEVICE did not appear within 600s; check 'iscsiadm -m session' and the agent log at /var/log/oracle-cloud-agent/plugins/oci-blockautoconfig/"
 }
 
 # blkid reporting a TYPE is the only thing standing between a re-run and an erased

scripts/terraform/oci-spx/cloud-init/setup-wan-bridge.sh

@@ -28,41 +28,56 @@ primary_mac() {
     cat "/sys/class/net/$dev/address"
 }
 
-# Prints shell assignments for the first VNIC that is not the primary, or exits 1.
-imds_secondary() {
+# One tab-separated record per VNIC that is not the primary.
+imds_secondaries() {
     curl -sf --max-time 10 -H 'Authorization: Bearer Oracle' "$IMDS/vnics/" |
         python3 -c '
 import ipaddress, json, sys
 
 primary = sys.argv[1].lower()
 for vnic in json.load(sys.stdin):
-    if vnic["macAddr"].lower() == primary:
+    mac = vnic["macAddr"].lower()
+    if mac == primary:
         continue
     net = ipaddress.ip_network(vnic["subnetCidrBlock"])
-    print("WAN_MAC=%s" % vnic["macAddr"].lower())
-    print("WAN_IP=%s" % vnic["privateIp"])
-    print("WAN_PREFIX=%d" % net.prefixlen)
-    print("WAN_CIDR=%s" % vnic["subnetCidrBlock"])
-    print("WAN_ROUTER=%s" % vnic["virtualRouterIp"])
-    sys.exit(0)
-sys.exit(1)
+    print("\t".join([mac, vnic["privateIp"], str(net.prefixlen),
+                     vnic["subnetCidrBlock"], vnic["virtualRouterIp"]]))
 ' "$1"
 }
 
+# Picks the secondary VNIC the kernel actually has an interface for, and sets the
+# WAN_* variables from it. Deliberately not "the first non-primary": IMDS order is
+# not documented, and on bare metal a VNIC sharing a port has no interface at all.
+select_secondary() {
+    local primary="$1" mac ip prefix cidr router iface
+    while IFS=$'\t' read -r mac ip prefix cidr router; do
+        [ -n "$mac" ] || continue
+        if iface="$(iface_for_mac "$mac")"; then
+            WAN_MAC="$mac"
+            WAN_IP="$ip"
+            WAN_PREFIX="$prefix"
+            WAN_CIDR="$cidr"
+            WAN_ROUTER="$router"
+            WAN_IFACE="$iface"
+            return 0
+        fi
+    done < <(imds_secondaries "$primary")
+    return 1
+}
+
 wait_for_secondary() {
-    local primary i out
+    local primary i
     primary="$(primary_mac)"
     log "primary VNIC MAC is $primary"
 
     for i in $(seq 1 60); do
-        if out="$(imds_secondary "$primary")" && [ -n "$out" ]; then
-            eval "$out"
-            log "secondary VNIC after $((i * 5))s: $WAN_MAC $WAN_IP/$WAN_PREFIX via $WAN_ROUTER"
+        if select_secondary "$primary"; then
+            log "secondary VNIC after $((i * 5))s: $WAN_MAC on $WAN_IFACE, $WAN_IP/$WAN_PREFIX via $WAN_ROUTER"
             return 0
         fi
         sleep 5
     done
-    die "no secondary VNIC appeared within 300s; check the VNIC attachment in OCI"
+    die "no secondary VNIC with an interface appeared within 300s; on bare metal check it is attached at nic_index 1, because a VNIC sharing the primary's port never becomes an interface"
 }
 
 # The physical interface for a MAC. Matching on the address rather than trusting a
@@ -184,16 +199,13 @@ PY
 
 main() {
     wait_for_secondary
-
-    local iface
-    iface="$(iface_for_mac "$WAN_MAC")" || die "no interface carries $WAN_MAC"
-    log "bridging $iface into $BRIDGE"
+    log "bridging $WAN_IFACE into $BRIDGE"
 
     strip_cloud_init_secondary
 
     local previous=""
     [ -f "$NETPLAN_FILE" ] && previous="$(cat "$NETPLAN_FILE")"
-    write_netplan "$iface"
+    write_netplan "$WAN_IFACE"
 
     if [ "$STRIPPED" = 0 ] && [ "$previous" = "$(cat "$NETPLAN_FILE")" ] && ip link show "$BRIDGE" >/dev/null 2>&1; then
         log "$BRIDGE is already configured and unchanged"

scripts/terraform/oci-spx/cloud-init/user-data.yaml.tftpl

@@ -21,15 +21,68 @@ write_files:
     content: |
       ${indent(6, service_ports_script)}
 
-  # The secondary VNIC is attached after the instance reaches RUNNING, so it can
+  # Staged rather than applied: spinifex.toml does not exist until spx admin init
+  # has run. Holds no secret under instance principal, which is the point of it --
+  # the instance's own certificate is the credential.
+  - path: /etc/spinifex/oci/external-pool.toml
+    owner: root:root
+    permissions: "0644"
+    content: |
+      ${indent(6, oci_pool_fragment)}
+
+  # A unit rather than a runcmd so a boot that raced the iSCSI login retries on the
+  # next one, and so the ports below can declare a dependency on it. The script
+  # polls for the device itself and is safe to re-run.
+  - path: /etc/systemd/system/spinifex-data-volume.service
+    owner: root:root
+    permissions: "0644"
+    content: |
+      [Unit]
+      Description=Spinifex data volume on the OCI block volume
+      Wants=network-online.target
+      After=network-online.target remote-fs.target
+      ConditionPathIsMountPoint=!${data_mountpoint}
+
+      [Service]
+      Type=oneshot
+      RemainAfterExit=yes
+      ExecStart=/usr/local/sbin/spinifex-mount-data-volume ${data_device} ${data_mountpoint}
+
+      [Install]
+      WantedBy=multi-user.target
+
+  # Requires= is the point: a node with no data volume has nothing behind these
+  # ports, so it must not publish them. systemd enforces that ordering, which a
+  # list of runcmd entries cannot -- runcmd continues past a failed step.
+  - path: /etc/systemd/system/spinifex-service-ports.service
+    owner: root:root
+    permissions: "0644"
+    content: |
+      [Unit]
+      Description=Spinifex node service ports
+      Requires=spinifex-data-volume.service
+      After=spinifex-data-volume.service
+
+      [Service]
+      Type=oneshot
+      RemainAfterExit=yes
+      ExecStart=/usr/local/sbin/spinifex-open-service-ports ${peer_cidr} ${service_ports}
+
+      [Install]
+      WantedBy=multi-user.target
+
+  # The external VNIC is attached after the instance reaches RUNNING, so it can
   # arrive late. This unit retries on the next boot if the first attempt missed it,
   # and stops running once the netplan file it writes exists.
+  #
+  # Deliberately independent of the data volume: losing the external datapath
+  # because a disk is late would make the node more broken, not safer.
   - path: /etc/systemd/system/spinifex-wan-bridge.service
     owner: root:root
     permissions: "0644"
     content: |
       [Unit]
-      Description=Spinifex WAN bridge over the secondary OCI VNIC
+      Description=Spinifex WAN bridge over the external OCI VNIC
       Wants=network-online.target
       After=network-online.target
       ConditionPathExists=!/etc/netplan/60-spinifex-wan.yaml
@@ -45,6 +98,6 @@ write_files:
       WantedBy=multi-user.target
 
 runcmd:
-  - [/usr/local/sbin/spinifex-mount-data-volume, "${data_device}", "${data_mountpoint}"]
-  - [/usr/local/sbin/spinifex-open-service-ports, "${peer_cidr}", ${service_ports}]
+  - [systemctl, enable, --now, spinifex-data-volume.service]
+  - [systemctl, enable, --now, spinifex-service-ports.service]
   - [systemctl, enable, --now, spinifex-wan-bridge.service]

scripts/terraform/oci-spx/compute.tf

@@ -30,6 +30,10 @@ locals {
   # one, so it is emitted only when the shape name says Flex.
   shape_is_flex = strcontains(var.compute_shape, "Flex")
 
+  # Bare metal presents VNICs differently from a VM, which decides where the
+  # external VNIC is attached. See the nic_index comment below.
+  shape_is_bare_metal = startswith(var.compute_shape, "BM.")
+
   # Round-robin, so three nodes land on three fault domains rather than wherever
   # OCI's placement happens to put them. Left to itself OCI spreads on a
   # best-effort basis and says nothing about the result, which is not something
@@ -41,12 +45,38 @@ locals {
   # after the boot disk (mulga-poc has oraclevda1 pointing at sdb1).
   data_device = "/dev/oracleoci/oraclevdb"
 
+  # adopt and create differ only in who creates the dynamic group and policy; a node
+  # authenticates identically either way, because the matching rule covers the whole
+  # compartment and so names no instance.
+  use_instance_principal = var.instance_principal != "off"
+
+  # The allocator's pool, staged on each node for spx admin init to pick up. Under
+  # instance principal there is no oci_config_file key, because naming one beside a
+  # certificate credential is a second source of truth that config load rejects.
+  oci_pool_fragment = join("\n", concat([
+    "# Generated by the oci-spx Terraform. Append to /etc/spinifex/spinifex.toml",
+    "# after spx admin init, then restart spinifex.target.",
+    "[[network.external_pools]]",
+    "name               = \"oci-public\"",
+    "source             = \"oci\"",
+    "oci_compartment_id = \"${var.compartment_ocid}\"",
+    "oci_vnic_iface     = \"${var.wan_bridge_name}\"",
+    ], local.use_instance_principal ? [
+    "oci_auth           = \"instance_principal\"",
+    ] : [
+    "oci_config_file    = \"/etc/spinifex/oci/config\"",
+    "oci_config_profile = \"spinifex\"",
+    ], [
+    "dns_servers        = [\"169.254.169.253\"]",
+  ]))
+
   # Identical on every node, so a change to it replaces all of them together.
   user_data = templatefile("${path.module}/cloud-init/user-data.yaml.tftpl", {
     data_device          = local.data_device
     data_mountpoint      = var.data_mountpoint
     peer_cidr            = var.vcn_cidr
-    service_ports        = join(", ", [for p in var.node_service_ports : "\"${p}\""])
+    service_ports        = join(" ", var.node_service_ports)
+    oci_pool_fragment    = local.oci_pool_fragment
     wan_bridge_name      = var.wan_bridge_name
     wan_bridge_mtu       = var.wan_bridge_mtu
     mount_script         = file("${path.module}/cloud-init/mount-data-volume.sh")
@@ -105,6 +135,11 @@ resource "oci_core_vnic_attachment" "mulgadc_external" {
   count       = var.node_count
   instance_id = oci_core_instance.mulgadc[count.index].id
 
+  # On bare metal only the first VNIC on a physical port becomes a Linux device;
+  # a second one is a tagged overlay with no device. The second port keeps this
+  # VNIC first, so it is untagged and present at boot exactly as it is on a VM.
+  nic_index = local.shape_is_bare_metal ? 1 : 0
+
   create_vnic_details {
     subnet_id        = oci_core_subnet.public.id
     display_name     = format("%s-node-%02d-external", var.deployment_name, count.index + 1)

scripts/terraform/oci-spx/e2e-oci.sh

@@ -0,0 +1,464 @@
+#!/bin/bash
+# e2e-oci.sh — prove a ref works on OCI, across every topology, and leave nothing
+# running. One entry point for a workstation, a Cloudflare-dispatched nightly and
+# the GitHub workflow, because three ways of starting it would drift into three
+# different answers about whether OCI works.
+#
+# Configured entirely by environment variables so a dispatch needs no arguments:
+#
+#   SPX_REF           Ref to prove. Default: the spinifex checkout's current branch.
+#   OCI_TOPOLOGIES    Space-separated. Default: "vm-single vm-multi bm
+#                     vm-single-principal". Set one name to test one thing: a
+#                     single VM topology is roughly a quarter of the full run.
+#   OCI_SOURCE        tree (build SPX_REF) or release (the published installer).
+#   OCI_VERSION       With OCI_SOURCE=release, install this exact release tag and
+#                     ignore OCI_CHANNEL. Preferred for anything repeatable: a tag
+#                     resolves by redirect, while the dev channel resolves through a
+#                     rate-limited GitHub API call that 404s when it trips.
+#   OCI_CHANNEL       With OCI_SOURCE=release, which published channel to install:
+#                     latest (default) or dev, the newest prerelease. dev is the
+#                     path our cloud contacts are given, so it is worth testing.
+#                     Default tree: a nightly exists to judge a ref, and the
+#                     published installer says nothing about one.
+#   WORKBOOKS         Workbooks to run on each topology. Default: the driver's own
+#                     list. Set to "" to skip the workbook phase entirely.
+#   OCI_INSTANCE_PRINCIPAL  Authenticate the node's allocator as the instance
+#                     principal. Default 1, and it needs the tenancy's dynamic
+#                     group to exist. The vm-single-principal topology ignores it
+#                     and always authenticates that way, which is its purpose.
+#
+#                     A tenancy where setup-identity.sh cannot run has no such
+#                     group, and then no principal run can pass: the node forms,
+#                     authenticates, and is refused every allocation. That now
+#                     fails at the allocator gate within a minute of the daemon
+#                     starting rather than an hour later inside a workbook, but
+#                     it still costs a full provision, so drop the topology from
+#                     OCI_TOPOLOGIES until the group exists.
+#   OCI_NO_EXTERNAL_POOL    1 to form with no allocator at all, for a tenancy whose
+#                     dynamic group does not exist yet. The allocator and every
+#                     workbook are then recorded SKIPPED, never PASS, so a green
+#                     run with this set is not a claim about guest networking.
+#                     Not applied to vm-single-principal, which the allocator is
+#                     the entire test of; that pairing is refused rather than run.
+#   OCI_SSH_PUBLIC_KEY / OCI_SSH_PRIVATE_KEY   Paths. Default ~/.ssh/oci-spx[.pub].
+#   OCI_CREDENTIAL_HOOK     Executable run on each topology after formation and
+#                     before the pool, as "hook <ssh-key> <host>...". An API-key
+#                     deployment installs its credential here; instance principal
+#                     needs none, so it is skipped unless the pool is key-based.
+#   OCI_PRINCIPAL_TENANCY_OCID, OCI_PRINCIPAL_USER_OCID,
+#   OCI_PRINCIPAL_FINGERPRINT, OCI_PRINCIPAL_PRIVATE_KEY, OCI_PRINCIPAL_REGION,
+#   OCI_PRINCIPAL_COMPARTMENT_OCID
+#                     The credential vm-single-principal builds with, replacing the
+#                     run's own for that one topology. The dynamic group is a
+#                     tenancy-root resource, so a tenancy that will not let
+#                     setup-identity.sh create one refuses every allocation however
+#                     correct the code is, and the principal route has to be proved
+#                     in a tenancy we administer while the rest of the run stays
+#                     where it is.
+#
+#                     All six or none. A partial set builds one tenancy's instances
+#                     against another's compartment, which is refused rather than
+#                     reported; with none set the topology is recorded SKIPPED
+#                     rather than costing a provision to fail. Running it in the
+#                     run's own tenancy is still available by pointing all six
+#                     there, which says so explicitly instead of by default.
+#   OCI_PRINCIPAL_SHAPE     Compute shape for vm-single-principal. Its tenancy
+#                     carries its own service limits, so the shape the other
+#                     topologies use can have zero quota there and fail the apply
+#                     before anything about the principal is exercised. Default
+#                     VM.Standard.E6.Flex, as the others.
+#   OCI_ARTIFACT_DIR  Where logs and the verdict land. Default ./.e2e-oci-<stamp>.
+#   OCI_KEEP_ON_FAIL  1 to leave a failed topology up for inspection. Off by
+#                     default: an OCI bare-metal host left overnight is expensive.
+#
+# The credential comes from OCI_TENANCY_OCID, OCI_USER_OCID, OCI_FINGERPRINT,
+# OCI_PRIVATE_KEY and OCI_REGION when they are set, and from ~/.oci/config when
+# they are not. scripts/oci_env.py owns that choice; nothing here reads a key.
+set -uo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SPINIFEX_ROOT="$(cd "$HERE/../../.." && pwd)"
+# On a workstation spinifex is a submodule of mulga; in CI the two are siblings
+# under the workspace. Deriving one from the other is only right in the first case.
+MULGA_ROOT="${MULGA_ROOT:-$SPINIFEX_ROOT/..}"
+
+TOPOLOGIES="${OCI_TOPOLOGIES:-vm-single vm-multi bm vm-single-principal}"
+SOURCE="${OCI_SOURCE:-tree}"
+CHANNEL="${OCI_CHANNEL:-latest}"
+INSTALL_VERSION="${OCI_VERSION:-}"
+REF="${SPX_REF:-$(git -C "$SPINIFEX_ROOT" rev-parse --abbrev-ref HEAD)}"
+SSH_PUBLIC_KEY="${OCI_SSH_PUBLIC_KEY:-$HOME/.ssh/oci-spx.pub}"
+SSH_PRIVATE_KEY="${OCI_SSH_PRIVATE_KEY:-$HOME/.ssh/oci-spx}"
+ARTIFACT_DIR="${OCI_ARTIFACT_DIR:-$HERE/.e2e-oci-$(date -u +%Y%m%dT%H%M%SZ)}"
+# Exported so validate-topology.sh agrees with the sweep about where state lives.
+# On a persistent runner this belongs outside the checkout: actions/checkout runs
+# git clean -ffdx, which would delete the state of a killed run before the sweep.
+export OCI_STATE_ROOT="${OCI_STATE_ROOT:-$HERE}"
+STATE_ROOT="$OCI_STATE_ROOT"
+SWEEP_ONLY=0
+SUMMARY_ONLY=0
+DRY_RUN=0
+
+log() { printf '[e2e-oci] %s\n' "$*"; }
+die() {
+    printf '[e2e-oci] ERROR: %s\n' "$*" >&2
+    exit 1
+}
+
+usage() {
+    cat >&2 <<EOF
+usage: ${0##*/} [--sweep-only] [--summary-only] [--dry-run]
+
+Everything else is an environment variable; see the header of this script.
+
+  --sweep-only   Destroy anything an earlier run left behind, then stop.
+  --summary-only Write the result tables from the state files, then stop. For
+                 a job that was cancelled or killed before it got there.
+  --dry-run      Print what each topology would do, build nothing.
+EOF
+    exit 2
+}
+
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --sweep-only) SWEEP_ONLY=1; shift ;;
+        --summary-only) SUMMARY_ONLY=1; shift ;;
+        --dry-run) DRY_RUN=1; shift ;;
+        -h | --help) usage ;;
+        *) die "unknown option $1" ;;
+    esac
+done
+
+case "$SOURCE" in
+    tree | release) ;;
+    *) die "OCI_SOURCE must be tree or release, got '$SOURCE'" ;;
+esac
+
+case "$CHANNEL" in
+    latest | dev) ;;
+    *) die "OCI_CHANNEL must be latest or dev, got '$CHANNEL'" ;;
+esac
+# Saying release+dev and tree at once is two different artifacts in one verdict.
+[ "$SOURCE" = tree ] && [ -n "$INSTALL_VERSION" ] \
+    && die "OCI_VERSION needs OCI_SOURCE=release: a tree build installs the ref, not a published tag"
+[ "$SOURCE" = tree ] && [ "$CHANNEL" != latest ] \
+    && die "OCI_CHANNEL=$CHANNEL needs OCI_SOURCE=release: a tree build installs the ref, not a channel"
+
+mkdir -p "$ARTIFACT_DIR"
+VERDICT="$ARTIFACT_DIR/verdict.txt"
+: > "$VERDICT"
+
+# vm-single-principal is the one topology that cannot run wherever the others do.
+# It needs the tenancy's dynamic group, and creating that is a tenancy-root write,
+# so the tenancy a shared test account gives us can never pass it. These let it
+# build somewhere else without moving the rest of the run.
+PRINCIPAL_CRED_VARS="TENANCY_OCID USER_OCID FINGERPRINT PRIVATE_KEY REGION COMPARTMENT_OCID"
+
+# principal_cred_state is none, all or partial. Partial is the one worth a stop:
+# one tenancy's OCIDs with another's compartment builds in a compartment nobody
+# chose, and the apply reports success doing it.
+principal_cred_state() {
+    local name var have=0 miss=0
+    for name in $PRINCIPAL_CRED_VARS; do
+        var="OCI_PRINCIPAL_$name"
+        if [ -n "${!var:-}" ]; then
+            have=$((have + 1))
+        else
+            miss=$((miss + 1))
+        fi
+    done
+    if [ "$have" = 0 ]; then
+        printf 'none\n'
+    elif [ "$miss" = 0 ]; then
+        printf 'all\n'
+    else
+        printf 'partial\n'
+    fi
+}
+
+# Into an array rather than a rendered string, because the private key is a
+# multi-line PEM and any line-based plumbing silently truncates it to its header.
+PRINCIPAL_ENV=()
+build_principal_env() {
+    local name var
+    PRINCIPAL_ENV=()
+    for name in $PRINCIPAL_CRED_VARS; do
+        var="OCI_PRINCIPAL_$name"
+        case "$name" in
+            COMPARTMENT_OCID) PRINCIPAL_ENV+=("TF_VAR_compartment_ocid=${!var}") ;;
+            *) PRINCIPAL_ENV+=("OCI_$name=${!var}") ;;
+        esac
+    done
+}
+
+# Here rather than at the topology, because the build runs first and takes minutes,
+# and a credential that cannot work is knowable before any of it. The loop checks
+# again, since it is what reads the variables.
+case " $TOPOLOGIES " in
+    *" vm-single-principal "*)
+        [ "$(principal_cred_state)" != partial ] \
+            || die "the OCI_PRINCIPAL_* credential is incomplete: set all of $PRINCIPAL_CRED_VARS or none of them"
+        ;;
+esac
+
+# The run page is the evidence, and its readers are not us: someone assessing
+# whether Spinifex works on their cloud should get the answer without opening a
+# log or knowing what this harness is. Markdown on stdout as well as to the step
+# summary, so a workstation run shows the same thing CI publishes.
+mark() {
+    case "$1" in
+        PASS) printf '✅ PASS' ;;
+        FAIL) printf '❌ FAIL' ;;
+        SKIPPED) printf '⏭️ SKIPPED' ;;
+        *) printf 'ℹ️ %s' "$1" ;;
+    esac
+}
+
+summary() {
+    printf '## Spinifex on OCI — %s\n\n' "$REF"
+    # Said, not implied. A tree build and a published release answer different
+    # questions, and a table headed "published release" over a build of somebody's
+    # branch is the one error here that would mislead a reader who trusted it.
+    if [ "$SOURCE" = tree ]; then
+        printf 'Built from this ref, region `%s`.\n\n' "${OCI_REGION:-ap-sydney-1}"
+    else
+        printf 'Published release `%s`, region `%s`.\n\n' \
+            "${INSTALL_VERSION:-$CHANNEL channel}" "${OCI_REGION:-ap-sydney-1}"
+    fi
+
+    # In the header, not left to a SKIPPED row further down. A run with no allocator
+    # is not evidence about public addressing, and that is precisely the claim a
+    # reader of a green table would otherwise take from it.
+    # Said per topology rather than once for the run: vm-single-principal pins its
+    # own credential, so a single claim over a mixed run is wrong for one of them.
+    if [ "${OCI_NO_EXTERNAL_POOL:-0}" = 1 ]; then
+        printf '> **No external address pool.** Public-address allocation and every workbook needing a public address were skipped, so nothing here speaks to guest ingress. `vm-single-principal`, if it ran, is the exception and did build one.\n\n'
+    elif [ "${OCI_INSTANCE_PRINCIPAL:-1}" = 1 ]; then
+        printf 'Allocator authenticated as the instance principal.\n\n'
+    else
+        printf 'Allocator authenticated with an API key, except `vm-single-principal` which authenticates as the instance.\n\n'
+    fi
+    # Said in the summary because the row is otherwise read as a result about the
+    # same tenancy as every row above it, which is the one thing it is not.
+    if [ "$(principal_cred_state)" = all ]; then
+        printf '`vm-single-principal` built in its own tenancy in `%s`, from the `OCI_PRINCIPAL_*` credential.\n\n' \
+            "$OCI_PRINCIPAL_REGION"
+    fi
+
+    local topology results workbooks gate status detail name secs
+    for topology in $TOPOLOGIES; do
+        results="$STATE_ROOT/.validate-$topology/results.tsv"
+        workbooks="$STATE_ROOT/.validate-$topology/workbooks.tsv"
+        printf '### `%s`\n\n' "$topology"
+        # A topology the loop never reached is said so rather than omitted: a
+        # reader takes an absent row for a passing one.
+        if [ ! -s "$results" ]; then
+            printf 'Did not run.\n\n'
+            continue
+        fi
+        printf '| Gate | Result | Detail |\n| --- | --- | --- |\n'
+        while IFS=$'\t' read -r gate status detail; do
+            [ -n "$gate" ] || continue
+            printf '| %s | %s | %s |\n' "$gate" "$(mark "$status")" "$detail"
+        done < "$results"
+        printf '\n'
+        if [ ! -s "$workbooks" ]; then
+            printf 'No published workbooks ran.\n\n'
+            continue
+        fi
+        printf '| Terraform workbook | Result | Duration |\n| --- | --- | --- |\n'
+        while IFS=$'\t' read -r name status secs; do
+            [ -n "$name" ] || continue
+            printf '| `%s` | %s | %ss |\n' "$name" "$(mark "$status")" "$secs"
+        done < "$workbooks"
+        printf '\n'
+    done
+}
+
+# GITHUB_STEP_SUMMARY is per step, so appending from both the run and the dedicated
+# publish step would put two copies of the tables on the run page. One owner: the run
+# writes the file and prints it, and only --summary-only publishes it.
+emit_summary() {
+    summary | tee "$ARTIFACT_DIR/summary.md"
+}
+
+publish_summary() {
+    emit_summary
+    if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
+        cat "$ARTIFACT_DIR/summary.md" >> "$GITHUB_STEP_SUMMARY"
+    fi
+}
+
+# A topology destroys itself from its own EXIT trap, so this only catches a run
+# the trap never reached: a killed process, a runner reclaimed mid-job, a reboot.
+# It works from the state files rather than by name filter, so it can only ever
+# destroy what one of these runs created.
+sweep() {
+    local found=0 dir topology
+    local sweep_env=(env)
+    for dir in "$STATE_ROOT"/.validate-*; do
+        [ -s "$dir/terraform.tfstate" ] || continue
+        python3 -c '
+import json, sys
+state = json.load(open(sys.argv[1]))
+sys.exit(0 if any(r["instances"] for r in state.get("resources", [])) else 1)
+' "$dir/terraform.tfstate" 2>/dev/null || continue
+        topology="${dir##*/.validate-}"
+        # The child reads OCI_STATE_ROOT from the environment, so it looks in the
+        # same place this loop found the state rather than beside its own script.
+        found=1
+        # The same credential the topology built with, or the destroy authenticates
+        # against a tenancy that never held these resources and reports a clean
+        # sweep over a deployment that is still billing.
+        sweep_env=(env)
+        if [ "$topology" = vm-single-principal ] && [ "$(principal_cred_state)" = all ]; then
+            build_principal_env
+            sweep_env=(env "${PRINCIPAL_ENV[@]}")
+        fi
+        log "sweep: $topology still holds resources, destroying"
+        if "${sweep_env[@]}" "$HERE/validate-topology.sh" --topology "$topology" --destroy-only \
+            >> "$ARTIFACT_DIR/sweep.log" 2>&1; then
+            log "sweep: $topology destroyed"
+        else
+            log "sweep: $topology WOULD NOT DESTROY -- it is still billing. See $ARTIFACT_DIR/sweep.log"
+            echo "sweep $topology: FAIL" >> "$VERDICT"
+        fi
+    done
+    [ "$found" = 0 ] && log "sweep: nothing left behind"
+}
+
+# Before the sweep, because the sweep is what destroys the state the tables are
+# read from. This is the path a cancelled job takes, so it has to work on whatever
+# the run got through rather than assuming a complete set.
+if [ "$SUMMARY_ONLY" = 1 ]; then
+    publish_summary
+    exit 0
+fi
+
+# Run the sweep before the topologies as well as after: a leftover cluster from a
+# killed run is both a bill and a second set of resources with our names on it.
+log "artifacts in $ARTIFACT_DIR"
+sweep
+
+if [ "$SWEEP_ONLY" = 1 ]; then
+    log "--sweep-only: done"
+    exit 0
+fi
+
+# Built once for every topology. A per-topology build would prove three different
+# sets of bytes and could not say which one failed.
+DISTRO_TARBALL=""
+if [ "$SOURCE" = tree ]; then
+    log "building $REF"
+    DISTRO_TARBALL="$ARTIFACT_DIR/spinifex-distro.tar.gz"
+    BUILDER="$MULGA_ROOT/scripts/tofu-cluster/build-install-artifacts.sh"
+    [ -x "$BUILDER" ] || die "no artifact builder at $BUILDER (set MULGA_ROOT)"
+    # It clones each repo from GitHub rather than using the checkout beside it, so a
+    # ref that exists only on this workstation silently falls back to dev and the
+    # run would report a verdict about the wrong code. Fail on that instead.
+    if ! git -C "$SPINIFEX_ROOT" rev-parse --verify --quiet "refs/remotes/origin/$REF" >/dev/null \
+        && ! git -C "$SPINIFEX_ROOT" ls-remote --exit-code --tags origin "$REF" >/dev/null 2>&1; then
+        die "$REF is not on origin: the builder clones from GitHub, so an unpushed ref would fall back to dev"
+    fi
+    if [ "$DRY_RUN" = 1 ]; then
+        log "dry run: would build $REF into $DISTRO_TARBALL"
+    else
+        # --skip-e2e-build: the Go suites are not run here. The workbook driver is a
+        # shell script, so building their binaries would add minutes and prove nothing.
+        "$BUILDER" "$REF" "$DISTRO_TARBALL" "$ARTIFACT_DIR/setup.sh" \
+            "$ARTIFACT_DIR/spinifex-tests.tar" --skip-e2e-build \
+            > "$ARTIFACT_DIR/build.log" 2>&1 \
+            || die "building $REF failed; see $ARTIFACT_DIR/build.log"
+        log "built $(du -h "$DISTRO_TARBALL" | cut -f1) from $REF"
+    fi
+fi
+
+RUN_RC=0
+for topology in $TOPOLOGIES; do
+    # The credential swap happens here and nowhere else, so everything downstream
+    # of the one validate call is unaware the topology built somewhere different.
+    # Always a bare `env` otherwise, because an empty array is an unbound variable
+    # under set -u on the bash 3.2 operators run this with.
+    run_env=(env)
+    if [ "$topology" = vm-single-principal ]; then
+        case "$(principal_cred_state)" in
+            none)
+                log "$topology: SKIPPED -- no OCI_PRINCIPAL_* credential, and the run's own tenancy cannot hold the dynamic group this topology needs"
+                echo "$topology: SKIPPED (no principal credential)" >> "$VERDICT"
+                continue
+                ;;
+            partial)
+                die "the OCI_PRINCIPAL_* credential is incomplete: set all of $PRINCIPAL_CRED_VARS or none of them"
+                ;;
+        esac
+        build_principal_env
+        run_env=(env "${PRINCIPAL_ENV[@]}")
+    fi
+    args=(--topology "$topology" --ssh-public-key "$SSH_PUBLIC_KEY" --ssh-private-key "$SSH_PRIVATE_KEY")
+    [ "$DRY_RUN" = 1 ] && args+=(--dry-run)
+    # Two separate choices. --no-external-pool is the one that forms without any
+    # allocator, for a tenancy whose dynamic group does not exist yet; it records
+    # the allocator and the workbooks as SKIPPED rather than passing them.
+    #
+    # vm-single-principal opts out of both: it sets its own credential, and a run
+    # of it without an allocator would exercise nothing. validate-topology.sh
+    # refuses that pairing, so skipping it here keeps a mixed run from aborting.
+    if [ "$topology" = vm-single-principal ]; then
+        :
+    elif [ "${OCI_NO_EXTERNAL_POOL:-0}" = 1 ]; then
+        args+=(--no-external-pool)
+    elif [ "${OCI_INSTANCE_PRINCIPAL:-1}" = 1 ]; then
+        args+=(--instance-principal)
+    fi
+    # Not under --dry-run, where the build was skipped and naming the tarball it
+    # would have produced fails the readable check, so every topology reported FAIL
+    # and the flag operators are told to run first could never pass. Still
+    # unconditional in a real run, so a build that produced nothing stays loud
+    # rather than quietly installing a release instead.
+    if [ -n "$DISTRO_TARBALL" ] && [ "$DRY_RUN" != 1 ]; then
+        args+=(--distro "$DISTRO_TARBALL" --setup-sh "$ARTIFACT_DIR/setup.sh")
+    fi
+    if [ "$SOURCE" = release ]; then
+        if [ -n "$INSTALL_VERSION" ]; then
+            args+=(--version "$INSTALL_VERSION")
+        else
+            args+=(--channel "$CHANNEL")
+        fi
+    fi
+    [ -n "${OCI_CREDENTIAL_HOOK:-}" ] && args+=(--credential-hook "$OCI_CREDENTIAL_HOOK")
+    [ "${OCI_KEEP_ON_FAIL:-0}" = 1 ] && args+=(--keep-on-fail)
+    [ -n "${WORKBOOKS+x}" ] && args+=(--workbooks "$WORKBOOKS")
+
+    # go test -v markers, so the same log feeds go-junit-report and the RCA bundle
+    # the Go suites produce. One testcase per topology.
+    tname="TestOCITopology_${topology//-/_}"
+    start=$SECONDS
+    echo "=== RUN   $tname"
+    if "${run_env[@]}" "$HERE/validate-topology.sh" "${args[@]}" 2>&1 | tee "$ARTIFACT_DIR/$topology.log"; then
+        printf -- '--- PASS: %s (%d.00s)\n' "$tname" "$((SECONDS - start))"
+        echo "$topology: PASS" >> "$VERDICT"
+    else
+        printf -- '--- FAIL: %s (%d.00s)\n' "$tname" "$((SECONDS - start))"
+        echo "$topology: FAIL" >> "$VERDICT"
+        RUN_RC=1
+    fi
+done
+
+# Again at the end, because a topology that failed its own teardown is exactly the
+# case that costs money, and the verdict should say so rather than the log.
+sweep
+
+emit_summary
+
+
+echo
+log "=== $REF on OCI ($SOURCE build$([ "$SOURCE" = release ] && { [ -n "$INSTALL_VERSION" ] && echo ", $INSTALL_VERSION" || echo ", $CHANNEL channel"; })) ==="
+cat "$VERDICT"
+if [ "$RUN_RC" = 0 ] && ! grep -q FAIL "$VERDICT"; then
+    log "PASS"
+    printf 'ok  \te2e-oci\t%d.000s\n' "$SECONDS"
+else
+    log "FAIL"
+    printf 'FAIL\te2e-oci\t%d.000s\n' "$SECONDS"
+fi
+exit "$RUN_RC"

scripts/terraform/oci-spx/eip-move-test.sh

@@ -17,7 +17,7 @@ set -euo pipefail
 
 KEY=${KEY:-$HOME/.ssh/oci-poc.pem}
 OCI_BIN=${OCI_BIN:-$HOME/bin/oci}
-OCI_PROFILE=${OCI_PROFILE:-apacanzset03child03}
+OCI_PROFILE=${OCI_PROFILE:-DEFAULT}
 SUBNET=${SUBNET:-ocid1.subnet.oc1.ap-sydney-1.aaaaaaaaimrhjgsqgrfg7bnka2d3nbx2n5ss7kurcwneon22rkclqf3ypfqa}
 
 # Public address -> node name. The control plane is reached through node01.

scripts/terraform/oci-spx/instance-principal.tf

@@ -0,0 +1,44 @@
+# Lets each node authenticate to the OCI API as itself, so the external-address
+# allocator needs no API key on disk. Without one of these two credential routes a
+# node forms and passes every health check, then refuses every launch that wants a
+# public address with InsufficientAddressCapacity and no stated cause.
+#
+# Only instance_principal = "create" builds these, and deliberately: a dynamic
+# group and a policy are tenancy-root resources, so creating them needs rights the
+# compartment-scoped user the rest of this configuration runs as does not have and
+# should not be given. Create them once with a tenancy admin, then every later
+# deployment uses "adopt", which references nothing and so needs no such rights.
+
+resource "oci_identity_dynamic_group" "nodes" {
+  provider       = oci.home
+  count          = var.instance_principal == "create" ? 1 : 0
+  compartment_id = var.tenancy_ocid
+  name           = "${var.deployment_name}-nodes"
+  description    = "Spinifex nodes in the ${var.deployment_name} deployment, authenticating as instance principals."
+
+  # Scoped to the compartment, not to instance OCIDs: nodes are replaced, and a
+  # rule listing them by OCID goes stale the first time one is.
+  matching_rule = "ALL {instance.compartment.id = '${var.compartment_ocid}'}"
+}
+
+# Narrower than it looks. The allocator creates, moves and deletes secondary
+# private IPs on a node's own VNIC and attaches public IPs to them, which is what
+# these four verbs cover; nothing here grants compute, storage or identity.
+resource "oci_identity_policy" "nodes" {
+  provider       = oci.home
+  count          = var.instance_principal == "create" ? 1 : 0
+  compartment_id = var.compartment_ocid
+  name           = "${var.deployment_name}-nodes-network"
+  description    = "Allows Spinifex nodes to manage the private and public IPs backing external addresses."
+
+  statements = [
+    "Allow dynamic-group ${oci_identity_dynamic_group.nodes[0].name} to use vnics in compartment id ${var.compartment_ocid}",
+    "Allow dynamic-group ${oci_identity_dynamic_group.nodes[0].name} to manage private-ips in compartment id ${var.compartment_ocid}",
+    "Allow dynamic-group ${oci_identity_dynamic_group.nodes[0].name} to manage public-ips in compartment id ${var.compartment_ocid}",
+
+    # CreatePrivateIp needs SUBNET_ATTACH and DeletePrivateIp needs SUBNET_DETACH,
+    # both of which only the subnets `use` verb carries. Without it every allocation
+    # is refused as a 404, which reads as a missing resource rather than a missing grant.
+    "Allow dynamic-group ${oci_identity_dynamic_group.nodes[0].name} to use subnets in compartment id ${var.compartment_ocid}",
+  ]
+}

scripts/terraform/oci-spx/outputs.tf

@@ -8,6 +8,13 @@ output "security_list_ocids" { value = { public = oci_core_security_list.public.
 output "compute_ocids" { value = [for instance in oci_core_instance.mulgadc : instance.id] }
 output "data_volume_ocids" { value = [for volume in oci_core_volume.mulgadc_data : volume.id] }
 
+# Read back rather than assumed: both can come from a tfvars file, so this is the
+# only place that knows what was actually built. The deploy driver skips the
+# credential handoff on instance_principal, so a wrong guess leaves a node with no
+# credential at all.
+output "compute_shape" { value = var.compute_shape }
+output "instance_principal" { value = var.instance_principal }
+
 # What the Spinifex deploy path consumes. The primary VNIC is the host plane, so
 # these are the addresses update-nodes.sh and install-node.sh connect to.
 output "nodes" {

scripts/terraform/oci-spx/provider.tf

@@ -1,10 +1,25 @@
+# A runner gets the key as a value, a workstation as a path. Passing the value
+# means CI never writes key material to disk, so there is nothing to leave behind
+# on a shared runner and nothing to clean up after a cancelled job.
 provider "oci" {
   tenancy_ocid     = var.tenancy_ocid
   user_ocid        = var.user_ocid
   fingerprint      = var.fingerprint
-  private_key_path = var.private_key_path
+  private_key      = var.private_key != "" ? var.private_key : null
+  private_key_path = var.private_key == "" ? var.private_key_path : null
   region           = var.region
 }
 
-# The aliased home-region provider is gone with the identity resources it served.
-# Everything here now lives in one region, in an existing compartment.
+# OCI serves IAM writes from the tenancy's home region, so the dynamic group and
+# policy go through this one. Defaults to var.region, which makes it identical to
+# the provider above until OCI_HOME_REGION says otherwise -- so a tenancy whose
+# home region is where we build is unaffected, and ours is not.
+provider "oci" {
+  alias            = "home"
+  tenancy_ocid     = var.tenancy_ocid
+  user_ocid        = var.user_ocid
+  fingerprint      = var.fingerprint
+  private_key      = var.private_key != "" ? var.private_key : null
+  private_key_path = var.private_key == "" ? var.private_key_path : null
+  region           = var.home_region != null && var.home_region != "" ? var.home_region : var.region
+}

scripts/terraform/oci-spx/requirements.txt

@@ -1,2 +0,0 @@
-# Dependencies used only by scripts/oci_env.py in its isolated .venv.
-oci>=2.160.0,<3.0.0

scripts/terraform/oci-spx/scripts/oci_env.py

@@ -1,5 +1,9 @@
 #!/usr/bin/env python3
-"""Prepare OCI-profile Terraform environment variables in this repo's .venv.
+"""Prepare OCI Terraform environment variables from a profile or the environment.
+
+Standard library only, deliberately. It once used the OCI SDK, which meant a
+virtualenv, which meant python3-venv on every host that runs it -- and banksia
+did not have it. Terraform's own provider makes the OCI calls.
 
 Examples:
   eval "$(python3 scripts/oci_env.py --shell)"
@@ -12,63 +16,144 @@ import argparse
 import configparser
 import os
 from pathlib import Path
+import re
 import shlex
 import subprocess
 import sys
-import venv
 
 
 REPOSITORY = Path(__file__).resolve().parent.parent
-VENV_PYTHON = REPOSITORY / ".venv" / "bin" / "python"
-REQUIREMENTS = REPOSITORY / "requirements.txt"
-REQUESTED_PROFILE = "apacanzset03child03"
-FALLBACK_PROFILE = "apacanzset03child3"
+# The OCI CLI's own name for "the profile when none is named", so a config written
+# by `oci setup config` works with no flag.
+DEFAULT_PROFILE = "DEFAULT"
 
 
-def ensure_venv() -> None:
-    """Create, populate, then re-execute from this repository's virtualenv."""
-    if Path(sys.executable).resolve() == VENV_PYTHON.resolve():
-        return
-    if not VENV_PYTHON.exists():
-        venv.EnvBuilder(with_pip=True).create(VENV_PYTHON.parent.parent)
-    subprocess.check_call([str(VENV_PYTHON), "-m", "pip", "install", "--quiet", "-r", str(REQUIREMENTS)])
-    os.execv(str(VENV_PYTHON), [str(VENV_PYTHON), str(Path(__file__).resolve()), *sys.argv[1:]])
+def default_profile() -> str:
+    """The profile to read when the caller named none.
+
+    OCI_CLI_PROFILE is the OCI CLI's own variable, so an operator who has already
+    selected a profile for `oci` does not select it twice.
+    """
+    return os.environ.get("OCI_CLI_PROFILE") or DEFAULT_PROFILE
+
+
+def has_profile(parser: configparser.RawConfigParser, name: str) -> bool:
+    """Whether the config holds this profile.
+
+    DEFAULT needs its own test: configparser treats it as the fallback for every
+    other section rather than as a section, so has_section and sections() both
+    deny it exists even when `oci setup config` wrote nothing else.
+    """
+    if name == DEFAULT_PROFILE:
+        return bool(parser.defaults())
+    return parser.has_section(name)
+
+
+def profile_names(parser: configparser.RawConfigParser) -> list[str]:
+    names = list(parser.sections())
+    if parser.defaults():
+        names.insert(0, DEFAULT_PROFILE)
+    return names
 
 
 def load_profile(config_path: Path, requested_profile: str) -> tuple[str, dict[str, str]]:
     parser = configparser.RawConfigParser()
     parser.read(config_path)
     profile = requested_profile
-    if not parser.has_section(profile):
-        if profile == REQUESTED_PROFILE and parser.has_section(FALLBACK_PROFILE):
-            print(
-                f"Profile {REQUESTED_PROFILE} not found; using {FALLBACK_PROFILE}.",
-                file=sys.stderr,
-            )
-            profile = FALLBACK_PROFILE
-        else:
-            raise ValueError(f"OCI profile {profile!r} was not found in {config_path}")
+    # No fallback to another profile. A missing one is an error because the only
+    # alternative is building a cluster in whichever tenancy a different profile
+    # happens to point at.
+    if not has_profile(parser, profile):
+        raise ValueError(
+            f"OCI profile {profile!r} was not found in {config_path}; "
+            f"profiles present: {', '.join(profile_names(parser)) or 'none'}. "
+            "Name one with --profile or OCI_CLI_PROFILE."
+        )
 
-    # Validate through the OCI SDK installed in the dedicated virtualenv.
-    import oci
-
-    values = oci.config.from_file(str(config_path), profile)
+    values = {key: value.strip() for key, value in parser.items(profile)}
     required = ("tenancy", "user", "fingerprint", "key_file", "region")
     missing = [key for key in required if not values.get(key)]
     if missing:
         raise ValueError(f"Missing OCI profile values: {', '.join(missing)}")
+    # ~ in key_file is the OCI CLI's own convention and configparser does not expand it.
+    values["key_file"] = str(Path(values["key_file"]).expanduser())
+    if not Path(values["key_file"]).is_file():
+        raise FileNotFoundError(f"key_file in profile {profile} does not exist: {values['key_file']}")
+    check_credential_shape(values, source=str(config_path), names=CONFIG_SOURCE)
     return profile, values
 
 
-def tenancy_home_region(values: dict[str, str]) -> str:
-    """Read the home region from OCI rather than assuming the profile region."""
-    import oci
+ENVIRONMENT_KEYS = ("OCI_TENANCY_OCID", "OCI_USER_OCID", "OCI_FINGERPRINT", "OCI_PRIVATE_KEY", "OCI_REGION")
 
-    subscriptions = oci.identity.IdentityClient(values).list_region_subscriptions(values["tenancy"]).data
-    home = next((item.region_name for item in subscriptions if item.is_home_region), None)
-    if not home:
-        raise ValueError("OCI did not return a tenancy home region")
-    return home
+
+def load_environment() -> dict[str, str] | None:
+    """Read the credential from the environment, the way a CI runner holds it.
+
+    Returns None when none of the keys are set, so a workstation still falls
+    through to ~/.oci/config. A partial set is an error rather than a fallback:
+    half a credential in the environment means a secret failed to reach the job,
+    and silently using a different one would validate the wrong tenancy.
+    """
+    present = [key for key in ENVIRONMENT_KEYS if os.environ.get(key)]
+    if not present:
+        return None
+    missing = [key for key in ENVIRONMENT_KEYS if not os.environ.get(key)]
+    if missing:
+        raise ValueError(
+            f"OCI credential in the environment is incomplete: {', '.join(present)} set, {', '.join(missing)} missing"
+        )
+    key = os.environ["OCI_PRIVATE_KEY"]
+    # Base64 is accepted because a PEM's newlines do not survive every secret
+    # store. Detected by shape rather than by a second variable to set wrongly.
+    if "-----BEGIN" not in key:
+        import base64
+
+        key = base64.b64decode(key).decode()
+    if "-----BEGIN" not in key:
+        raise ValueError("OCI_PRIVATE_KEY is neither a PEM nor base64-encoded PEM")
+    # Stripped because `echo` into a secret store appends a newline, and the SDK
+    # then calls the value malformed without saying which one or why.
+    values = {
+        "tenancy": os.environ["OCI_TENANCY_OCID"].strip(),
+        "user": os.environ["OCI_USER_OCID"].strip(),
+        "fingerprint": os.environ["OCI_FINGERPRINT"].strip(),
+        "key_content": key.strip() + "\n",
+        "region": os.environ["OCI_REGION"].strip(),
+    }
+    check_credential_shape(values, source="the environment", names=ENVIRONMENT_SOURCE)
+    return values
+
+
+# Which input to go and fix, named the way the person reading the failure set it.
+ENVIRONMENT_SOURCE = {
+    "tenancy": "OCI_TENANCY_OCID",
+    "user": "OCI_USER_OCID",
+    "fingerprint": "OCI_FINGERPRINT",
+    "key_content": "OCI_PRIVATE_KEY",
+    "region": "OCI_REGION",
+}
+CONFIG_SOURCE = {key: key for key in ENVIRONMENT_SOURCE} | {"key_content": "key_file"}
+FINGERPRINT = re.compile(r"^[0-9a-f]{2}(:[0-9a-f]{2}){15}$")
+
+
+def check_credential_shape(values: dict[str, str], source: str, names: dict[str, str]) -> None:
+    """Reject a malformed credential here rather than as an OCI 401 later.
+
+    Every fault names the input the reader set, because the alternative is a
+    401 forty seconds into a Terraform run, which says nothing about which of
+    five values is wrong. A newline from `echo` into a secret store is the
+    common one, and it is stripped before this runs.
+    """
+    faults = []
+    for field in ("tenancy", "user"):
+        if not values.get(field, "").startswith("ocid1."):
+            faults.append(f"{names[field]} is not an OCID")
+    if not FINGERPRINT.match(values.get("fingerprint", "")):
+        faults.append(f"{names['fingerprint']} is not a 16-byte hex fingerprint")
+    if not values.get("region"):
+        faults.append(f"{names['region']} is empty")
+    if faults:
+        raise ValueError(f"the OCI credential in {source} is not usable: {', '.join(faults)}")
 
 
 def terraform_environment(
@@ -79,14 +164,22 @@ def terraform_environment(
     ssh_private_key_path: str | None,
 ) -> dict[str, str]:
     environment = {
-        "OCI_CLI_PROFILE": profile,
         "TF_VAR_tenancy_ocid": values["tenancy"],
         "TF_VAR_user_ocid": values["user"],
         "TF_VAR_fingerprint": values["fingerprint"],
-        "TF_VAR_private_key_path": values["key_file"],
         "TF_VAR_region": region_override or values["region"],
-        "TF_VAR_home_region": tenancy_home_region(values),
+        # A tenancy's home region is a fixed property of the tenancy, so it is an
+        # input rather than something to rediscover on every run. IAM resources
+        # have to be created against it; see the phase 12 note in the plan.
+        "TF_VAR_home_region": os.environ.get("OCI_HOME_REGION") or region_override or values["region"],
     }
+    # One of the two, never both: the provider rejects a key given twice, and the
+    # variable validation in variables.tf says so before OCI is called at all.
+    if values.get("key_content"):
+        environment["TF_VAR_private_key"] = values["key_content"]
+    else:
+        environment["OCI_CLI_PROFILE"] = profile
+        environment["TF_VAR_private_key_path"] = values["key_file"]
     if ssh_public_key_path:
         key_path = Path(ssh_public_key_path).expanduser().resolve()
         if not key_path.is_file():
@@ -102,7 +195,11 @@ def terraform_environment(
 
 def main() -> int:
     arg_parser = argparse.ArgumentParser()
-    arg_parser.add_argument("--profile", default=REQUESTED_PROFILE)
+    arg_parser.add_argument(
+        "--profile",
+        default=default_profile(),
+        help="Profile in ~/.oci/config. Defaults to $OCI_CLI_PROFILE, else DEFAULT.",
+    )
     arg_parser.add_argument("--region")
     arg_parser.add_argument(
         "--ssh-public-key-path",
@@ -118,11 +215,21 @@ def main() -> int:
     if args.shell and args.command:
         arg_parser.error("--shell and a command cannot be used together")
 
-    ensure_venv()
-    config_path = Path(os.environ.get("OCI_CONFIG_FILE", Path.home() / ".oci" / "config"))
-    if not config_path.is_file():
-        raise FileNotFoundError(f"OCI configuration was not found at {config_path}")
-    profile, values = load_profile(config_path, args.profile)
+    values = load_environment()
+    if values is not None:
+        profile = "environment"
+        # Checked here, before anything calls OCI: --shell prints to stdout, which
+        # in CI is the job log, and this route holds the PEM itself.
+        if args.shell:
+            raise ValueError("--shell cannot be used with a credential from the environment: it would print the key")
+    else:
+        config_path = Path(os.environ.get("OCI_CONFIG_FILE", Path.home() / ".oci" / "config"))
+        if not config_path.is_file():
+            raise FileNotFoundError(
+                f"OCI configuration was not found at {config_path} and the environment holds no credential"
+                f" (set {', '.join(ENVIRONMENT_KEYS)})"
+            )
+        profile, values = load_profile(config_path, args.profile)
     environment = terraform_environment(
         profile,
         values,
@@ -137,7 +244,7 @@ def main() -> int:
         return 0
 
     if not args.command:
-        print(f"Loaded OCI profile {profile} for region {environment['TF_VAR_region']}; virtualenv: {VENV_PYTHON.parent.parent}")
+        print(f"Loaded OCI profile {profile} for region {environment['TF_VAR_region']}")
         return 0
 
     command = args.command[1:] if args.command[0] == "--" else args.command
@@ -145,7 +252,12 @@ def main() -> int:
         raise ValueError("A command is required after --")
     child_environment = os.environ.copy()
     child_environment.update(environment)
-    print(f"Running with OCI profile {profile} in {environment['TF_VAR_region']}: {' '.join(map(shlex.quote, command))}")
+    # Stderr, because stdout belongs to the wrapped command: a banner there is
+    # indistinguishable from output and corrupts anything that captures it.
+    print(
+        f"Running with OCI profile {profile} in {environment['TF_VAR_region']}: {' '.join(map(shlex.quote, command))}",
+        file=sys.stderr,
+    )
     return subprocess.call(command, cwd=REPOSITORY, env=child_environment)
 
 

scripts/terraform/oci-spx/setup-identity.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+# setup-identity.sh -- create the dynamic group and policy that let Spinifex nodes
+# authenticate to the OCI API as themselves, so no API key is installed on any node.
+#
+# Run once per tenancy, with a tenancy-admin principal. Every deployment afterwards
+# passes instance_principal = "adopt", which references these by nothing at all: the
+# matching rule covers the whole compartment, so it names no instance and needs no
+# OCID from here.
+#
+# Its own state file, deliberately. validate-topology.sh destroys its state at the
+# end of every run, and these two resources outlive every topology -- holding them
+# in a per-topology state would let a nightly teardown delete the tenancy's policy.
+#
+# MUST MAINTAIN BASH 3.X COMPATABILITY. Operators run this on macOS, which ships
+# bash 3.2: no declare -A, no mapfile, and under set -u an empty "${arr[@]}" is an
+# unbound variable, so guard every such expansion with ${#arr[@]}.
+set -uo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+STATE_DIR="${OCI_STATE_ROOT:-$HERE}/.identity"
+SSH_PUBLIC_KEY="${OCI_SSH_PUBLIC_KEY:-$HOME/.ssh/oci-spx.pub}"
+DESTROY=0
+DRY_RUN=0
+
+log() { printf '[setup-identity] %s\n' "$*"; }
+die() {
+    printf '[setup-identity] ERROR: %s\n' "$*" >&2
+    exit 1
+}
+
+usage() {
+    cat >&2 <<EOF
+usage: ${0##*/} [--dry-run] [--destroy]
+
+Creates the OCI dynamic group and policy for instance-principal authentication.
+Needs a tenancy-admin OCI credential; the compartment-scoped user the rest of this
+configuration runs as cannot create either resource.
+
+  --dry-run   Show the plan, change nothing.
+  --destroy   Remove them again. Every node using "adopt" loses its credential.
+EOF
+    exit 2
+}
+
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --dry-run) DRY_RUN=1; shift ;;
+        --destroy) DESTROY=1; shift ;;
+        -h | --help) usage ;;
+        *) die "unknown option $1" ;;
+    esac
+done
+
+[ -r "$SSH_PUBLIC_KEY" ] || die "no readable SSH public key at $SSH_PUBLIC_KEY (set OCI_SSH_PUBLIC_KEY)"
+mkdir -p "$STATE_DIR"
+
+# The instance and network resources are not targeted, so nothing else in this
+# configuration is created or read. A tenancy admin running this does not need to
+# hold the compartment rights the rest of it uses.
+targets=(-target oci_identity_dynamic_group.nodes -target oci_identity_policy.nodes)
+
+tf() {
+    python3 "$HERE/scripts/oci_env.py" --ssh-public-key-path "$SSH_PUBLIC_KEY" -- \
+        terraform -chdir="$HERE" "$@"
+}
+
+tf init -input=false -upgrade >/dev/null || die "terraform init failed"
+
+if [ "$DESTROY" = 1 ]; then
+    log "destroying the dynamic group and policy"
+    tf destroy -auto-approve -var 'instance_principal=create' \
+        -state "$STATE_DIR/terraform.tfstate" "${targets[@]}" \
+        || die "destroy failed"
+    log "removed. Any node configured for instance principal can no longer allocate an address."
+    exit 0
+fi
+
+args=(-var 'instance_principal=create' -state "$STATE_DIR/terraform.tfstate" "${targets[@]}")
+
+if [ "$DRY_RUN" = 1 ]; then
+    log "dry run: planning the dynamic group and policy only"
+    tf plan "${args[@]}" || die "plan failed"
+    exit 0
+fi
+
+log "creating the dynamic group and policy (tenancy-root resources)"
+tf apply -auto-approve "${args[@]}" || die "apply failed -- a tenancy-admin principal is required"
+
+log "done. State in $STATE_DIR/terraform.tfstate -- keep it, it is the only record."
+log "Deployments can now use instance_principal = \"adopt\" with a compartment-scoped user."

scripts/terraform/oci-spx/spx-oci-config.sh

@@ -0,0 +1,268 @@
+#!/usr/bin/env bash
+# Installs the OCI credential the external-address allocator authenticates with
+# on every node, as /etc/spinifex/oci/{config,oci_api_key.pem}.
+#
+# Usage: spx-oci-config.sh [--dry-run] <ssh-private-key> <host>...
+#
+# The argument shape is validate-topology.sh's credential-hook contract, so this
+# drops straight into --credential-hook or $OCI_CREDENTIAL_HOOK.
+#
+# MUST MAINTAIN BASH 3.X COMPATABILITY. Operators run this on macOS, which ships
+# bash 3.2: no declare -A, no mapfile, and under set -u an empty "${arr[@]}" is an
+# unbound variable, so guard every such expansion with ${#arr[@]}.
+set -euo pipefail
+
+HERE=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
+SELF="$HERE/$(basename -- "$0")"
+
+# What the allocator reads, from the pool fragment Terraform stages on each node.
+# The config path and the profile name are the contract; the key filename is only
+# ever named by key_file inside that config.
+REMOTE_DIR=/etc/spinifex/oci
+REMOTE_CONFIG="$REMOTE_DIR/config"
+REMOTE_PEM="$REMOTE_DIR/oci_api_key.pem"
+REMOTE_PROFILE=spinifex
+
+SPX_KEY="${OCI_SPX_KEY:-$HOME/.oci/oci_api_key_spx.pem}"
+SPX_PROFILE="${OCI_SPX_PROFILE:-spinifex}"
+SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+          -o LogLevel=ERROR -o BatchMode=yes -o ConnectTimeout=15)
+DRY_RUN=0
+
+die() { echo "$(basename -- "$0"): $*" >&2; exit 1; }
+log() { echo "[spx-oci-config] $*"; }
+
+usage() {
+    sed -n '2,8p' "$SELF" | sed 's/^# \{0,1\}//'
+    cat <<'EOF'
+
+Options:
+  --dry-run   Resolve and validate the credential, print what would be written,
+              and touch no host.
+  -h, --help  This text.
+
+The credential is resolved in this order, and the first one that is complete wins:
+
+  1. $OCI_SPX_PRIVATE_KEY with $OCI_SPX_USER_OCID, $OCI_SPX_FINGERPRINT,
+     $OCI_SPX_TENANCY_OCID and $OCI_SPX_REGION. A dedicated narrow credential,
+     which is the shape a CI secret store holds. A partial set is an error.
+  2. The [spinifex] profile in ~/.oci/config, or $OCI_SPX_PROFILE.
+  3. ~/.oci/oci_api_key_spx.pem, or $OCI_SPX_KEY, with the user, tenancy and
+     region of the profile Terraform itself uses. The fingerprint is derived
+     from the key rather than read from the config.
+  4. The credential Terraform builds with. This works and is wider than the
+     allocator needs, so it warns.
+
+Rules 2 to 4 resolve the Terraform profile through scripts/oci_env.py, so the
+precedence is the same one the apply used and the two cannot disagree.
+EOF
+}
+
+while [ "$#" -gt 0 ]; do
+    case "$1" in
+        --dry-run) DRY_RUN=1; shift ;;
+        -h|--help) usage; exit 0 ;;
+        --) shift; break ;;
+        -*) die "unknown option: $1" ;;
+        *) break ;;
+    esac
+done
+
+# oci_env.py owns credential resolution for this directory: the environment route
+# a runner uses, ~/.oci/config otherwise, a partial set treated as an error, and
+# the shape checks. Re-exec under it rather than reimplement any of that here.
+if [ -z "${SPX_OCI_ENV_READY:-}" ]; then
+    [ -x "$HERE/scripts/oci_env.py" ] || [ -r "$HERE/scripts/oci_env.py" ] \
+        || die "scripts/oci_env.py was not found next to $SELF"
+    if [ "$DRY_RUN" = 1 ]; then
+        exec env SPX_OCI_ENV_READY=1 python3 "$HERE/scripts/oci_env.py" -- "$SELF" --dry-run "$@"
+    fi
+    exec env SPX_OCI_ENV_READY=1 python3 "$HERE/scripts/oci_env.py" -- "$SELF" "$@"
+fi
+
+# The same argument shape in both modes, so a --dry-run rehearsal and the real run
+# differ by one flag. Only the checks that need a host are skipped.
+ssh_key="${1:?usage: $(basename -- "$0") [--dry-run] <ssh-private-key> <host>...}"
+shift
+if [ "$DRY_RUN" != 1 ]; then
+    [ "$#" -gt 0 ] || die "no hosts given"
+    [ -r "$ssh_key" ] || die "SSH private key is not readable: $ssh_key"
+fi
+
+# A key's fingerprint is a property of the key. Deriving it beats trusting a
+# config field, because a stale fingerprint beside a good key is a 401 that names
+# neither.
+fingerprint_of() {
+    openssl rsa -pubout -outform DER -in "$1" 2>/dev/null \
+        | openssl md5 -c 2>/dev/null | awk '{print $NF}'
+}
+
+require_complete() {
+    local source="$1" missing="" name
+    shift
+    for name in "$@"; do
+        [ -n "${!name:-}" ] || missing="$missing $name"
+    done
+    [ -z "$missing" ] || die "$source is incomplete, missing:$missing"
+}
+
+key_pem=""      # the PEM itself, never a path, so nothing is read twice
+oci_user=""
+oci_fingerprint=""
+oci_tenancy=""
+oci_region=""
+source_name=""
+
+if [ -n "${OCI_SPX_PRIVATE_KEY:-}${OCI_SPX_USER_OCID:-}${OCI_SPX_FINGERPRINT:-}${OCI_SPX_TENANCY_OCID:-}${OCI_SPX_REGION:-}" ]; then
+    require_complete "the OCI_SPX_* credential in the environment" \
+        OCI_SPX_PRIVATE_KEY OCI_SPX_USER_OCID OCI_SPX_FINGERPRINT \
+        OCI_SPX_TENANCY_OCID OCI_SPX_REGION
+    # Base64 accepted by shape, because a PEM's newlines do not survive every
+    # secret store, and detected rather than selected by a second variable.
+    case "$OCI_SPX_PRIVATE_KEY" in
+        *-----BEGIN*) key_pem="$OCI_SPX_PRIVATE_KEY" ;;
+        *) key_pem=$(printf '%s' "$OCI_SPX_PRIVATE_KEY" | tr -d ' \n' | base64 -d 2>/dev/null) || true ;;
+    esac
+    case "$key_pem" in
+        *-----BEGIN*) ;;
+        *) die "OCI_SPX_PRIVATE_KEY is neither a PEM nor a base64-encoded PEM" ;;
+    esac
+    oci_user="$OCI_SPX_USER_OCID"
+    oci_fingerprint="$OCI_SPX_FINGERPRINT"
+    oci_tenancy="$OCI_SPX_TENANCY_OCID"
+    oci_region="$OCI_SPX_REGION"
+    source_name="the OCI_SPX_* environment credential"
+
+elif [ -r "$HOME/.oci/config" ] && python3 - "$SPX_PROFILE" <<'PY'
+import configparser, pathlib, sys
+cfg = configparser.RawConfigParser()
+cfg.read(pathlib.Path.home() / ".oci" / "config")
+sys.exit(0 if cfg.has_section(sys.argv[1]) else 1)
+PY
+then
+    # One pipe-separated line read into named variables, rather than eval of
+    # generated shell: none of these fields can contain a pipe.
+    profile_fields=$(python3 - "$SPX_PROFILE" <<'PY'
+import configparser, pathlib, sys
+cfg = configparser.RawConfigParser()
+cfg.read(pathlib.Path.home() / ".oci" / "config")
+section = dict(cfg[sys.argv[1]])
+fields = ("user", "fingerprint", "tenancy", "region", "key_file")
+missing = [f for f in fields if not section.get(f)]
+if missing:
+    sys.exit(f"profile {sys.argv[1]} has no {', '.join(missing)}")
+print("|".join(section[f] for f in fields))
+PY
+) || die "could not read the [$SPX_PROFILE] profile from ~/.oci/config"
+    IFS='|' read -r profile_user profile_fingerprint profile_tenancy \
+        profile_region profile_key_file <<EOF
+$profile_fields
+EOF
+    profile_key=$(printf '%s' "$profile_key_file" | sed "s|^~|$HOME|")
+    [ -r "$profile_key" ] || die "key_file in [$SPX_PROFILE] is not readable: $profile_key"
+    key_pem=$(cat "$profile_key")
+    oci_user="$profile_user"
+    oci_fingerprint="$profile_fingerprint"
+    oci_tenancy="$profile_tenancy"
+    oci_region="$profile_region"
+    source_name="the [$SPX_PROFILE] profile in ~/.oci/config"
+
+elif [ -r "$SPX_KEY" ]; then
+    require_complete "the credential oci_env.py resolved" \
+        TF_VAR_user_ocid TF_VAR_tenancy_ocid TF_VAR_region
+    key_pem=$(cat "$SPX_KEY")
+    oci_user="${TF_VAR_user_ocid:-}"
+    oci_fingerprint=$(fingerprint_of "$SPX_KEY")
+    oci_tenancy="${TF_VAR_tenancy_ocid:-}"
+    oci_region="${TF_VAR_region:-}"
+    source_name="$SPX_KEY with the user and tenancy of the Terraform credential"
+    [ -n "$oci_fingerprint" ] || die "could not read an RSA public key out of $SPX_KEY"
+
+else
+    require_complete "the credential oci_env.py resolved" \
+        TF_VAR_user_ocid TF_VAR_fingerprint TF_VAR_tenancy_ocid TF_VAR_region
+    if [ -n "${TF_VAR_private_key:-}" ]; then
+        key_pem="$TF_VAR_private_key"
+    elif [ -n "${TF_VAR_private_key_path:-}" ] && [ -r "$TF_VAR_private_key_path" ]; then
+        key_pem=$(cat "$TF_VAR_private_key_path")
+    else
+        die "no node credential found: set OCI_SPX_PRIVATE_KEY, add a [$SPX_PROFILE] profile to ~/.oci/config, or create $SPX_KEY"
+    fi
+    oci_user="${TF_VAR_user_ocid:-}"
+    oci_fingerprint="${TF_VAR_fingerprint:-}"
+    oci_tenancy="${TF_VAR_tenancy_ocid:-}"
+    oci_region="${TF_VAR_region:-}"
+    source_name="the credential Terraform builds with"
+    log "WARNING: falling back to $source_name, which is wider than the allocator needs."
+    log "WARNING: see the OCI guide for a user scoped to vnics, private-ips and public-ips."
+fi
+
+case "$oci_user$oci_tenancy" in
+    ocid1.*) ;;
+    *) die "$source_name did not yield OCIDs for the user and tenancy" ;;
+esac
+echo "$oci_fingerprint" | grep -Eq '^[0-9a-f]{2}(:[0-9a-f]{2}){15}$' \
+    || die "$source_name has no 16-byte hex fingerprint: '$oci_fingerprint'"
+
+# The fingerprint and the key have to be the same key, and nothing upstream checks
+# that they are. Unchecked it is an OCI 401 at the first allocation, long after the
+# deploy reports success.
+pem_tmp=$(mktemp); trap 'rm -f "$pem_tmp"' EXIT
+( umask 077; printf '%s\n' "$key_pem" > "$pem_tmp" )
+derived=$(fingerprint_of "$pem_tmp")
+[ -n "$derived" ] || die "$source_name does not hold a usable RSA private key"
+[ "$derived" = "$oci_fingerprint" ] \
+    || die "the key from $source_name has fingerprint $derived, but the credential names $oci_fingerprint"
+
+log "credential: $source_name"
+log "user: $oci_user"
+log "fingerprint: $oci_fingerprint"
+log "region: $oci_region"
+
+if [ "$DRY_RUN" = 1 ]; then
+    log "would write $REMOTE_CONFIG and $REMOTE_PEM as 0640 root:spinifex on:${*:-" (no hosts given)"}"
+    exit 0
+fi
+
+# base64 so the PEM survives two shells, and carried in the stdin stream rather
+# than in the command: an argument is readable from ps on the node for the length
+# of the call, on a host that runs other people's guests.
+key_b64=$(printf '%s\n' "$key_pem" | base64 | tr -d '\n')
+
+install_on() {
+    # Unquoted heredoc on purpose: the key and the four credential fields are
+    # expanded here, so nothing in it is expanded on the node.
+    # shellcheck disable=SC2087
+    ssh -i "$ssh_key" "${SSH_OPTS[@]}" "ubuntu@$1" bash -s <<REMOTE
+set -eu
+getent group spinifex >/dev/null \
+    || { echo "no spinifex group yet, so this ran before the install" >&2; exit 3; }
+sudo install -d -m 0750 -o root -g spinifex $REMOTE_DIR
+# install /dev/null first, then write: the file never exists at a wider mode, and
+# sudo's umask is not the caller's.
+sudo install -m 0640 -o root -g spinifex /dev/null $REMOTE_PEM
+base64 -d <<'PEM_B64' | sudo tee $REMOTE_PEM >/dev/null
+$key_b64
+PEM_B64
+sudo install -m 0640 -o root -g spinifex /dev/null $REMOTE_CONFIG
+sudo tee $REMOTE_CONFIG >/dev/null <<'CONFIG'
+[$REMOTE_PROFILE]
+user=$oci_user
+fingerprint=$oci_fingerprint
+tenancy=$oci_tenancy
+region=$oci_region
+key_file=$REMOTE_PEM
+CONFIG
+REMOTE
+}
+
+# Every host is attempted and the failures are named. Stopping at the first leaves
+# a half-credentialled cluster and a log that does not say which half.
+failed=""
+for host in "$@"; do
+    log "installing the OCI credential on $host"
+    install_on "$host" || failed="$failed $host"
+done
+
+[ -z "$failed" ] || die "failed on:$failed"
+log "credential installed on $# host(s)"

scripts/terraform/oci-spx/validate-topology.sh

@@ -0,0 +1,853 @@
+#!/bin/bash
+# Builds one OCI topology from nothing, proves Spinifex works on it, and destroys
+# it again. Three topologies, because each breaks differently: bare metal presents
+# VNICs unlike a VM, a single node has no Geneve underlay to get wrong, and only a
+# cluster exercises RAFT, the gateway chassis and cross-node allocation.
+#
+# Every run is build-prove-destroy. A topology that cannot be torn down is half
+# proved, so the teardown decides the verdict and runs even on failure.
+#
+# MUST MAINTAIN BASH 3.X COMPATABILITY. Operators run this on macOS, which ships
+# bash 3.2: no declare -A, no mapfile, and under set -u an empty "${arr[@]}" is an
+# unbound variable, so guard every such expansion with ${#arr[@]}.
+set -euo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$HERE/../../.." && pwd)"
+
+TOPOLOGY=""
+SSH_PUBLIC_KEY="$HOME/.ssh/oci-spx.pub"
+SSH_PRIVATE_KEY="$HOME/.ssh/oci-spx"
+KEEP=0
+KEEP_ON_FAIL=0
+SKIP_WORKLOAD=0
+SKIP_POOL=0
+DRY_RUN=0
+DESTROY_ONLY=0
+INSTANCE_PRINCIPAL=0
+CREDENTIAL_HOOK="${OCI_CREDENTIAL_HOOK:-}"
+# Empty means oci_env.py chooses, which is $OCI_CLI_PROFILE, then the reference
+# tenancy, then DEFAULT. Named here so a deployment can say which tenancy it is in.
+OCI_PROFILE="${OCI_PROFILE:-}"
+DISTRO=""
+SETUP_SH=""
+# Empty means the driver's own default list. Unset is distinguishable from empty,
+# so --workbooks "" can deliberately mean "run none".
+WORKBOOKS_SET=0
+CHANNEL=latest
+INSTALL_VERSION=""
+WORKBOOKS=""
+# Declared here, not at the apply that fills it: the EXIT trap reads it, and under
+# set -u an unset array is an error rather than an empty one.
+HOSTS=()
+
+TOPOLOGIES="bm vm-single vm-multi vm-single-principal"
+
+# What a topology is worth when nothing says otherwise, so a CI run of a named
+# topology is the same every time. Supplied as TF_VAR_, which terraform.auto.tfvars
+# outranks, so a deployment sizes itself in that file and no flag has to carry it.
+#
+# TOPO_FORCE_PRINCIPAL is the exception: vm-single-principal exists to exercise
+# instance-principal authentication, so the credential is the topology rather than
+# a separate switch that could be left at the other value.
+#
+# A case rather than an associative array: macOS ships bash 3.2, where declare -A
+# is an indexed assignment and the key is read as an arithmetic variable.
+topo_defaults() {
+    TOPO_FORCE_PRINCIPAL=0
+    case "$1" in
+        bm)        TOPO_DEFAULT_SHAPE="BM.Standard.E2.64";  TOPO_DEFAULT_NODES=1 ;;
+        vm-single) TOPO_DEFAULT_SHAPE="VM.Standard.E6.Flex"; TOPO_DEFAULT_NODES=1 ;;
+        vm-multi)  TOPO_DEFAULT_SHAPE="VM.Standard.E6.Flex"; TOPO_DEFAULT_NODES=3 ;;
+        vm-single-principal)
+            # Its own tenancy carries its own service limits, so the shape every
+            # other topology uses can have zero quota there and fail the apply
+            # before a single thing about the principal has been proved.
+            TOPO_DEFAULT_SHAPE="${OCI_PRINCIPAL_SHAPE:-VM.Standard.E6.Flex}"
+            TOPO_DEFAULT_NODES=1
+            TOPO_FORCE_PRINCIPAL=1 ;;
+        *) return 1 ;;
+    esac
+}
+
+log() { printf '[validate-%s] %s\n' "${TOPOLOGY:-?}" "$*"; }
+die() {
+    printf '[validate-%s] ERROR: %s\n' "${TOPOLOGY:-?}" "$*" >&2
+    exit 1
+}
+
+usage() {
+    cat >&2 <<EOF
+usage: ${0##*/} --topology <bm|vm-single|vm-multi|vm-single-principal> [options]
+
+Builds the topology, installs Spinifex, forms the cluster, runs a Terraform
+workbook against it, then destroys everything.
+
+  --topology <name>       Required. No default: a command aimed at the wrong
+                          topology is the easiest expensive mistake here.
+                          vm-single-principal is vm-single authenticating as the
+                          instance, and implies --instance-principal.
+  --ssh-public-key PATH   Public half installed on each node. Default $SSH_PUBLIC_KEY
+  --ssh-private-key PATH  Private half used to reach them. Default $SSH_PRIVATE_KEY
+  --instance-principal    Configure the pool with oci_auth="instance_principal"
+                          instead of a key file. Needs the dynamic group and
+                          policy to exist already; see instance-principal.tf.
+                          Not needed when instance_principal is set in your
+                          tfvars, which this reads; the flag outranks it.
+  --credential-hook PATH  Executable run after formation, before the pool, as
+                          "hook <ssh-key> <host>...". Where an API-key deployment
+                          installs its credential. Default \$OCI_CREDENTIAL_HOOK.
+  --oci-profile NAME      Profile in ~/.oci/config that Terraform builds with.
+                          Default \$OCI_PROFILE, else \$OCI_CLI_PROFILE, else the
+                          reference tenancy, else DEFAULT. A name that is passed
+                          and absent is an error, never a fallback.
+  --workbooks LIST        Space-separated workbooks for the published driver to
+                          run on the cluster. Empty means run none; omitted means
+                          the driver's own default list.
+  --version TAG           Install this exact release tag. Preferred over
+                          --channel dev for anything repeatable: a tag resolves
+                          by redirect, while the dev channel resolves through a
+                          rate-limited GitHub API call that 404s when it trips.
+  --distro PATH           Install this distro tarball instead of the published
+                          release, so what is proved is the ref it was built from.
+  --setup-sh PATH         setup.sh to pair with --distro. Both or neither.
+  --no-external-pool      Form without an OCI credential: no external pool and no
+                          allocator gate. Implies --skip-workload, because a guest
+                          with no public address proves nothing a customer wants.
+  --skip-workload         Form and verify only; launch no guests.
+  --keep                  Leave the infrastructure up. Implies no verdict.
+  --keep-on-fail          Destroy on success, leave a failure up to inspect.
+  --destroy-only          Destroy whatever this topology's state holds, then stop.
+  --dry-run               Print the plan and the per-node steps, change nothing.
+EOF
+    exit 2
+}
+
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --topology) TOPOLOGY="${2:?}"; shift 2 ;;
+        --ssh-public-key) SSH_PUBLIC_KEY="${2:?}"; shift 2 ;;
+        --ssh-private-key) SSH_PRIVATE_KEY="${2:?}"; shift 2 ;;
+        --workbooks) WORKBOOKS_SET=1; WORKBOOKS="${2-}"; shift 2 ;;
+        --channel) CHANNEL="${2-}"; shift 2 ;;
+        --version) INSTALL_VERSION="${2:?}"; shift 2 ;;
+        --distro) DISTRO="${2:?}"; shift 2 ;;
+        --setup-sh) SETUP_SH="${2:?}"; shift 2 ;;
+        --instance-principal) INSTANCE_PRINCIPAL=1; shift ;;
+        --credential-hook) CREDENTIAL_HOOK="${2:?}"; shift 2 ;;
+        --oci-profile) OCI_PROFILE="${2:?}"; shift 2 ;;
+        --no-external-pool) SKIP_POOL=1; SKIP_WORKLOAD=1; shift ;;
+        --skip-workload) SKIP_WORKLOAD=1; shift ;;
+        --keep) KEEP=1; shift ;;
+        --keep-on-fail) KEEP_ON_FAIL=1; shift ;;
+        --destroy-only) DESTROY_ONLY=1; shift ;;
+        --dry-run) DRY_RUN=1; shift ;;
+        -h | --help) usage ;;
+        *) die "unknown option $1" ;;
+    esac
+done
+
+[ -n "$TOPOLOGY" ] || usage
+topo_defaults "$TOPOLOGY" || die "unknown topology $TOPOLOGY; one of: $TOPOLOGIES"
+[ -r "$SSH_PUBLIC_KEY" ] || die "no readable SSH public key at $SSH_PUBLIC_KEY"
+[ -r "$SSH_PRIVATE_KEY" ] || die "no readable SSH private key at $SSH_PRIVATE_KEY"
+# Both or neither: a distro with the published setup.sh installs one ref's bytes
+# with another's layout, and that is a cluster nobody can reason about.
+if [ -n "$DISTRO" ] || [ -n "$SETUP_SH" ]; then
+    [ -r "$DISTRO" ] || die "--distro is not readable: '$DISTRO'"
+    [ -r "$SETUP_SH" ] || die "--setup-sh is not readable: '$SETUP_SH'"
+fi
+
+if [ "$TOPO_FORCE_PRINCIPAL" = 1 ]; then
+    INSTANCE_PRINCIPAL=1
+    # The pool is the whole point of this topology: with no allocator built, the
+    # credential is never used and a green run says nothing about it. That is
+    # exactly how the auth branch stayed broken through passing nightlies.
+    [ "$SKIP_POOL" != 1 ] \
+        || die "--no-external-pool makes $TOPOLOGY meaningless: the allocator is what exercises the instance principal, so a run without it proves nothing"
+fi
+
+# Terraform's own order, as far as this script needs it: the last assignment across
+# terraform.tfvars then *.auto.tfvars alphabetically is the one that applies. Read
+# because the credential-hook gate has to run before the apply spends anything; the
+# apply's own output is checked against this afterwards.
+tfvars_instance_principal() {
+    local file mode="" found=""
+    for file in "$HERE/terraform.tfvars" "$HERE"/*.auto.tfvars; do
+        [ -r "$file" ] || continue
+        mode=$(sed -n 's/^[[:space:]]*instance_principal[[:space:]]*=[[:space:]]*"\([a-z]*\)".*/\1/p' \
+            "$file" | tail -1)
+        [ -n "$mode" ] && found="$mode"
+    done
+    [ -n "$found" ] || return 1
+    printf '%s\n' "$found"
+}
+
+# adopt, never create: this script's state is destroyed at the end of every run, and
+# the dynamic group and policy outlive every topology. setup-identity.sh owns them.
+#
+# A flag goes in as -var and outranks the tfvars, because it is a choice about this
+# run. Without one the deployment's own tfvars decides, so a guide can set the
+# credential in the file it already writes and no flag has to carry it.
+tf_principal_var=()
+if [ "$INSTANCE_PRINCIPAL" = 1 ]; then
+    PRINCIPAL_MODE=adopt
+    tf_principal_var=(-var "instance_principal=$PRINCIPAL_MODE")
+else
+    PRINCIPAL_MODE="$(tfvars_instance_principal || echo off)"
+fi
+
+# What the apply actually built, filled in from the hosts_file output once it has.
+# Until then these are what the topology asked for, which is all there is to say.
+SHAPE="$TOPO_DEFAULT_SHAPE"
+NODES="$TOPO_DEFAULT_NODES"
+# Outside the checkout on a persistent runner, because actions/checkout runs
+# git clean -ffdx: it would delete the Terraform state of a killed run before the
+# sweep could read it, leaving instances running that nothing can find.
+STATE_ROOT="${OCI_STATE_ROOT:-$HERE}"
+mkdir -p "$STATE_ROOT"
+STATE_DIR="$STATE_ROOT/.validate-$TOPOLOGY"
+RESULTS="$STATE_DIR/results.txt"
+
+# Tab-separated siblings of the human block, for the run-page tables. Written here
+# rather than parsed out of the log later: the status of a gate is then a field
+# this script sets, not a regex over prose that reads fine and matches wrong.
+RESULTS_TSV="$STATE_DIR/results.tsv"
+WORKBOOKS_TSV="$STATE_DIR/workbooks.tsv"
+
+# A separate state directory per topology, so two topologies can be built from one
+# checkout without one destroying the other's instances.
+tf() {
+    # The topology's shape and count go in as TF_VAR_, which is the weakest source
+    # Terraform reads: a terraform.auto.tfvars in the checkout outranks it, so a
+    # deployment sizes itself in that file and the topology only supplies the
+    # default a CI run wants.
+    TF_VAR_compute_shape="$TOPO_DEFAULT_SHAPE" TF_VAR_node_count="$TOPO_DEFAULT_NODES" \
+        python3 "$HERE/scripts/oci_env.py" --ssh-public-key-path "$SSH_PUBLIC_KEY" \
+        ${OCI_PROFILE:+--profile "$OCI_PROFILE"} -- \
+        terraform -chdir="$HERE" "$@"
+}
+
+# terraform output takes -state but not -var, so the two sets are kept apart. They
+# were one set once, and the -var made output fail into the default state file.
+tf_state=(-state "$STATE_DIR/terraform.tfstate")
+tf_vars=("${tf_state[@]}")
+# Expanded only when it holds something. bash before 4.4, which is what macOS
+# ships, calls "${arr[@]}" on an empty array an unbound variable under set -u, and
+# tf_principal_var is empty on every run that did not pass --instance-principal.
+if [ "${#tf_principal_var[@]}" -gt 0 ]; then
+    tf_vars=("${tf_principal_var[@]}" "${tf_vars[@]}")
+fi
+
+ssh_node() {
+    local host="$1"
+    shift
+    # LogLevel=ERROR: the host-key warning is unavoidable with a throwaway known-hosts
+    # file, and it lands in the middle of whatever the remote command reported.
+    ssh -i "$SSH_PRIVATE_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+        -o LogLevel=ERROR -o BatchMode=yes -o ConnectTimeout=15 "ubuntu@$host" "$@"
+}
+
+# record <gate> <PASS|FAIL|SKIPPED|INFO> [detail]
+# The node's own account of a failed run. A workbook's log says which API call
+# failed; only the journal says why, and the nodes are destroyed minutes later.
+# Warnings first because that is where a swallowed error surfaces, then a bounded
+# tail for context -- unbounded, a multi-hour suite's journal dwarfs the artifact.
+capture_journals() {
+    local host
+    # A failure before the apply leaves no hosts, and on bash before 4.4 iterating
+    # the empty array is an unbound-variable error rather than zero passes.
+    [ "${#HOSTS[@]}" -gt 0 ] || return 0
+    for host in "${HOSTS[@]}"; do
+        ssh_node "$host" '
+            echo "=== spinifex, warning and above ==="
+            sudo journalctl -u "spinifex-*" --since -4h --priority=warning --no-pager | tail -2000
+            echo "=== spinifex, all priorities, last 2000 lines ==="
+            sudo journalctl -u "spinifex-*" --since -4h --no-pager | tail -2000
+        ' > "$STATE_DIR/journal-$host.log" 2>&1 || log "could not collect the journal from $host"
+    done
+}
+
+record() {
+    local gate="$1" status="$2" detail="${3-}"
+    printf '%s: %s%s\n' "$gate" "$status" "${detail:+ $detail}" >> "$RESULTS"
+    printf '%s\t%s\t%s\n' "$gate" "$status" "$detail" >> "$RESULTS_TSV"
+}
+
+# The driver's per-workbook lines, which are go test's own shape because
+# go-junit-report consumes them downstream. A workbook the driver never reached
+# is absent from its log and so absent here, which the summary renders as a row
+# rather than dropping — a missing row reads as a pass.
+record_workbooks() {
+    local log="$1"
+    [ -r "$log" ] || return 0
+    sed -n 's/^--- \(PASS\|FAIL\): TestTofuWorkbook_\([A-Za-z0-9_]*\) (\([0-9]*\)\.[0-9]*s)$/\2\t\1\t\3/p' \
+        "$log" | tr '_' '-' > "$WORKBOOKS_TSV"
+}
+
+# Teardown is the last assertion, not cleanup: a workbook or a topology that
+# cannot be destroyed is a defect, and it has been one before.
+DESTROY_RC=""
+destroy_topology() {
+    log "destroying"
+    if tf destroy -auto-approve -no-color "${tf_vars[@]}" > "$STATE_DIR/destroy.log" 2>&1; then
+        DESTROY_RC=0
+        record destroy PASS
+    else
+        DESTROY_RC=1
+        record destroy FAIL "see $STATE_DIR/destroy.log"
+        log "TEARDOWN FAILED — resources may still be billing. $STATE_DIR/destroy.log"
+    fi
+}
+
+cleanup() {
+    local rc=$?
+    if [ "$KEEP" = 1 ]; then
+        log "--keep: leaving the infrastructure up, no verdict recorded"
+        # The command to get in, printed rather than left to be assembled from an
+        # address elsewhere in the log. HOSTS is empty if we never got as far as
+        # the apply, which is why this is guarded rather than assumed.
+        if [ "${#HOSTS[@]}" -gt 0 ]; then
+            # ~ rather than the expanded path, so the line is the same on every
+            # machine and reads as something a person would type.
+            local key="$SSH_PRIVATE_KEY"
+            case "$key" in "$HOME"/*) key="~${key#"$HOME"}" ;; esac
+            log "ssh in with: ssh -i $key ubuntu@${HOSTS[0]}"
+            log "destroy it with: $0 --topology $TOPOLOGY --destroy-only"
+        fi
+        return
+    fi
+    # Only on a failure, and said out loud: the cost of forgetting is an OCI bare
+    # metal host running overnight.
+    if [ "$KEEP_ON_FAIL" = 1 ] && [ "$rc" != 0 ]; then
+        log "--keep-on-fail: $TOPOLOGY FAILED and is being left up. IT IS STILL BILLING."
+        log "destroy it with: $0 --topology $TOPOLOGY --destroy-only"
+        verdict "$rc"
+        return
+    fi
+    [ "$DRY_RUN" = 1 ] && return
+    destroy_topology
+    verdict "$rc"
+}
+
+verdict() {
+    local run_rc="$1"
+    echo
+    log "=== $TOPOLOGY ($SHAPE, $NODES node(s)) ==="
+    [ -r "$RESULTS" ] && cat "$RESULTS"
+    if [ "$run_rc" = 0 ] && [ "$DESTROY_RC" = 0 ]; then
+        log "PASS"
+    else
+        log "FAIL"
+    fi
+}
+
+mkdir -p "$STATE_DIR"
+
+if [ "$DESTROY_ONLY" = 1 ]; then
+    log "destroy only: whatever $STATE_DIR/terraform.tfstate holds"
+    destroy_topology
+    [ "$DESTROY_RC" = 0 ] || exit 1
+    exit 0
+fi
+
+: > "$RESULTS"
+: > "$RESULTS_TSV"
+: > "$WORKBOOKS_TSV"
+
+# Last run's logs go before this one's first line, so what is left in here always
+# describes the run that is starting. Without this a topology keeps logs that read
+# as current, and anything collecting the directory publishes them as this run's.
+# Named globs rather than a find: the tfstate is how the sweep finds hosts to
+# destroy, so what is removed here has to be readable at a glance.
+# :? so an unset STATE_DIR stops the shell rather than expanding to /*.log.
+rm -f "${STATE_DIR:?}"/*.log "${STATE_DIR:?}"/nodes.txt "${STATE_DIR:?}"/cloudinit-*.txt
+
+log "$SHAPE, $NODES node(s), state in $STATE_DIR"
+
+# Checked here, not where the hook is run: nothing about it depends on the apply, and
+# the run is otherwise forty minutes and a bare-metal bill from discovering that the
+# path in a CI variable does not exist on this runner.
+if [ "$SKIP_POOL" != 1 ] && [ "$PRINCIPAL_MODE" = off ]; then
+    [ -n "$CREDENTIAL_HOOK" ] \
+        || die "this deployment authenticates with an API key and no credential hook is set, so no node could reach the OCI API; set --credential-hook or OCI_CREDENTIAL_HOOK, or use an instance principal"
+    [ -x "$CREDENTIAL_HOOK" ] || die "credential hook is not executable: $CREDENTIAL_HOOK"
+fi
+
+# The directory's own terraform.tfstate belongs to hand-driven runs, not to a
+# topology, and an instance left in it is both a bill and a name that collides
+# with ours in the console. Say so rather than letting it be a surprise.
+if [ -s "$HERE/terraform.tfstate" ]; then
+    stray="$(python3 -c '
+import json, sys
+state = json.load(open(sys.argv[1]))
+for res in state.get("resources", []):
+    if res["type"] != "oci_core_instance":
+        continue
+    for inst in res["instances"]:
+        attrs = inst["attributes"]
+        if attrs.get("state") not in ("TERMINATED", None):
+            print(attrs.get("display_name"), attrs.get("shape"), attrs.get("state"), attrs.get("public_ip"))
+' "$HERE/terraform.tfstate" 2>/dev/null || true)"
+    [ -n "$stray" ] && log "NOTE: the hand-driven state in $HERE still holds: $stray"
+fi
+
+# Terraform ranks a *.auto.tfvars file above TF_VAR_*, so a leftover one wins
+# silently over an operator targeting a different tenancy by environment. The
+# guide tells every operator to write one, which is what makes this likely.
+assert_no_tfvars_override() {
+    local f want got
+    want="${TF_VAR_compartment_ocid:-}"
+    [ -n "$want" ] || return 0
+    for f in "$HERE"/*.auto.tfvars "$HERE"/terraform.tfvars; do
+        [ -f "$f" ] || continue
+        got=$(sed -n 's/^[[:space:]]*compartment_ocid[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' "$f" | tail -1)
+        [ -n "$got" ] || continue
+        [ "$got" = "$want" ] && continue
+        die "${f##*/} sets compartment_ocid=$got, which Terraform ranks above the TF_VAR_compartment_ocid=$want this run was given, so it would build in a compartment nobody chose. Remove that file or make it agree."
+    done
+}
+assert_no_tfvars_override
+
+if [ "$DRY_RUN" = 1 ]; then
+    # Whole plan to a file, summary to the terminal. A dry run is here to catch a
+    # plan aimed at the wrong compartment, and `tail` dropped the resource count
+    # and every compartment and shape line along with it, leaving only the outputs
+    # block -- so the one check it exists for could not be made from its output.
+    mkdir -p "$STATE_DIR"
+    tf plan -no-color "${tf_vars[@]}" > "$STATE_DIR/plan.txt" 2>&1 \
+        || { tail -30 "$STATE_DIR/plan.txt"; die "plan failed; see $STATE_DIR/plan.txt"; }
+    grep -E '^Plan: |^(No changes)' "$STATE_DIR/plan.txt" || true
+    printf 'compartment and shape as planned:\n'
+    # Terraform aligns the = to the widest key in each block, so the same value
+    # appears several times at several indents unless the spacing is normalised.
+    grep -hoE '(compartment_id|shape|availability_domain) += +"[^"]*"' "$STATE_DIR/plan.txt" \
+        | sed -E 's/ +=/ =/' | sort -u | sed 's/^/  /'
+    log "full plan in $STATE_DIR/plan.txt"
+    plan="install Spinifex on $NODES node(s), form the cluster"
+    [ "$SKIP_POOL" = 1 ] && plan="$plan, configure no external pool"
+    if [ "$SKIP_WORKLOAD" = 1 ]; then
+        plan="$plan, launch no guests"
+    elif [ "$WORKBOOKS_SET" = 1 ]; then
+        plan="$plan, run the workbooks '$WORKBOOKS'"
+    else
+        plan="$plan, run the driver's default workbooks"
+    fi
+    log "dry run: would $plan, then destroy"
+    exit 0
+fi
+
+trap cleanup EXIT
+
+tf init -input=false > "$STATE_DIR/init.log" 2>&1 || die "terraform init failed; see $STATE_DIR/init.log"
+
+log "building"
+tf apply -auto-approve -no-color "${tf_vars[@]}" > "$STATE_DIR/apply.log" 2>&1 \
+    || die "terraform apply failed; see $STATE_DIR/apply.log"
+record build PASS
+
+# A read loop rather than mapfile, which bash 3.2 on macOS does not have. The
+# output is captured first so a failed terraform fails here, not silently as an
+# empty host list.
+hosts_raw=$(tf output -raw "${tf_state[@]}" hosts_file) \
+    || die "could not read the hosts_file output from $STATE_DIR/terraform.tfstate"
+HOSTS=()
+while IFS= read -r line; do
+    if [ -n "$line" ]; then HOSTS+=("$line"); fi
+done <<EOF
+$hosts_raw
+EOF
+[ "${#HOSTS[@]}" -ge 1 ] || die "the hosts_file output named no hosts, so the apply built nothing to install on"
+for host in "${HOSTS[@]}"; do
+    [[ "$host" =~ ^[0-9]+(\.[0-9]+){3}$ ]] \
+        || die "the hosts_file output is not an address: '$host' -- something on the wrapper's stdout is in the capture"
+done
+
+# Written for every topology, not just the multi-node one that passes it to
+# install-node.sh: it is also how an operator gets the addresses afterwards
+# without a terraform invocation and the right -state path.
+printf '%s\n' "${HOSTS[@]}" > "$STATE_DIR/hosts"
+
+# node_count can come from a tfvars file, so the hosts the apply produced are the
+# authority on how many there are. Every later count check reads this.
+if [ "${#HOSTS[@]}" != "$NODES" ]; then
+    log "note: $NODES node(s) is the $TOPOLOGY default, and the apply built ${#HOSTS[@]}; taking the apply's"
+    NODES="${#HOSTS[@]}"
+fi
+SHAPE=$(tf output -raw "${tf_state[@]}" compute_shape) \
+    || die "could not read the compute_shape output from $STATE_DIR/terraform.tfstate"
+
+# Terraform is the authority on which credential was built, and the reader above is
+# a guess made before the apply. They have to agree: the hook is skipped on the
+# strength of that guess, and a node left with no credential at all would otherwise
+# fail much later as an allocator that cannot resolve its VNIC.
+built_principal=$(tf output -raw "${tf_state[@]}" instance_principal) \
+    || die "could not read the instance_principal output from $STATE_DIR/terraform.tfstate"
+[ "$built_principal" = "$PRINCIPAL_MODE" ] \
+    || die "read instance_principal='$PRINCIPAL_MODE' from the tfvars and the apply built '$built_principal'; set it in one place"
+
+log "hosts: ${HOSTS[*]}"
+log "$SHAPE, $NODES node(s), oci_auth $([ "$PRINCIPAL_MODE" = off ] && echo "from a key file" || echo "as the instance principal")"
+
+# cloud-init owns the volume, the ports and br-wan, and all three are units now, so
+# the question is whether they converged rather than whether a runcmd happened to
+# run. This is the gate the bare-metal findings were about.
+log "waiting for cloud-init on each node"
+for host in "${HOSTS[@]}"; do
+    for _ in $(seq 1 120); do
+        if ssh_node "$host" 'cloud-init status --wait >/dev/null 2>&1 || true; test -f /var/lib/cloud/instance/boot-finished' 2>/dev/null; then
+            break
+        fi
+        sleep 10
+    done
+    ssh_node "$host" '
+        set -e
+        for u in spinifex-data-volume spinifex-service-ports spinifex-wan-bridge; do
+            printf "%s=%s " "$u" "$(systemctl is-active "$u" 2>&1)"
+        done
+        echo
+        findmnt --noheadings --output SOURCE,TARGET /var/lib/spinifex
+        ip -br addr show br-wan
+    ' > "$STATE_DIR/cloudinit-$host.txt" 2>&1 \
+        || die "cloud-init did not converge on $host; see $STATE_DIR/cloudinit-$host.txt"
+    log "$host cloud-init: $(head -1 "$STATE_DIR/cloudinit-$host.txt")"
+done
+record "cloud-init units" PASS
+
+# With --distro the bytes are the ref under test; without it they are the published
+# release, which answers a different question -- what a customer following the
+# guide gets today. A nightly judging a branch must pass --distro.
+log "installing Spinifex on each node ($([ -n "$DISTRO" ] && echo "$(basename "$DISTRO")" || echo "published release"))"
+for host in "${HOSTS[@]}"; do
+    if [ -n "$DISTRO" ]; then
+        scp -i "$SSH_PRIVATE_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+            -o LogLevel=ERROR -q "$DISTRO" "$SETUP_SH" "ubuntu@$host:/tmp/" \
+            || die "could not copy the distro to $host"
+    fi
+    # cloud-init being finished does not mean apt is: the apt-daily timers and
+    # unattended-upgrades run on their own schedule and hold the dpkg lock, which
+    # the installer then fails on. Wait for the lock rather than fight it.
+    ssh_node "$host" "DISTRO_NAME='$(basename "${DISTRO:-}")' SETUP_NAME='$(basename "${SETUP_SH:-}")' CHANNEL='$CHANNEL' INSTALL_VERSION='$INSTALL_VERSION' bash -s" \
+        > "$STATE_DIR/install-$host.log" 2>&1 <<'REMOTE' \
+        || die "install failed on $host; see $STATE_DIR/install-$host.log"
+set -e
+for _ in $(seq 1 120); do
+    sudo fuser /var/lib/dpkg/lock-frontend /var/lib/apt/lists/lock >/dev/null 2>&1 || break
+    sleep 5
+done
+sudo fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 \
+    && { echo "the dpkg lock was still held after 10 minutes"; exit 1; }
+if [ -n "${DISTRO_NAME:-}" ]; then
+    # The production setup.sh against the ref's own tarball, which is what the
+    # release and the ISO both run, so there is no separate dev install layout.
+    # No INSTALL_SPINIFEX_SKIP_APT/SKIP_AWS here, unlike the CI hypervisors: an OCI
+    # stock image is not pre-baked, so the dependency stages have to run.
+    sudo env INSTALL_SPINIFEX_TARBALL="/tmp/$DISTRO_NAME" bash "/tmp/$SETUP_NAME"
+else
+    # The real customer path, including the checksum step a local tarball skips.
+    # The environment variable rather than --channel: the installer served by
+    # install.mulgadc.com is the latest release's, so it predates the flag.
+    #
+    # A tag goes in as VERSION, not CHANNEL. Only the dev channel resolves through
+    # an unauthenticated GitHub API call, which is rate limited per source address
+    # and returns 404 once it trips -- indistinguishable from a missing asset. A
+    # tagged path is a plain redirect, so it does not have that failure mode.
+    if [ -n "$INSTALL_VERSION" ]; then
+        curl -sfL https://install.mulgadc.com | sudo env INSTALL_SPINIFEX_VERSION="$INSTALL_VERSION" bash
+    else
+        curl -sfL https://install.mulgadc.com | sudo env INSTALL_SPINIFEX_CHANNEL="$CHANNEL" bash
+    fi
+fi
+sudo /usr/local/share/spinifex/setup-ovn.sh --management --nat-uplink
+REMOTE
+    INSTALLED_VERSION="$(ssh_node "$host" 'spx version' 2>&1 | head -1)"
+    log "$host installed: $INSTALLED_VERSION"
+done
+record install PASS "$INSTALLED_VERSION"
+
+# A single node is its own documented path: install-node.sh refuses fewer than two
+# hosts, because there is nothing to join. Both branches use the flags the guide
+# names, so what is validated is what the guide tells a customer to type.
+if [ "$NODES" = 1 ]; then
+    log "initializing the single node"
+    ssh_node "${HOSTS[0]}" '
+        set -e
+        sudo spx admin init --node "$(hostname -s)" --nodes 1 \
+            --region ap-southeast-2 --az ap-southeast-2a \
+            --external-mode=nat --ipsec=false
+        sudo systemctl start spinifex.target
+    ' > "$STATE_DIR/form.log" 2>&1 \
+        || die "single-node init failed; see $STATE_DIR/form.log"
+else
+    log "forming the cluster"
+    "$REPO_ROOT/scripts/install-node.sh" \
+        --hosts-file "$STATE_DIR/hosts" \
+        --user ubuntu \
+        --identity "$SSH_PRIVATE_KEY" \
+        --external-mode nat \
+        --ipsec off \
+        --yes > "$STATE_DIR/form.log" 2>&1 \
+        || die "formation failed; see $STATE_DIR/form.log"
+fi
+record formation PASS
+
+# Runs after formation and before the pool is configured, which is the only window
+# where a node has /etc/spinifex but has not yet started the allocator. Nothing in
+# this repository knows what it does: an API-key deployment needs a credential on
+# each node, and a credential belongs to the operator, not to a checked-in script.
+# Skipped under instance principal, which needs no handoff at all.
+if [ "$SKIP_POOL" != 1 ] && [ "$PRINCIPAL_MODE" = off ]; then
+    log "running the credential hook"
+    # Arguments, not a file: the hook is told where the nodes are and how to reach
+    # them, and decides for itself what to put there.
+    "$CREDENTIAL_HOOK" "$SSH_PRIVATE_KEY" "${HOSTS[@]}" > "$STATE_DIR/credential-hook.log" 2>&1 \
+        || die "the credential hook failed; see $STATE_DIR/credential-hook.log"
+    log "credential hook ok"
+fi
+
+# The IMDS remap and the pool are both set-before-first-start, and the remap is the
+# half that is invisible when missing: without it the cloud's metadata service is
+# shadowed by Spinifex's own endpoints and the allocator cannot resolve its VNIC.
+log "configuring the IMDS remap$([ "$SKIP_POOL" = 1 ] && echo " (no external pool)" || echo " and the external pool")"
+for host in "${HOSTS[@]}"; do
+    ssh_node "$host" "SKIP_POOL=$SKIP_POOL bash -s" > "$STATE_DIR/pool-$host.log" 2>&1 <<'REMOTE' \
+        || die "pool configuration failed on $host; see $STATE_DIR/pool-$host.log"
+set -e
+sudo cp /etc/spinifex/spinifex.toml /etc/spinifex/spinifex.toml.bak-prepool
+# subn, not sub: with no [network] section to match, sub rewrites the file unchanged
+# and exits 0, leaving the remap silently unapplied. That is the half of this that is
+# invisible when missing, so the substitution count is checked rather than assumed.
+grep -q imds_host_meta_ip /etc/spinifex/spinifex.toml || sudo python3 - <<"PY"
+import re, sys
+path = "/etc/spinifex/spinifex.toml"
+text, n = re.subn(r"(?m)^\[network\]$",
+                  "[network]\nimds_host_meta_ip = \"169.254.42.254\"\nimds_host_dns_ip  = \"169.254.42.253\"",
+                  open(path).read(), count=1)
+if n != 1:
+    sys.exit("no [network] section in %s, so the IMDS remap was not applied" % path)
+open(path, "w").write(text)
+PY
+if [ "${SKIP_POOL:-0}" != 1 ]; then
+    grep -q 'name               = "oci-public"' /etc/spinifex/spinifex.toml \
+        || sudo tee -a /etc/spinifex/spinifex.toml < /etc/spinifex/oci/external-pool.toml >/dev/null
+fi
+# spx has no config subcommand, so the `spx config validate` that used to be here
+# could only ever fail, and was swallowed. Parsing the file is the check that was
+# wanted: both edits above are textual, and a broken result would otherwise surface
+# as every service failing to start with nothing saying why.
+sudo python3 -c 'import sys, tomllib; tomllib.load(open(sys.argv[1], "rb"))' \
+    /etc/spinifex/spinifex.toml \
+    || { echo "spinifex.toml is not valid TOML after the IMDS remap and the pool append"; exit 1; }
+sudo systemctl restart spinifex.target
+REMOTE
+done
+
+# Everything needed to tell the three allocator faults apart, and deliberately no
+# credential: the config file and the PEM are reported as mode and size only, which
+# distinguishes absent from unreadable without copying either into a log that is
+# attached to a CI run.
+allocator_diagnostics() {
+    local host="$1"
+    ssh_node "$host" '
+        echo "=== spinifex-daemon: the allocator (ocinet) lives here ==="
+        sudo journalctl -u spinifex-daemon --since -20min --no-pager | grep -i "ocinet\|external VNIC\|allocator\|oci" | tail -40
+        echo "=== spinifex-vpcd: consumes the addresses, does not allocate them ==="
+        sudo journalctl -u spinifex-vpcd --since -20min --no-pager | tail -40
+        echo "=== external_pools as configured ==="
+        sudo sed -n "/\[\[network.external_pools\]\]/,\$p" /etc/spinifex/spinifex.toml
+        echo "=== credential files (mode and size, never content) ==="
+        sudo stat -c "%n %A %s bytes owner=%U:%G" \
+            /etc/spinifex/oci/config /etc/spinifex/oci/oci_api_key.pem 2>&1
+        echo "=== the bridge MAC the allocator matches on ==="
+        ip -br link show
+        echo "=== MACs IMDS reports for this instance VNICs ==="
+        curl -sf -H "Authorization: Bearer Oracle" \
+            http://169.254.169.254/opc/v2/vnics/ | tr "," "\n" | grep -i "macAddr\|privateIp" || \
+            echo "IMDS unreachable -- check the remap, Spinifex claims 169.254.169.254 itself"
+    ' 2>&1
+}
+
+# credential authorised to allocate is the line that proves the credential is
+# accepted, in the right compartment, and holds every permission an allocation
+# needs. A node missing it accepts allocate-address and then fails it.
+#
+# "resolved the external VNIC" is not enough and was what this used to wait for.
+# That line comes from the metadata service and needs no IAM policy at all, so a
+# node with an unauthorised credential passed here and failed hours later inside
+# a workbook, reported as capacity.
+ALLOCATOR_READY='ocinet credential authorised to allocate'
+ALLOCATOR_DENIED='ocinet credential cannot allocate an external address'
+if [ "$SKIP_POOL" = 1 ]; then
+    log "--no-external-pool: no allocator to check"
+    record "oci allocator" SKIPPED "no external pool configured"
+else
+    log "checking the allocator came up"
+    for host in "${HOSTS[@]}"; do
+        found=0
+        for _ in $(seq 1 30); do
+            # spinifex-daemon, not spinifex-vpcd: the allocator is built in the
+            # daemon, and vpcd only consumes the addresses it hands out. Gating on
+            # vpcd's journal failed a node whose allocator was working.
+            if ssh_node "$host" "sudo journalctl -u spinifex-daemon --since -10min --no-pager | grep -q '$ALLOCATOR_READY'" 2>/dev/null; then
+                found=1
+                break
+            fi
+            # A refusal is final, so waiting out the remaining five minutes only
+            # delays a verdict the node has already reached.
+            if denial=$(ssh_node "$host" "sudo journalctl -u spinifex-daemon --since -10min --no-pager | grep -m1 '$ALLOCATOR_DENIED'" 2>/dev/null) && [ -n "$denial" ]; then
+                allocator_diagnostics "$host" > "$STATE_DIR/allocator-$host.log" 2>&1 || true
+                die "$host refused its own OCI credential: ${denial#*$ALLOCATOR_DENIED}. Diagnostics: $STATE_DIR/allocator-$host.log"
+            fi
+            sleep 10
+        done
+        if [ "$found" != 1 ]; then
+            # Before the teardown, because the teardown is what destroys the only
+            # copy. Three unrelated faults land here -- a credential the SDK would
+            # not load, a bridge MAC matching no VNIC, and a pool vpcd never read --
+            # and they are indistinguishable from the missing log line alone.
+            allocator_diagnostics "$host" > "$STATE_DIR/allocator-$host.log" 2>&1 || true
+            die "$host never logged '$ALLOCATOR_READY'; the OCI allocator is not up, so no guest can get a public address. Diagnostics: $STATE_DIR/allocator-$host.log"
+        fi
+        log "$host allocator ready"
+    done
+    record "oci allocator" PASS
+fi
+
+# Parse the STATUS column, never grep for Ready: NotReady contains it, and that
+# read passed a single-node run whose only node was NotReady on 0.0.0.0. The table
+# is colourised, so the escapes come off first.
+count_ready() {
+    sed 's/\x1b\[[0-9;]*m//g' "$STATE_DIR/nodes.txt" \
+        | awk -F'|' 'NR > 1 { gsub(/^[ \t]+|[ \t]+$/, "", $2); if ($2 == "Ready") n++ } END { print n + 0 }'
+}
+
+log "cluster membership"
+ready=0
+for _ in $(seq 1 30); do
+    ssh_node "${HOSTS[0]}" 'sudo spx get nodes' > "$STATE_DIR/nodes.txt" 2>&1 || true
+    ready="$(count_ready)"
+    [ "$ready" = "$NODES" ] && break
+    sleep 10
+done
+cat "$STATE_DIR/nodes.txt"
+[ "$ready" = "$NODES" ] || die "expected $NODES Ready node(s), got $ready; see $STATE_DIR/nodes.txt"
+record membership PASS "$ready Ready"
+
+if [ "$SKIP_WORKLOAD" = 1 ]; then
+    log "--skip-workload: stopping before the workbook"
+    record workbooks SKIPPED "--skip-workload"
+    exit 0
+fi
+
+# The same driver the GitHub nightly runs (cell 17), on the same published
+# workbooks, so a workbook that passes on a hypervisor and fails on OCI is a
+# difference in OCI and not in the test. It owns its own per-workbook assertions
+# and destroys each one it builds.
+# The suite used to run with stdout redirected to a file, so a run wedged inside a
+# workbook printed nothing at all until it finished. Streamed now, and each of the
+# driver's own RUN/PASS/FAIL lines is stamped with elapsed seconds and the workbook.
+stream_workbooks() {
+    local start=$SECONDS line current="(starting)"
+    while IFS= read -r line; do
+        case "$line" in
+            '=== RUN'*)
+                current="${line##*RUN   }"
+                printf '[validate-%s] %5ds workbook START %s\n' "$TOPOLOGY" "$((SECONDS - start))" "$current" ;;
+            '--- PASS'*|'--- FAIL'*|'--- SKIP'*)
+                printf '[validate-%s] %5ds workbook %s\n' "$TOPOLOGY" "$((SECONDS - start))" "$line" ;;
+            *)
+                printf '%s\n' "$line" ;;
+        esac
+    done
+}
+
+log "running the published workbooks"
+DRIVER="$REPO_ROOT/tests/e2e/run-tofu-examples-e2e.sh"
+[ -r "$DRIVER" ] || die "no workbook driver at $DRIVER"
+scp -i "$SSH_PRIVATE_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+    -o LogLevel=ERROR -q "$DRIVER" "ubuntu@${HOSTS[0]}:~/run-tofu-examples-e2e.sh" \
+    || die "could not copy the workbook driver"
+scp -i "$SSH_PRIVATE_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+    -o LogLevel=ERROR -qr "$REPO_ROOT/docs/terraform-workbooks" "ubuntu@${HOSTS[0]}:~/workbooks" \
+    || die "could not copy the workbooks"
+
+# Every AMI the workbooks need, imported one at a time so predastore is not asked
+# to absorb parallel uploads. Three are appliances rather than distros: RDS, ECS
+# and EKS each boot their own, found by a spinifex:managed-by tag and never by name.
+log "importing the images the workbooks need"
+ssh_node "${HOSTS[0]}" '
+    set -e
+    for img in ubuntu-26.04-x86_64 spinifex-rds-postgres spinifex-ecs-node spinifex-eks-node; do
+        sudo spx admin images import --name "$img" --config /etc/spinifex/spinifex.toml >/dev/null
+    done
+' > "$STATE_DIR/images.log" 2>&1 || die "image import failed; see $STATE_DIR/images.log"
+
+# Every workbook that is a workbook. The shared driver's own default is five, which
+# leaves ECS and all three EKS variants untested on every platform -- their
+# assertions exist and nothing was running them.
+#
+# demo-app is absent because it is not a workbook: it has no .tf at all, being the
+# container image the EKS workbooks' nested workloads/ modules deploy. Listing it
+# here would fail on a missing root module rather than test anything.
+OCI_WORKBOOKS="nginx-alb bastion-private-subnet nginx-webserver s3-webapp rds-quickstart"
+OCI_WORKBOOKS="$OCI_WORKBOOKS ecs-quickstart eks-quickstart eks-https-ingress eks-gitops-argocd"
+[ "$WORKBOOKS_SET" = 1 ] || WORKBOOKS="$OCI_WORKBOOKS"
+workbook_env="WORKBOOK_DIR=\$HOME/workbooks $(printf 'WORKBOOKS=%q' "$WORKBOOKS")"
+
+# Whether a node can reach a public address inside its own VCN. Informational: the
+# remedy below is right either way, because a customer reaches a guest from outside.
+#
+# Needs two hosts. With one, HOSTS[0] and HOSTS[-1] are the same address and the node
+# connects to itself, which always succeeds and measures nothing — it reported
+# "present" on vm-single and bm for exactly that reason before this guard.
+#
+# A negative result is only interpretable when the security list admits the source,
+# so the record says which it was: narrowing node_client_cidr_allow_list makes a
+# node's own public address a non-permitted source, and the refusal that follows
+# looks identical to a VCN that does not hairpin.
+if [ "${#HOSTS[@]}" -lt 2 ]; then
+    record "vcn hairpin" SKIPPED "needs two nodes; one node can only probe itself"
+elif ssh_node "${HOSTS[0]}" \
+    "bash -c 'exec 3<>/dev/tcp/${HOSTS[${#HOSTS[@]}-1]}/22' 2>/dev/null"; then
+    record "vcn hairpin" INFO "present: a node reached another node's public address"
+else
+    record "vcn hairpin" INFO "not reached, which is a VCN without hairpin or a security list that excludes the source"
+fi
+
+# Remote dynamic forward: the node gets a SOCKS5 proxy on this port whose egress
+# is this host, outside the VCN. It gives the driver the customer's vantage point
+# for public addresses without moving the driver off the node, where its ovn-nbctl,
+# journalctl and spx diagnostics have to run. The subnet's own allow list does not
+# need a new entry: ssh from here already works, and its rules are per-CIDR for
+# every protocol.
+PUBLIC_PROXY_PORT="${PUBLIC_PROXY_PORT:-1080}"
+workbook_env="$workbook_env E2E_PUBLIC_PROXY=127.0.0.1:$PUBLIC_PROXY_PORT"
+
+# ExitOnForwardFailure, so a port already in use stops the run here instead of
+# handing the driver a proxy that is not there. Keepalives because this one
+# connection carries the proxy for the whole suite, and a dropped forward would
+# read as every remaining public address going dark at once.
+#
+# if !, not a $? read after the fact: under set -e a failing ssh never reaches the
+# next line, which is the one that prints the driver's own diagnostics.
+if ssh -i "$SSH_PRIVATE_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+    -o LogLevel=ERROR -o BatchMode=yes -o ConnectTimeout=15 \
+    -o ServerAliveInterval=30 -o ServerAliveCountMax=6 \
+    -o ExitOnForwardFailure=yes -R "$PUBLIC_PROXY_PORT" "ubuntu@${HOSTS[0]}" \
+    "chmod +x ~/run-tofu-examples-e2e.sh; $workbook_env ~/run-tofu-examples-e2e.sh" \
+    2>&1 | tee "$STATE_DIR/workbooks.log" | stream_workbooks; then
+    record_workbooks "$STATE_DIR/workbooks.log"
+    passed="$(grep -c '^--- PASS' "$STATE_DIR/workbooks.log" || true)"
+    ran="$(grep -c '^=== RUN' "$STATE_DIR/workbooks.log" || true)"
+    record workbooks PASS "$passed of $ran"
+else
+    # Before die, so a failed suite still gets its per-workbook table: which four
+    # passed is most of what the fifth failing means.
+    record_workbooks "$STATE_DIR/workbooks.log"
+    failed="$(grep -c '^--- FAIL' "$STATE_DIR/workbooks.log" || true)"
+    record workbooks FAIL "$failed failed, see $STATE_DIR/workbooks.log"
+    capture_journals
+    tail -60 "$STATE_DIR/workbooks.log"
+    grep -E '^--- (PASS|FAIL)' "$STATE_DIR/workbooks.log" | sed 's/^/  /' || true
+    die "the published workbooks failed on $TOPOLOGY; see $STATE_DIR/workbooks.log"
+fi

scripts/terraform/oci-spx/variables.tf

@@ -1,17 +1,34 @@
 variable "tenancy_ocid" { type = string }
 variable "user_ocid" { type = string }
 variable "fingerprint" { type = string }
-variable "private_key_path" { type = string }
+variable "private_key_path" {
+  type    = string
+  default = ""
+}
+
+# The PEM itself, for a CI runner that holds it as a secret in the environment.
+# Exactly one of this and private_key_path is set; the check lives here so a run
+# fails on the input rather than on an OCI 401 forty seconds later.
+variable "private_key" {
+  type      = string
+  default   = ""
+  sensitive = true
+  validation {
+    condition     = (var.private_key == "") != (var.private_key_path == "")
+    error_message = "Set exactly one of private_key (the PEM, for CI) or private_key_path (a file, for a workstation)."
+  }
+}
 # OCI's own region name. Spinifex's region is a separate setting that follows AWS
 # naming, so ap-sydney-1 here beside ap-southeast-2 in spinifex.toml is correct.
 variable "region" {
   type    = string
   default = "ap-sydney-1"
 }
-# Unused. Kept declared because scripts/oci_env.py discovers and exports it, and
-# there are no identity resources left that need the home-region endpoint.
+# OCI serves IAM writes from the tenancy home region, which is not necessarily
+# where we build: the dynamic group and policy in instance-principal.tf go through
+# the oci.home provider. Unset means the two are the same region.
 variable "home_region" {
-  description = "Unused. The tenancy home region, exported by the Python environment tool."
+  description = "Tenancy home region, for IAM writes. Defaults to var.region; set OCI_HOME_REGION when they differ."
   type        = string
   default     = null
   nullable    = true
@@ -37,6 +54,20 @@ variable "deployment_name" {
     error_message = "deployment_name must be 1-15 lowercase alphanumeric characters starting with a letter, because it is used as the VCN dns_label."
   }
 }
+# Three values rather than a bool, because using an instance principal and being
+# allowed to create one are different rights. See instance-principal.tf for what
+# the policy grants and why it is narrow.
+variable "instance_principal" {
+  description = "off: nodes use an API key. adopt: nodes authenticate as themselves against a dynamic group and policy that already exist. create: also create those two, which needs a tenancy-admin principal."
+  type        = string
+  default     = "off"
+
+  validation {
+    condition     = contains(["off", "adopt", "create"], var.instance_principal)
+    error_message = "instance_principal must be off, adopt or create."
+  }
+}
+
 variable "vcn_cidr" {
   description = "VCN address space. A /22 leaves room for the per-node secondary private IPs Spinifex allocates for external addresses."
   type        = string

spinifex/accountteardown/engine.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"log/slog"
+	"slices"
 	"sort"
 	"time"
 )
@@ -270,12 +271,9 @@ func (e *Engine) drainPoll() time.Duration {
 }
 
 func containsResource(resources []Resource, want Resource) bool {
-	for _, resource := range resources {
-		if resource.Kind == want.Kind && resource.ID == want.ID {
-			return true
-		}
-	}
-	return false
+	return slices.ContainsFunc(resources, func(resource Resource) bool {
+		return resource.Kind == want.Kind && resource.ID == want.ID
+	})
 }
 
 // SortReapers orders reapers by stage while preserving the caller's order

spinifex/accountteardown/reapers_account.go

@@ -96,6 +96,7 @@ type IAMAccounts struct{ svc handlers_iam.IAMService }
 
 var _ AccountStore = (*IAMAccounts)(nil)
 
+// NewIAMAccounts wraps svc as the engine's AccountStore.
 func NewIAMAccounts(svc handlers_iam.IAMService) *IAMAccounts {
 	return &IAMAccounts{svc: svc}
 }

spinifex/accountteardown/reapers_ec2.go

@@ -3,6 +3,7 @@ package accountteardown
 import (
 	"context"
 	"fmt"
+	"slices"
 	"strings"
 	"time"
 
@@ -664,12 +665,9 @@ func isAlreadyGone(err error) bool {
 		return true
 	}
 	message := err.Error()
-	for _, marker := range notFoundMarkers {
-		if strings.Contains(message, marker) {
-			return true
-		}
-	}
-	return false
+	return slices.ContainsFunc(notFoundMarkers, func(marker string) bool {
+		return strings.Contains(message, marker)
+	})
 }
 
 // isNotAssociated reports the benign case of disassociating something that was

spinifex/admin/admin.go

@@ -1,3 +1,6 @@
+// Package admin holds the node administration helpers behind spx admin: config
+// and certificate generation, credential bootstrap, image import and host
+// network discovery.
 package admin
 
 import (
@@ -19,6 +22,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"testing"
 	"text/template"
 	"time"
 
@@ -42,6 +46,8 @@ type RemoteNode struct {
 	NorthstarConfigPath string
 }
 
+// ConfigSettings is the template data rendered into every generated config file
+// (spinifex.toml, predastore.toml, NATS and the rest) by GenerateConfigFile.
 type ConfigSettings struct {
 	AccessKey string
 	SecretKey string
@@ -173,12 +179,16 @@ type PredastoreNodeConfig struct {
 	Host string
 }
 
+// ConfigFile names one config file to render: Template is the text/template
+// source and Path the destination it is written to.
 type ConfigFile struct {
 	Name     string
 	Path     string
 	Template string
 }
 
+// GenerateConfigFiles renders each of configs with configSettings, printing a
+// line per file and stopping at the first failure.
 func GenerateConfigFiles(configs []ConfigFile, configSettings ConfigSettings) error {
 	for _, cfg := range configs {
 		if err := GenerateConfigFile(cfg.Path, cfg.Template, configSettings); err != nil {
@@ -309,6 +319,8 @@ func TenantCACertPath(configDir string) string {
 	return filepath.Join(configDir, tenantCACertFilename)
 }
 
+// TenantCAKeyPath returns the tenant private CA key path under configDir; see
+// TenantCACertPath.
 func TenantCAKeyPath(configDir string) string {
 	return filepath.Join(configDir, tenantCAKeyFilename)
 }
@@ -341,6 +353,8 @@ func PredastoreDataDir(spxRoot string) string {
 	return filepath.Join(spxRoot, "predastore", "cluster")
 }
 
+// CreateServiceDirectories creates the per-service directories under spxRoot.
+// A directory that cannot be created is warned about on stderr, not returned.
 func CreateServiceDirectories(spxRoot string) {
 	dirs := []string{
 		filepath.Join(spxRoot, "images"),
@@ -367,6 +381,8 @@ func CreateServiceDirectories(spxRoot string) {
 	fmt.Printf("✅ Directory structure created in %s\n", spxRoot)
 }
 
+// FileExists reports whether path can be stat'd. Any stat error, including
+// permission denied, reads as false.
 func FileExists(path string) bool {
 	_, err := os.Stat(path)
 	return err == nil
@@ -516,33 +532,111 @@ func SetServiceOwnership() error {
 	return ownershipErr
 }
 
+// awsINIOptions keeps AWS nested settings (an empty key followed by indented
+// lines) intact across a rewrite; without it they are flattened into the
+// enclosing section and lost.
+var awsINIOptions = ini.LoadOptions{AllowNestedValues: true}
+
 // UpdateAWSINIFile updates or creates an AWS INI file section with the given key-value pairs.
 func UpdateAWSINIFile(path, section string, values map[string]string) error {
-	var cfg *ini.File
-	var err error
-
-	if FileExists(path) {
-		cfg, err = ini.Load(path)
-		if err != nil {
-			return fmt.Errorf("failed to load INI file: %w", err)
-		}
-	} else {
-		cfg = ini.Empty()
-	}
-
-	sec, err := cfg.NewSection(section)
+	cfg, err := loadAWSINIFile(path)
 	if err != nil {
-		// Section already exists, get it.
-		sec, err = cfg.GetSection(section)
-		if err != nil {
-			return fmt.Errorf("failed to get section: %w", err)
-		}
+		return err
+	}
+	sec, err := awsINISection(cfg, section)
+	if err != nil {
+		return err
 	}
-
 	for key, value := range values {
 		sec.Key(key).SetValue(value)
 	}
+	return saveAWSINIFile(cfg, path)
+}
 
+// WriteAWSProfileConfig points an AWS CLI profile at this cluster. One profile
+// endpoint_url cannot send S3 to the predastore gate and everything else to the
+// gateway, so S3 goes in the profile's services section, which takes precedence.
+func WriteAWSProfileConfig(path, profileName, region, certPath, endpointHost string) error {
+	cfg, err := loadAWSINIFile(path)
+	if err != nil {
+		return err
+	}
+	profileSection := profileName
+	if profileName != "default" {
+		profileSection = "profile " + profileName
+	}
+	profile, err := awsINISection(cfg, profileSection)
+	if err != nil {
+		return err
+	}
+	for key, value := range map[string]string{
+		"region":       region,
+		"endpoint_url": "https://" + net.JoinHostPort(endpointHost, "9999"),
+		"ca_bundle":    certPath,
+		"output":       "json",
+		"services":     profileName,
+	} {
+		profile.Key(key).SetValue(value)
+	}
+	services, err := awsINISection(cfg, "services "+profileName)
+	if err != nil {
+		return err
+	}
+	s3Endpoint := "https://" + net.JoinHostPort(endpointHost, strconv.Itoa(predastoreGatePort))
+	if err := setNestedAWSSetting(services, "s3", "endpoint_url", s3Endpoint); err != nil {
+		return err
+	}
+	return saveAWSINIFile(cfg, path)
+}
+
+// setNestedAWSSetting sets name inside the nested block under key, keeping any
+// other settings an operator nested there.
+func setNestedAWSSetting(sec *ini.Section, key, name, value string) error {
+	var lines []string
+	if sec.HasKey(key) {
+		for _, line := range sec.Key(key).NestedValues() {
+			if setting, _, _ := strings.Cut(line, "="); strings.TrimSpace(setting) != name {
+				lines = append(lines, line)
+			}
+		}
+		sec.DeleteKey(key)
+	}
+	nested, err := sec.NewKey(key, "")
+	if err != nil {
+		return fmt.Errorf("failed to create key %s: %w", key, err)
+	}
+	for _, line := range append(lines, name+" = "+value) {
+		if err := nested.AddNestedValue(line); err != nil {
+			return fmt.Errorf("failed to set %s.%s: %w", key, name, err)
+		}
+	}
+	return nil
+}
+
+func loadAWSINIFile(path string) (*ini.File, error) {
+	if !FileExists(path) {
+		return ini.Empty(awsINIOptions), nil
+	}
+	cfg, err := ini.LoadSources(awsINIOptions, path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load INI file: %w", err)
+	}
+	return cfg, nil
+}
+
+func awsINISection(cfg *ini.File, name string) (*ini.Section, error) {
+	sec, err := cfg.NewSection(name)
+	if err != nil {
+		// Section already exists, get it.
+		sec, err = cfg.GetSection(name)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get section: %w", err)
+		}
+	}
+	return sec, nil
+}
+
+func saveAWSINIFile(cfg *ini.File, path string) error {
 	// Write atomically: ini.SaveTo uses os.Create (world-readable); render to a
 	// sibling temp file (0600) and rename to avoid briefly exposing secrets.
 	dir := filepath.Dir(path)
@@ -647,6 +741,16 @@ func GenerateNATSToken() (string, error) {
 // keeps the 4096-bit default.
 var certKeyBits = 4096
 
+// ShortenCertKeyBitsForTest generates certificate keys at bits for the rest of
+// one test and restores the previous size afterwards. There is one size per
+// process, so the test must not run in parallel with anything generating keys.
+func ShortenCertKeyBitsForTest(tb testing.TB, bits int) {
+	tb.Helper()
+	prev := certKeyBits
+	certKeyBits = bits
+	tb.Cleanup(func() { certKeyBits = prev })
+}
+
 // GenerateCACert generates a Certificate Authority certificate and key.
 func GenerateCACert(caCertPath, caKeyPath string) error {
 	caPrivateKey, err := rsa.GenerateKey(rand.Reader, certKeyBits)
@@ -936,22 +1040,12 @@ func SetupAWSCredentials(accessKey, secretKey, region, certPath, bindIP string)
 		}
 	}
 
-	configSection := profileName
-	if profileName != "default" {
-		configSection = "profile " + profileName
-	}
-
 	endpointHost := bindIP
 	if endpointHost == "" || endpointHost == "0.0.0.0" {
 		endpointHost = "localhost"
 	}
 
-	if err := UpdateAWSINIFile(configPath, configSection, map[string]string{
-		"region":       region,
-		"endpoint_url": "https://" + net.JoinHostPort(endpointHost, "9999"),
-		"ca_bundle":    certPath,
-		"output":       "json",
-	}); err != nil {
+	if err := WriteAWSProfileConfig(configPath, profileName, region, certPath, endpointHost); err != nil {
 		return err
 	}
 

spinifex/admin/admin_test.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"testing"
 
+	awsconfig "github.com/aws/aws-sdk-go-v2/config"
 	"github.com/mulgadc/spinifex/spinifex/config"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -234,6 +235,79 @@ func TestUpdateAWSINIFile_AddNewSection(t *testing.T) {
 	assert.Contains(t, content, "spinifex-val")
 }
 
+// --- WriteAWSProfileConfig ---
+
+// resolveProfileEndpoint resolves sdkID's endpoint through the Go SDK's own
+// shared-config loader, the way a client does: the service's entry in the
+// profile's services section, else the profile-wide endpoint_url.
+func resolveProfileEndpoint(t *testing.T, configPath, profile, sdkID string) string {
+	t.Helper()
+	shared, err := awsconfig.LoadSharedConfigProfile(t.Context(), profile, func(o *awsconfig.LoadSharedConfigOptions) {
+		o.ConfigFiles = []string{configPath}
+		o.CredentialsFiles = []string{}
+	})
+	require.NoError(t, err)
+	endpoint, found, err := shared.GetServiceBaseEndpoint(t.Context(), sdkID)
+	require.NoError(t, err)
+	if found {
+		return endpoint
+	}
+	return shared.BaseEndpoint
+}
+
+func TestWriteAWSProfileConfig_SendsS3ToPredastoreAndTheRestToTheGateway(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "config")
+	require.NoError(t, WriteAWSProfileConfig(path, "spinifex", "ap-southeast-2", "/ca.pem", "10.11.12.1"))
+
+	assert.Equal(t, "https://10.11.12.1:8443", resolveProfileEndpoint(t, path, "spinifex", "S3"))
+	assert.Equal(t, "https://10.11.12.1:9999", resolveProfileEndpoint(t, path, "spinifex", "EC2"))
+	assert.Equal(t, "https://10.11.12.1:9999", resolveProfileEndpoint(t, path, "spinifex", "STS"))
+}
+
+// A rewrite must not flatten nested settings an operator added, which is how a
+// hand-written s3 block used to be lost on every re-init.
+func TestWriteAWSProfileConfig_KeepsNestedSettingsAcrossARewrite(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "config")
+	require.NoError(t, os.WriteFile(path, []byte(`[profile spinifex]
+s3 =
+  max_concurrent_requests = 20
+services = spinifex
+
+[services spinifex]
+s3 =
+  addressing_style = path
+  endpoint_url = https://stale:8443
+`), 0600))
+
+	require.NoError(t, WriteAWSProfileConfig(path, "spinifex", "ap-southeast-2", "/ca.pem", "localhost"))
+	require.NoError(t, WriteAWSProfileConfig(path, "spinifex", "ap-southeast-2", "/ca.pem", "localhost"))
+
+	assert.Equal(t, "https://localhost:8443", resolveProfileEndpoint(t, path, "spinifex", "S3"))
+	shared, err := awsconfig.LoadSharedConfigProfile(t.Context(), "spinifex", func(o *awsconfig.LoadSharedConfigOptions) {
+		o.ConfigFiles = []string{path}
+		o.CredentialsFiles = []string{}
+	})
+	require.NoError(t, err)
+	assert.Equal(t, map[string]string{"addressing_style": "path", "endpoint_url": "https://localhost:8443"},
+		shared.Services.ServiceValues["s3"])
+
+	data, err := os.ReadFile(path)
+	require.NoError(t, err)
+	assert.Contains(t, string(data), "max_concurrent_requests = 20")
+	assert.NotContains(t, string(data), "stale")
+}
+
+// Account profiles each get their own services section, so a tenant profile
+// resolves S3 the same way as the operator's without sharing its section.
+func TestWriteAWSProfileConfig_ProfilesDoNotShareAServicesSection(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "config")
+	require.NoError(t, WriteAWSProfileConfig(path, "spinifex", "ap-southeast-2", "/ca.pem", "localhost"))
+	require.NoError(t, WriteAWSProfileConfig(path, "spinifex-acme", "ap-southeast-2", "/ca.pem", "10.0.0.5"))
+
+	assert.Equal(t, "https://localhost:8443", resolveProfileEndpoint(t, path, "spinifex", "S3"))
+	assert.Equal(t, "https://10.0.0.5:8443", resolveProfileEndpoint(t, path, "spinifex-acme", "S3"))
+}
+
 // --- SetupAWSCredentials ---
 
 func TestSetupAWSCredentials_CreatesFiles(t *testing.T) {
@@ -279,9 +353,9 @@ func TestSetupAWSCredentials_UsesBindIP(t *testing.T) {
 	err := SetupAWSCredentials("AKIATEST123", "secret123", "us-east-1", "/ca.pem", "10.11.12.1")
 	require.NoError(t, err)
 
-	configData, _ := os.ReadFile(filepath.Join(dir, ".aws", "config"))
-	assert.Contains(t, string(configData), "https://10.11.12.1:9999")
-	assert.NotContains(t, string(configData), "localhost")
+	configPath := filepath.Join(dir, ".aws", "config")
+	assert.Equal(t, "https://10.11.12.1:9999", resolveProfileEndpoint(t, configPath, "spinifex", "EC2"))
+	assert.Equal(t, "https://10.11.12.1:8443", resolveProfileEndpoint(t, configPath, "spinifex", "S3"))
 }
 
 func TestSetupAWSCredentials_FallsBackToLocalhostForWildcard(t *testing.T) {

spinifex/admin/ipsec.go

@@ -162,8 +162,9 @@ var charonRereadCAs = func() error {
 	return nil
 }
 
-// Exported so runtime services that mint from the cluster CA share one loader
-// rather than each re-implementing the PEM/PKCS8 handling.
+// LoadCAKeyPair reads the cluster CA certificate and its PKCS8 RSA key from PEM
+// files. Exported so runtime services that mint from the cluster CA share one
+// loader rather than each re-implementing the PEM/PKCS8 handling.
 func LoadCAKeyPair(caCertPath, caKeyPath string) (*x509.Certificate, *rsa.PrivateKey, error) {
 	certPEM, err := os.ReadFile(caCertPath)
 	if err != nil {

spinifex/admin/network_detect.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net"
 	"os/exec"
+	"slices"
 	"strings"
 
 	"github.com/mulgadc/bluebottle/pkg/safecast"
@@ -190,10 +191,7 @@ func isVirtualInterface(name string, defaultRouteDev string) bool {
 		"docker", "br-", "veth", "ovs", "virbr",
 		"tun", "tap", "tailscale", "wg", "lo",
 	}
-	for _, p := range prefixes {
-		if strings.HasPrefix(name, p) {
-			return true
-		}
-	}
-	return false
+	return slices.ContainsFunc(prefixes, func(p string) bool {
+		return strings.HasPrefix(name, p)
+	})
 }

spinifex/arn/acm.go

@@ -23,11 +23,11 @@ func FormatACMCertificate(region, accountID, id string) string {
 // The id keeps everything after the first "/". Truncating at the last one would
 // let a grant on certificate/<a> authorize a request naming certificate/<a>/<b>.
 func ParseACMCertificateID(certARN string) (string, bool) {
-	parts := strings.SplitN(certARN, ":", 6)
-	if len(parts) != 6 || parts[0] != "arn" || parts[2] != "acm" {
+	service, _, _, resource, ok := Split(certARN)
+	if !ok || service != "acm" {
 		return "", false
 	}
-	kind, id, found := strings.Cut(parts[5], "/")
+	kind, id, found := strings.Cut(resource, "/")
 	if !found || ACMResourceType(kind) != ACMCertificate || id == "" {
 		return "", false
 	}

spinifex/arn/arn.go

@@ -0,0 +1,14 @@
+package arn
+
+import "strings"
+
+// Split breaks an arn:aws ARN into its service, region, account and resource
+// components. The resource keeps any further colons or slashes verbatim. ok is
+// false for anything outside the aws partition, the only one this stack builds.
+func Split(s string) (service, region, account, resource string, ok bool) {
+	parts := strings.SplitN(s, ":", 6)
+	if len(parts) != 6 || parts[0] != "arn" || parts[1] != "aws" {
+		return "", "", "", "", false
+	}
+	return parts[2], parts[3], parts[4], parts[5], true
+}

spinifex/arn/arn_test.go

@@ -0,0 +1,81 @@
+package arn_test
+
+import (
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/arn"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSplit(t *testing.T) {
+	tests := []struct {
+		name     string
+		value    string
+		service  string
+		region   string
+		account  string
+		resource string
+		wantOK   bool
+	}{
+		{
+			name:     "regional ARN",
+			value:    "arn:aws:ecs:ap-southeast-2:123456789012:cluster/web",
+			service:  "ecs",
+			region:   "ap-southeast-2",
+			account:  "123456789012",
+			resource: "cluster/web",
+			wantOK:   true,
+		},
+		{
+			name:     "global ARN has an empty region",
+			value:    "arn:aws:iam::123456789012:root",
+			service:  "iam",
+			account:  "123456789012",
+			resource: "root",
+			wantOK:   true,
+		},
+		{
+			name:     "resource keeps its own colons",
+			value:    "arn:aws:rds:ap-southeast-2:123456789012:snapshot:rds:orders-2026-09-30",
+			service:  "rds",
+			region:   "ap-southeast-2",
+			account:  "123456789012",
+			resource: "snapshot:rds:orders-2026-09-30",
+			wantOK:   true,
+		},
+		{
+			name:    "empty resource",
+			value:   "arn:aws:ecs:ap-southeast-2:123456789012:",
+			service: "ecs",
+			region:  "ap-southeast-2",
+			account: "123456789012",
+			wantOK:  true,
+		},
+		{
+			name:  "another partition",
+			value: "arn:aws-cn:ecs:cn-north-1:123456789012:cluster/web",
+		},
+		{
+			name:  "not an ARN",
+			value: "urn:aws:ecs:ap-southeast-2:123456789012:cluster/web",
+		},
+		{
+			name:  "too few components",
+			value: "arn:aws:ecs:ap-southeast-2:cluster/web",
+		},
+		{
+			name: "empty",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			service, region, account, resource, ok := arn.Split(tt.value)
+			assert.Equal(t, tt.wantOK, ok)
+			assert.Equal(t, tt.service, service)
+			assert.Equal(t, tt.region, region)
+			assert.Equal(t, tt.account, account)
+			assert.Equal(t, tt.resource, resource)
+		})
+	}
+}

spinifex/arn/elbv2.go

@@ -70,18 +70,18 @@ type ParsedELBv2 struct {
 // arn:aws:elasticloadbalancing ARN naming one of the four resource types: an
 // ARN this stack never builds has no resource it can correctly stand for.
 func ParseELBv2(value string) (ParsedELBv2, bool) {
-	parts := strings.SplitN(value, ":", 6)
-	if len(parts) != 6 || parts[0] != "arn" || parts[1] != "aws" || parts[2] != "elasticloadbalancing" {
+	service, region, account, rest, ok := Split(value)
+	if !ok || service != "elasticloadbalancing" {
 		return ParsedELBv2{}, false
 	}
-	rawKind, resource, found := strings.Cut(parts[5], "/")
+	rawKind, resource, found := strings.Cut(rest, "/")
 	if !found || rawKind == "" {
 		return ParsedELBv2{}, false
 	}
 	kind := ELBv2ResourceType(rawKind)
 	switch kind {
 	case ELBv2LoadBalancer, ELBv2TargetGroup, ELBv2Listener, ELBv2ListenerRule:
-		return ParsedELBv2{Region: parts[3], AccountID: parts[4], Kind: kind, Resource: resource}, true
+		return ParsedELBv2{Region: region, AccountID: account, Kind: kind, Resource: resource}, true
 	default:
 		return ParsedELBv2{}, false
 	}

spinifex/awsec2query/parser.go

@@ -1,3 +1,5 @@
+// Package awsec2query decodes AWS EC2 Query-protocol request parameters into
+// aws-sdk-go input structs.
 package awsec2query
 
 import (
@@ -14,6 +16,9 @@ import (
 // Callers should map this to AWS's MalformedQueryString error code.
 var ErrSliceTooLarge = errors.New("list parameter exceeds maximum entries")
 
+// QueryParamsToStruct fills the struct out points to from flattened EC2 Query
+// params, matching each field by name or locationName tag and recursing into
+// dotted prefixes for nested structs and lists. out must be a struct pointer.
 func QueryParamsToStruct(params map[string]string, out any) error {
 	v := reflect.ValueOf(out)
 

spinifex/awsec2query/parser_fuzz_test.go

@@ -0,0 +1,151 @@
+package awsec2query
+
+import (
+	"fmt"
+	"net/url"
+	"reflect"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/elbv2"
+	"github.com/aws/aws-sdk-go/service/rds"
+)
+
+// Input shapes covering each list serialisation: EC2 Prefix.N, ELBv2
+// Prefix.member.N, RDS locationNameList, and deep nesting under a struct.
+var fuzzTargets = []func() any{
+	func() any { return &ec2.DescribeInstancesInput{} },
+	func() any { return &ec2.RunInstancesInput{} },
+	func() any { return &ec2.CreateLaunchTemplateInput{} },
+	func() any { return &elbv2.RegisterTargetsInput{} },
+	func() any { return &elbv2.CreateListenerInput{} },
+	func() any { return &rds.CreateDBInstanceInput{} },
+}
+
+// queryParams builds the params map the way the gateway does from a request
+// body: url.ParseQuery, keeping the last value of a repeated key.
+func queryParams(body string) (map[string]string, bool) {
+	values, err := url.ParseQuery(body)
+	if err != nil {
+		return nil, false
+	}
+	params := make(map[string]string, len(values))
+	for key, vs := range values {
+		params[key] = vs[len(vs)-1]
+	}
+	return params, true
+}
+
+// countSliceElems tallies materialised slice elements by nesting depth,
+// failing on any slice past MaxSliceLen.
+func countSliceElems(t *testing.T, v reflect.Value, depth int, counts map[int]int) {
+	switch v.Kind() {
+	case reflect.Pointer, reflect.Interface:
+		if !v.IsNil() {
+			countSliceElems(t, v.Elem(), depth, counts)
+		}
+	case reflect.Struct:
+		for i := range v.NumField() {
+			if v.Type().Field(i).IsExported() {
+				countSliceElems(t, v.Field(i), depth, counts)
+			}
+		}
+	case reflect.Slice:
+		if v.Type().Elem().Kind() == reflect.Uint8 {
+			return
+		}
+		if v.Len() > MaxSliceLen {
+			t.Fatalf("slice of %d entries exceeds MaxSliceLen %d", v.Len(), MaxSliceLen)
+		}
+		counts[depth] += v.Len()
+		for i := range v.Len() {
+			countSliceElems(t, v.Index(i), depth+1, counts)
+		}
+	}
+}
+
+// FuzzQueryParamsToStruct pins the properties the parser owes untrusted input:
+// it never panics, and on success every list level holds no more entries than
+// the request had parameters, so allocation is bounded by the input.
+func FuzzQueryParamsToStruct(f *testing.F) {
+	for _, seed := range []string{
+		"Filter.1.Name=instance-type&Filter.1.Value.1=t2.micro&Filter.3.Name=x&Filter.3.Value.1=y",
+		"Filter.2.Name=skipped&Filter.2.Value.1=x",
+		"Filter.999999.Name=ignored",
+		"Filter.01.Name=a&Filter.+1.Value.1=b&Filter.-1.Name=c",
+		"Filter.1.Value.item.1=a&Filter.1.Value.2=b",
+		"InstanceId.1=i-1&InstanceId.2=i-2&MaxResults=5&DryRun=true",
+		"MaxResults=99999999999999999999&DryRun=maybe",
+		"ImageId=ami-1&MinCount=1&MaxCount=1&UserData=aGVsbG8=&BlockDeviceMapping.1.DeviceName=/dev/sda1&BlockDeviceMapping.1.Ebs.VolumeSize=20",
+		"TagSpecification.1.ResourceType=instance&TagSpecification.1.Tag.1.Key=k&TagSpecification.1.Tag.1.Value=v",
+		"LaunchTemplateData.NetworkInterface.1.PrivateIpAddresses.1.PrivateIpAddress=10.0.0.1&LaunchTemplateData.InstanceMarketOptions.SpotOptions.ValidUntil=2026-07-10T12:00:00Z",
+		"TargetGroupArn=arn&Targets.member.1.Id=i-1&Targets.member.1.Port=80&Targets.member.2.Id=i-2",
+		"DefaultActions.member.1.Type=forward&DefaultActions.member.1.ForwardConfig.TargetGroups.member.1.Weight=1",
+		"DBInstanceIdentifier=db&Tags.Tag.1.Key=k&Tags.Tag.1.Value=v&Tags.member.2.Key=x",
+	} {
+		f.Add(seed)
+	}
+
+	f.Fuzz(func(t *testing.T, body string) {
+		params, ok := queryParams(body)
+		if !ok {
+			return
+		}
+		for _, target := range fuzzTargets {
+			out := target()
+			if err := QueryParamsToStruct(params, out); err != nil {
+				continue
+			}
+			counts := map[int]int{}
+			countSliceElems(t, reflect.ValueOf(out), 0, counts)
+			for depth, n := range counts {
+				if n > len(params) {
+					t.Fatalf("%T: %d list entries at depth %d from %d params: %q", out, n, depth, len(params), body)
+				}
+			}
+		}
+	})
+}
+
+// FuzzQueryParamsToStruct_EntryPastGapIsIgnored pins AWS's gap rule on
+// arbitrary input: adding an entry one past the first gap in any list leaves
+// the parsed request unchanged.
+func FuzzQueryParamsToStruct_EntryPastGapIsIgnored(f *testing.F) {
+	for _, seed := range []string{
+		"Filter.1.Name=a&Filter.1.Value.1=b&Filter.3.Name=c",
+		"Filter.2.Name=a",
+		"Filter.1.Name=a&Filter.1.Value.2=b",
+		"InstanceId.1=i-1&InstanceId.3=i-3",
+		"Filter.01.Name=a&Filter.1.Value.1=b",
+		"Filter.1.Value.item.1=a",
+	} {
+		f.Add(seed)
+	}
+
+	f.Fuzz(func(t *testing.T, body string) {
+		params, ok := queryParams(body)
+		if !ok {
+			return
+		}
+		want := &ec2.DescribeInstancesInput{}
+		if err := QueryParamsToStruct(params, want); err != nil {
+			return
+		}
+
+		pastGap := len(want.Filters) + 2
+		params[fmt.Sprintf("Filter.%d.Name", pastGap)] = "past-gap"
+		params[fmt.Sprintf("Filter.%d.Value.1", pastGap)] = "past-gap"
+		params[fmt.Sprintf("InstanceId.%d", len(want.InstanceIds)+2)] = "past-gap"
+		for i, filter := range want.Filters {
+			params[fmt.Sprintf("Filter.%d.Value.%d", i+1, len(filter.Values)+2)] = "past-gap"
+		}
+
+		got := &ec2.DescribeInstancesInput{}
+		if err := QueryParamsToStruct(params, got); err != nil {
+			t.Fatalf("entry past the gap turned success into %v: %q", err, body)
+		}
+		if !reflect.DeepEqual(want, got) {
+			t.Fatalf("entry past the gap changed the parse of %q:\nwant %s\ngot  %s", body, want.String(), got.String())
+		}
+	})
+}

spinifex/awserrors/awserrors.go

@@ -1,3 +1,6 @@
+// Package awserrors defines the AWS error codes Spinifex returns, their HTTP
+// status and default message, and helpers to attach a code to an error and
+// resolve it again at the API boundary.
 package awserrors
 
 import (
@@ -8,6 +11,8 @@ import (
 	"time"
 )
 
+// ErrorMessage is the HTTP status and default message returned for an AWS
+// error code, as resolved by LookupErrorMessage.
 type ErrorMessage struct {
 	HTTPCode int
 	Message  string
@@ -314,7 +319,7 @@ var (
 	ErrorInvalidVpcPeeringConnectionIDNotFound                = "InvalidVpcPeeringConnectionID.NotFound"
 	ErrorInvalidVpcPeeringConnectionIdMalformed               = "InvalidVpcPeeringConnectionId.Malformed"
 	ErrorInvalidVpcPeeringConnectionStateDnsHostnamesDisabled = "InvalidVpcPeeringConnectionState.DnsHostnamesDisabled"
-	ErrorInvalidVpcRange                                      = "InvalidVpcRange"
+	ErrorInvalidVpcRange                                      = "InvalidVpc.Range"
 	ErrorInvalidVpcState                                      = "InvalidVpcState"
 	ErrorInvalidVpnConnectionInvalidState                     = "InvalidVpnConnection.InvalidState"
 	ErrorInvalidVpnConnectionInvalidType                      = "InvalidVpnConnection.InvalidType"
@@ -388,6 +393,9 @@ var (
 	// real ECS both use the restJson1 wire code "InvalidParameterException",
 	// so this gives EKS call sites their own name for the same registered code.
 	ErrorEKSInvalidParameter = ErrorECSInvalidParameter
+	// ErrorECRInvalidParameter aliases ErrorECSInvalidParameter: every ECR
+	// operation declares the AWS JSON 1.1 wire code "InvalidParameterException".
+	ErrorECRInvalidParameter = ErrorECSInvalidParameter
 	// ErrorACMResourceInUse aliases ErrorEKSResourceInUse: real ACM's
 	// DeleteCertificate and real EKS's CreateCluster both legitimately use the
 	// wire code "ResourceInUseException", so this is a distinct, self-documenting
@@ -517,6 +525,7 @@ var (
 	ErrorLayersNotFound           = "LayersNotFoundException"
 	ErrorReferencedImagesNotFound = "ReferencedImagesNotFoundException"
 	ErrorTooManyTags              = "TooManyTagsException"
+	ErrorInvalidTagParameter      = "InvalidTagParameterException"
 	ErrorOperationNotSupported    = "OperationNotSupportedException"
 
 	// ELBv2-specific error codes.
@@ -577,7 +586,7 @@ func ValidErrorCode(code string) string {
 
 // ResolveErrorCode returns the first registered AWS error code in err's unwrap tree.
 func ResolveErrorCode(err error) (string, bool) {
-	code, _, ok := resolveErrorDetail(err)
+	code, _, ok := ResolveErrorDetail(err)
 	return code, ok
 }
 
@@ -585,12 +594,6 @@ func ResolveErrorCode(err error) (string, bool) {
 // plus the message the producing call site attached via Errorf, if any. A
 // generic %w wrapper added purely for internal context carries no message.
 func ResolveErrorDetail(err error) (code, message string, ok bool) {
-	return resolveErrorDetail(err)
-}
-
-// resolveErrorDetail is the shared unwrap-tree walk behind ResolveErrorCode
-// and ResolveErrorDetail.
-func resolveErrorDetail(err error) (code, message string, ok bool) {
 	if err == nil {
 		return "", "", false
 	}
@@ -604,14 +607,14 @@ func resolveErrorDetail(err error) (code, message string, ok bool) {
 
 	if joined, isJoined := err.(interface{ Unwrap() []error }); isJoined {
 		for _, inner := range joined.Unwrap() {
-			if c, m, found := resolveErrorDetail(inner); found {
+			if c, m, found := ResolveErrorDetail(inner); found {
 				return c, m, true
 			}
 		}
 		return "", "", false
 	}
 	if wrapped, isWrapped := err.(interface{ Unwrap() error }); isWrapped {
-		return resolveErrorDetail(wrapped.Unwrap())
+		return ResolveErrorDetail(wrapped.Unwrap())
 	}
 	return "", "", false
 }
@@ -640,6 +643,18 @@ func Errorf(code, format string, args ...any) error {
 	return outer
 }
 
+// IDNotFound returns code with EC2's not-found message for one resource ID,
+// "The <kind> ID '<id>' does not exist".
+func IDNotFound(code, kind, id string) error {
+	return Errorf(code, "The %s ID '%s' does not exist", kind, id)
+}
+
+// HasDependencies returns DependencyViolation with EC2's message for a
+// resource that cannot be deleted while others depend on it.
+func HasDependencies(kind, id string) error {
+	return Errorf(ErrorDependencyViolation, "The %s '%s' has dependencies and cannot be deleted.", kind, id)
+}
+
 // retryableError wraps a registered code with a suggested Retry-After
 // duration, for a 503 that a client can expect to clear on its own within a
 // bounded window (e.g. a warm-up race) rather than an open-ended outage.
@@ -658,7 +673,7 @@ func RetryAfter(code string, d time.Duration) error {
 }
 
 // ResolveRetryAfter returns the Retry-After duration attached to err via
-// RetryAfter, if any, walking the same unwrap tree resolveErrorDetail does.
+// RetryAfter, if any, walking the same unwrap tree ResolveErrorDetail does.
 func ResolveRetryAfter(err error) (time.Duration, bool) {
 	if err == nil {
 		return 0, false
@@ -1012,7 +1027,7 @@ var ErrorLookup = map[string]ErrorMessage{
 	ErrorInvalidNetworkLoadBalancerArnNotFound:                 {HTTPCode: 404, Message: "The specified Network Load Balancer ARN does not exist."},
 	ErrorInvalidNextToken:                                      {HTTPCode: 400, Message: "The specified NextToken is not valid."},
 	ErrorInvalidOptionConflict:                                 {HTTPCode: 409, Message: "A VPN connection between the virtual private gateway and the customer gateway already exists."},
-	ErrorInvalidPaginationToken:                                {HTTPCode: 403, Message: "The specified pagination token is not valid or is expired."},
+	ErrorInvalidPaginationToken:                                {HTTPCode: 400, Message: "The specified pagination token is not valid or is expired."},
 	ErrorInvalidParameter:                                      {HTTPCode: 400, Message: "A parameter specified in a request is not valid, is unsupported, or cannot be used. The returned message provides an explanation of the error value. For example, if you are launching an instance, you can't specify a security group and subnet that are in different VPCs."},
 	ErrorInvalidParameterCombination:                           {HTTPCode: 400, Message: "Indicates an incorrect combination of parameters, or a missing parameter. For example, trying to terminate an instance without specifying the instance ID."},
 	ErrorInvalidParameterDependency:                            {HTTPCode: 400, Message: "Indicates an incorrect combination of parameters, or a missing parameter. For example, trying to terminate an instance without specifying the instance ID."},
@@ -1069,7 +1084,7 @@ var ErrorLookup = map[string]ErrorMessage{
 	ErrorInvalidSubnetConflict:                                 {HTTPCode: 409, Message: "The specified CIDR block conflicts with that of another subnet in your VPC."},
 	ErrorInvalidSubnetRange:                                    {HTTPCode: 400, Message: "The CIDR block you've specified for the subnet is not valid. The allowed block size is between a /28 netmask and /16 netmask."},
 	ErrorInvalidSubnetIDMalformed:                              {HTTPCode: 400, Message: "The specified subnet ID is malformed. Ensure that you specify the ID in the form subnet-xxxxxxxxxxxxxxxxx"},
-	ErrorInvalidSubnetIDNotFound:                               {HTTPCode: 404, Message: "or InvalidSubnetId.NotFound \tThe specified subnet does not exist."},
+	ErrorInvalidSubnetIDNotFound:                               {HTTPCode: 404, Message: "The specified subnet does not exist."},
 	ErrorInvalidTagKeyMalformed:                                {HTTPCode: 400, Message: "The specified tag key is not valid. Tag keys cannot be empty or null, and cannot start with aws:."},
 	ErrorInvalidTargetArnUnknown:                               {HTTPCode: 404, Message: "The specified ARN for the specified user or role is not valid or does not exist."},
 	ErrorInvalidTargetException:                                {HTTPCode: 404, Message: "The specified TargetId is not valid, does not exist, or is in another organization. You can only generate an account status report for declarative policies in your own organization. Ensure that you specify the TargetId in one of the following forms: r-xxxx, ou-xxxx-xxxxxxxx, or a 12-digit account ID in the form xxxxxxxxxxxx."},
@@ -1097,7 +1112,7 @@ var ErrorLookup = map[string]ErrorMessage{
 	ErrorInvalidVpcPeeringConnectionIDNotFound:                 {HTTPCode: 404, Message: "The specified VPC peering connection ID does not exist."},
 	ErrorInvalidVpcPeeringConnectionIdMalformed:                {HTTPCode: 400, Message: "The specified VPC peering connection ID is malformed. Ensure that you provide the ID in the form pcx-xxxxxxxxxxxxxxxxx."},
 	ErrorInvalidVpcPeeringConnectionStateDnsHostnamesDisabled:  {HTTPCode: 400, Message: "To enable DNS hostname resolution for the VPC peering connection, DNS hostname support must be enabled for the VPCs."},
-	ErrorInvalidVpcRange:                                       {HTTPCode: 400, Message: "The specified CIDR block range is not valid. The block range must be between a /28 netmask and /16 netmask. For more information, see VPC CIDR blocks."},
+	ErrorInvalidVpcRange:                                       {HTTPCode: 400, Message: "The CIDR block is invalid."},
 	ErrorInvalidVpcState:                                       {HTTPCode: 400, Message: "The specified VPC already has a virtual private gateway attached to it."},
 	ErrorInvalidVpnConnectionInvalidState:                      {HTTPCode: 400, Message: "The VPN connection must be in the available state to complete the request."},
 	ErrorInvalidVpnConnectionInvalidType:                       {HTTPCode: 400, Message: "The specified VPN connection does not support static routes."},
@@ -1210,7 +1225,7 @@ var ErrorLookup = map[string]ErrorMessage{
 	ErrorUnfulfillableCapacity:                                 {HTTPCode: 400, Message: "At this time there isn't enough spare capacity to fulfill your request for Spot Instances. You can wait a few minutes to see whether capacity becomes available for your request. Alternatively, create a more flexible request. For example, include additional instance types, include additional Availability Zones, or use the capacity-optimized allocation strategy."},
 	ErrorUnknownParameter:                                      {HTTPCode: 404, Message: "An unknown or unrecognized parameter was supplied. Requests that could cause this error include supplying a misspelled parameter or a parameter that is not supported for the specified API version."},
 	ErrorUnknownPrincipalTypeUnsupported:                       {HTTPCode: 400, Message: "The principal type is not supported."},
-	ErrorUnknownVolumeType:                                     {HTTPCode: 400, Message: "The specified volume type is unsupported. The supported volume types are gp2, io1, st1, sc1, and standard."},
+	ErrorUnknownVolumeType:                                     {HTTPCode: 400, Message: "Unsupported volume type for volume creation."},
 	ErrorUnsupported:                                           {HTTPCode: 400, Message: "The specified request is unsupported. For example, you might be trying to launch an instance in an Availability Zone that currently has constraints on that instance type. The returned message provides details of the unsupported request."},
 	ErrorUnsupportedException:                                  {HTTPCode: 400, Message: "Capacity Reservations are not supported for this Region."},
 	ErrorUnsupportedHibernationConfiguration:                   {HTTPCode: 400, Message: "The instance could not be launched because one or more parameter values do not meet the prerequisites for enabling hibernation. For more information, see Hibernation Prerequisites. Alternatively, the instance could not be hibernated because it is not enabled for hibernation."},
@@ -1277,6 +1292,7 @@ var ErrorLookup = map[string]ErrorMessage{
 	ErrorImageDigestDoesNotMatch:  {HTTPCode: 400, Message: "The specified image digest does not match the digest of the supplied image manifest."},
 	ErrorLayersNotFound:           {HTTPCode: 400, Message: "The specified layers could not be found, or the specified layer is not valid for this repository."},
 	ErrorReferencedImagesNotFound: {HTTPCode: 400, Message: "The manifest list is referencing an image that does not exist."},
+	ErrorInvalidTagParameter:      {HTTPCode: 400, Message: "Tag parameters are invalid"},
 	ErrorTooManyTags:              {HTTPCode: 400, Message: "The list of tags on the repository is over the limit. The maximum number of tags that can be applied to a repository is 50."},
 	ErrorOperationNotSupported:    {HTTPCode: 400, Message: "The specified operation is not supported in this registry."},