mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-24 05:04:03 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 joshsiv-mulga
|
fa8f35b764 |
+6
-6
@@ -6,10 +6,10 @@
|
||||
# Set the matching branch protection rule on `dev` and `main` to
|
||||
# "Require review from Code Owners" or this file is advisory only.
|
||||
|
||||
# Network layer is governed by its invariant tests.
|
||||
# Network layer is governed by ADR-0006 invariant tests.
|
||||
# Changes here must respect those clauses
|
||||
# see spinifex/domains/network/invariants/
|
||||
# mulga docs/development/proposals/region-local-architecture/0006-spinifex-network-layer-contract.md (proposal, not accepted)
|
||||
/spinifex/domains/network/ @joshsiv-mulga
|
||||
/spinifex/runtime/roles/vpcd/ @joshsiv-mulga
|
||||
/spinifex/domains/network/invariants/ @joshsiv-mulga
|
||||
# see spinifex/network/invariants/
|
||||
# docs/development/proposals/az-local-architecture/0006-spinifex-network-layer-contract.md
|
||||
/spinifex/network/ @joshsiv-mulga
|
||||
/spinifex/vpcd/ @joshsiv-mulga
|
||||
/spinifex/network/invariants/ @joshsiv-mulga
|
||||
|
||||
@@ -8,7 +8,7 @@ Actions job summary and emits one `::error` annotation per failed test case.
|
||||
```yaml
|
||||
- name: Publish test summary
|
||||
if: always()
|
||||
uses: $/.github/actions/junit-report
|
||||
uses: ./spinifex/.github/actions/junit-report # or your own checkout path
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
title: "Go E2E results"
|
||||
|
||||
@@ -27,7 +27,7 @@
|
||||
<line x1="510.0" y1="212.0" x2="510.0" y2="220.0" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
|
||||
<rect x="28.0" y="224.0" width="964.0" height="94.0" rx="13" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
|
||||
<text x="48.0" y="251.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#EE4B12" font-weight="700" text-anchor="start" letter-spacing="1.5">AWS GATEWAY</text>
|
||||
<text x="48.0" y="272.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">spinifex/runtime/roles/awsgw · TLS termination · SigV4 verification · IAM policy eval · NATS routing</text>
|
||||
<text x="48.0" y="272.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">spinifex/services/awsgw · TLS termination · SigV4 verification · IAM policy eval · NATS routing</text>
|
||||
<rect x="46.0" y="287.0" width="68.2" height="22.0" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="80.1" y="301.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">TLS 1.3</text>
|
||||
<rect x="123.2" y="287.0" width="88.0" height="22.0" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
|
||||
|
Before Width: | Height: | Size: 20 KiB After Width: | Height: | Size: 20 KiB |
@@ -51,7 +51,7 @@
|
||||
<!-- ═══ AWS GATEWAY ════════════════════════════════════ -->
|
||||
<rect x="16" y="154" width="868" height="104" rx="8" fill="url(#gw-grad)" stroke="#DC2626" stroke-width="1.3" stroke-opacity="0.75"/>
|
||||
<text x="32" y="176" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#F87171" font-weight="700" letter-spacing="1">AWS GATEWAY · port 9999</text>
|
||||
<text x="876" y="176" font-family="'SF Mono', ui-monospace, monospace" font-size="9" fill="#71717a" text-anchor="end">spinifex/runtime/roles/awsgw/awsgw.go</text>
|
||||
<text x="876" y="176" font-family="'SF Mono', ui-monospace, monospace" font-size="9" fill="#71717a" text-anchor="end">spinifex/services/awsgw/awsgw.go</text>
|
||||
|
||||
<!-- Two-column feature bullets -->
|
||||
<g font-family="'Segoe UI', sans-serif" font-size="10.5" fill="#f5f5f5">
|
||||
|
||||
|
Before Width: | Height: | Size: 12 KiB After Width: | Height: | Size: 12 KiB |
@@ -1,79 +0,0 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="1020" height="836" viewBox="0 0 1020 836" role="img" aria-label="Spinifex on a single Exoscale instance: the instance sits on the zone's public network with one interface, eth0, holding its own address and the default route. Manual Elastic IPs are delivered to eth0 and routed by the host into OVN through the spx-nat transit veth, and an Exoscale block storage volume backs /var/lib/spinifex.">
|
||||
<rect x="0" y="0" width="1020" height="836" rx="20" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="28" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2.5">SPINIFEX ON EXOSCALE — SINGLE NODE</text>
|
||||
<text x="992" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="end">one instance · routed NAT · Elastic IPs</text>
|
||||
<text x="28" y="72" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="23" fill="#F5F3EE" font-weight="800" text-anchor="start">One Exoscale instance, single Spinifex node</text>
|
||||
<rect x="28" y="92" width="964" height="76" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="48" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">INTERNET</text>
|
||||
<text x="972" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">AWS SDKs, SSH and guest egress all arrive here</text>
|
||||
<rect x="46" y="126" width="260.0" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="176.0" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">aws-cli / Terraform / boto3 → :9999</text>
|
||||
<rect x="315.0" y="126" width="171.6" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="400.8" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">ssh → guest Elastic IP</text>
|
||||
<rect x="495.6" y="126" width="205.6" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="598.4" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">guest egress ← same address</text>
|
||||
<line x1="510" y1="170" x2="510" y2="186" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
|
||||
<rect x="28" y="188" width="964" height="582" rx="16" fill="#EE4B12" fill-opacity="0.022" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.45" stroke-dasharray="6 5"/>
|
||||
<text x="50" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2">EXOSCALE ZONE DE-FRA-1 · PUBLIC NETWORK 91.92.116.0/23</text>
|
||||
<text x="970" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">gateway 91.92.116.1 · security group open to all</text>
|
||||
<rect x="48" y="232" width="924" height="520" rx="14" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
|
||||
<text x="68" y="266" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="16" fill="#F5F3EE" font-weight="800" text-anchor="start">standard.huge</text>
|
||||
<text x="212" y="266" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="400" text-anchor="start">spinifex-node-01 · 8 vCPU / 32 GiB · Ubuntu 26.04</text>
|
||||
<text x="952" y="266" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">one host = one Spinifex region</text>
|
||||
<line x1="68" y1="282" x2="952" y2="282" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
|
||||
<rect x="68" y="298" width="434" height="108" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="88" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">eth0 — PUBLIC INTERFACE</text>
|
||||
<text x="482" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">the only NIC</text>
|
||||
<text x="88" y="356" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">91.92.116.250/23 · holds the default route</text>
|
||||
<text x="88" y="378" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">advertise IP · API :9999 · SSH · host masquerade</text>
|
||||
<rect x="518" y="298" width="434" height="108" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="538" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">ELASTIC IPs — MANUAL</text>
|
||||
<text x="932" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">attached to the instance</text>
|
||||
<text x="538" y="356" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">89.145.163.1/32 → delivered to eth0</text>
|
||||
<text x="538" y="378" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">the host routes each /32 into OVN, never owns it</text>
|
||||
<line x1="735" y1="406" x2="735" y2="422" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
|
||||
<rect x="68" y="424" width="884" height="40" rx="11" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
|
||||
<text x="88" y="449" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#EE4B12" font-weight="600" text-anchor="start">spx-nat-host ⇄ spx-nat-ovs</text>
|
||||
<text x="296" y="449" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#9AA1A8" font-weight="400" text-anchor="start">transit 100.127.0.0/24 · EIP/32 routed in · masquerade out eth0 for the rest</text>
|
||||
<line x1="510" y1="464" x2="510" y2="480" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
|
||||
<rect x="68" y="482" width="884" height="100" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="88" y="512" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">SPINIFEX CONTROL PLANE</text>
|
||||
<text x="932" y="512" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">every service is one binary: spx service <name></text>
|
||||
<rect x="86" y="530" width="93.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="132.5" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">awsgw :9999</text>
|
||||
<rect x="188.0" y="530" width="58.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="217.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">daemon</text>
|
||||
<rect x="255.0" y="530" width="44.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="277.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">vpcd</text>
|
||||
<rect x="308.0" y="530" width="44.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="330.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">NATS</text>
|
||||
<rect x="361.0" y="530" width="86.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="404.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">predastore</text>
|
||||
<rect x="456.0" y="530" width="86.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="499.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">viperblock</text>
|
||||
<rect x="551.0" y="530" width="79.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="590.5" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">northstar</text>
|
||||
<rect x="639.0" y="530" width="72.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="675.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">ui :3000</text>
|
||||
<text x="86" y="568" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">SigV4 in, NATS in the middle, QEMU/KVM (nested) · OVN/OVS · Predastore · Viperblock underneath</text>
|
||||
<line x1="248" y1="582" x2="248" y2="598" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
|
||||
<line x1="772" y1="582" x2="772" y2="598" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
|
||||
<rect x="68" y="600" width="532" height="130" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="88" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">TENANT VPCs — OVN br-int</text>
|
||||
<text x="580" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">geneve-local</text>
|
||||
<rect x="86" y="644" width="496" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="100" y="666" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">vpc-a 172.31.0.0/16</text>
|
||||
<text x="300" y="666" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-… 172.31.0.4 → 89.145.163.1</text>
|
||||
<rect x="86" y="686" width="496" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="100" y="708" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">vpc-b 10.0.0.0/16</text>
|
||||
<text x="300" y="708" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-… 10.0.1.9 · private, egress only</text>
|
||||
<rect x="616" y="600" width="336" height="130" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="636" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">BLOCK STORAGE VOLUME</text>
|
||||
<text x="932" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">256 GiB+</text>
|
||||
<text x="636" y="658" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">virtio-blk → /dev/vdb</text>
|
||||
<text x="636" y="680" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">/var/lib/spinifex</text>
|
||||
<text x="636" y="700" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11.5" fill="#5B636C" font-weight="400" text-anchor="start">found by-id, defaults,nofail</text>
|
||||
<text x="636" y="718" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11.5" fill="#5B636C" font-weight="400" text-anchor="start">viperblock · predastore · JetStream</text>
|
||||
<text x="28" y="796" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">A guest's public address is a manual Exoscale Elastic IP, delivered to eth0 addressed to itself — no private/public pair to translate.</text>
|
||||
<text x="28" y="814" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">The host routes it into OVN, so inbound and outbound are the same address and the guest sees its private IP only.</text>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 14 KiB |
@@ -1,123 +0,0 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="1020" height="796" viewBox="0 0 1020 796" role="img" aria-label="Spinifex across three Exoscale instances in one zone, spread by an anti-affinity group. Each has a public eth0 carrying its own Elastic IPs, a private-network eth1 for the cluster mesh, its own block storage volume and its share of the guests. Geneve, NATS JetStream, predastore erasure coding and the OVN raft join them east-west over the private network. This topology is not yet tested.">
|
||||
<rect x="0" y="0" width="1020" height="796" rx="20" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="28" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2.5">SPINIFEX ON EXOSCALE — THREE NODES</text>
|
||||
<text x="992" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="end">anti-affinity group · one zone · not yet tested</text>
|
||||
<text x="28" y="72" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="23" fill="#F5F3EE" font-weight="800" text-anchor="start">Three Exoscale instances, redundant multi-node install</text>
|
||||
<rect x="28" y="92" width="964" height="76" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="48" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">INTERNET</text>
|
||||
<text x="972" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">one endpoint, whichever node answers</text>
|
||||
<rect x="46" y="126" width="260.0" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="176.0" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">aws-cli / Terraform / boto3 → :9999</text>
|
||||
<rect x="315.0" y="126" width="171.6" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="400.8" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">ssh → guest Elastic IP</text>
|
||||
<rect x="495.6" y="126" width="205.6" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="598.4" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">guest egress ← same address</text>
|
||||
<line x1="510" y1="170" x2="510" y2="186" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
|
||||
<rect x="28" y="188" width="964" height="440" rx="16" fill="#EE4B12" fill-opacity="0.022" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.45" stroke-dasharray="6 5"/>
|
||||
<text x="50" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2">EXOSCALE ZONE DE-FRA-1 · PUBLIC NETWORK + PRIVATE NETWORK</text>
|
||||
<text x="970" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">one security group · one anti-affinity group</text>
|
||||
<rect x="48" y="232" width="300" height="380" rx="14" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
|
||||
<text x="64" y="260" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-01</text>
|
||||
<text x="332" y="260" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">standard.huge</text>
|
||||
<line x1="64" y1="272" x2="332" y2="272" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
|
||||
<rect x="64" y="284" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="78" y="305" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth0 91.92.116.250</text>
|
||||
<text x="78" y="323" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">public · default route · API</text>
|
||||
<rect x="64" y="342" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="78" y="363" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth1 10.10.0.11</text>
|
||||
<text x="78" y="381" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">private network · mesh · advertise · encap</text>
|
||||
<rect x="64" y="400" width="268" height="50" rx="10" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
|
||||
<text x="78" y="421" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start">EIP 89.145.163.1</text>
|
||||
<text x="78" y="439" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">→ eth0 · /32 via spx-nat-host</text>
|
||||
<rect x="64" y="462" width="51.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="89.5" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">awsgw</text>
|
||||
<rect x="124.0" y="462" width="57.6" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="152.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">daemon</text>
|
||||
<rect x="190.6" y="462" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="212.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">vpcd</text>
|
||||
<rect x="64" y="492" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="86.2" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">nats</text>
|
||||
<rect x="117.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="159.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">predastore</text>
|
||||
<rect x="210.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="252.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">viperblock</text>
|
||||
<rect x="64" y="526" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="76" y="548" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-23d2… 172.31.0.4 → 89.145.163.1</text>
|
||||
<rect x="64" y="568" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="76" y="590" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">volume /dev/vdb → /var/lib/spinifex</text>
|
||||
<rect x="360" y="232" width="300" height="380" rx="14" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
|
||||
<text x="376" y="260" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-02</text>
|
||||
<text x="644" y="260" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">standard.huge</text>
|
||||
<line x1="376" y1="272" x2="644" y2="272" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
|
||||
<rect x="376" y="284" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="390" y="305" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth0 91.92.117.14</text>
|
||||
<text x="390" y="323" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">public · default route · API</text>
|
||||
<rect x="376" y="342" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="390" y="363" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth1 10.10.0.12</text>
|
||||
<text x="390" y="381" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">private network · mesh · advertise · encap</text>
|
||||
<rect x="376" y="400" width="268" height="50" rx="10" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
|
||||
<text x="390" y="421" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start">EIP 89.145.160.107</text>
|
||||
<text x="390" y="439" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">→ eth0 · /32 via spx-nat-host</text>
|
||||
<rect x="376" y="462" width="51.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="401.5" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">awsgw</text>
|
||||
<rect x="436.0" y="462" width="57.6" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="464.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">daemon</text>
|
||||
<rect x="502.6" y="462" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="524.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">vpcd</text>
|
||||
<rect x="376" y="492" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="398.2" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">nats</text>
|
||||
<rect x="429.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="471.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">predastore</text>
|
||||
<rect x="522.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="564.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">viperblock</text>
|
||||
<rect x="376" y="526" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="388" y="548" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-6b91… 172.31.0.9 → 89.145.160.107</text>
|
||||
<rect x="376" y="568" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="388" y="590" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">volume /dev/vdb → /var/lib/spinifex</text>
|
||||
<rect x="672" y="232" width="300" height="380" rx="14" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
|
||||
<text x="688" y="260" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-03</text>
|
||||
<text x="956" y="260" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">standard.huge</text>
|
||||
<line x1="688" y1="272" x2="956" y2="272" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
|
||||
<rect x="688" y="284" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="702" y="305" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth0 91.92.116.88</text>
|
||||
<text x="702" y="323" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">public · default route · API</text>
|
||||
<rect x="688" y="342" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="702" y="363" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth1 10.10.0.13</text>
|
||||
<text x="702" y="381" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">private network · mesh · advertise · encap</text>
|
||||
<rect x="688" y="400" width="268" height="50" rx="10" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
|
||||
<text x="702" y="421" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start">EIP 89.145.162.131</text>
|
||||
<text x="702" y="439" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">→ eth0 · /32 via spx-nat-host</text>
|
||||
<rect x="688" y="462" width="51.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="713.5" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">awsgw</text>
|
||||
<rect x="748.0" y="462" width="57.6" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="776.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">daemon</text>
|
||||
<rect x="814.6" y="462" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="836.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">vpcd</text>
|
||||
<rect x="688" y="492" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="710.2" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">nats</text>
|
||||
<rect x="741.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="783.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">predastore</text>
|
||||
<rect x="834.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="876.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">viperblock</text>
|
||||
<rect x="688" y="526" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="700" y="548" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-0ae4… 172.31.1.6 → 89.145.162.131</text>
|
||||
<rect x="688" y="568" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="700" y="590" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">volume /dev/vdb → /var/lib/spinifex</text>
|
||||
<line x1="510" y1="628" x2="510" y2="644" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
|
||||
<rect x="28" y="646" width="964" height="70" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="48" y="676" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">EAST–WEST, OVER THE EXOSCALE PRIVATE NETWORK</text>
|
||||
<text x="972" y="676" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">no Elastic IP is involved · MTU 1500</text>
|
||||
<rect x="46" y="684" width="124.0" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="108.0" y="698.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">Geneve UDP 6081</text>
|
||||
<rect x="179.0" y="684" width="137.6" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="247.8" y="698.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">NATS JetStream R3</text>
|
||||
<rect x="325.6" y="684" width="144.4" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="397.8" y="698.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">predastore RS(2,1)</text>
|
||||
<rect x="479.0" y="684" width="151.2" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="554.6" y="698.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">OVN Northbound raft</text>
|
||||
<rect x="639.2" y="684" width="56.0" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="667.2" y="698.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">OVSDB</text>
|
||||
<text x="28" y="744" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Guest-to-guest traffic rides Geneve between the nodes' private-network interfaces — it never touches eth0 or an Elastic IP.</text>
|
||||
<text x="28" y="762" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Each node carries the Elastic IPs of its own guests. Moving one between nodes is an Exoscale API call, not yet automated.</text>
|
||||
<text x="28" y="780" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Three nodes is the smallest cluster that tolerates losing one; the anti-affinity group keeps them on different hypervisors.</text>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 22 KiB |
@@ -1,80 +0,0 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="1020" height="836" viewBox="0 0 1020 836" role="img" aria-label="Spinifex on a single OCI instance: the instance sits inside the VCN public subnet 10.200.0.0/23; its primary VNIC carries the default route and the cluster mesh, its secondary VNIC is the br-wan bridge holding registered secondary private IPs paired with reserved public IPs; guests reach them through the spx-nat transit veth, and an iSCSI block volume backs /var/lib/spinifex.">
|
||||
<defs><linearGradient id="nats" x1="0" y1="0" x2="1" y2="0"><stop offset="0" stop-color="#F5652E"/><stop offset="1" stop-color="#C63C0E"/></linearGradient></defs>
|
||||
<rect x="0" y="0" width="1020" height="836" rx="20" fill="#14181D" stroke="#2A2F36" stroke-width="1"/>
|
||||
<text x="28" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2.5">SPINIFEX ON OCI — SINGLE NODE</text>
|
||||
<text x="992" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="end">one instance · routed NAT · real public addresses</text>
|
||||
<text x="28" y="72" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="23" fill="#F5F3EE" font-weight="800" text-anchor="start">One OCI instance, single Spinifex node</text>
|
||||
<rect x="28" y="92" width="964" height="76" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="48" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">INTERNET</text>
|
||||
<text x="972" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">AWS SDKs, SSH and guest egress all arrive here</text>
|
||||
<rect x="46" y="126" width="272.7" height="22" rx="7" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="182.3" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">aws-cli / Terraform / boto3 → :9999</text>
|
||||
<rect x="327.7" y="126" width="177.2" height="22" rx="7" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="416.3" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">ssh → guest public IP</text>
|
||||
<rect x="513.92" y="126" width="218.1" height="22" rx="7" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="623.0" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">guest egress ← same address</text>
|
||||
<line x1="510.0" y1="170" x2="510.0" y2="186" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
|
||||
<rect x="28" y="188" width="964" height="582" rx="16" fill="#EE4B12" fill-opacity="0.022" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.45" stroke-dasharray="6 5"/>
|
||||
<text x="50" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2">OCI VCN 10.200.0.0/16 · PUBLIC SUBNET 10.200.0.0/23</text>
|
||||
<text x="970" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">virtual router 10.200.0.1 · internet gateway</text>
|
||||
<rect x="48" y="232" width="924" height="520" rx="14" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
|
||||
<text x="68" y="266" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="16" fill="#F5F3EE" font-weight="800" text-anchor="start">BM.Standard.E2.64</text>
|
||||
<text x="260" y="266" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="400" text-anchor="start">spinifex-node-01 · FAULT-DOMAIN-1</text>
|
||||
<text x="952" y="266" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">one host = one Spinifex region</text>
|
||||
<line x1="68" y1="282" x2="952" y2="282" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
|
||||
<rect x="68" y="298" width="434.0" height="108" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="88" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">VNIC 0 — PRIMARY</text>
|
||||
<text x="482.0" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">enp0s9</text>
|
||||
<text x="88" y="356" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">10.200.0.252/23 · holds the default route</text>
|
||||
<text x="88" y="378" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">advertise IP · ovn-encap-ip · NATS · API :9999</text>
|
||||
<rect x="518.0" y="298" width="434.0" height="108" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="538.0" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">VNIC 1 — SECONDARY</text>
|
||||
<text x="932.0" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">br-wan (Linux bridge)</text>
|
||||
<text x="538.0" y="356" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">10.200.1.233/32 + ip rule → table 200</text>
|
||||
<text x="538.0" y="378" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">secondary private IPs ⇄ reserved public IPs</text>
|
||||
<line x1="735.0" y1="406" x2="735.0" y2="422" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
|
||||
<rect x="68" y="424" width="884" height="40" rx="11" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
|
||||
<text x="88" y="449" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#EE4B12" font-weight="600" text-anchor="start">spx-nat-host ⇄ spx-nat-ovs</text>
|
||||
<text x="296" y="449" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#9AA1A8" font-weight="400" text-anchor="start">transit 100.127.0.0/24 · nft chain spinifex-nat-egress masquerades out br-wan</text>
|
||||
<line x1="510.0" y1="464" x2="510.0" y2="480" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
|
||||
<rect x="68" y="482" width="884" height="100" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="88" y="512" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">SPINIFEX CONTROL PLANE</text>
|
||||
<text x="932" y="512" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">every service is one binary: spx service <name></text>
|
||||
<rect x="86" y="530" width="109.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="140.5" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">awsgw :9999</text>
|
||||
<rect x="204.01999999999998" y="530" width="74.9" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="241.5" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">daemon</text>
|
||||
<rect x="287.94" y="530" width="61.3" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="318.6" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">vpcd</text>
|
||||
<rect x="358.22" y="530" width="61.3" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="388.9" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">NATS</text>
|
||||
<rect x="428.5" y="530" width="102.2" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="479.6" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">predastore</text>
|
||||
<rect x="539.7" y="530" width="102.2" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="590.8" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">viperblock</text>
|
||||
<rect x="650.9000000000001" y="530" width="95.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="698.6" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">northstar</text>
|
||||
<rect x="755.2800000000001" y="530" width="88.6" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="799.6" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">ui :3000</text>
|
||||
<text x="86" y="568" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">SigV4 in, NATS in the middle, QEMU/KVM · OVN/OVS · Predastore · Viperblock underneath</text>
|
||||
<line x1="248" y1="582" x2="248" y2="598" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
|
||||
<line x1="772" y1="582" x2="772" y2="598" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
|
||||
<rect x="68" y="600" width="532" height="130" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="88" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">TENANT VPCs — OVN br-int</text>
|
||||
<text x="580" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">geneve-local</text>
|
||||
<rect x="86" y="644" width="496" height="34" rx="9" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="100" y="666" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">vpc-a 10.0.0.0/16</text>
|
||||
<text x="310" y="666" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-… 10.0.1.4 → 150.230.13.131</text>
|
||||
<rect x="86" y="686" width="496" height="34" rx="9" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="100" y="708" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">vpc-b 172.31.0.0/16</text>
|
||||
<text x="310" y="708" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-… 172.31.4.9 · private, egress only</text>
|
||||
<rect x="616" y="600" width="336" height="130" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="636" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">BLOCK VOLUME — iSCSI</text>
|
||||
<text x="932" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">256 GB+</text>
|
||||
<text x="636" y="658" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">169.254.2.2:3260 → /dev/sdb</text>
|
||||
<text x="636" y="680" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">/var/lib/spinifex</text>
|
||||
<text x="636" y="700" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11.5" fill="#5B636C" font-weight="400" text-anchor="start">mounted by UUID, defaults,_netdev,nofail</text>
|
||||
<text x="636" y="718" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11.5" fill="#5B636C" font-weight="400" text-anchor="start">viperblock · predastore · JetStream</text>
|
||||
<text x="28" y="796" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">A guest's public address is a real OCI reserved public IP, attached to a secondary private IP on br-wan.</text>
|
||||
<text x="28" y="814" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Inbound and outbound are therefore the same address, and OCI's per-VNIC source check passes.</text>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 15 KiB |
@@ -1,116 +0,0 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="1020" height="796" viewBox="0 0 1020 796" role="img" aria-label="Spinifex across three OCI instances: all three sit inside the VCN public subnet 10.200.0.0/23, one per fault domain. Each carries a primary VNIC for the cluster mesh and a br-wan secondary VNIC holding its reserved public addresses, its own iSCSI block volume, and its share of the guests. Geneve, NATS JetStream, predastore erasure coding and the OVN raft join them east-west.">
|
||||
<defs><linearGradient id="nats" x1="0" y1="0" x2="1" y2="0"><stop offset="0" stop-color="#F5652E"/><stop offset="1" stop-color="#C63C0E"/></linearGradient></defs>
|
||||
<rect x="0" y="0" width="1020" height="796" rx="20" fill="#14181D" stroke="#2A2F36" stroke-width="1"/>
|
||||
<text x="28" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2.5">SPINIFEX ON OCI — THREE NODES</text>
|
||||
<text x="992" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="end">three fault domains · one region · one API endpoint</text>
|
||||
<text x="28" y="72" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="23" fill="#F5F3EE" font-weight="800" text-anchor="start">Three OCI instances, redundant multi-node install</text>
|
||||
<rect x="28" y="92" width="964" height="76" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="48" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">INTERNET</text>
|
||||
<text x="972" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">one endpoint, whichever node answers</text>
|
||||
<rect x="46" y="126" width="272.7" height="22" rx="7" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="182.3" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">aws-cli / Terraform / boto3 → :9999</text>
|
||||
<rect x="327.7" y="126" width="177.2" height="22" rx="7" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="416.3" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">ssh → guest public IP</text>
|
||||
<rect x="513.92" y="126" width="218.1" height="22" rx="7" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="623.0" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">guest egress ← same address</text>
|
||||
<line x1="510.0" y1="170" x2="510.0" y2="186" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
|
||||
<rect x="28" y="188" width="964" height="464" rx="16" fill="#EE4B12" fill-opacity="0.022" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.45" stroke-dasharray="6 5"/>
|
||||
<text x="50" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2">OCI VCN 10.200.0.0/16 · PUBLIC SUBNET 10.200.0.0/23</text>
|
||||
<text x="970" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">virtual router 10.200.0.1 · internet gateway</text>
|
||||
<rect x="48.0" y="232" width="297.3" height="272" rx="13" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
|
||||
<text x="66.0" y="262" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-01</text>
|
||||
<text x="327.3333333333333" y="262" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#EE4B12" font-weight="400" text-anchor="end" fill-opacity="0.85">FAULT-DOMAIN-1</text>
|
||||
<text x="66.0" y="280" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">BM.Standard.E2.64</text>
|
||||
<line x1="66.0" y1="292" x2="327.3333333333333" y2="292" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
|
||||
<text x="66.0" y="314" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">VNIC 0 10.200.0.252</text>
|
||||
<text x="66.0" y="330" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11" fill="#5B636C" font-weight="400" text-anchor="start">mesh · advertise · encap</text>
|
||||
<text x="66.0" y="352" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">VNIC 1 br-wan 10.200.1.233/32</text>
|
||||
<text x="66.0" y="368" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11" fill="#5B636C" font-weight="400" text-anchor="start">public IP pairs · table 200</text>
|
||||
<rect x="66.0" y="382" width="51.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="91.5" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">awsgw</text>
|
||||
<rect x="123.0" y="382" width="57.2" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="151.6" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">daemon</text>
|
||||
<rect x="186.2" y="382" width="44.8" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="208.6" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">vpcd</text>
|
||||
<rect x="66.0" y="408" width="44.8" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="88.4" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">nats</text>
|
||||
<rect x="116.8" y="408" width="82.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="157.8" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">predastore</text>
|
||||
<rect x="204.8" y="408" width="82.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="245.8" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">viperblock</text>
|
||||
<rect x="66.0" y="438" width="261.3" height="24" rx="8" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="76.0" y="454" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-23d2… 10.0.1.4 → 150.230.13.131</text>
|
||||
<rect x="66.0" y="466" width="261.3" height="24" rx="8" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="76.0" y="482" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">iSCSI /dev/sdb → /var/lib/spinifex</text>
|
||||
<line x1="345.3333333333333" y1="364" x2="361.3333333333333" y2="364" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.5" stroke-dasharray="2 4"/>
|
||||
<rect x="361.3333333333333" y="232" width="297.3" height="272" rx="13" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
|
||||
<text x="379.3333333333333" y="262" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-02</text>
|
||||
<text x="640.6666666666666" y="262" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#EE4B12" font-weight="400" text-anchor="end" fill-opacity="0.85">FAULT-DOMAIN-2</text>
|
||||
<text x="379.3333333333333" y="280" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">BM.Standard.E2.64</text>
|
||||
<line x1="379.3333333333333" y1="292" x2="640.6666666666666" y2="292" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
|
||||
<text x="379.3333333333333" y="314" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">VNIC 0 10.200.0.118</text>
|
||||
<text x="379.3333333333333" y="330" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11" fill="#5B636C" font-weight="400" text-anchor="start">mesh · advertise · encap</text>
|
||||
<text x="379.3333333333333" y="352" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">VNIC 1 br-wan 10.200.1.96/32</text>
|
||||
<text x="379.3333333333333" y="368" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11" fill="#5B636C" font-weight="400" text-anchor="start">public IP pairs · table 200</text>
|
||||
<rect x="379.3333333333333" y="382" width="51.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="404.8" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">awsgw</text>
|
||||
<rect x="436.3333333333333" y="382" width="57.2" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="464.9" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">daemon</text>
|
||||
<rect x="499.5333333333333" y="382" width="44.8" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="521.9" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">vpcd</text>
|
||||
<rect x="379.3333333333333" y="408" width="44.8" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="401.7" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">nats</text>
|
||||
<rect x="430.1333333333333" y="408" width="82.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="471.1" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">predastore</text>
|
||||
<rect x="518.1333333333333" y="408" width="82.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="559.1" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">viperblock</text>
|
||||
<rect x="379.3333333333333" y="438" width="261.3" height="24" rx="8" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="389.3333333333333" y="454" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-6b91… 10.0.1.9 → 137.23.9.125</text>
|
||||
<rect x="379.3333333333333" y="466" width="261.3" height="24" rx="8" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="389.3333333333333" y="482" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">iSCSI /dev/sdb → /var/lib/spinifex</text>
|
||||
<line x1="658.6666666666666" y1="364" x2="674.6666666666666" y2="364" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.5" stroke-dasharray="2 4"/>
|
||||
<rect x="674.6666666666666" y="232" width="297.3" height="272" rx="13" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
|
||||
<text x="692.6666666666666" y="262" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-03</text>
|
||||
<text x="954.0" y="262" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#EE4B12" font-weight="400" text-anchor="end" fill-opacity="0.85">FAULT-DOMAIN-3</text>
|
||||
<text x="692.6666666666666" y="280" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">BM.Standard.E2.64</text>
|
||||
<line x1="692.6666666666666" y1="292" x2="954.0" y2="292" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
|
||||
<text x="692.6666666666666" y="314" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">VNIC 0 10.200.0.41</text>
|
||||
<text x="692.6666666666666" y="330" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11" fill="#5B636C" font-weight="400" text-anchor="start">mesh · advertise · encap</text>
|
||||
<text x="692.6666666666666" y="352" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">VNIC 1 br-wan 10.200.1.12/32</text>
|
||||
<text x="692.6666666666666" y="368" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11" fill="#5B636C" font-weight="400" text-anchor="start">public IP pairs · table 200</text>
|
||||
<rect x="692.6666666666666" y="382" width="51.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="718.2" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">awsgw</text>
|
||||
<rect x="749.6666666666666" y="382" width="57.2" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="778.3" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">daemon</text>
|
||||
<rect x="812.8666666666667" y="382" width="44.8" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="835.3" y="394.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">vpcd</text>
|
||||
<rect x="692.6666666666666" y="408" width="44.8" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="715.1" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">nats</text>
|
||||
<rect x="743.4666666666666" y="408" width="82.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="784.5" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">predastore</text>
|
||||
<rect x="831.4666666666666" y="408" width="82.0" height="20" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="872.5" y="420.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">viperblock</text>
|
||||
<rect x="692.6666666666666" y="438" width="261.3" height="24" rx="8" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="702.6666666666666" y="454" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-0ae4… 10.0.2.6 → 150.230.4.88</text>
|
||||
<rect x="692.6666666666666" y="466" width="261.3" height="24" rx="8" fill="none" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="702.6666666666666" y="482" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">iSCSI /dev/sdb → /var/lib/spinifex</text>
|
||||
<rect x="48" y="524" width="924" height="104" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="68" y="554" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">EAST–WEST, INSIDE THE VCN</text>
|
||||
<text x="952" y="554" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">no public address is involved</text>
|
||||
<rect x="66" y="574" width="136.3" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="134.2" y="588.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">Geneve UDP 6081</text>
|
||||
<rect x="211.3" y="574" width="149.9" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="286.3" y="588.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">NATS JetStream R3</text>
|
||||
<rect x="370.24" y="574" width="156.8" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="448.6" y="588.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">predastore RS(2,1)</text>
|
||||
<rect x="536.0" y="574" width="163.6" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="617.8" y="588.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">OVN Northbound raft</text>
|
||||
<rect x="708.58" y="574" width="68.1" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="742.6" y="588.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">OVSDB</text>
|
||||
<text x="66" y="612" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Guest-to-guest traffic rides Geneve between the nodes' primary VNICs — it never leaves the subnet and never touches br-wan.</text>
|
||||
<rect x="28" y="668" width="964" height="46" rx="12" fill="#EE4B12" fill-opacity="0.07" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.55"/>
|
||||
<text x="48" y="688" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="13" fill="#F5F3EE" font-weight="700" text-anchor="start">A guest that moves takes its address with it.</text>
|
||||
<text x="48" y="705" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#9AA1A8" font-weight="400" text-anchor="start">The reserved public IP follows the secondary private IP to whichever node's br-wan the guest lands on — the tenant sees no change.</text>
|
||||
<text x="28" y="756" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Three nodes is the smallest cluster that tolerates losing one: NATS JetStream keeps quorum, predastore RS(2,1)</text>
|
||||
<text x="28" y="774" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">keeps the data readable, and the OVN Northbound raft keeps a leader. One node per fault domain is what makes that real.</text>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 21 KiB |
+6
-26
@@ -23,6 +23,11 @@ updates:
|
||||
timezone: Australia/Sydney
|
||||
cooldown:
|
||||
default-days: 3
|
||||
ignore:
|
||||
- dependency-name: "*"
|
||||
update-types:
|
||||
- "version-update:semver-minor"
|
||||
- "version-update:semver-patch"
|
||||
groups:
|
||||
github-actions:
|
||||
patterns:
|
||||
@@ -30,7 +35,7 @@ updates:
|
||||
|
||||
- package-ecosystem: npm
|
||||
target-branch: dev
|
||||
directory: /spinifex/runtime/roles/spinifexui/frontend
|
||||
directory: /spinifex/services/spinifexui/frontend
|
||||
schedule:
|
||||
interval: weekly
|
||||
day: sunday
|
||||
@@ -45,28 +50,3 @@ updates:
|
||||
npm-dependencies:
|
||||
patterns:
|
||||
- "*"
|
||||
|
||||
# Base images are pinned by digest, so this is what moves them. The golang
|
||||
# version is deliberate and changes with go.mod; only its digest moves here.
|
||||
- package-ecosystem: docker
|
||||
target-branch: dev
|
||||
directories:
|
||||
- /build
|
||||
- /scripts
|
||||
- /docs/terraform-workbooks/demo-app
|
||||
schedule:
|
||||
interval: weekly
|
||||
day: sunday
|
||||
timezone: Australia/Sydney
|
||||
cooldown:
|
||||
default-days: 3
|
||||
ignore:
|
||||
- dependency-name: golang
|
||||
update-types:
|
||||
- "version-update:semver-major"
|
||||
- "version-update:semver-minor"
|
||||
- "version-update:semver-patch"
|
||||
groups:
|
||||
docker-images:
|
||||
patterns:
|
||||
- "*"
|
||||
|
||||
@@ -143,52 +143,5 @@
|
||||
"extra": "",
|
||||
"suite": "storagefault",
|
||||
"timeout_minutes": 100
|
||||
},
|
||||
{
|
||||
"cell": 30,
|
||||
"install": "binary",
|
||||
"topo": "real-multi",
|
||||
"net": "nat",
|
||||
"host": "debian-13",
|
||||
"arch": "x86_64",
|
||||
"runner": "ci-multi",
|
||||
"extra": "nat-multi",
|
||||
"suite": ""
|
||||
},
|
||||
{
|
||||
"cell": 31,
|
||||
"install": "binary",
|
||||
"topo": "real-multi",
|
||||
"net": "static",
|
||||
"host": "debian-13",
|
||||
"arch": "x86_64",
|
||||
"runner": "ci-multi",
|
||||
"extra": "",
|
||||
"suite": "instancerecovery",
|
||||
"timeout_minutes": 105
|
||||
},
|
||||
{
|
||||
"cell": 32,
|
||||
"install": "binary",
|
||||
"topo": "real-multi",
|
||||
"net": "static",
|
||||
"host": "debian-13",
|
||||
"arch": "x86_64",
|
||||
"runner": "ci-multi",
|
||||
"extra": "",
|
||||
"suite": "lbrecovery",
|
||||
"timeout_minutes": 90
|
||||
},
|
||||
{
|
||||
"cell": 33,
|
||||
"install": "binary",
|
||||
"topo": "real-multi",
|
||||
"net": "static",
|
||||
"host": "debian-13",
|
||||
"arch": "x86_64",
|
||||
"runner": "ci-multi",
|
||||
"extra": "",
|
||||
"suite": "kvquorum",
|
||||
"timeout_minutes": 55
|
||||
}
|
||||
]
|
||||
|
||||
@@ -5,17 +5,3 @@
|
||||
## Changes
|
||||
|
||||
<!-- The notable changes, as bullets. -->
|
||||
|
||||
## Account-scoped lifecycle impact
|
||||
|
||||
<!-- Complete this section only when the PR changes account-scoped resource lifecycle, account termination, resource cleanup, or a lifecycle boundary. Otherwise write: N/A. -->
|
||||
|
||||
- Lifecycle owner and resource family:
|
||||
- Owner-local ordering, recovery, retained-state, or force impact:
|
||||
- Cross-domain dependency or ownership impact:
|
||||
- Evidence added or rechecked:
|
||||
- ADR-0008 lifecycle register / ADR-0001 migration-record / bead updates:
|
||||
|
||||
## Community Contribution License
|
||||
|
||||
- [ ] All community contributions in this pull request are licensed to Mulga under the terms of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). By creating this pull request I represent that I have the right to license these contributions to Mulga under the Apache License 2.0.
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Resolve the tag of each sibling repo (viperblock, predastore, northstar,
|
||||
# bluebottle) a release build checks out. A release ships what go.mod requires,
|
||||
# so each sibling is built at the version go.mod names, never a branch tip. A
|
||||
# pseudo-version has no tag to check out, so it fails the release rather than
|
||||
# silently building something else.
|
||||
#
|
||||
# Takes the path to spinifex's go.mod and writes viperblock=<tag>,
|
||||
# predastore=<tag>, northstar=<tag> and bluebottle=<tag> to $GITHUB_OUTPUT for
|
||||
# the checkout steps to consume.
|
||||
set -euo pipefail
|
||||
|
||||
gomod="${1:?usage: $0 <path to spinifex go.mod>}"
|
||||
: "${GITHUB_OUTPUT:?GITHUB_OUTPUT must be set}"
|
||||
|
||||
for repo in viperblock predastore northstar bluebottle; do
|
||||
module="github.com/mulgadc/${repo}"
|
||||
version=$(awk -v m="$module" '$1 == m { print $2; exit }' "$gomod")
|
||||
if [[ -z "$version" ]]; then
|
||||
echo "::error::${gomod} does not require ${module}"
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! "$version" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ || "$version" =~ [0-9]{14}-[0-9a-f]{12}$ ]]; then
|
||||
echo "::error::${module} is required at ${version}, which is not a release tag; tag ${repo} and bump go.mod before releasing"
|
||||
exit 1
|
||||
fi
|
||||
echo "${repo}: building against ${version}"
|
||||
echo "${repo}=${version}" >>"$GITHUB_OUTPUT"
|
||||
done
|
||||
@@ -40,24 +40,7 @@
|
||||
E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota storagefault"
|
||||
|
||||
# Suites runnable against a multi-node environment.
|
||||
#
|
||||
# instancerecovery is multi-only and needs three: recovery is inert below that,
|
||||
# because there is no survivor to move a guest to that is not the node that
|
||||
# failed. Like storagefault it takes a node away and freezes a cluster-wide
|
||||
# service, so it runs last and owns its environment.
|
||||
#
|
||||
# lbrecovery is multi-only and needs three for two reasons rather than one: a
|
||||
# survivor to move the load balancer to, and a node for its backends that is not
|
||||
# the load balancer's, so taking that node away does not take the backends with
|
||||
# it. It runs beside instancerecovery at the end because it also takes a node
|
||||
# away and owns its environment while it does.
|
||||
#
|
||||
# kvquorum is multi-only and needs three, for the same reason as those two: a
|
||||
# surviving majority. It is what proves the control plane's own state is
|
||||
# replicated rather than sitting on one node, so it runs before the pair that
|
||||
# take a node away — a cluster whose buckets are already wrong should report
|
||||
# that, not report a recovery failing.
|
||||
E2E_SUITES_MULTI="multinode lb cert quota kvquorum lbrecovery instancerecovery storagefault"
|
||||
E2E_SUITES_MULTI="multinode lb cert quota storagefault"
|
||||
|
||||
# grep -xE alternation form, for narrowing a requested set to the eligible one.
|
||||
E2E_SUITES_SINGLE_RE="$(printf '%s' "${E2E_SUITES_SINGLE}" | tr ' ' '|')"
|
||||
@@ -81,10 +64,6 @@ E2E_SUITES_MULTI_RE="$(printf '%s' "${E2E_SUITES_MULTI}" | tr ' ' '|')"
|
||||
# after a failed seal do not exist on one node, and they are the assertions
|
||||
# worth a nightly.
|
||||
E2E_SUITES_NIGHTLY_SINGLE="single cert iam"
|
||||
#
|
||||
# The multi-node nightly subset carries kvquorum's read-only half through
|
||||
# `multinode`, which sweeps every KV bucket rather than the three the daemon
|
||||
# owns. The node-loss half is too slow for a 35-minute cell and has its own.
|
||||
E2E_SUITES_NIGHTLY_MULTI="multinode cert lb"
|
||||
|
||||
# How long a suite gets, for the same reason the lists above live here: the
|
||||
@@ -109,23 +88,6 @@ e2e_suite_timeout() {
|
||||
# SIGSTOPped for seven minutes to prove a guest survives it. A full pass on
|
||||
# dev-prod is ~63 minutes, so the 30m default truncated it mid-suite.
|
||||
storagefault) echo "90m" ;;
|
||||
# instancerecovery is almost entirely waiting, and none of it is avoidable:
|
||||
# a dead owner has to be seen stale twice, a guest has to boot, and the
|
||||
# storage case waits for a pause measured at 4m47s and then holds four
|
||||
# minutes to prove nothing moved. Three tests, each taking a node away and
|
||||
# bringing it back, and the partition case adds a lease validity, a lease
|
||||
# TTL and a boot on top of everything the other two wait for.
|
||||
instancerecovery) echo "90m" ;;
|
||||
# lbrecovery pays for a load balancer twice: once to stand one up with two
|
||||
# backends placed around it, and once for the recovery itself — a stale
|
||||
# owner, a lease, a boot, HAProxy's own start, and its health checks against
|
||||
# the backends before it forwards anything. Then the node comes back and has
|
||||
# to be shown not to answer for an address it no longer holds.
|
||||
lbrecovery) echo "75m" ;;
|
||||
# kvquorum audits every bucket in seconds and then spends the rest of its
|
||||
# budget on a node stopping and starting: seven units down, a raft election,
|
||||
# and the whole service stack back up with every stream caught up.
|
||||
kvquorum) echo "40m" ;;
|
||||
*) echo "30m" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
@@ -1,137 +0,0 @@
|
||||
# This file is machine-generated by `gh actions-lock`.
|
||||
# Do not edit by hand; run `gh actions-lock` to update.
|
||||
# Docs: https://gh.io/actions-lockfile
|
||||
version: 'v0.0.3'
|
||||
workflows:
|
||||
'.github/workflows/apt-packages.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
'.github/workflows/dev-release.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/setup-go@v7.0.0'
|
||||
- 'docker/setup-buildx-action@v4.4.1'
|
||||
'.github/workflows/e2e-baremetal.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/create-github-app-token@v3.3.0'
|
||||
- 'actions/upload-artifact@v7.0.2'
|
||||
'.github/workflows/e2e-cloudvendor-nightly.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/create-github-app-token@v3.3.0'
|
||||
- 'actions/upload-artifact@v7.0.2'
|
||||
- 'hashicorp/setup-terraform@v4.0.1'
|
||||
'.github/workflows/e2e-ddil.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/create-github-app-token@v3.3.0'
|
||||
- 'actions/upload-artifact@v7.0.2'
|
||||
'.github/workflows/e2e-nightly.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/create-github-app-token@v3.3.0'
|
||||
- 'actions/download-artifact@v8.0.2'
|
||||
- 'actions/upload-artifact@v7.0.2'
|
||||
'.github/workflows/e2e-suites.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/create-github-app-token@v3.3.0'
|
||||
- 'actions/setup-go@v7.0.0'
|
||||
- 'actions/upload-artifact@v7.0.2'
|
||||
'.github/workflows/e2e.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/create-github-app-token@v3.3.0'
|
||||
- 'actions/download-artifact@v8.0.2'
|
||||
- 'actions/setup-go@v7.0.0'
|
||||
- 'actions/upload-artifact@v7.0.2'
|
||||
'.github/workflows/gpu-e2e.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/setup-go@v7.0.0'
|
||||
- 'actions/upload-artifact@v7.0.2'
|
||||
'.github/workflows/image-scripts.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/setup-go@v7.0.0'
|
||||
'.github/workflows/perf-3node.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/create-github-app-token@v3.3.0'
|
||||
- 'actions/setup-go@v7.0.0'
|
||||
- 'actions/upload-artifact@v7.0.2'
|
||||
'.github/workflows/publish-system-ami.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/setup-go@v7.0.0'
|
||||
'.github/workflows/release.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/create-github-app-token@v3.3.0'
|
||||
- 'actions/download-artifact@v8.0.2'
|
||||
- 'actions/setup-go@v7.0.0'
|
||||
- 'actions/upload-artifact@v7.0.2'
|
||||
- 'docker/setup-buildx-action@v4.4.1'
|
||||
- 'softprops/action-gh-release@v3.0.3'
|
||||
'.github/workflows/segscan-oracle.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/create-github-app-token@v3.3.0'
|
||||
- 'actions/setup-go@v7.0.0'
|
||||
'.github/workflows/spinifex.yml':
|
||||
- 'actions/cache@v6.1.0'
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/setup-go@v7.0.0'
|
||||
- 'golangci/golangci-lint-action@v9.3.0'
|
||||
'.github/workflows/ui.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/setup-node@v7.1.0'
|
||||
- 'pnpm/action-setup@v6.1.0'
|
||||
dependencies:
|
||||
'actions/cache@v6.1.0':
|
||||
ref: 'v6.1.0'
|
||||
commit: 'sha1-55cc8345863c7cc4c66a329aec7e433d2d1c52a9'
|
||||
owner_id: 44036562
|
||||
repo_id: 215566462
|
||||
'actions/checkout@v7.0.1':
|
||||
ref: 'v7.0.1'
|
||||
commit: 'sha1-3d3c42e5aac5ba805825da76410c181273ba90b1'
|
||||
owner_id: 44036562
|
||||
repo_id: 197814629
|
||||
'actions/create-github-app-token@v3.3.0':
|
||||
ref: 'v3.3.0'
|
||||
commit: 'sha1-64ab729a6294000f5aa4a4bc0b73d1f32bc45510'
|
||||
owner_id: 44036562
|
||||
repo_id: 642580244
|
||||
'actions/download-artifact@v8.0.2':
|
||||
ref: 'v8.0.2'
|
||||
commit: 'sha1-9000827ccba6bdab643e8b6fd33ac0654aef8333'
|
||||
owner_id: 44036562
|
||||
repo_id: 192626254
|
||||
'actions/setup-go@v7.0.0':
|
||||
ref: 'v7.0.0'
|
||||
commit: 'sha1-b7ad1dad31e06c5925ef5d2fc7ad053ef454303e'
|
||||
owner_id: 44036562
|
||||
repo_id: 192624594
|
||||
'actions/setup-node@v7.1.0':
|
||||
ref: 'v7.1.0'
|
||||
commit: 'sha1-949feb2413d6458794dcd2491c4babbbce0c15c1'
|
||||
owner_id: 44036562
|
||||
repo_id: 189476904
|
||||
'actions/upload-artifact@v7.0.2':
|
||||
ref: 'v7.0.2'
|
||||
commit: 'sha1-cf430e030ddbb5b0abf93d22962f4752f3646cd9'
|
||||
owner_id: 44036562
|
||||
repo_id: 192625955
|
||||
'docker/setup-buildx-action@v4.4.1':
|
||||
ref: 'v4.4.1'
|
||||
commit: 'sha1-f87e5991a6d7451dcb8d9637bfbc97413f497069'
|
||||
owner_id: 5429470
|
||||
repo_id: 288485773
|
||||
'golangci/golangci-lint-action@v9.3.0':
|
||||
ref: 'v9.3.0'
|
||||
commit: 'sha1-ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a'
|
||||
owner_id: 35628013
|
||||
repo_id: 232539879
|
||||
'hashicorp/setup-terraform@v4.0.1':
|
||||
ref: 'v4.0.1'
|
||||
commit: 'sha1-dfe3c3f87815947d99a8997f908cb6525fc44e9e'
|
||||
owner_id: 761456
|
||||
repo_id: 258271226
|
||||
'pnpm/action-setup@v6.1.0':
|
||||
ref: 'v6.1.0'
|
||||
commit: 'sha1-ea17c68df8912ef543352723c149a84f56e3d413'
|
||||
owner_id: 21320719
|
||||
repo_id: 262289747
|
||||
'softprops/action-gh-release@v3.0.3':
|
||||
ref: 'v3.0.3'
|
||||
commit: 'sha1-efb35369e0ad2afab669f228072c1b0d510eae64'
|
||||
owner_id: 2242
|
||||
repo_id: 204253808
|
||||
@@ -40,7 +40,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Resolve runtime package list
|
||||
run: ./scripts/lint-apt-packages.sh ${{ matrix.image }}
|
||||
|
||||
@@ -1,185 +0,0 @@
|
||||
name: Dev Prerelease
|
||||
|
||||
# Publishes the dev channel that install.mulgadc.com already serves. setup.sh has
|
||||
# understood INSTALL_SPINIFEX_CHANNEL=dev since it was written and the install
|
||||
# Worker resolves that channel to the newest GitHub prerelease, but nothing has
|
||||
# ever produced one -- so /download/dev/amd64 returned 404. This is the producer.
|
||||
#
|
||||
# A separate workflow rather than conditionals in release.yml, because a dev build
|
||||
# is a different artifact and not a release with flags off: amd64 only, no ISO, no
|
||||
# AMI refresh, no bare-metal gate, and nothing that touches a self-hosted runner.
|
||||
on:
|
||||
push:
|
||||
branches: [dev]
|
||||
# Nothing here changes the binary, and a prerelease per docs commit would
|
||||
# churn the channel our contacts are installing from.
|
||||
paths-ignore:
|
||||
- "docs/**"
|
||||
- "**/*.md"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
# Supersede rather than queue: on a run of merges to dev only the newest commit is
|
||||
# worth publishing, and the older build would land after it and become the channel.
|
||||
concurrency:
|
||||
group: dev-release
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
prerelease:
|
||||
runs-on: ubuntu-26.04
|
||||
steps:
|
||||
# Full history and tags, because the tag name is git describe and a shallow
|
||||
# clone has no tag to describe against.
|
||||
- name: Checkout Spinifex
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
path: spinifex
|
||||
fetch-depth: 0
|
||||
|
||||
# dev, not each repository's default branch, which is what release.yml takes.
|
||||
# A dev build of spinifex against main of the others is a pairing nobody runs.
|
||||
- name: Checkout Predastore
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
ref: dev
|
||||
path: predastore
|
||||
|
||||
- name: Checkout Viperblock
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/viperblock
|
||||
ref: dev
|
||||
path: viperblock
|
||||
|
||||
- name: Checkout Northstar
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/northstar
|
||||
ref: dev
|
||||
path: northstar
|
||||
|
||||
- name: Checkout Bluebottle
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/bluebottle
|
||||
ref: dev
|
||||
path: bluebottle
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4.4.1
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7.0.0
|
||||
with:
|
||||
go-version-file: spinifex/go.mod
|
||||
cache: false
|
||||
|
||||
# The tag carries the describe string so the commit is recoverable from the
|
||||
# version alone, and the -dev suffix is what release.yml's own convention
|
||||
# already treats as a prerelease. VERSION must equal the tag: the install
|
||||
# Worker derives the asset name from the tag it resolves.
|
||||
- name: Resolve the version
|
||||
id: version
|
||||
working-directory: spinifex
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Describing against a dev tag would nest it, growing the name every release.
|
||||
TAG="$(git describe --tags --always --exclude '*-dev')-dev"
|
||||
case "$TAG" in
|
||||
v*) ;;
|
||||
*) echo "::error::git describe gave '$TAG', which is not a v-prefixed version"; exit 1 ;;
|
||||
esac
|
||||
if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
|
||||
echo "::notice::$TAG already published; nothing to do"
|
||||
echo "exists=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
echo "tag=$TAG" >> "$GITHUB_OUTPUT"
|
||||
echo "Publishing $TAG"
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
|
||||
- name: Build microVM artifacts (lb-agent + kernel + initramfs)
|
||||
if: steps.version.outputs.exists != 'true'
|
||||
run: cd spinifex && make build-microvm-image
|
||||
|
||||
- name: Build amd64 Tarball
|
||||
if: steps.version.outputs.exists != 'true'
|
||||
env:
|
||||
VERSION: ${{ steps.version.outputs.tag }}
|
||||
run: cd spinifex && make distro-amd64 VERSION="$VERSION"
|
||||
|
||||
# --prerelease on create, never a later promote step. release.yml sets it in
|
||||
# a job that runs after its verification gates, which leaves a window where
|
||||
# the release is not flagged; GitHub's releases/latest excludes prereleases
|
||||
# only, so in that window a dev build is what every customer curl installs.
|
||||
- name: Publish the prerelease
|
||||
if: steps.version.outputs.exists != 'true'
|
||||
working-directory: spinifex
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
TAG: ${{ steps.version.outputs.tag }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# A file, not an inline string: the notes are Markdown, and leading
|
||||
# whitespace from indenting a shell literal turns them into code blocks.
|
||||
cat > "$RUNNER_TEMP/notes.md" <<EOF
|
||||
Automated dev-channel build of \`$GITHUB_SHA\`.
|
||||
|
||||
**Not a release.** amd64 only, no ISO and no system images, and it has not been through the release verification gates. Install it with:
|
||||
|
||||
\`\`\`
|
||||
curl -sfL https://install.mulgadc.com | sudo bash -s -- --channel dev
|
||||
\`\`\`
|
||||
EOF
|
||||
sed -i 's/^ //' "$RUNNER_TEMP/notes.md"
|
||||
gh release create "$TAG" \
|
||||
--repo "$GITHUB_REPOSITORY" \
|
||||
--target "$GITHUB_SHA" \
|
||||
--prerelease \
|
||||
--title "$TAG" \
|
||||
--notes-file "$RUNNER_TEMP/notes.md" \
|
||||
"dist/spinifex-$TAG-linux-amd64.tar.gz" \
|
||||
"dist/spinifex-$TAG-linux-amd64.tar.gz.sha256" \
|
||||
"scripts/setup.sh"
|
||||
|
||||
# The assertion that matters. A prerelease must never become the latest
|
||||
# release, because latest is what an unqualified curl | bash installs -- so
|
||||
# this checks the published state rather than trusting the flag we passed.
|
||||
- name: Assert the stable channel is untouched
|
||||
if: steps.version.outputs.exists != 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
TAG: ${{ steps.version.outputs.tag }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
latest="$(gh api "repos/$GITHUB_REPOSITORY/releases/latest" --jq .tag_name)"
|
||||
if [ "$latest" = "$TAG" ]; then
|
||||
echo "::error::$TAG became the latest release: every customer curl | bash would install it"
|
||||
gh release edit "$TAG" --repo "$GITHUB_REPOSITORY" --prerelease
|
||||
exit 1
|
||||
fi
|
||||
echo "latest is still $latest"
|
||||
dev="$(gh api "repos/$GITHUB_REPOSITORY/releases?per_page=20" --jq 'map(select(.prerelease))[0].tag_name')"
|
||||
[ "$dev" = "$TAG" ] || { echo "::error::the dev channel resolves to $dev, not $TAG"; exit 1; }
|
||||
|
||||
# Kept rather than unbounded: the Worker reads the first page of releases to
|
||||
# resolve the channel, so enough prereleases would eventually push the real
|
||||
# ones off it. Five leaves a few recent builds to roll back to.
|
||||
- name: Prune old dev prereleases
|
||||
if: steps.version.outputs.exists != 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
gh api "repos/$GITHUB_REPOSITORY/releases?per_page=100" \
|
||||
--jq 'map(select(.prerelease and (.tag_name | endswith("-dev")))) | .[5:] | .[].tag_name' \
|
||||
> stale.txt
|
||||
while read -r tag; do
|
||||
[ -n "$tag" ] || continue
|
||||
echo "deleting $tag"
|
||||
gh release delete "$tag" --repo "$GITHUB_REPOSITORY" --yes --cleanup-tag
|
||||
done < stale.txt
|
||||
@@ -77,18 +77,18 @@ jobs:
|
||||
if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi
|
||||
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
|
||||
- uses: actions/create-github-app-token@v3.3.0
|
||||
- uses: actions/create-github-app-token@v3
|
||||
id: app-token
|
||||
with:
|
||||
client-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
repositories: mulga
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/mulga
|
||||
token: ${{ steps.app-token.outputs.token }}
|
||||
@@ -123,7 +123,7 @@ jobs:
|
||||
INSTALLER_BIN="${RUNNER_TEMP}/bm-bins/spinifex-installer" \
|
||||
SPINIFEX_SRC="${GITHUB_WORKSPACE}" \
|
||||
VIPERBLOCK_SRC="${RUNNER_TEMP}/bm-bins/viperblock" \
|
||||
VERSION="$(cd "$GITHUB_WORKSPACE" && git describe --tags --always --exclude '*-dev' 2>/dev/null || echo dev)"
|
||||
VERSION="$(cd "$GITHUB_WORKSPACE" && git describe --tags --always 2>/dev/null || echo dev)"
|
||||
|
||||
- name: Write ePDU credentials
|
||||
run: |
|
||||
@@ -151,8 +151,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Install go-junit-report on runner
|
||||
# Version from go.mod's tool directive, checked against go.sum.
|
||||
run: GOWORK=off go install github.com/jstemmer/go-junit-report/v2
|
||||
run: go install github.com/jstemmer/go-junit-report/v2@latest
|
||||
|
||||
# The eks suite's CreateCluster boots a control-plane VM from the eks-node
|
||||
# AMI. Built here because libguestfs needs source and network, which the
|
||||
@@ -222,20 +221,20 @@ jobs:
|
||||
- name: Ship test results to Elasticsearch
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
uses: $/.github/actions/junit-to-es
|
||||
uses: ./.github/actions/junit-to-es
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
cell: ${{ env.BM_CELL }}
|
||||
|
||||
- name: Analyze e2e failures
|
||||
if: always()
|
||||
uses: $/.github/actions/e2e-analyze
|
||||
uses: ./.github/actions/e2e-analyze
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
log-dir: ${{ env.ARTIFACT_DIR }}
|
||||
title: "E2E failure analysis (cell ${{ env.BM_CELL }})"
|
||||
|
||||
- uses: actions/upload-artifact@v7.0.2
|
||||
- uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: e2e-artifacts-cell-${{ env.BM_CELL }}-${{ github.run_id }}
|
||||
|
||||
@@ -1,287 +0,0 @@
|
||||
name: E2E Cloud Vendor Nightly
|
||||
|
||||
# Dispatched by the same Cloudflare Worker that drives e2e-nightly.yml, once per
|
||||
# ref, rather than by a `schedule:` here. GitHub registers a schedule only from
|
||||
# the default branch, so a cron in this file could never run `dev` and a feature
|
||||
# branch as separate nightlies, and it would always execute the default branch's
|
||||
# copy of this file -- leaving a fix to the workflow itself untestable until it
|
||||
# had already shipped. A dispatch takes both the file and the source from its ref.
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
topologies:
|
||||
description: "Space-separated topologies (blank = vm-single vm-multi bm vm-single-principal). One name runs one topology."
|
||||
required: false
|
||||
workbooks:
|
||||
description: "Space-separated workbooks (blank = the driver's default list)"
|
||||
required: false
|
||||
source:
|
||||
description: "tree (build this ref) or release (the published installer)"
|
||||
required: false
|
||||
default: tree
|
||||
channel:
|
||||
description: "With source=release: latest, or dev for the newest prerelease"
|
||||
required: false
|
||||
default: latest
|
||||
keep_on_fail:
|
||||
description: "Leave a failed topology up to inspect. It keeps billing."
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
auth:
|
||||
description: "How nodes reach the OCI API: instance_principal (needs the tenancy dynamic group) or api_key (needs the credential hook). The vm-single-principal topology always uses instance_principal."
|
||||
required: false
|
||||
default: instance_principal
|
||||
type: choice
|
||||
options: [instance_principal, api_key]
|
||||
no_external_pool:
|
||||
description: "Form without the OCI allocator. Skips every workbook needing a public address. Not applied to vm-single-principal."
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
runner:
|
||||
description: "Runner label (blank = the OCI_E2E_RUNNER variable, else ubuntu-26.04)"
|
||||
required: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# Every topology is built in one OCI compartment under one set of resource names,
|
||||
# so two runs at once would each see the other's instances and the sweep in
|
||||
# e2e-oci.sh could destroy a cluster the other run was still testing. Queue the
|
||||
# second rather than cancelling it: a dispatch is usually a verification run.
|
||||
concurrency:
|
||||
group: e2e-oci
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
oci:
|
||||
# Hosted by default: this job only drives Terraform and SSH, and the work
|
||||
# happens on OCI, so nothing here needs a hypervisor. Overridable because the
|
||||
# argument for a per-provider self-hosted runner is control, not capacity --
|
||||
# a dedicated label keeps a two-hour cloud wait off the hypervisor labels and
|
||||
# lifts the six-hour cap a hosted job carries.
|
||||
runs-on: ${{ inputs.runner || vars.OCI_E2E_RUNNER || 'ubuntu-26.04' }}
|
||||
# Three topologies including a bare-metal build, each with its own install,
|
||||
# formation and workbook pass. The sweep still runs if this fires.
|
||||
timeout-minutes: 300
|
||||
steps:
|
||||
# Set here rather than in a job-level env block: the runner context is not
|
||||
# available at job level, only from a step onwards, and a workflow that
|
||||
# references it there is rejected as invalid before any job starts.
|
||||
#
|
||||
# Outside github.workspace on purpose. The checkout wipes untracked files, so
|
||||
# on a persistent runner it would delete the Terraform state of a killed run
|
||||
# before the next sweep could read it, leaving instances nothing can find.
|
||||
- name: Put the Terraform state outside the checkout
|
||||
run: echo "OCI_STATE_ROOT=$RUNNER_WORKSPACE/oci-state" >> "$GITHUB_ENV"
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
- uses: actions/create-github-app-token@v3.3.0
|
||||
id: app-token
|
||||
with:
|
||||
client-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
# mulga alone. It is the only private repository of the six, and the
|
||||
# five that build-install-artifacts.sh clones are public, so the builder
|
||||
# needs no credential and this token is not widened to reach them.
|
||||
repositories: mulga
|
||||
|
||||
# build-install-artifacts.sh lives in mulga. The workbooks and their driver
|
||||
# are in spinifex, so only the one directory is needed here.
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/mulga
|
||||
path: mulga
|
||||
token: ${{ steps.app-token.outputs.token }}
|
||||
sparse-checkout: scripts/tofu-cluster
|
||||
|
||||
- uses: hashicorp/setup-terraform@v4.0.1
|
||||
with:
|
||||
terraform_wrapper: false
|
||||
|
||||
# Written to the runner's own filesystem and nowhere else. The OCI API
|
||||
# credential is not here at all: it reaches Terraform as OCI_PRIVATE_KEY in
|
||||
# the environment, so no key file is ever created.
|
||||
- name: Install the throwaway SSH key
|
||||
env:
|
||||
OCI_E2E_SSH_PRIVATE_KEY: ${{ secrets.OCI_E2E_SSH_PRIVATE_KEY }}
|
||||
OCI_E2E_SSH_PUBLIC_KEY: ${{ secrets.OCI_E2E_SSH_PUBLIC_KEY }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p ~/.ssh
|
||||
printf '%s\n' "$OCI_E2E_SSH_PRIVATE_KEY" > ~/.ssh/oci-spx
|
||||
printf '%s\n' "$OCI_E2E_SSH_PUBLIC_KEY" > ~/.ssh/oci-spx.pub
|
||||
chmod 600 ~/.ssh/oci-spx
|
||||
ssh-keygen -y -f ~/.ssh/oci-spx > /dev/null
|
||||
|
||||
- name: Run the OCI topologies
|
||||
working-directory: spinifex/scripts/terraform/oci-spx
|
||||
env:
|
||||
OCI_TENANCY_OCID: ${{ secrets.OCI_TENANCY_OCID }}
|
||||
OCI_USER_OCID: ${{ secrets.OCI_USER_OCID }}
|
||||
OCI_FINGERPRINT: ${{ secrets.OCI_FINGERPRINT }}
|
||||
OCI_PRIVATE_KEY: ${{ secrets.OCI_PRIVATE_KEY }}
|
||||
OCI_REGION: ${{ vars.OCI_REGION || 'ap-sydney-1' }}
|
||||
TF_VAR_compartment_ocid: ${{ secrets.OCI_COMPARTMENT_OCID }}
|
||||
MULGA_ROOT: ${{ github.workspace }}/mulga
|
||||
SPX_REF: ${{ github.ref_name }}
|
||||
OCI_TOPOLOGIES: ${{ inputs.topologies }}
|
||||
OCI_SOURCE: ${{ inputs.source }}
|
||||
OCI_CHANNEL: ${{ inputs.channel }}
|
||||
# Independent of the pool, deliberately. These were one switch, which made
|
||||
# the combination we actually need -- an API key with the allocator running
|
||||
# -- impossible to ask for, so public-IP allocation went untested.
|
||||
OCI_INSTANCE_PRINCIPAL: ${{ inputs.auth == 'api_key' && '0' || '1' }}
|
||||
# Skipping the pool records the allocator and every public-address workbook
|
||||
# as SKIPPED, so a green run with this set is not a claim about guest
|
||||
# networking. It exists for a formation-only check, not as a way past a
|
||||
# missing credential.
|
||||
OCI_NO_EXTERNAL_POOL: ${{ inputs.no_external_pool && '1' || '0' }}
|
||||
# Only an API-key run needs one: an instance principal is the node's own
|
||||
# certificate, so there is nothing to hand over. Unset is not an error, and
|
||||
# the allocator gate then fails rather than passing a node that cannot
|
||||
# authenticate.
|
||||
OCI_CREDENTIAL_HOOK: ${{ inputs.auth == 'api_key' && vars.OCI_CREDENTIAL_HOOK || '' }}
|
||||
# vm-single-principal builds in its own tenancy. The dynamic group it needs
|
||||
# is a tenancy-root resource, and the shared test tenancy grants us only
|
||||
# compartment rights, so that topology can never pass there however correct
|
||||
# the code is. Unset skips it rather than spending a provision to fail.
|
||||
OCI_PRINCIPAL_TENANCY_OCID: ${{ secrets.OCI_PRINCIPAL_TENANCY_OCID }}
|
||||
OCI_PRINCIPAL_USER_OCID: ${{ secrets.OCI_PRINCIPAL_USER_OCID }}
|
||||
OCI_PRINCIPAL_FINGERPRINT: ${{ secrets.OCI_PRINCIPAL_FINGERPRINT }}
|
||||
OCI_PRINCIPAL_PRIVATE_KEY: ${{ secrets.OCI_PRINCIPAL_PRIVATE_KEY }}
|
||||
OCI_PRINCIPAL_COMPARTMENT_OCID: ${{ secrets.OCI_PRINCIPAL_COMPARTMENT_OCID }}
|
||||
OCI_PRINCIPAL_REGION: ${{ vars.OCI_PRINCIPAL_REGION }}
|
||||
# Service limits are per tenancy, so the shape the other topologies use
|
||||
# can have zero quota in this one. Ours has no E6 cores at all, and the
|
||||
# apply then fails before anything about the principal is exercised.
|
||||
OCI_PRINCIPAL_SHAPE: ${{ vars.OCI_PRINCIPAL_SHAPE }}
|
||||
OCI_ARTIFACT_DIR: ${{ runner.temp }}/e2e-oci
|
||||
OCI_KEEP_ON_FAIL: ${{ inputs.keep_on_fail && '1' || '0' }}
|
||||
# Through the environment, never interpolated into the script below: a
|
||||
# dispatch input is attacker-controlled text as far as the shell is
|
||||
# concerned, and a quote in it would otherwise end the string.
|
||||
WORKBOOKS_INPUT: ${{ inputs.workbooks }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
# WORKBOOKS is exported only when the dispatch named some: the harness
|
||||
# distinguishes unset (the driver's default list) from empty (run none).
|
||||
if [ -n "$WORKBOOKS_INPUT" ]; then export WORKBOOKS="$WORKBOOKS_INPUT"; fi
|
||||
./e2e-oci.sh 2>&1 | tee "$RUNNER_TEMP/e2e-oci.log"
|
||||
exit "${PIPESTATUS[0]}"
|
||||
|
||||
# Before the sweep, which destroys the state the tables are read from, and
|
||||
# always(): a run whose verdict is a failure is exactly the one somebody
|
||||
# wants a table for. The successful path writes the same tables itself, so
|
||||
# this step is for the run that did not get there.
|
||||
#
|
||||
# The markdown lives in the script rather than here. A heredoc in a YAML
|
||||
# string is the one place in this repository nothing can lint or run.
|
||||
- name: Publish the result tables
|
||||
if: always()
|
||||
working-directory: spinifex/scripts/terraform/oci-spx
|
||||
env:
|
||||
OCI_REGION: ${{ vars.OCI_REGION || 'ap-sydney-1' }}
|
||||
OCI_ARTIFACT_DIR: ${{ runner.temp }}/e2e-oci
|
||||
OCI_TOPOLOGIES: ${{ inputs.topologies }}
|
||||
OCI_SOURCE: ${{ inputs.source }}
|
||||
OCI_CHANNEL: ${{ inputs.channel }}
|
||||
SPX_REF: ${{ github.ref_name }}
|
||||
# The same two the run step passes. The header states what was and was not
|
||||
# tested, so a summary rendered without them would claim more than the run.
|
||||
OCI_INSTANCE_PRINCIPAL: ${{ inputs.auth == 'api_key' && '0' || '1' }}
|
||||
OCI_NO_EXTERNAL_POOL: ${{ inputs.no_external_pool && '1' || '0' }}
|
||||
run: ./e2e-oci.sh --summary-only
|
||||
|
||||
# The sweep below deletes the per-topology state directories, so anything
|
||||
# still only in one of them at that point is unrecoverable -- and a failing
|
||||
# workbook writes its assertion output to exactly there. Copying into the
|
||||
# artifact directory first is what makes a failure diagnosable by whoever
|
||||
# reads the run page, rather than only on the runner that produced it.
|
||||
- name: Collect the per-topology logs
|
||||
if: always()
|
||||
run: |
|
||||
set -uo pipefail
|
||||
dest="$RUNNER_TEMP/e2e-oci"
|
||||
mkdir -p "$dest"
|
||||
# The topologies this run drove, taken from the per-topology log the
|
||||
# harness writes rather than from the dispatch input: a run that names
|
||||
# none gets the harness's own default, and reading that back here would
|
||||
# mean a second copy of the list to disagree with the first. A state
|
||||
# directory left by an earlier run is then not collected, so the artifact
|
||||
# cannot show a topology this run never attempted.
|
||||
topologies=""
|
||||
for log in "$dest"/*.log; do
|
||||
[ -f "$log" ] || continue
|
||||
name="${log##*/}"; name="${name%.log}"
|
||||
[ -d "$OCI_STATE_ROOT/.validate-$name" ] && topologies="$topologies $name"
|
||||
done
|
||||
echo "collecting:${topologies:- nothing}"
|
||||
for topology in $topologies; do
|
||||
for file in "$OCI_STATE_ROOT/.validate-$topology"/*.log \
|
||||
"$OCI_STATE_ROOT/.validate-$topology"/*.txt \
|
||||
"$OCI_STATE_ROOT/.validate-$topology"/*.tsv; do
|
||||
[ -f "$file" ] || continue
|
||||
# The hook handles a private key and the one thing known about an
|
||||
# operator's executable is that it is not ours to publish.
|
||||
case "${file##*/}" in credential-hook.log) continue ;; esac
|
||||
cp -- "$file" "$dest/$topology-${file##*/}"
|
||||
done
|
||||
done
|
||||
ls -l "$dest"
|
||||
|
||||
# always(), and before the artifact upload: an OCI bare-metal host left
|
||||
# running by a cancelled or timed-out job is the most expensive way this
|
||||
# workflow can fail. The sweep works from the Terraform state files, so it
|
||||
# can only destroy what this run created.
|
||||
- name: Sweep anything left behind
|
||||
if: always() && !inputs.keep_on_fail
|
||||
working-directory: spinifex/scripts/terraform/oci-spx
|
||||
env:
|
||||
OCI_TENANCY_OCID: ${{ secrets.OCI_TENANCY_OCID }}
|
||||
OCI_USER_OCID: ${{ secrets.OCI_USER_OCID }}
|
||||
OCI_FINGERPRINT: ${{ secrets.OCI_FINGERPRINT }}
|
||||
OCI_PRIVATE_KEY: ${{ secrets.OCI_PRIVATE_KEY }}
|
||||
OCI_REGION: ${{ vars.OCI_REGION || 'ap-sydney-1' }}
|
||||
TF_VAR_compartment_ocid: ${{ secrets.OCI_COMPARTMENT_OCID }}
|
||||
# The sweep destroys by state file, so it needs whichever tenancy each
|
||||
# topology built in. Without these a leftover principal deployment is
|
||||
# swept against a tenancy that never held it and reported clean.
|
||||
OCI_PRINCIPAL_TENANCY_OCID: ${{ secrets.OCI_PRINCIPAL_TENANCY_OCID }}
|
||||
OCI_PRINCIPAL_USER_OCID: ${{ secrets.OCI_PRINCIPAL_USER_OCID }}
|
||||
OCI_PRINCIPAL_FINGERPRINT: ${{ secrets.OCI_PRINCIPAL_FINGERPRINT }}
|
||||
OCI_PRINCIPAL_PRIVATE_KEY: ${{ secrets.OCI_PRINCIPAL_PRIVATE_KEY }}
|
||||
OCI_PRINCIPAL_COMPARTMENT_OCID: ${{ secrets.OCI_PRINCIPAL_COMPARTMENT_OCID }}
|
||||
OCI_PRINCIPAL_REGION: ${{ vars.OCI_PRINCIPAL_REGION }}
|
||||
OCI_ARTIFACT_DIR: ${{ runner.temp }}/e2e-oci-sweep
|
||||
run: ./e2e-oci.sh --sweep-only
|
||||
|
||||
# Explicit, not left to the runner's own cleanup: a hosted runner is
|
||||
# discarded after the job, but a self-hosted one keeps $HOME, so the key
|
||||
# would outlive the run that needed it and sit there between nightlies.
|
||||
- name: Remove the throwaway SSH key
|
||||
if: always()
|
||||
run: rm -f ~/.ssh/oci-spx ~/.ssh/oci-spx.pub
|
||||
|
||||
# Named extensions rather than a directory glob: the artifact directory also
|
||||
# holds the distro tarball and the test archive, which are hundreds of
|
||||
# megabytes and reproducible from the ref. Everything here is a file the
|
||||
# collect step above chose, so terraform.tfstate cannot reach an artifact.
|
||||
- name: Upload the run
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
with:
|
||||
name: e2e-oci-${{ github.run_id }}
|
||||
path: |
|
||||
${{ runner.temp }}/e2e-oci.log
|
||||
${{ runner.temp }}/e2e-oci/*.log
|
||||
${{ runner.temp }}/e2e-oci/verdict.txt
|
||||
${{ runner.temp }}/e2e-oci/summary.md
|
||||
${{ runner.temp }}/e2e-oci/*.txt
|
||||
${{ runner.temp }}/e2e-oci/*.tsv
|
||||
${{ runner.temp }}/e2e-oci-sweep/*.log
|
||||
retention-days: 14
|
||||
@@ -38,19 +38,19 @@ jobs:
|
||||
- name: Clean Workspace
|
||||
run: sudo rm -rf "$GITHUB_WORKSPACE"/*
|
||||
|
||||
- uses: actions/create-github-app-token@v3.3.0
|
||||
- uses: actions/create-github-app-token@v3
|
||||
id: app-token
|
||||
with:
|
||||
client-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
repositories: mulga
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
name: Checkout Spinifex
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
name: Checkout Mulga (tofu-cluster)
|
||||
with:
|
||||
repository: mulgadc/mulga
|
||||
@@ -81,7 +81,7 @@ jobs:
|
||||
# Adds ci.job_id alongside ci.cell, before Bootstrap bakes the attrs into
|
||||
# the VMs, so a telemetry doc points back at this job's log directly.
|
||||
- name: Resolve CI job id
|
||||
uses: $/.github/actions/otel-job-id
|
||||
uses: ./spinifex/.github/actions/otel-job-id
|
||||
with:
|
||||
job-name: DDIL Real Multi-Node
|
||||
|
||||
@@ -175,7 +175,7 @@ jobs:
|
||||
|
||||
- name: Publish test summary
|
||||
if: always()
|
||||
uses: $/.github/actions/junit-report
|
||||
uses: ./spinifex/.github/actions/junit-report
|
||||
with:
|
||||
junit-glob: ${{ runner.temp }}/ddil-results/junit-*.xml
|
||||
title: "DDIL scenario results"
|
||||
@@ -185,7 +185,7 @@ jobs:
|
||||
- name: Ship test results to Elasticsearch
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
uses: $/.github/actions/junit-to-es
|
||||
uses: ./spinifex/.github/actions/junit-to-es
|
||||
with:
|
||||
junit-glob: ${{ runner.temp }}/ddil-results/junit-*.xml
|
||||
cell: ddil
|
||||
@@ -214,7 +214,7 @@ jobs:
|
||||
|
||||
- name: Upload DDIL Results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: ddil-results-${{ github.run_id }}
|
||||
path: ${{ runner.temp }}/ddil-results
|
||||
@@ -234,7 +234,7 @@ jobs:
|
||||
|
||||
- name: Upload tofu log
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: tofu-log-ddil-${{ github.run_id }}
|
||||
path: ${{ env.TOFU_LOG }}
|
||||
|
||||
@@ -10,7 +10,7 @@ on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
cells:
|
||||
description: "Comma-separated cell numbers to run, no spaces (blank = all). 1-3, 8-10, 17-19 tofu permutation matrix, 20 baremetal, 21-22 ISO, 23 predastore throughput, 24 predastore fault injection, 25-27 service suites (eks, ecs, rds), 28 guest disk performance, 29 storage fault injection, 30 routed NAT on three nodes, 31 instance recovery on three nodes, 32 load balancer recovery on three nodes, 33 KV bucket quorum under node loss on three nodes"
|
||||
description: "Comma-separated cell numbers to run, no spaces (blank = all). 1-3, 8-10, 17-19 tofu permutation matrix, 20 baremetal, 21-22 ISO, 23 predastore throughput, 24 predastore fault injection, 25-27 service suites (eks, ecs, rds), 28 guest disk performance, 29 storage fault injection"
|
||||
required: false
|
||||
|
||||
permissions:
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
cells: ${{ steps.pick.outputs.cells }}
|
||||
any: ${{ steps.pick.outputs.any }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
sparse-checkout: |
|
||||
.github/e2e-cells.json
|
||||
@@ -128,21 +128,21 @@ jobs:
|
||||
if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi
|
||||
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
|
||||
|
||||
- uses: actions/create-github-app-token@v3.3.0
|
||||
- uses: actions/create-github-app-token@v3
|
||||
id: app-token
|
||||
with:
|
||||
client-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
repositories: mulga
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/mulga
|
||||
token: ${{ steps.app-token.outputs.token }}
|
||||
sparse-checkout: scripts/tofu-cluster
|
||||
path: mulga
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
@@ -151,7 +151,7 @@ jobs:
|
||||
# links each cell's telemetry to the job log that produced it. Runs
|
||||
# before Bootstrap bakes SPINIFEX_OTEL_ATTRS into the VM's telemetry.env.
|
||||
- name: Resolve CI job id
|
||||
uses: $/.github/actions/otel-job-id
|
||||
uses: ./spinifex/.github/actions/otel-job-id
|
||||
with:
|
||||
job-name: "cell-${{ matrix.cell }} (${{ matrix.topo }}/${{ matrix.net }}/${{ matrix.host }}${{ matrix.extra && format('/{0}', matrix.extra) || '' }})"
|
||||
|
||||
@@ -278,31 +278,8 @@ jobs:
|
||||
rc=\${PIPESTATUS[0]}; echo '::endgroup::'; \
|
||||
if [ \$rc -ne 0 ]; then echo 'natuplink: FAILED' >> '${ARTIFACT_DIR}/suite-status.txt'; fi; exit \$rc"
|
||||
|
||||
# natuplink proves a guest with a public IP reaches the internet. The two
|
||||
# surfaces it never touches are a private subnet's isolation and its NAT
|
||||
# gateway, and TestVPCEgressPaths is the authority on both — it skipped in
|
||||
# nat mode until the fixture learned that routed NAT carries EIPs too.
|
||||
- name: Run VPC Egress Paths in NAT mode
|
||||
if: matrix.extra == 'nat-single'
|
||||
working-directory: mulga/scripts/tofu-cluster
|
||||
run: |
|
||||
INVENTORY=$(tofu output -json inventory)
|
||||
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
|
||||
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
|
||||
SSH_KEY="${SSH_KEY/#\~/$HOME}"
|
||||
ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
|
||||
"set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; \
|
||||
echo '::group::suite single/TestVPCEgressPaths'; \
|
||||
date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-vpcegress-nat.start'; \
|
||||
GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
|
||||
ARTIFACT_DIR='${ARTIFACT_DIR}' \
|
||||
./single.test -test.v -test.timeout=30m -test.run '^TestVPCEgressPaths\$' \
|
||||
2>&1 | tee '${ARTIFACT_DIR}/test-vpcegress-nat.log'; \
|
||||
rc=\${PIPESTATUS[0]}; echo '::endgroup::'; \
|
||||
if [ \$rc -ne 0 ]; then echo 'vpcegress-nat: FAILED' >> '${ARTIFACT_DIR}/suite-status.txt'; fi; exit \$rc"
|
||||
|
||||
- name: Run Multi Node E2E
|
||||
if: matrix.topo == 'real-multi' && matrix.suite == '' && matrix.extra == ''
|
||||
if: matrix.topo == 'real-multi' && matrix.suite == ''
|
||||
working-directory: mulga/scripts/tofu-cluster
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -337,40 +314,6 @@ jobs:
|
||||
if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
|
||||
done; exit \$FAIL"
|
||||
|
||||
# The same suite as the nat-single cell, run against three nodes: there it
|
||||
# proves routed NAT works at all, here that each node delivers its own
|
||||
# guests' public IPs and no others — which one node cannot show.
|
||||
- name: Run NAT Uplink Multi E2E
|
||||
if: matrix.extra == 'nat-multi'
|
||||
working-directory: mulga/scripts/tofu-cluster
|
||||
run: |
|
||||
set -euo pipefail
|
||||
INVENTORY=$(tofu output -json inventory)
|
||||
NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
|
||||
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
|
||||
SSH_KEY="${SSH_KEY/#\~/$HOME}"
|
||||
|
||||
NODES_CSV=""
|
||||
NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
|
||||
for i in $(seq 0 $((NODE_COUNT - 1))); do
|
||||
ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
|
||||
NODES_CSV="${NODES_CSV:+$NODES_CSV,}$ip"
|
||||
done
|
||||
|
||||
ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
|
||||
"set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; \
|
||||
if [ ! -f natuplink.test ]; then echo 'natuplink.test not in tests tar (branch predates suite) — skipping'; exit 0; fi; \
|
||||
echo '::group::suite natuplink'; \
|
||||
date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-natuplink.start'; \
|
||||
GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
|
||||
SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \
|
||||
SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
|
||||
ARTIFACT_DIR='${ARTIFACT_DIR}' \
|
||||
./natuplink.test -test.v -test.timeout=30m \
|
||||
2>&1 | tee '${ARTIFACT_DIR}/test-natuplink.log'; \
|
||||
rc=\${PIPESTATUS[0]}; echo '::endgroup::'; \
|
||||
if [ \$rc -ne 0 ]; then echo 'natuplink: FAILED' >> '${ARTIFACT_DIR}/suite-status.txt'; fi; exit \$rc"
|
||||
|
||||
# The dedicated suite cells only. Resolved once here rather than in each
|
||||
# step below, which would repeat the same three lines five times.
|
||||
- name: Resolve VM SSH details
|
||||
@@ -795,8 +738,7 @@ jobs:
|
||||
- name: Install go-junit-report
|
||||
if: always()
|
||||
run: |
|
||||
# Version from go.mod's tool directive, checked against go.sum.
|
||||
GOWORK=off go -C spinifex install github.com/jstemmer/go-junit-report/v2
|
||||
go install github.com/jstemmer/go-junit-report/v2@latest
|
||||
echo "$HOME/go/bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Convert test logs to JUnit XML
|
||||
@@ -813,20 +755,20 @@ jobs:
|
||||
- name: Ship test results to Elasticsearch
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
uses: $/.github/actions/junit-to-es
|
||||
uses: ./spinifex/.github/actions/junit-to-es
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
cell: ${{ matrix.cell }}
|
||||
|
||||
- name: Analyze e2e failures
|
||||
if: always()
|
||||
uses: $/.github/actions/e2e-analyze
|
||||
uses: ./spinifex/.github/actions/e2e-analyze
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
log-dir: ${{ env.ARTIFACT_DIR }}
|
||||
title: "E2E failure analysis (cell ${{ matrix.cell }})"
|
||||
|
||||
- uses: actions/upload-artifact@v7.0.2
|
||||
- uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: e2e-artifacts-cell-${{ matrix.cell }}-${{ github.run_id }}
|
||||
@@ -838,7 +780,7 @@ jobs:
|
||||
# pulling nineteen journals it has no use for.
|
||||
- name: Upload junit only
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: junit-cell-${{ matrix.cell }}-${{ github.run_id }}
|
||||
path: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
@@ -869,7 +811,7 @@ jobs:
|
||||
|
||||
- name: Upload tofu log
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: tofu-log-cell-${{ matrix.cell }}-${{ github.run_id }}
|
||||
path: ${{ env.TOFU_LOG }}
|
||||
@@ -974,20 +916,20 @@ jobs:
|
||||
{ sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
|
||||
echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
|
||||
|
||||
- uses: actions/create-github-app-token@v3.3.0
|
||||
- uses: actions/create-github-app-token@v3
|
||||
id: app-token
|
||||
with:
|
||||
client-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
repositories: mulga
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/mulga
|
||||
token: ${{ steps.app-token.outputs.token }}
|
||||
path: mulga
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ env.GIT_BRANCH }}
|
||||
path: mulga/spinifex
|
||||
@@ -999,25 +941,25 @@ jobs:
|
||||
run: bash .github/resolve-sibling-refs.sh
|
||||
working-directory: mulga/spinifex
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/viperblock
|
||||
ref: ${{ steps.siblings.outputs.viperblock }}
|
||||
path: mulga/viperblock
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
ref: ${{ steps.siblings.outputs.predastore }}
|
||||
path: mulga/predastore
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/northstar
|
||||
ref: ${{ steps.siblings.outputs.northstar }}
|
||||
path: mulga/northstar
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/bluebottle
|
||||
ref: ${{ steps.siblings.outputs.bluebottle }}
|
||||
@@ -1152,7 +1094,7 @@ jobs:
|
||||
/tmp/smoke-$GITHUB_RUN_ID.log | tail -1)
|
||||
[ -n "$qemu" ] && cp "$qemu" /tmp/cell21-artifacts-$GITHUB_RUN_ID/qemu.log || true
|
||||
|
||||
- uses: actions/upload-artifact@v7.0.2
|
||||
- uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: e2e-artifacts-cell-21-${{ github.run_id }}
|
||||
@@ -1221,20 +1163,20 @@ jobs:
|
||||
# disks, which is what the tmpfs assertion below is about.
|
||||
echo "WORK_DIR=$E2E_ROOT/work/cell22-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
|
||||
|
||||
- uses: actions/create-github-app-token@v3.3.0
|
||||
- uses: actions/create-github-app-token@v3
|
||||
id: app-token
|
||||
with:
|
||||
client-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
repositories: mulga
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/mulga
|
||||
token: ${{ steps.app-token.outputs.token }}
|
||||
path: mulga
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ env.GIT_BRANCH }}
|
||||
path: mulga/spinifex
|
||||
@@ -1246,25 +1188,25 @@ jobs:
|
||||
run: bash .github/resolve-sibling-refs.sh
|
||||
working-directory: mulga/spinifex
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/viperblock
|
||||
ref: ${{ steps.siblings.outputs.viperblock }}
|
||||
path: mulga/viperblock
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
ref: ${{ steps.siblings.outputs.predastore }}
|
||||
path: mulga/predastore
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/northstar
|
||||
ref: ${{ steps.siblings.outputs.northstar }}
|
||||
path: mulga/northstar
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/bluebottle
|
||||
ref: ${{ steps.siblings.outputs.bluebottle }}
|
||||
@@ -1318,7 +1260,7 @@ jobs:
|
||||
cp "$WORK_DIR"/*-serial.log /tmp/cell22-artifacts-$GITHUB_RUN_ID/ 2>/dev/null ||
|
||||
echo "no per-node serial logs — the run passed and cleaned up after itself"
|
||||
|
||||
- uses: actions/upload-artifact@v7.0.2
|
||||
- uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: e2e-artifacts-cell-22-${{ github.run_id }}
|
||||
@@ -1370,7 +1312,7 @@ jobs:
|
||||
echo "PERF_RESULTS_ROOT=$E2E_ROOT/predastore-perf-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Checkout Spinifex
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
@@ -1385,7 +1327,7 @@ jobs:
|
||||
working-directory: spinifex
|
||||
|
||||
- name: Checkout Predastore
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
ref: ${{ steps.siblings.outputs.predastore }}
|
||||
@@ -1438,7 +1380,7 @@ jobs:
|
||||
# Report-only. The baseline comparison and the -10% gate are mulga-wpjsx;
|
||||
# until the run-to-run spread is known from real nights, a threshold would
|
||||
# be a guess, and a gate that cries wolf gets muted.
|
||||
- uses: actions/upload-artifact@v7.0.2
|
||||
- uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: predastore-perf-${{ github.run_id }}
|
||||
@@ -1497,7 +1439,7 @@ jobs:
|
||||
echo "STRESS_RESULTS_ROOT=$E2E_ROOT/predastore-stress-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Checkout Spinifex
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
@@ -1509,7 +1451,7 @@ jobs:
|
||||
working-directory: spinifex
|
||||
|
||||
- name: Checkout Predastore
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
ref: ${{ steps.siblings.outputs.predastore }}
|
||||
@@ -1564,7 +1506,7 @@ jobs:
|
||||
mkdir -p "$STRESS_RESULTS_ROOT"
|
||||
make e2e-stress 2>&1 | tee "$STRESS_RESULTS_ROOT/e2e-stress.log"
|
||||
|
||||
- uses: actions/upload-artifact@v7.0.2
|
||||
- uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: predastore-stress-${{ github.run_id }}
|
||||
@@ -1601,13 +1543,13 @@ jobs:
|
||||
if: always()
|
||||
runs-on: ubuntu-26.04
|
||||
steps:
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
sparse-checkout: .github/scripts/e2e-run-summary.py
|
||||
sparse-checkout-cone-mode: false
|
||||
|
||||
- name: Collect junit
|
||||
uses: actions/download-artifact@v8.0.2
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: junit-cell-*-${{ github.run_id }}
|
||||
path: junit
|
||||
|
||||
@@ -103,25 +103,25 @@ jobs:
|
||||
| sort -r | xargs -r -I{} sudo umount -l -R {} || true
|
||||
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
|
||||
|
||||
- uses: actions/create-github-app-token@v3.3.0
|
||||
- uses: actions/create-github-app-token@v3
|
||||
id: app-token
|
||||
with:
|
||||
client-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
repositories: mulga
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/mulga
|
||||
token: ${{ steps.app-token.outputs.token }}
|
||||
sparse-checkout: scripts/tofu-cluster
|
||||
path: mulga
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
- uses: actions/setup-go@v7.0.0
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: spinifex/go.mod
|
||||
cache: false
|
||||
@@ -157,7 +157,7 @@ jobs:
|
||||
# the env: the suite legs share this env and its baked telemetry.env, so
|
||||
# the provisioner is the only job the daemon telemetry can belong to.
|
||||
- name: Resolve CI job id
|
||||
uses: $/.github/actions/otel-job-id
|
||||
uses: ./spinifex/.github/actions/otel-job-id
|
||||
with:
|
||||
job-name: Single env · provision
|
||||
|
||||
@@ -300,7 +300,7 @@ jobs:
|
||||
|
||||
- name: Upload tofu log
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: tofu-log-single-${{ github.run_id }}
|
||||
path: ${{ env.TOFU_LOG }}
|
||||
@@ -323,19 +323,18 @@ jobs:
|
||||
SSH_KEY: ${{ needs.single_provision.outputs.ssh_key }}
|
||||
ARTIFACT_DIR: /tmp/spinifex-e2e-single-${{ github.run_id }}-${{ matrix.suite }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
- uses: actions/setup-go@v7.0.0
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: spinifex/go.mod
|
||||
cache: false
|
||||
|
||||
- name: Install go-junit-report
|
||||
run: |
|
||||
# Version from go.mod's tool directive, checked against go.sum.
|
||||
GOWORK=off go -C spinifex install github.com/jstemmer/go-junit-report/v2
|
||||
go install github.com/jstemmer/go-junit-report/v2@latest
|
||||
echo "$HOME/go/bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Run ${{ matrix.suite }}
|
||||
@@ -391,7 +390,7 @@ jobs:
|
||||
|
||||
- name: Publish test summary
|
||||
if: always()
|
||||
uses: $/.github/actions/junit-report
|
||||
uses: ./spinifex/.github/actions/junit-report
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
title: "E2E ${{ matrix.suite }} results"
|
||||
@@ -401,20 +400,20 @@ jobs:
|
||||
- name: Ship test results to Elasticsearch
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
uses: $/.github/actions/junit-to-es
|
||||
uses: ./spinifex/.github/actions/junit-to-es
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
cell: single
|
||||
|
||||
- name: Analyze e2e failures
|
||||
if: always()
|
||||
uses: $/.github/actions/e2e-analyze
|
||||
uses: ./spinifex/.github/actions/e2e-analyze
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
log-dir: ${{ env.ARTIFACT_DIR }}
|
||||
title: "E2E ${{ matrix.suite }} failure analysis"
|
||||
|
||||
- uses: actions/upload-artifact@v7.0.2
|
||||
- uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: e2e-single-${{ matrix.suite }}-${{ github.run_id }}
|
||||
@@ -445,7 +444,7 @@ jobs:
|
||||
# Reuse the persisted mulga/ tofu-cluster workspace from single_provision.
|
||||
# Re-checkout only spinifex (for ci-fmt + diag scripts); checking out mulga
|
||||
# again would wipe the local tofu state and orphan the VM.
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
@@ -468,7 +467,7 @@ jobs:
|
||||
| tar -C "${ARTIFACT_DIR}/lb-console" -xf - 2>/dev/null
|
||||
true
|
||||
|
||||
- uses: actions/upload-artifact@v7.0.2
|
||||
- uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: e2e-single-diag-${{ github.run_id }}
|
||||
@@ -524,7 +523,7 @@ jobs:
|
||||
|
||||
- name: Upload tofu log
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: tofu-log-single-td-${{ github.run_id }}
|
||||
path: ${{ env.TOFU_LOG }}
|
||||
@@ -572,25 +571,25 @@ jobs:
|
||||
| sort -r | xargs -r -I{} sudo umount -l -R {} || true
|
||||
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
|
||||
|
||||
- uses: actions/create-github-app-token@v3.3.0
|
||||
- uses: actions/create-github-app-token@v3
|
||||
id: app-token
|
||||
with:
|
||||
client-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
repositories: mulga
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/mulga
|
||||
token: ${{ steps.app-token.outputs.token }}
|
||||
sparse-checkout: scripts/tofu-cluster
|
||||
path: mulga
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
- uses: actions/setup-go@v7.0.0
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: spinifex/go.mod
|
||||
cache: false
|
||||
@@ -607,7 +606,7 @@ jobs:
|
||||
|
||||
# See the matching note on single_provision.
|
||||
- name: Resolve CI job id
|
||||
uses: $/.github/actions/otel-job-id
|
||||
uses: ./spinifex/.github/actions/otel-job-id
|
||||
with:
|
||||
job-name: Multi env · provision
|
||||
|
||||
@@ -715,7 +714,7 @@ jobs:
|
||||
|
||||
- name: Upload tofu log
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: tofu-log-multi-${{ github.run_id }}
|
||||
path: ${{ env.TOFU_LOG }}
|
||||
@@ -738,19 +737,18 @@ jobs:
|
||||
NODES_CSV: ${{ needs.multi_provision.outputs.nodes_csv }}
|
||||
ARTIFACT_DIR: /tmp/spinifex-e2e-multi-${{ github.run_id }}-${{ matrix.suite }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
- uses: actions/setup-go@v7.0.0
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: spinifex/go.mod
|
||||
cache: false
|
||||
|
||||
- name: Install go-junit-report
|
||||
run: |
|
||||
# Version from go.mod's tool directive, checked against go.sum.
|
||||
GOWORK=off go -C spinifex install github.com/jstemmer/go-junit-report/v2
|
||||
go install github.com/jstemmer/go-junit-report/v2@latest
|
||||
echo "$HOME/go/bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Run ${{ matrix.suite }}
|
||||
@@ -806,7 +804,7 @@ jobs:
|
||||
|
||||
- name: Publish test summary
|
||||
if: always()
|
||||
uses: $/.github/actions/junit-report
|
||||
uses: ./spinifex/.github/actions/junit-report
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
title: "Multi-Node E2E ${{ matrix.suite }} results"
|
||||
@@ -816,20 +814,20 @@ jobs:
|
||||
- name: Ship test results to Elasticsearch
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
uses: $/.github/actions/junit-to-es
|
||||
uses: ./spinifex/.github/actions/junit-to-es
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
cell: multi
|
||||
|
||||
- name: Analyze e2e failures
|
||||
if: always()
|
||||
uses: $/.github/actions/e2e-analyze
|
||||
uses: ./spinifex/.github/actions/e2e-analyze
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
log-dir: ${{ env.ARTIFACT_DIR }}
|
||||
title: "Multi-Node E2E ${{ matrix.suite }} failure analysis"
|
||||
|
||||
- uses: actions/upload-artifact@v7.0.2
|
||||
- uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: e2e-multi-${{ matrix.suite }}-${{ github.run_id }}
|
||||
@@ -856,7 +854,7 @@ jobs:
|
||||
SSH_KEY: ${{ needs.multi_provision.outputs.ssh_key }}
|
||||
NODES_CSV: ${{ needs.multi_provision.outputs.nodes_csv }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
@@ -910,7 +908,7 @@ jobs:
|
||||
|
||||
- name: Upload tofu log
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: tofu-log-multi-td-${{ github.run_id }}
|
||||
path: ${{ env.TOFU_LOG }}
|
||||
|
||||
+25
-27
@@ -54,7 +54,7 @@ jobs:
|
||||
single_timeout: ${{ steps.budgets.outputs.single_timeout }}
|
||||
multi_timeout: ${{ steps.budgets.outputs.multi_timeout }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
path: spinifex
|
||||
sparse-checkout: .github/scripts
|
||||
@@ -142,25 +142,25 @@ jobs:
|
||||
# stale suite-status.txt / analysis dirs can't leak into this attempt.
|
||||
rm -rf "${ARTIFACT_DIR}"
|
||||
|
||||
- uses: actions/create-github-app-token@v3.3.0
|
||||
- uses: actions/create-github-app-token@v3
|
||||
id: app-token
|
||||
with:
|
||||
client-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
repositories: mulga
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/mulga
|
||||
token: ${{ steps.app-token.outputs.token }}
|
||||
sparse-checkout: scripts/tofu-cluster
|
||||
path: mulga
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
- uses: actions/setup-go@v7.0.0
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: spinifex/go.mod
|
||||
cache: false
|
||||
@@ -178,7 +178,7 @@ jobs:
|
||||
# Adds ci.job_id alongside ci.cell, before Bootstrap bakes the attrs into
|
||||
# the VM, so a telemetry doc points back at this job's log directly.
|
||||
- name: Resolve CI job id
|
||||
uses: $/.github/actions/otel-job-id
|
||||
uses: ./spinifex/.github/actions/otel-job-id
|
||||
with:
|
||||
job-name: Single-Node E2E
|
||||
|
||||
@@ -372,8 +372,7 @@ jobs:
|
||||
|
||||
- name: Install go-junit-report on runner
|
||||
run: |
|
||||
# Version from go.mod's tool directive, checked against go.sum.
|
||||
GOWORK=off go -C spinifex install github.com/jstemmer/go-junit-report/v2
|
||||
go install github.com/jstemmer/go-junit-report/v2@latest
|
||||
echo "$HOME/go/bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Ship test binaries to VM
|
||||
@@ -513,7 +512,7 @@ jobs:
|
||||
|
||||
- name: Publish test summary
|
||||
if: always()
|
||||
uses: $/.github/actions/junit-report
|
||||
uses: ./spinifex/.github/actions/junit-report
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
title: "E2E results"
|
||||
@@ -523,20 +522,20 @@ jobs:
|
||||
- name: Ship test results to Elasticsearch
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
uses: $/.github/actions/junit-to-es
|
||||
uses: ./spinifex/.github/actions/junit-to-es
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
cell: single
|
||||
|
||||
- name: Analyze e2e failures
|
||||
if: always()
|
||||
uses: $/.github/actions/e2e-analyze
|
||||
uses: ./spinifex/.github/actions/e2e-analyze
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
log-dir: ${{ env.ARTIFACT_DIR }}
|
||||
title: "E2E failure analysis"
|
||||
|
||||
- uses: actions/upload-artifact@v7.0.2
|
||||
- uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: e2e-artifacts-${{ github.run_id }}
|
||||
@@ -580,7 +579,7 @@ jobs:
|
||||
|
||||
- name: Upload tofu log
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: tofu-log-single-${{ github.run_id }}
|
||||
path: ${{ env.TOFU_LOG }}
|
||||
@@ -643,25 +642,25 @@ jobs:
|
||||
# stale suite-status.txt / analysis dirs can't leak into this attempt.
|
||||
rm -rf "${ARTIFACT_DIR}"
|
||||
|
||||
- uses: actions/create-github-app-token@v3.3.0
|
||||
- uses: actions/create-github-app-token@v3
|
||||
id: app-token
|
||||
with:
|
||||
client-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
repositories: mulga
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/mulga
|
||||
token: ${{ steps.app-token.outputs.token }}
|
||||
sparse-checkout: scripts/tofu-cluster
|
||||
path: mulga
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
- uses: actions/setup-go@v7.0.0
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: spinifex/go.mod
|
||||
cache: false
|
||||
@@ -678,7 +677,7 @@ jobs:
|
||||
|
||||
# See the matching note on e2e_single.
|
||||
- name: Resolve CI job id
|
||||
uses: $/.github/actions/otel-job-id
|
||||
uses: ./spinifex/.github/actions/otel-job-id
|
||||
with:
|
||||
job-name: Multi-Node E2E
|
||||
|
||||
@@ -759,8 +758,7 @@ jobs:
|
||||
|
||||
- name: Install go-junit-report on runner
|
||||
run: |
|
||||
# Version from go.mod's tool directive, checked against go.sum.
|
||||
GOWORK=off go -C spinifex install github.com/jstemmer/go-junit-report/v2
|
||||
go install github.com/jstemmer/go-junit-report/v2@latest
|
||||
echo "$HOME/go/bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Ship test binaries to node1
|
||||
@@ -967,7 +965,7 @@ jobs:
|
||||
|
||||
- name: Publish test summary
|
||||
if: always()
|
||||
uses: $/.github/actions/junit-report
|
||||
uses: ./spinifex/.github/actions/junit-report
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
title: "Multi-Node E2E results"
|
||||
@@ -977,20 +975,20 @@ jobs:
|
||||
- name: Ship test results to Elasticsearch
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
uses: $/.github/actions/junit-to-es
|
||||
uses: ./spinifex/.github/actions/junit-to-es
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
cell: multi
|
||||
|
||||
- name: Analyze e2e failures
|
||||
if: always()
|
||||
uses: $/.github/actions/e2e-analyze
|
||||
uses: ./spinifex/.github/actions/e2e-analyze
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
log-dir: ${{ env.ARTIFACT_DIR }}
|
||||
title: "Multi-Node E2E failure analysis"
|
||||
|
||||
- uses: actions/upload-artifact@v7.0.2
|
||||
- uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: e2e-multi-artifacts-${{ github.run_id }}
|
||||
@@ -1037,7 +1035,7 @@ jobs:
|
||||
|
||||
- name: Upload tofu log
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: tofu-log-multi-${{ github.run_id }}
|
||||
path: ${{ env.TOFU_LOG }}
|
||||
@@ -1107,14 +1105,14 @@ jobs:
|
||||
# the aggregation script below treats an absent suite-status.txt as
|
||||
# "no failures from that leg".
|
||||
- name: Download single-node artifacts
|
||||
uses: actions/download-artifact@v8.0.2
|
||||
uses: actions/download-artifact@v8
|
||||
continue-on-error: true
|
||||
with:
|
||||
name: e2e-artifacts-${{ github.run_id }}
|
||||
path: ${{ github.workspace }}/es-summary-single
|
||||
|
||||
- name: Download multi-node artifacts
|
||||
uses: actions/download-artifact@v8.0.2
|
||||
uses: actions/download-artifact@v8
|
||||
continue-on-error: true
|
||||
with:
|
||||
name: e2e-multi-artifacts-${{ github.run_id }}
|
||||
|
||||
@@ -95,10 +95,10 @@ jobs:
|
||||
if: steps.gate.outputs.paused != 'true'
|
||||
run: sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
if: steps.gate.outputs.paused != 'true'
|
||||
|
||||
- uses: actions/setup-go@v7.0.0
|
||||
- uses: actions/setup-go@v7
|
||||
if: steps.gate.outputs.paused != 'true'
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
@@ -187,8 +187,7 @@ jobs:
|
||||
- name: Install go-junit-report on runner
|
||||
if: steps.gate.outputs.paused != 'true'
|
||||
run: |
|
||||
# Version from go.mod's tool directive, checked against go.sum.
|
||||
GOWORK=off go install github.com/jstemmer/go-junit-report/v2
|
||||
go install github.com/jstemmer/go-junit-report/v2@latest
|
||||
echo "$HOME/go/bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Ship GPU e2e test binary to wattle
|
||||
@@ -244,7 +243,7 @@ jobs:
|
||||
|
||||
- name: Publish test summary
|
||||
if: always() && steps.gate.outputs.paused != 'true'
|
||||
uses: $/.github/actions/junit-report
|
||||
uses: ./.github/actions/junit-report
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
title: "GPU E2E results"
|
||||
@@ -252,20 +251,20 @@ jobs:
|
||||
- name: Ship test results to Elasticsearch
|
||||
if: always() && steps.gate.outputs.paused != 'true'
|
||||
continue-on-error: true
|
||||
uses: $/.github/actions/junit-to-es
|
||||
uses: ./.github/actions/junit-to-es
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
cell: gpu
|
||||
|
||||
- name: Analyze e2e failures
|
||||
if: always() && steps.gate.outputs.paused != 'true'
|
||||
uses: $/.github/actions/e2e-analyze
|
||||
uses: ./.github/actions/e2e-analyze
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
log-dir: ${{ env.ARTIFACT_DIR }}
|
||||
title: "GPU E2E failure analysis"
|
||||
|
||||
- uses: actions/upload-artifact@v7.0.2
|
||||
- uses: actions/upload-artifact@v7
|
||||
if: always() && steps.gate.outputs.paused != 'true'
|
||||
with:
|
||||
name: gpu-e2e-artifacts-${{ github.run_id }}
|
||||
|
||||
@@ -29,16 +29,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Run image script tests + shellcheck
|
||||
run: make test-images
|
||||
|
||||
eks_mkosi_image:
|
||||
name: Build mkosi EKS image
|
||||
# A fork PR's code must never run on a persistent self-hosted runner: it
|
||||
# shares disk, caches and credentials with the release jobs.
|
||||
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
|
||||
runs-on: [self-hosted, ci-single]
|
||||
concurrency:
|
||||
group: eks-mkosi-image-${{ github.ref }}
|
||||
@@ -46,10 +43,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7.0.0
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: "go.mod"
|
||||
# Self-hosted: the module + build cache already persists on local
|
||||
|
||||
@@ -80,25 +80,25 @@ jobs:
|
||||
echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
|
||||
echo "PERF_RESULTS_ROOT=$E2E_ROOT/predastore-perf-3node-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
|
||||
|
||||
- uses: actions/create-github-app-token@v3.3.0
|
||||
- uses: actions/create-github-app-token@v3
|
||||
id: app-token
|
||||
with:
|
||||
client-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
repositories: mulga
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/mulga
|
||||
token: ${{ steps.app-token.outputs.token }}
|
||||
sparse-checkout: scripts/tofu-cluster
|
||||
path: mulga
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
- uses: actions/setup-go@v7.0.0
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: spinifex/go.mod
|
||||
cache: false
|
||||
@@ -116,7 +116,7 @@ jobs:
|
||||
run: bash .github/resolve-sibling-refs.sh
|
||||
working-directory: spinifex
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
id: predastore
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
@@ -366,7 +366,7 @@ jobs:
|
||||
# Report-only, matching cell-23: the baseline comparison and any
|
||||
# regression gate are future work once run-to-run spread is known from
|
||||
# real bare-metal runs.
|
||||
- uses: actions/upload-artifact@v7.0.2
|
||||
- uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: predastore-perf-3node-${{ github.run_id }}
|
||||
@@ -407,7 +407,7 @@ jobs:
|
||||
|
||||
- name: Upload tofu log
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: tofu-log-perf3node-${{ github.run_id }}
|
||||
path: ${{ env.TOFU_LOG }}
|
||||
|
||||
@@ -17,12 +17,28 @@ name: Publish System AMIs
|
||||
# spx admin images import --name spinifex-rds-postgres
|
||||
# spx admin images import --name spinifex-rds-mariadb
|
||||
#
|
||||
# Publishing overwrites the mirror artifact, so it runs only from release.yml
|
||||
# once a tag has passed its gates, with siblings at the tags go.mod requires.
|
||||
# The `variant` input scopes which images build — the Ubuntu GPU legs are 16G
|
||||
# and much slower (NVIDIA driver download + DKMS build).
|
||||
# Publishing overwrites the mirror artifact, so it is a deliberate action, not
|
||||
# something every push should trigger: it runs manually, or from release.yml
|
||||
# once a tag has passed its gates. The `variant` input scopes which images
|
||||
# build — the Ubuntu GPU legs are 16G and much slower (NVIDIA driver download +
|
||||
# DKMS build), so publishing only the CPU AMIs stays cheap.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
variant:
|
||||
description: "Which images to build"
|
||||
type: choice
|
||||
default: all
|
||||
options:
|
||||
- all
|
||||
- cpu
|
||||
- gpu
|
||||
- rds
|
||||
dry_run:
|
||||
description: "Build + convert only; skip the R2 upload"
|
||||
type: boolean
|
||||
default: false
|
||||
workflow_call:
|
||||
inputs:
|
||||
variant:
|
||||
@@ -78,44 +94,40 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout Spinifex
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
- name: Resolve sibling release tags from go.mod
|
||||
id: siblings
|
||||
run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
|
||||
|
||||
- name: Checkout Viperblock
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/viperblock
|
||||
ref: ${{ steps.siblings.outputs.viperblock }}
|
||||
ref: dev
|
||||
path: viperblock
|
||||
|
||||
- name: Checkout Predastore
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
ref: ${{ steps.siblings.outputs.predastore }}
|
||||
ref: dev
|
||||
path: predastore
|
||||
|
||||
- name: Checkout Northstar
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/northstar
|
||||
ref: ${{ steps.siblings.outputs.northstar }}
|
||||
ref: dev
|
||||
path: northstar
|
||||
|
||||
- name: Checkout Bluebottle
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/bluebottle
|
||||
ref: ${{ steps.siblings.outputs.bluebottle }}
|
||||
ref: dev
|
||||
path: bluebottle
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v7.0.0
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: "spinifex/go.mod"
|
||||
# Self-hosted runner: the Go module + build cache already persists on
|
||||
|
||||
@@ -5,57 +5,48 @@ on:
|
||||
tags: [ "v*" ]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
distro-amd64:
|
||||
runs-on: ubuntu-26.04
|
||||
steps:
|
||||
- name: Checkout Spinifex
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
- name: Resolve sibling release tags from go.mod
|
||||
id: siblings
|
||||
run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
|
||||
|
||||
- name: Checkout Predastore
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
ref: ${{ steps.siblings.outputs.predastore }}
|
||||
path: predastore
|
||||
|
||||
- name: Checkout Viperblock
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/viperblock
|
||||
ref: ${{ steps.siblings.outputs.viperblock }}
|
||||
path: viperblock
|
||||
|
||||
- name: Checkout Northstar
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/northstar
|
||||
ref: ${{ steps.siblings.outputs.northstar }}
|
||||
path: northstar
|
||||
|
||||
- name: Checkout Bluebottle
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/bluebottle
|
||||
ref: ${{ steps.siblings.outputs.bluebottle }}
|
||||
path: bluebottle
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4.4.1
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7.0.0
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: spinifex/go.mod
|
||||
cache: false
|
||||
go-version: "1.27.0"
|
||||
|
||||
- name: Build microVM artifacts (lb-agent + kernel + initramfs)
|
||||
run: cd spinifex && make build-microvm-image
|
||||
@@ -66,7 +57,7 @@ jobs:
|
||||
run: cd spinifex && make distro-amd64 VERSION="$VERSION"
|
||||
|
||||
- name: Upload amd64 Artifacts
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: distro-amd64
|
||||
path: |
|
||||
@@ -78,44 +69,36 @@ jobs:
|
||||
runs-on: ubuntu-26.04-arm
|
||||
steps:
|
||||
- name: Checkout Spinifex
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
- name: Resolve sibling release tags from go.mod
|
||||
id: siblings
|
||||
run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
|
||||
|
||||
- name: Checkout Predastore
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
ref: ${{ steps.siblings.outputs.predastore }}
|
||||
path: predastore
|
||||
|
||||
- name: Checkout Viperblock
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/viperblock
|
||||
ref: ${{ steps.siblings.outputs.viperblock }}
|
||||
path: viperblock
|
||||
|
||||
- name: Checkout Northstar
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/northstar
|
||||
ref: ${{ steps.siblings.outputs.northstar }}
|
||||
path: northstar
|
||||
|
||||
- name: Checkout Bluebottle
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/bluebottle
|
||||
ref: ${{ steps.siblings.outputs.bluebottle }}
|
||||
path: bluebottle
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4.4.1
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Build arm64 Tarball
|
||||
env:
|
||||
@@ -123,7 +106,7 @@ jobs:
|
||||
run: cd spinifex && make distro-arm64 VERSION="$VERSION"
|
||||
|
||||
- name: Upload arm64 Artifacts
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: distro-arm64
|
||||
path: |
|
||||
@@ -133,23 +116,21 @@ jobs:
|
||||
release:
|
||||
needs: [ distro-amd64, distro-arm64 ]
|
||||
runs-on: ubuntu-26.04
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Download amd64 Artifacts
|
||||
uses: actions/download-artifact@v8.0.2
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: distro-amd64
|
||||
path: artifacts
|
||||
|
||||
- name: Download arm64 Artifacts
|
||||
uses: actions/download-artifact@v8.0.2
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: distro-arm64
|
||||
path: artifacts
|
||||
|
||||
- name: Create Draft GitHub Release
|
||||
uses: softprops/action-gh-release@v3.0.3
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
draft: true
|
||||
files: |
|
||||
@@ -159,7 +140,7 @@ jobs:
|
||||
generate_release_notes: true
|
||||
|
||||
- name: Upload Tarball for Verify Job
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: release-assets
|
||||
path: artifacts/
|
||||
@@ -186,14 +167,14 @@ jobs:
|
||||
| sort -r | xargs -r -I{} sudo umount -l -R {} || true
|
||||
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
|
||||
|
||||
- uses: actions/create-github-app-token@v3.3.0
|
||||
- uses: actions/create-github-app-token@v3
|
||||
id: app-token
|
||||
with:
|
||||
client-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
repositories: mulga
|
||||
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/mulga
|
||||
token: ${{ steps.app-token.outputs.token }}
|
||||
@@ -202,13 +183,13 @@ jobs:
|
||||
|
||||
# Only the shared shell helpers: this job builds nothing from source, but
|
||||
# it drives the same libvirt provider the e2e cells retry around.
|
||||
- uses: actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
sparse-checkout: .github/scripts
|
||||
path: spinifex
|
||||
|
||||
- name: Download Release Assets
|
||||
uses: actions/download-artifact@v8.0.2
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: release-assets
|
||||
path: release-assets
|
||||
@@ -250,8 +231,6 @@ jobs:
|
||||
promote:
|
||||
needs: [ verify, baremetal ]
|
||||
runs-on: ubuntu-26.04
|
||||
permissions:
|
||||
contents: write
|
||||
env:
|
||||
TAG: ${{ github.ref_name }}
|
||||
REPO: ${{ github.repository }}
|
||||
@@ -284,8 +263,6 @@ jobs:
|
||||
needs: [ verify, baremetal ]
|
||||
if: failure()
|
||||
runs-on: ubuntu-26.04
|
||||
permissions:
|
||||
contents: write
|
||||
env:
|
||||
TAG: ${{ github.ref_name }}
|
||||
REPO: ${{ github.repository }}
|
||||
@@ -303,8 +280,6 @@ jobs:
|
||||
iso:
|
||||
needs: release
|
||||
runs-on: [ self-hosted, ci-single ]
|
||||
permissions:
|
||||
contents: write
|
||||
# ISO build (~60-90m) overlaps the parallel baremetal run (~120m); the
|
||||
# upload gate then waits for baremetal to finish, so the job must outlast it.
|
||||
timeout-minutes: 180
|
||||
@@ -319,7 +294,7 @@ jobs:
|
||||
sudo umount -l "${ROOTFS}/proc" 2>/dev/null || true
|
||||
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
|
||||
|
||||
- uses: actions/create-github-app-token@v3.3.0
|
||||
- uses: actions/create-github-app-token@v3
|
||||
id: app-token
|
||||
with:
|
||||
client-id: ${{ secrets.APP_ID }}
|
||||
@@ -327,45 +302,37 @@ jobs:
|
||||
repositories: mulga
|
||||
|
||||
- name: Checkout Spinifex
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
path: spinifex
|
||||
ref: ${{ github.ref }}
|
||||
|
||||
- name: Resolve sibling release tags from go.mod
|
||||
id: siblings
|
||||
run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
|
||||
|
||||
- name: Checkout Predastore
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
ref: ${{ steps.siblings.outputs.predastore }}
|
||||
path: predastore
|
||||
|
||||
- name: Checkout Viperblock
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/viperblock
|
||||
ref: ${{ steps.siblings.outputs.viperblock }}
|
||||
path: viperblock
|
||||
|
||||
- name: Checkout Northstar
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/northstar
|
||||
ref: ${{ steps.siblings.outputs.northstar }}
|
||||
path: northstar
|
||||
|
||||
- name: Checkout Bluebottle
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/bluebottle
|
||||
ref: ${{ steps.siblings.outputs.bluebottle }}
|
||||
path: bluebottle
|
||||
|
||||
- name: Checkout Mulga iso-builder
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/mulga
|
||||
token: ${{ steps.app-token.outputs.token }}
|
||||
@@ -376,7 +343,7 @@ jobs:
|
||||
run: sudo make -C mulga/scripts/iso-builder clean
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7.0.0
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: spinifex/go.mod
|
||||
cache: false
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
#
|
||||
# tests/integration/segscan_oracle_test.go needs the mulga umbrella repo's
|
||||
# scripts/segscan checked out alongside spinifex (it is a separate Go module,
|
||||
# `package main`, and cannot be imported — see internal/testkit/segscanoracle).
|
||||
# `package main`, and cannot be imported — see spinifex/testutil/segscanoracle).
|
||||
# spinifex.yml's jobs check out only spinifex/viperblock/predastore and run on
|
||||
# every PR; adding a second-repo checkout there would make every spinifex PR
|
||||
# depend on mulga main being healthy and pay that checkout cost on every run,
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout Spinifex
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
path: spinifex
|
||||
fetch-depth: 2
|
||||
@@ -38,34 +38,34 @@ jobs:
|
||||
working-directory: spinifex
|
||||
|
||||
- name: Checkout Viperblock
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/viperblock
|
||||
ref: ${{ steps.siblings.outputs.viperblock }}
|
||||
path: viperblock
|
||||
|
||||
- name: Checkout Predastore
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
ref: ${{ steps.siblings.outputs.predastore }}
|
||||
path: predastore
|
||||
|
||||
- name: Checkout Northstar
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/northstar
|
||||
ref: ${{ steps.siblings.outputs.northstar }}
|
||||
path: northstar
|
||||
|
||||
- name: Checkout Bluebottle
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/bluebottle
|
||||
ref: ${{ steps.siblings.outputs.bluebottle }}
|
||||
path: bluebottle
|
||||
|
||||
- uses: actions/create-github-app-token@v3.3.0
|
||||
- uses: actions/create-github-app-token@v3
|
||||
id: app-token
|
||||
with:
|
||||
client-id: ${{ secrets.APP_ID }}
|
||||
@@ -73,7 +73,7 @@ jobs:
|
||||
repositories: mulga
|
||||
|
||||
- name: Checkout Mulga (scripts/segscan only)
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/mulga
|
||||
token: ${{ steps.app-token.outputs.token }}
|
||||
@@ -81,7 +81,7 @@ jobs:
|
||||
path: mulga
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v7.0.0
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
cache-dependency-path: "spinifex/go.sum"
|
||||
go-version-file: "spinifex/go.mod"
|
||||
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout Spinifex
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
path: spinifex
|
||||
fetch-depth: 2
|
||||
@@ -40,28 +40,28 @@ jobs:
|
||||
working-directory: spinifex
|
||||
|
||||
- name: Checkout Viperblock
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/viperblock
|
||||
ref: ${{ steps.siblings.outputs.viperblock }}
|
||||
path: viperblock
|
||||
|
||||
- name: Checkout Predastore
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
ref: ${{ steps.siblings.outputs.predastore }}
|
||||
path: predastore
|
||||
|
||||
- name: Checkout Northstar
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/northstar
|
||||
ref: ${{ steps.siblings.outputs.northstar }}
|
||||
path: northstar
|
||||
|
||||
- name: Checkout Bluebottle
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/bluebottle
|
||||
ref: ${{ steps.siblings.outputs.bluebottle }}
|
||||
@@ -71,7 +71,7 @@ jobs:
|
||||
# the last go.sum change and never refreshed it. Both caches are managed
|
||||
# below instead.
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v7.0.0
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: "spinifex/go.mod"
|
||||
cache: false
|
||||
@@ -80,7 +80,7 @@ jobs:
|
||||
# setup-go hashed spinifex/go.sum alone, which left the sibling modules'
|
||||
# dependencies out of the key and re-downloading on every run.
|
||||
- name: Cache Go modules
|
||||
uses: actions/cache@v6.1.0
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/go/pkg/mod
|
||||
key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
|
||||
@@ -93,7 +93,7 @@ jobs:
|
||||
# a PR reads the base branch's entry without spending a minute uploading
|
||||
# one only that PR would ever read.
|
||||
- name: Restore Go build cache
|
||||
uses: actions/cache/restore@v6.1.0
|
||||
uses: actions/cache/restore@v6
|
||||
with:
|
||||
path: ~/.cache/go-build
|
||||
key: gobuild-${{ github.job }}-${{ github.run_id }}
|
||||
@@ -150,7 +150,7 @@ jobs:
|
||||
|
||||
- name: Save Go build cache
|
||||
if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
|
||||
uses: actions/cache/save@v6.1.0
|
||||
uses: actions/cache/save@v6
|
||||
with:
|
||||
path: ~/.cache/go-build
|
||||
key: gobuild-${{ github.job }}-${{ github.run_id }}
|
||||
@@ -161,7 +161,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout Spinifex
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
@@ -173,28 +173,28 @@ jobs:
|
||||
working-directory: spinifex
|
||||
|
||||
- name: Checkout Viperblock
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/viperblock
|
||||
ref: ${{ steps.siblings.outputs.viperblock }}
|
||||
path: viperblock
|
||||
|
||||
- name: Checkout Predastore
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
ref: ${{ steps.siblings.outputs.predastore }}
|
||||
path: predastore
|
||||
|
||||
- name: Checkout Northstar
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/northstar
|
||||
ref: ${{ steps.siblings.outputs.northstar }}
|
||||
path: northstar
|
||||
|
||||
- name: Checkout Bluebottle
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/bluebottle
|
||||
ref: ${{ steps.siblings.outputs.bluebottle }}
|
||||
@@ -204,7 +204,7 @@ jobs:
|
||||
# the last go.sum change and never refreshed it. Both caches are managed
|
||||
# below instead.
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v7.0.0
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: "spinifex/go.mod"
|
||||
cache: false
|
||||
@@ -213,7 +213,7 @@ jobs:
|
||||
# setup-go hashed spinifex/go.sum alone, which left the sibling modules'
|
||||
# dependencies out of the key and re-downloading on every run.
|
||||
- name: Cache Go modules
|
||||
uses: actions/cache@v6.1.0
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/go/pkg/mod
|
||||
key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
|
||||
@@ -226,7 +226,7 @@ jobs:
|
||||
# a PR reads the base branch's entry without spending a minute uploading
|
||||
# one only that PR would ever read.
|
||||
- name: Restore Go build cache
|
||||
uses: actions/cache/restore@v6.1.0
|
||||
uses: actions/cache/restore@v6
|
||||
with:
|
||||
path: ~/.cache/go-build
|
||||
key: gobuild-${{ github.job }}-${{ github.run_id }}
|
||||
@@ -251,7 +251,7 @@ jobs:
|
||||
|
||||
- name: Save Go build cache
|
||||
if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
|
||||
uses: actions/cache/save@v6.1.0
|
||||
uses: actions/cache/save@v6
|
||||
with:
|
||||
path: ~/.cache/go-build
|
||||
key: gobuild-${{ github.job }}-${{ github.run_id }}
|
||||
@@ -262,7 +262,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout Spinifex
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
path: spinifex
|
||||
fetch-depth: 2
|
||||
@@ -275,28 +275,28 @@ jobs:
|
||||
working-directory: spinifex
|
||||
|
||||
- name: Checkout Viperblock
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/viperblock
|
||||
ref: ${{ steps.siblings.outputs.viperblock }}
|
||||
path: viperblock
|
||||
|
||||
- name: Checkout Predastore
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
ref: ${{ steps.siblings.outputs.predastore }}
|
||||
path: predastore
|
||||
|
||||
- name: Checkout Northstar
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/northstar
|
||||
ref: ${{ steps.siblings.outputs.northstar }}
|
||||
path: northstar
|
||||
|
||||
- name: Checkout Bluebottle
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: mulgadc/bluebottle
|
||||
ref: ${{ steps.siblings.outputs.bluebottle }}
|
||||
@@ -306,7 +306,7 @@ jobs:
|
||||
# the last go.sum change and never refreshed it. Both caches are managed
|
||||
# below instead.
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v7.0.0
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: "spinifex/go.mod"
|
||||
cache: false
|
||||
@@ -315,7 +315,7 @@ jobs:
|
||||
# setup-go hashed spinifex/go.sum alone, which left the sibling modules'
|
||||
# dependencies out of the key and re-downloading on every run.
|
||||
- name: Cache Go modules
|
||||
uses: actions/cache@v6.1.0
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/go/pkg/mod
|
||||
key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
|
||||
@@ -328,7 +328,7 @@ jobs:
|
||||
# a PR reads the base branch's entry without spending a minute uploading
|
||||
# one only that PR would ever read.
|
||||
- name: Restore Go build cache
|
||||
uses: actions/cache/restore@v6.1.0
|
||||
uses: actions/cache/restore@v6
|
||||
with:
|
||||
path: ~/.cache/go-build
|
||||
key: gobuild-${{ github.job }}-${{ github.run_id }}
|
||||
@@ -338,9 +338,9 @@ jobs:
|
||||
run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
|
||||
|
||||
- name: Lint
|
||||
uses: golangci/golangci-lint-action@v9.3.0
|
||||
uses: golangci/golangci-lint-action@v9
|
||||
with:
|
||||
version: v2.14.0
|
||||
version: v2.13.0
|
||||
working-directory: spinifex
|
||||
|
||||
- name: Govulncheck
|
||||
@@ -363,7 +363,7 @@ jobs:
|
||||
|
||||
- name: Save Go build cache
|
||||
if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
|
||||
uses: actions/cache/save@v6.1.0
|
||||
uses: actions/cache/save@v6
|
||||
with:
|
||||
path: ~/.cache/go-build
|
||||
key: gobuild-${{ github.job }}-${{ github.run_id }}
|
||||
|
||||
+12
-21
@@ -6,14 +6,14 @@ on:
|
||||
- main
|
||||
- dev
|
||||
paths:
|
||||
- spinifex/runtime/roles/spinifexui/frontend/**
|
||||
- spinifex/services/spinifexui/frontend/**
|
||||
- .github/workflows/ui.yml
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- dev
|
||||
paths:
|
||||
- spinifex/runtime/roles/spinifexui/frontend/**
|
||||
- spinifex/services/spinifexui/frontend/**
|
||||
- .github/workflows/ui.yml
|
||||
workflow_dispatch:
|
||||
|
||||
@@ -26,25 +26,25 @@ jobs:
|
||||
runs-on: ubuntu-26.04
|
||||
defaults:
|
||||
run:
|
||||
working-directory: spinifex/runtime/roles/spinifexui/frontend
|
||||
working-directory: spinifex/services/spinifexui/frontend
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v7.0.1
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
sparse-checkout: spinifex/runtime/roles/spinifexui/frontend
|
||||
sparse-checkout: spinifex/services/spinifexui/frontend
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v6.1.0
|
||||
uses: pnpm/action-setup@v6
|
||||
with:
|
||||
package_json_file: spinifex/runtime/roles/spinifexui/frontend/package.json
|
||||
package_json_file: spinifex/services/spinifexui/frontend/package.json
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v7.1.0
|
||||
uses: actions/setup-node@v7
|
||||
with:
|
||||
cache: "pnpm"
|
||||
cache-dependency-path: spinifex/runtime/roles/spinifexui/frontend/pnpm-lock.yaml
|
||||
node-version-file: spinifex/runtime/roles/spinifexui/frontend/.nvmrc
|
||||
cache-dependency-path: spinifex/services/spinifexui/frontend/pnpm-lock.yaml
|
||||
node-version-file: spinifex/services/spinifexui/frontend/.nvmrc
|
||||
|
||||
- name: Install Dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
@@ -58,14 +58,5 @@ jobs:
|
||||
- name: Build Frontend
|
||||
run: pnpm build
|
||||
|
||||
# spx embeds the committed dist/, so a bundle that differs from a clean
|
||||
# build of this source would ship JavaScript CI never built. Porcelain
|
||||
# rather than git diff, so added and deleted chunks count too.
|
||||
- name: Check for generated file drift
|
||||
run: |
|
||||
drift=$(git status --porcelain -- src/routeTree.gen.ts src/lib/cn-tables.gen.ts dist)
|
||||
if [ -n "$drift" ]; then
|
||||
echo "::error::Generated files differ from a clean build. Run pnpm build and commit the result."
|
||||
echo "$drift"
|
||||
exit 1
|
||||
fi
|
||||
- name: Check for generated route tree drift
|
||||
run: git diff --exit-code -- src/routeTree.gen.ts
|
||||
|
||||
+1
-1
@@ -19,7 +19,7 @@ bin/
|
||||
dist/
|
||||
build/microvm/vmlinuz
|
||||
build/microvm/initramfs.cpio.gz
|
||||
!spinifex/runtime/roles/spinifexui/frontend/dist/
|
||||
!spinifex/services/spinifexui/frontend/dist/
|
||||
tmp/
|
||||
*.test
|
||||
*.out
|
||||
|
||||
+7
-7
@@ -1,8 +1,5 @@
|
||||
version: "2"
|
||||
|
||||
run:
|
||||
allow-serial-runners: true
|
||||
|
||||
issues:
|
||||
max-same-issues: 10
|
||||
|
||||
@@ -71,18 +68,17 @@ linters:
|
||||
- pkg: math/rand$
|
||||
desc: Use math/rand/v2 instead, see https://go.dev/blog/randv2
|
||||
|
||||
# The control plane reaches block storage through the EBS provider, not
|
||||
# The EC2 control plane reaches block storage through ebsprovider, not
|
||||
# by importing the engine. viperblock.New starts a chunk uploader, so
|
||||
# every construction is a writer against a volume viperblockd serves.
|
||||
# No domain or legacy handler may import viperblock, in any form, tests
|
||||
# Nothing under handlers/ may import viperblock, in any form, tests
|
||||
# included. A types-only import counts: one became an engine import.
|
||||
ec2-control-plane:
|
||||
files:
|
||||
- '**/spinifex/domains/**'
|
||||
- '**/spinifex/handlers/**'
|
||||
deny:
|
||||
- pkg: github.com/mulgadc/viperblock
|
||||
desc: Use the providers/ebs EBSProvider interface; the control plane must not construct or name viperblock types.
|
||||
desc: Use the ebsprovider.EBSProvider interface; the control plane must not construct or name viperblock types.
|
||||
|
||||
embeddedstructfieldcheck:
|
||||
forbid-mutex: true
|
||||
@@ -146,10 +142,14 @@ linters:
|
||||
staticcheck:
|
||||
checks:
|
||||
- all
|
||||
- -ST1000 # package comment
|
||||
- -ST1003 # naming conventions (Id→ID etc.) — too noisy to retrofit
|
||||
- -ST1012 # error var naming — errname covers this + error type suffixes
|
||||
- -ST1013 # HTTP status constants — usestdlibvars is broader
|
||||
- -ST1016 # receiver name consistency (too noisy to retrofit)
|
||||
- -ST1020 # godoc on exported types — not enforcing yet
|
||||
- -ST1021 # godoc on exported type groups — not enforcing yet
|
||||
- -ST1022 # godoc on exported vars/consts — not enforcing yet
|
||||
- -SA4006 # value overwritten — ineffassign catches strictly more of these
|
||||
- -SA9005 # marshaling structs with no exported fields — intended {} for empty API-response types
|
||||
|
||||
|
||||
@@ -1,97 +0,0 @@
|
||||
# How to Contribute
|
||||
|
||||
We welcome contributions to Spinifex!
|
||||
|
||||
Report security vulnerabilities privately through [SECURITY.md](SECURITY.md), not as a public issue.
|
||||
|
||||
**AWS parity is the design rule.** When Spinifex's behaviour is in question, the answer is what AWS does. If you believe Spinifex diverges from AWS, state what AWS does and show it with a test or a reproduction against a running cluster.
|
||||
|
||||
## Understanding the Codebase
|
||||
|
||||
Every AWS call enters through a SigV4-authenticated gateway, is published to NATS, and is answered by the daemon that claims it. Start with these:
|
||||
|
||||
| Doc | Covers |
|
||||
|---|---|
|
||||
| [`docs/DESIGN.md`](docs/DESIGN.md) | Architecture, request flow, daemons, storage integration, file layout. |
|
||||
| [`docs/COMMANDS.md`](docs/COMMANDS.md) | The `spx admin` CLI and the systemd units it manages. |
|
||||
| [`docs/coverage/`](docs/coverage/README.md) | Which AWS API operations each service implements. |
|
||||
| [`scripts/README.md`](scripts/README.md) | Install, dev-environment, and image-build scripts. |
|
||||
| [`docs/PACKAGE_BOUNDARY_MIGRATION.md`](docs/PACKAGE_BOUNDARY_MIGRATION.md) | ADR-0001 implementation inventory and incremental source moves. |
|
||||
|
||||
User-facing docs are published at [docs.mulgadc.com](https://docs.mulgadc.com).
|
||||
|
||||
## Development Setup
|
||||
|
||||
These steps build Spinifex from source and run it as a single-node cluster, following the [Source Install](https://docs.mulgadc.com/docs/install-source) guide.
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- **Ubuntu 26.04** or **Debian 13**, on x86_64 or aarch64
|
||||
- A host with hardware virtualisation (KVM) and `sudo` access
|
||||
- The host's WAN interface enslaved to a Linux bridge named `br-wan`, with the host IP, default route and DHCP on the bridge rather than the bare NIC (see [VPC Networking → Bridge Setup](https://docs.mulgadc.com/docs/vpc-networking#bridge-setup-physical-network-wiring))
|
||||
|
||||
Check the bridge before continuing:
|
||||
|
||||
```bash
|
||||
ip -br link show br-wan # bridge exists and is UP
|
||||
ip route # default route's dev is br-wan
|
||||
```
|
||||
|
||||
### Install and Run
|
||||
|
||||
```bash
|
||||
mkdir -p ~/Development/mulga && cd ~/Development/mulga
|
||||
git clone https://github.com/mulgadc/spinifex.git
|
||||
sudo make -C spinifex quickinstall # system packages, Go 1.27, AWS CLI v2
|
||||
export PATH=$PATH:/usr/local/go/bin
|
||||
|
||||
cd spinifex
|
||||
./scripts/clone-deps.sh # sibling repos: viperblock, predastore, northstar, bluebottle
|
||||
./scripts/dev-install.sh # build, install, init a one-node cluster, start services
|
||||
|
||||
AWS_PROFILE=spinifex aws ec2 describe-instance-types
|
||||
```
|
||||
|
||||
A list of instance types means the cluster is up.
|
||||
|
||||
`make preflight` also needs `shellcheck` and `golangci-lint`
|
||||
|
||||
```bash
|
||||
sudo apt-get install -y shellcheck
|
||||
curl -sSfL https://golangci-lint.run/install.sh | sh -s -- -b "$(go env GOPATH)/bin" v2.14.0
|
||||
```
|
||||
|
||||
The web console additionally needs Node.js 24 and `pnpm`.
|
||||
|
||||
## Development Workflow
|
||||
|
||||
| Command | Use it to |
|
||||
|---|---|
|
||||
| `make deploy` | Rebuild, install `spx`, and restart `spinifex.target` after a code change. |
|
||||
| `make reinstall` | Re-run `dev-install.sh` after changing systemd units, helper scripts, or logrotate config. |
|
||||
| `./scripts/reset-dev-env.sh` | Wipe all cluster state and reinstall from scratch, keeping your network topology. |
|
||||
| `make test` | Run unit tests (`make test-race` for the race detector). |
|
||||
| `make test-integration` | Run the real gateway router against embedded NATS JetStream. |
|
||||
| `make preflight` | Run the pre-commit gate: lint, `govulncheck`, coverage, and manifest and script checks. Must pass. |
|
||||
|
||||
Follow the services with `journalctl -u 'spinifex-*' -f`.
|
||||
|
||||
The web console lives in `spinifex/runtime/roles/spinifexui/frontend/` (`pnpm install`, `pnpm dev`, `pnpm lint`, `pnpm test`). `spx` embeds its checked-in `dist/`, so run `make build-ui` and commit the rebuilt `dist/` with your change.
|
||||
|
||||
## Creating a Pull Request
|
||||
|
||||
1. Fork [mulgadc/spinifex](https://github.com/mulgadc/spinifex) and clone your fork into the same parent directory as the sibling repositories, branching off `dev`:
|
||||
|
||||
```bash
|
||||
cd ~/Development/mulga
|
||||
git clone git@github.com:YOUR_USERNAME/spinifex.git && cd spinifex
|
||||
git remote add upstream git@github.com:mulgadc/spinifex.git
|
||||
git fetch upstream && git checkout -b feat/my-feature upstream/dev
|
||||
```
|
||||
|
||||
2. Make your change, then verify it:
|
||||
- `make preflight` passes.
|
||||
- `go mod tidy` leaves only intended changes in `go.mod` and `go.sum`, with none of the `clone-deps.sh` `replace` lines.
|
||||
- Runtime changes are exercised on your dev cluster with `make deploy`.
|
||||
|
||||
3. Commit using [Conventional Commits](https://www.conventionalcommits.org/) (`feat:`, `fix:`, `refactor:`, `docs:`, `chore:`), push, and open a pull request against `dev`, filling in the template.
|
||||
@@ -54,12 +54,12 @@ build: go_build build-installer build-lb-agent generate-aws-model-coverage
|
||||
# Build spinifex-ui frontend (requires pnpm)
|
||||
build-ui:
|
||||
@echo -e "\n....Building spinifex-ui frontend...."
|
||||
cd spinifex/runtime/roles/spinifexui/frontend && pnpm build
|
||||
cd spinifex/services/spinifexui/frontend && pnpm build
|
||||
|
||||
# GO commands
|
||||
VERSION ?= $(shell git describe --tags --always --dirty --exclude '*-dev')
|
||||
VERSION ?= $(shell git describe --tags --always --dirty)
|
||||
COMMIT ?= $(shell git rev-parse --short HEAD)
|
||||
LDFLAGS := -s -w -X github.com/mulgadc/spinifex/spinifex/operator/cli.Version=$(VERSION) -X github.com/mulgadc/spinifex/spinifex/operator/cli.Commit=$(COMMIT)
|
||||
LDFLAGS := -s -w -X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Version=$(VERSION) -X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Commit=$(COMMIT)
|
||||
|
||||
go_build:
|
||||
@echo -e "\n....Building $(GO_PROJECT_NAME)"
|
||||
@@ -153,35 +153,21 @@ install-microvm: $(MICROVM_ARTIFACTS) ## Install microVM artifacts to /usr/share
|
||||
# -j2 overlaps the only two heavy gates, lint and test-cover: alone each leaves
|
||||
# most of the machine idle, so pairing them cuts a cold run by about a quarter.
|
||||
# -Otarget keeps each gate's output in one block instead of interleaving them.
|
||||
# manifest-check manifest-lint test-package-check removed, can add back later
|
||||
preflight:
|
||||
@$(MAKE) --no-print-directory -j2 -Otarget QUIET=1 lint govulncheck test-cover diff-coverage test-harness test-build-scripts manifest-check manifest-lint
|
||||
@$(MAKE) --no-print-directory -j2 -Otarget QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-package-check test-harness test-build-scripts
|
||||
@echo -e "\n ✅ Preflight passed — safe to commit."
|
||||
|
||||
# Shell suites + shellcheck for build/scripts/, the systemd-unit helpers that
|
||||
# ship on every node (unlike scripts/images/, kept in preflight: a wrong
|
||||
# restart decision here is an outage, not asset churn).
|
||||
#
|
||||
# NODE_SCRIPTS are the four that setup.sh installs into
|
||||
# /usr/local/share/spinifex on every node. They are covered here for the same
|
||||
# reason: between them they install the software, wire OVN, erase the state and
|
||||
# remove the install, so a shellcheck warning in one is an outage or a wrong
|
||||
# deletion, not a style note.
|
||||
NODE_SCRIPTS := scripts/setup.sh scripts/setup-ovn.sh scripts/install-node.sh scripts/smoke-test.sh scripts/node-reset.sh scripts/uninstall-spx.sh
|
||||
|
||||
test-build-scripts:
|
||||
@echo -e "\n....Running build/scripts/**/*_test.sh...."
|
||||
@for t in $$(find build/scripts -name '*_test.sh' | sort); do \
|
||||
echo "-- $$t"; \
|
||||
sh "$$t" || exit 1; \
|
||||
done
|
||||
@echo -e "\n....Running scripts/*_test.sh...."
|
||||
@for t in $$(find scripts -maxdepth 1 -name '*_test.sh' | sort); do \
|
||||
echo "-- $$t"; \
|
||||
sh "$$t" || exit 1; \
|
||||
done
|
||||
@echo -e "\n....Running shellcheck over build/scripts/**/*.sh + the node scripts...."
|
||||
shellcheck -S warning $$(find build/scripts -name '*.sh' | sort) $(NODE_SCRIPTS)
|
||||
@echo -e "\n....Running shellcheck over build/scripts/**/*.sh...."
|
||||
shellcheck -S warning $$(find build/scripts -name '*.sh' | sort)
|
||||
@echo " test-build-scripts ok"
|
||||
|
||||
# E2E harness unit tests. Build-tagged `e2e` so they're skipped by the
|
||||
@@ -203,12 +189,10 @@ AWS_MODEL_OPERATION_COVERAGE_DIR ?= $(CURDIR)/docs/coverage
|
||||
# -count=1 is load-bearing: the conformance report is written by the test binary,
|
||||
# so a cached pass skips the run, leaves no report, and the cat below fails the
|
||||
# target. It also keeps the conformance gate honest — a cached result would mean
|
||||
# the check never ran against this commit. The generated request sweep adds
|
||||
# about 15s, most of it EC2 fan-outs waiting out their window for instances
|
||||
# no daemon-lite owns.
|
||||
# the check never ran against this commit.
|
||||
test-integration:
|
||||
@echo -e "\n....Running in-process integration tests...."
|
||||
$(_Q)LOG_IGNORE=1 AWS_MODEL_CONFORMANCE_MODE=$(AWS_MODEL_CONFORMANCE_MODE) AWS_MODEL_CONFORMANCE_REPORT=$(AWS_MODEL_CONFORMANCE_REPORT) go test -count=1 -tags=integration -timeout 120s ./tests/integration/... $(_RACEQ)
|
||||
$(_Q)LOG_IGNORE=1 AWS_MODEL_CONFORMANCE_MODE=$(AWS_MODEL_CONFORMANCE_MODE) AWS_MODEL_CONFORMANCE_REPORT=$(AWS_MODEL_CONFORMANCE_REPORT) go test -count=1 -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ)
|
||||
@cat $(AWS_MODEL_CONFORMANCE_REPORT)
|
||||
@$(MAKE) --no-print-directory generate-aws-model-coverage
|
||||
@echo "AWS operation coverage: $(AWS_MODEL_OPERATION_COVERAGE_DIR)"
|
||||
@@ -220,7 +204,7 @@ aws-model-coverage: generate-aws-model-coverage
|
||||
@go run ./cmd/aws-model-coverage
|
||||
|
||||
# Segscan storage oracle: needs the mulga umbrella repo's scripts/segscan
|
||||
# checked out alongside spinifex (see internal/testkit/segscanoracle), which
|
||||
# checked out alongside spinifex (see spinifex/testutil/segscanoracle), which
|
||||
# is not the default local or CI layout, so this is a separate target from
|
||||
# test-integration rather than folded into it. Skips itself when segscan's
|
||||
# source isn't found.
|
||||
@@ -248,7 +232,7 @@ manifest-lint-update:
|
||||
# Run unit tests
|
||||
test:
|
||||
@echo -e "\n....Running tests for $(GO_PROJECT_NAME)...."
|
||||
LOG_IGNORE=1 go test -timeout 180s ./spinifex/... ./cmd/... ./internal/... ./contracts/... ./architecture/...
|
||||
LOG_IGNORE=1 go test -timeout 180s ./spinifex/... ./cmd/... ./internal/...
|
||||
|
||||
# Empty locally, where reusing a cached result between runs is the point.
|
||||
# CI passes -count=1 so a green run means the suite executed against that commit
|
||||
@@ -259,13 +243,13 @@ GOTESTFLAGS ?=
|
||||
COVERPROFILE ?= coverage.out
|
||||
test-cover:
|
||||
@echo -e "\n....Running tests with coverage for $(GO_PROJECT_NAME)...."
|
||||
$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -timeout 180s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... ./cmd/... ./internal/... ./contracts/... ./architecture/... $(_COVQ)
|
||||
$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -timeout 180s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... ./cmd/... ./internal/... $(_COVQ)
|
||||
@scripts/check-coverage.sh $(COVERPROFILE) $(QUIET)
|
||||
|
||||
# Run unit tests with race detector
|
||||
test-race:
|
||||
@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
|
||||
$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... ./contracts/... ./architecture/... $(_RACEQ)
|
||||
$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... $(_RACEQ)
|
||||
|
||||
# Unit tests for in-repo GitHub Actions (e.g. .github/actions/e2e-analyze).
|
||||
# Kept out of `test-cover` so coverage % isn't diluted by CI-only tooling.
|
||||
@@ -318,17 +302,6 @@ bench:
|
||||
@echo -e "\n....Running benchmarks for $(GO_PROJECT_NAME)...."
|
||||
LOG_IGNORE=1 go test -benchmem -run=. -bench=. ./...
|
||||
|
||||
# Fuzz every Fuzz* target for FUZZTIME each; -fuzz takes one target per run.
|
||||
# Plain `go test` already replays the seeds and testdata/fuzz on preflight.
|
||||
FUZZTIME ?= 30s
|
||||
fuzz:
|
||||
@set -e; grep -rlE '^func Fuzz\w+\(' --include='*_test.go' spinifex cmd internal | while read -r file; do \
|
||||
for name in $$(grep -oE '^func Fuzz\w+' "$$file" | cut -d' ' -f2); do \
|
||||
echo -e "\n....Fuzzing $$name ($$(dirname $$file)) for $(FUZZTIME)...."; \
|
||||
LOG_IGNORE=1 go test -run='^$$' -fuzz="^$$name$$" -fuzztime=$(FUZZTIME) ./$$(dirname $$file); \
|
||||
done; \
|
||||
done
|
||||
|
||||
# Fast iteration: build + install binary + restart all services.
|
||||
# Microvm artifacts are reinstalled when they already exist on disk — the rule's
|
||||
# input timestamps drive a rebuild only if anything actually changed. On a fresh
|
||||
@@ -365,7 +338,7 @@ reinstall:
|
||||
|
||||
clean:
|
||||
rm -f ./bin/$(GO_PROJECT_NAME)
|
||||
rm -rf spinifex/runtime/roles/spinifexui/frontend/dist
|
||||
rm -rf spinifex/services/spinifexui/frontend/dist
|
||||
|
||||
install-system:
|
||||
@echo -e "\n....Installing system dependencies for $(ARCH)...."
|
||||
@@ -379,9 +352,9 @@ install-system:
|
||||
ovn-central ovn-host openvswitch-switch dhcpcd-base
|
||||
|
||||
install-go:
|
||||
@echo -e "\n....Installing Go 1.27.3 for $(ARCH) ($(GO_ARCH))...."
|
||||
@echo -e "\n....Installing Go 1.27.0 for $(ARCH) ($(GO_ARCH))...."
|
||||
@if [ ! -d "/usr/local/go" ]; then \
|
||||
curl -L https://go.dev/dl/go1.27.3.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
|
||||
curl -L https://go.dev/dl/go1.27.0.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
|
||||
else \
|
||||
echo "Go already installed in /usr/local/go"; \
|
||||
fi
|
||||
@@ -464,7 +437,7 @@ distro-arm64:
|
||||
distro-clean:
|
||||
rm -rf dist/
|
||||
|
||||
.PHONY: test-package-check build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
|
||||
.PHONY: test-package-check build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
|
||||
deploy reinstall clean \
|
||||
install-system install-go install-aws quickinstall \
|
||||
lint fix govulncheck nilaway \
|
||||
|
||||
@@ -4,136 +4,223 @@
|
||||
|
||||
<p align="center">
|
||||
<a href="https://go.dev"><img src="https://img.shields.io/badge/Go-1.27+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
|
||||
<a href="https://pkg.go.dev/github.com/mulgadc/spinifex"><img src="https://img.shields.io/badge/pkg.go.dev-Reference-007D9C?style=flat-square&logo=go&logoColor=white" alt="Go Reference"></a>
|
||||
<a href="LICENSE"><img src="https://img.shields.io/badge/License-AGPL--3.0-3fb950?style=flat-square" alt="License"></a>
|
||||
<a href="https://mulgadc.com"><img src="https://img.shields.io/badge/Site-mulga-orange?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTE2LjcxOCA4Ljg5MWMtMS4yODUgMS4zNy0zLjE3OCAyLjMxNy00LjY5NSAzLjQ0LS44NTQuNjMtMy4wODkgMi4yNy0xLjYxIDMuMzQ0IDEuNzg4IDEuMjk4IDYuMjQzLjE2NCA3Ljc4NS0xLjI2Ljg0LS43NzUuODE0LTEuODIyLS4zMjgtMi4yNTktMS4yMzUtLjQ3Mi0yLjY2LS4xMTEtMy45MTMuMDg3LS4wNDIuMDA3LS4xMzEuMDU1LS4xMjMtLjAyLjUyMy0uMzM2Ljk5My0uNzUgMS41MDMtMS4xMDIuNDkyLS4zNCAxLjA5Ny0uODI3IDEuNy0uODM4IDEuODcxLS4wMzQgMy43OTkuODkgNC4yODcgMi44MTUuODExIDMuMjAzLTMuMDA2IDUuNzE1LTUuNzg0IDUuOTQybDEuNjE0LS43NDdjLjYwNS0uMzYgMS4yMTctLjczNCAxLjc1Mi0xLjE5Ni4xMzMtLjExNS4yMy0uMjYzLjM0Mi0uMzczLjAyNy0uMDI2LjMwMi0uMTQ0LjE0NC0uMTUtMS40NDEuOTQ1LTMuMTI3IDEuNTcyLTQuODEzIDEuOTMyLS41OC4xMjMtMS4xOTUuMjQyLTEuNzg1LjI1Ny4wMTUuMDguMDg3LjA2NC4xNC4wOC40NzYuMTU0IDEuMDIuMjQ1IDEuNTE2LjMyLjA2OC4wNTItLjAwNi4wNzQtLjA2LjA4MS0uNDQ4LjA1OC0uOTIzLjE0My0xLjM3LjE2Ni0xLjI3LjA2NC0yLjU1LS4wNjgtMy43NzctLjM4bC0uNjktMS45NTdjLS4wNi4wMDYtLjA2My4wNi0uMDc5LjEwMy0uMDYuMTctLjMwNSAxLjQxNy0uMzg3IDEuNDM1LS42Ni0uMi0xLjI1MS0uNjQ3LTEuNzM2LTEuMTMybDEuMTUtMi4wNi0xLjgzNSAxLjAxMWMtLjI1OC0uNTgzLS4zMDUtMS4yNDMtLjIxOS0xLjg3bDIuMzc1LTEuMThjLS43MzktLjA0OS0xLjQ4Ni4wNjgtMi4yMi4xNDUuMTI1LS4zNi4yNzctLjcxOC40NjItMS4wNTEuMDY3LS4xMjIuNDI2LS43MjEuNTItLjcyNC44Ni4xNjQgMS43ODYuMjQxIDIuNjQ4LjA3NGwtMS44Ny0uNzcxLS4wNjktLjA5M2MuMzgyLS4zNi43Ny0uNzE3IDEuMTk5LTEuMDI0LjgwNC4yMzQgMS42My40MjIgMi40NzMuNDMzTDkuMzUgOS4zNDJhNyA3IDAgMCAxIC41NjctLjQwMWMuMTYtLjEwMy42MDktLjQwMi43NzItLjM4LjY1LjIyMSAxLjMyNC4zNzggMi4wMS40MzMtLjMxMy0uMzI1LS45MDktLjU2NC0xLjIxLS44NjgtLjAzMy0uMDM0LS4wNTgtLjAzNi0uMDQzLS4wOTguNzg3LS40NTIgMS42NjItLjkwMSAyLjM0LTEuNTE3LjY5LS42MjkgMS40MjEtMS42NTYuOTI0LTIuNjA1LjU3My4xODIgMS4wNjcuOTA2IDEuMDU0IDEuNTEyLS4wMzQgMS42NzYtMS44MjIgMy4xNy0yLjk0MyA0LjIyMiAxLjczMi0uNzI4IDMuNzE0LTIuMjMgMy43MS00LjMwNS0uMDAzLTEuNjg4LTEuNTQtMi4zNjUtMi45OTMtMi41MThhMy4yIDMuMiAwIDAgMS0uMzg1IDEuMDg3Yy0uNDI4LjcxOS0xLjMwMiAxLjE2OC0xLjc1OCAxLjkxNS0uMzExLjUxLS4zNyAxLjE5NS0xLjAzMSAxLjM5LS4yNy4wOC0uNjE3LjA5My0uODk3LjA5NGwuNjE1LS4zMjVjLjY4OS0uNTA2LjY3Ny0xLjQyIDEuMDk5LTIuMS0uMDUtLjA3LS41NDUtLjItLjY2LS4yMjktLjUxLS4xMjUtMS4zNzUtLjI4NS0xLjg4Ni0uMjUtLjE1Ny4wMS0uODY5LjEyMS0uOTMuMjQyLS4wODguMTc3LjM0MS45My40NjggMS4xMDEuMDI1LjAzNS4wNzcuMDI2LjA4LjAzLjAyNC4wMzEuMDA3LjEwNS0uMDYuMDhhNSA1IDAgMCAxLS40MDQtLjI4M2MtLjUwMy0uMzg4LTEuMDc4LS45NzQtMS40OTktMS40NDctLjA2OC0uMTY2LS4xNi0uMzUuMDAyLS40ODcuODg4LS41NSAxLjc2OC0xLjEyNiAyLjY3LTEuNjUxIDIuNjcxLTEuNTU4IDUuNzExLTMuMzE4IDguMjY3LS40NjUgMi4xMTkgMi4zNjYgMS41MTQgNS4yMTItLjUxMiA3LjM3MnptLTguMjI0LTUuMzhjLjY5OS0uMTIyIDIuMDE4LjU3MiAyLjIzNS0uNDU2LjAyMS0uMS0uMDM2LS4xNTcuMDItLjI1NS4wNDMtLjA3My4yODYtLjI1LjM3LS4zMTguMjctLjIxNy41NzctLjM4Ny44NDItLjYxMi0xLjAyOS4xNjYtMi4wNjUuNzU2LTIuOTY0IDEuMjc3LS4xNDkuMDg2LS4zMS4xNzYtLjQ1MS4yNzMtLjAzNy4wMjUtLjA3NS0uMDA2LS4wNTMuMDltLTQuODMgMTAuODQ0Yy0xLjcxNSAxLjg4MS0xLjMyNSA0LjU3LjQ5NCA2LjIxNyAxLjgxMSAxLjY0MSA0LjY2IDIuMjIzIDcuMDQ4IDIuMTA4IDIuMTUyLS4xMDMgNC4zMzctLjgxMiA2LjQ2LS4wOTEuODI1LjI4IDEuNTQ2LjgwNSAyLjE2IDEuNDExLS4wODMtLjQtLjMyNC0uODE5LS41NTgtMS4xNTktMi4xMy0zLjA5NS02LjI3LTEuOTM1LTkuNDI2LTIuNTUzLTEuNzExLS4zMzUtMy40OTEtMS4xMTgtNC41MzMtMi41NjctLjkwMS0xLjI1My0xLjA0Mi0yLjczLS41OTUtNC4xOTQtLjA5MS0uMDktLjk1Mi43MjEtMS4wNS44MjhtNi4zNjYtOC42NjdjLS4zODIuMzM5LS43ODcuNjYtMS4yMTIuOTQ4bC0xLjIwNy41OWMxLjAyMi4wMzkgMi4wOC0uNTQ2IDIuNDItMS41MzciLz48L3N2Zz4=" alt="mulgadc.com"></a>
|
||||
<a href="https://docs.mulgadc.com"><img src="https://img.shields.io/badge/Docs-spinifex-0969da?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTYuNSAySDIwYTEgMSAwIDAgMSAxIDF2MTQuNWExIDEgMCAwIDEtMSAxSDYuNWExLjUgMS41IDAgMCAwIDAgM0gyMWExIDEgMCAwIDEgMCAySDYuNUEzLjUgMy41IDAgMCAxIDMgMjBWNS41QTMuNSAzLjUgMCAwIDEgNi41IDJ6bTEgNGExIDEgMCAwIDAgMCAyaDlhMSAxIDAgMCAwIDAtMmgtOXptMCA0YTEgMSAwIDAgMCAwIDJoOWExIDEgMCAwIDAgMC0yaC05eiIvPjwvc3ZnPg==" alt="docs.mulgadc.com"></a>
|
||||
<a href="https://mulgadc.com"><img src="https://img.shields.io/badge/home-mulga-orange?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTE2LjcxOCA4Ljg5MWMtMS4yODUgMS4zNy0zLjE3OCAyLjMxNy00LjY5NSAzLjQ0LS44NTQuNjMtMy4wODkgMi4yNy0xLjYxIDMuMzQ0IDEuNzg4IDEuMjk4IDYuMjQzLjE2NCA3Ljc4NS0xLjI2Ljg0LS43NzUuODE0LTEuODIyLS4zMjgtMi4yNTktMS4yMzUtLjQ3Mi0yLjY2LS4xMTEtMy45MTMuMDg3LS4wNDIuMDA3LS4xMzEuMDU1LS4xMjMtLjAyLjUyMy0uMzM2Ljk5My0uNzUgMS41MDMtMS4xMDIuNDkyLS4zNCAxLjA5Ny0uODI3IDEuNy0uODM4IDEuODcxLS4wMzQgMy43OTkuODkgNC4yODcgMi44MTUuODExIDMuMjAzLTMuMDA2IDUuNzE1LTUuNzg0IDUuOTQybDEuNjE0LS43NDdjLjYwNS0uMzYgMS4yMTctLjczNCAxLjc1Mi0xLjE5Ni4xMzMtLjExNS4yMy0uMjYzLjM0Mi0uMzczLjAyNy0uMDI2LjMwMi0uMTQ0LjE0NC0uMTUtMS40NDEuOTQ1LTMuMTI3IDEuNTcyLTQuODEzIDEuOTMyLS41OC4xMjMtMS4xOTUuMjQyLTEuNzg1LjI1Ny4wMTUuMDguMDg3LjA2NC4xNC4wOC40NzYuMTU0IDEuMDIuMjQ1IDEuNTE2LjMyLjA2OC4wNTItLjAwNi4wNzQtLjA2LjA4MS0uNDQ4LjA1OC0uOTIzLjE0My0xLjM3LjE2Ni0xLjI3LjA2NC0yLjU1LS4wNjgtMy43NzctLjM4bC0uNjktMS45NTdjLS4wNi4wMDYtLjA2My4wNi0uMDc5LjEwMy0uMDYuMTctLjMwNSAxLjQxNy0uMzg3IDEuNDM1LS42Ni0uMi0xLjI1MS0uNjQ3LTEuNzM2LTEuMTMybDEuMTUtMi4wNi0xLjgzNSAxLjAxMWMtLjI1OC0uNTgzLS4zMDUtMS4yNDMtLjIxOS0xLjg3bDIuMzc1LTEuMThjLS43MzktLjA0OS0xLjQ4Ni4wNjgtMi4yMi4xNDUuMTI1LS4zNi4yNzctLjcxOC40NjItMS4wNTEuMDY3LS4xMjIuNDI2LS43MjEuNTItLjcyNC44Ni4xNjQgMS43ODYuMjQxIDIuNjQ4LjA3NGwtMS44Ny0uNzcxLS4wNjktLjA5M2MuMzgyLS4zNi43Ny0uNzE3IDEuMTk5LTEuMDI0LjgwNC4yMzQgMS42My40MjIgMi40NzMuNDMzTDkuMzUgOS4zNDJhNyA3IDAgMCAxIC41NjctLjQwMWMuMTYtLjEwMy42MDktLjQwMi43NzItLjM4LjY1LjIyMSAxLjMyNC4zNzggMi4wMS40MzMtLjMxMy0uMzI1LS45MDktLjU2NC0xLjIxLS44NjgtLjAzMy0uMDM0LS4wNTgtLjAzNi0uMDQzLS4wOTguNzg3LS40NTIgMS42NjItLjkwMSAyLjM0LTEuNTE3LjY5LS42MjkgMS40MjEtMS42NTYuOTI0LTIuNjA1LjU3My4xODIgMS4wNjcuOTA2IDEuMDU0IDEuNTEyLS4wMzQgMS42NzYtMS44MjIgMy4xNy0yLjk0MyA0LjIyMiAxLjczMi0uNzI4IDMuNzE0LTIuMjMgMy43MS00LjMwNS0uMDAzLTEuNjg4LTEuNTQtMi4zNjUtMi45OTMtMi41MThhMy4yIDMuMiAwIDAgMS0uMzg1IDEuMDg3Yy0uNDI4LjcxOS0xLjMwMiAxLjE2OC0xLjc1OCAxLjkxNS0uMzExLjUxLS4zNyAxLjE5NS0xLjAzMSAxLjM5LS4yNy4wOC0uNjE3LjA5My0uODk3LjA5NGwuNjE1LS4zMjVjLjY4OS0uNTA2LjY3Ny0xLjQyIDEuMDk5LTIuMS0uMDUtLjA3LS41NDUtLjItLjY2LS4yMjktLjUxLS4xMjUtMS4zNzUtLjI4NS0xLjg4Ni0uMjUtLjE1Ny4wMS0uODY5LjEyMS0uOTMuMjQyLS4wODguMTc3LjM0MS45My40NjggMS4xMDEuMDI1LjAzNS4wNzcuMDI2LjA4LjAzLjAyNC4wMzEuMDA3LjEwNS0uMDYuMDhhNSA1IDAgMCAxLS40MDQtLjI4M2MtLjUwMy0uMzg4LTEuMDc4LS45NzQtMS40OTktMS40NDctLjA2OC0uMTY2LS4xNi0uMzUuMDAyLS40ODcuODg4LS41NSAxLjc2OC0xLjEyNiAyLjY3LTEuNjUxIDIuNjcxLTEuNTU4IDUuNzExLTMuMzE4IDguMjY3LS40NjUgMi4xMTkgMi4zNjYgMS41MTQgNS4yMTItLjUxMiA3LjM3MnptLTguMjI0LTUuMzhjLjY5OS0uMTIyIDIuMDE4LjU3MiAyLjIzNS0uNDU2LjAyMS0uMS0uMDM2LS4xNTcuMDItLjI1NS4wNDMtLjA3My4yODYtLjI1LjM3LS4zMTguMjctLjIxNy41NzctLjM4Ny44NDItLjYxMi0xLjAyOS4xNjYtMi4wNjUuNzU2LTIuOTY0IDEuMjc3LS4xNDkuMDg2LS4zMS4xNzYtLjQ1MS4yNzMtLjAzNy4wMjUtLjA3NS0uMDA2LS4wNTMuMDltLTQuODMgMTAuODQ0Yy0xLjcxNSAxLjg4MS0xLjMyNSA0LjU3LjQ5NCA2LjIxNyAxLjgxMSAxLjY0MSA0LjY2IDIuMjIzIDcuMDQ4IDIuMTA4IDIuMTUyLS4xMDMgNC4zMzctLjgxMiA2LjQ2LS4wOTEuODI1LjI4IDEuNTQ2LjgwNSAyLjE2IDEuNDExLS4wODMtLjQtLjMyNC0uODE5LS41NTgtMS4xNTktMi4xMy0zLjA5NS02LjI3LTEuOTM1LTkuNDI2LTIuNTUzLTEuNzExLS4zMzUtMy40OTEtMS4xMTgtNC41MzMtMi41NjctLjkwMS0xLjI1My0xLjA0Mi0yLjczLS41OTUtNC4xOTQtLjA5MS0uMDktLjk1Mi43MjEtMS4wNS44MjhtNi4zNjYtOC42NjdjLS4zODIuMzM5LS43ODcuNjYtMS4yMTIuOTQ4bC0xLjIwNy41OWMxLjAyMi4wMzkgMi4wOC0uNTQ2IDIuNDItMS41MzciLz48L3N2Zz4=" alt="mulgadc.com"></a>
|
||||
<a href="https://docs.mulgadc.com"><img src="https://img.shields.io/badge/docs-spinifex-0969da?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTYuNSAySDIwYTEgMSAwIDAgMSAxIDF2MTQuNWExIDEgMCAwIDEtMSAxSDYuNWExLjUgMS41IDAgMCAwIDAgM0gyMWExIDEgMCAwIDEgMCAySDYuNUEzLjUgMy41IDAgMCAxIDMgMjBWNS41QTMuNSAzLjUgMCAwIDEgNi41IDJ6bTEgNGExIDEgMCAwIDAgMCAyaDlhMSAxIDAgMCAwIDAtMmgtOXptMCA0YTEgMSAwIDAgMCAwIDJoOWExIDEgMCAwIDAgMC0yaC05eiIvPjwvc3ZnPg==" alt="docs.mulgadc.com"></a>
|
||||
</p>
|
||||
|
||||
|
||||
<p align="center">
|
||||
<a href="#try-it-live">Live Demo</a> ·
|
||||
<a href="#services">Services</a> ·
|
||||
<a href="#deploy-anywhere">Deploy</a> ·
|
||||
<a href="#install">Install</a> ·
|
||||
<a href="#architecture">Architecture</a>
|
||||
<a href="#what-is-spinifex">What is Spinifex?</a> ·
|
||||
<a href="#the-platform">Platform</a> ·
|
||||
<a href="#three-ways-to-deploy">Deploy</a> ·
|
||||
<a href="#aws-compatibility">AWS Compatibility</a> ·
|
||||
<a href="#live-demo">Live Demo</a> ·
|
||||
<a href="#core-components">Components</a> ·
|
||||
<a href="#architecture-at-a-glance">Architecture</a> ·
|
||||
<a href="#self-hosted-installation">Installation</a> ·
|
||||
<a href="https://docs.mulgadc.com">Docs</a>
|
||||
</p>
|
||||
|
||||
---
|
||||
|
||||
# Spinifex: the open, AWS-compatible cloud you run yourself
|
||||
|
||||
Spinifex, built by [Mulga](https://mulgadc.com), recreates the AWS services your software already uses on infrastructure you control. Move workloads off the hyperscalers, run AI at a fraction of the cost, or take the cloud to the edge, without rewriting a line.
|
||||
Spinifex, built by [Mulga](https://mulgadc.com), recreates the AWS services your software already uses (EC2, EBS, S3, VPC, IAM) on infrastructure you control. Move workloads off the hyperscalers, run AI at a fraction of the cost, or take the cloud to the edge, without rewriting a line. The exact build you ship to AWS, Spinifex serves on your own hardware: same APIs, same Terraform, same SDKs.
|
||||
|
||||
- **Your AWS tooling, unchanged.** The AWS CLI, SDKs, and Terraform work as they are.
|
||||
- **Full control of the stack.** Your hardware, data, and keys, with no external control plane, running through disconnection.
|
||||
- **Open core under AGPL-3.0.** Auditable, yours to keep, with no lock-in.
|
||||
## What is Spinifex?
|
||||
|
||||
Spinifex is an open-source infrastructure platform that recreates the AWS service surface (EC2, EBS, S3, VPC, IAM) on bare-metal, edge, and on-premises environments. Run cloud-native software without a hyperscaler.
|
||||
|
||||
Built for teams that need:
|
||||
|
||||
- The AWS tooling you already use (AWS CLI, SDKs, Terraform), with nothing to rewrite
|
||||
- Full control of the stack: your hardware, your network, your data, your keys
|
||||
- A cloud that keeps running offline, through disconnection, with no external control plane
|
||||
- An open core (AGPL-3.0), auditable and yours to keep, with no lock-in
|
||||
|
||||
You change *where* your software runs, not *what* it is.
|
||||
|
||||
## Try It Live
|
||||
|
||||
Get a real Spinifex endpoint in under a minute, with no card and no install. The sandbox is yours for 72 hours, with access keys, the web console, and EC2, EBS, S3, VPC, IAM, EKS, and RDS ready to call. Point your existing AWS CLI or Terraform straight at it.
|
||||
|
||||
**[Create your sandbox →](https://mulgadc.com/signup)**
|
||||
|
||||
## Same Commands, Your Cloud
|
||||
|
||||
```bash
|
||||
export AWS_PROFILE=spinifex
|
||||
aws ec2 describe-images
|
||||
aws ec2 run-instances --image-id ami-xxxxxxxx --instance-type t3.micro
|
||||
aws s3 mb s3://my-bucket
|
||||
```
|
||||
|
||||
For Terraform, start from the [example workbooks](https://docs.mulgadc.com/docs/nginx-webserver).
|
||||
|
||||
## Why Not LocalStack or Floci?
|
||||
|
||||
[LocalStack](https://github.com/localstack/localstack) and [Floci](https://github.com/floci-io/floci) are emulators for local development and CI testing. An emulator returns a plausible response; Spinifex boots the machine. A call launches a real KVM guest, attaches a replicated Viperblock volume, or writes an erasure-coded object into Predastore, so Spinifex can run your production workloads on hardware you own. Test against an emulator, then run in production on Spinifex.
|
||||
|
||||
## Services
|
||||
|
||||
The platform serves **392 AWS API operations** across the services below. See exactly which ones in the [coverage report](https://docs.mulgadc.com/coverage).
|
||||
|
||||
| Service | What it is | Built on | Status |
|
||||
| --- | --- | --- | --- |
|
||||
| **EC2** | Compute | QEMU/KVM, with spot instances and GPU passthrough | Available |
|
||||
| **EBS** | Block storage | [Viperblock](https://github.com/mulgadc/viperblock): replicated, with a WAL on local NVMe | Available |
|
||||
| **S3** | Object storage | [Predastore](https://github.com/mulgadc/predastore): Reed-Solomon erasure coding | Available |
|
||||
| **VPC** | Networking | OVN, with security groups | Available |
|
||||
| **IAM** | Identity & auth | SigV4 across every service | Available |
|
||||
| **ALB / NLB** | Load balancers | HAProxy (L7), nginx `stream` (L4), with ACM certificates | Available |
|
||||
| **EKS** | Kubernetes | k3s, with HA control planes and managed node groups | Available |
|
||||
| **ECR** | Container registry | Predastore, deduplicated per account | Available |
|
||||
| **ECS** | Container service | Spinifex instances | Available |
|
||||
| **RDS** | Databases | PostgreSQL and MariaDB, with automated backups | Available |
|
||||
| **Bedrock** | AI deployment | vLLM for text generation and Hugging Face TEI for embeddings and reranking | Q4 2026 |
|
||||
| **Lambda** | Serverless functions | | Q1 2027 |
|
||||
|
||||
Roadmap services ship under the same AWS API surface, so code written for AWS today works the moment they land. [Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
|
||||
|
||||
## Deploy Anywhere
|
||||
## The Platform
|
||||
|
||||
<p align="center">
|
||||
<img src=".github/assets/platform.svg" alt="The Spinifex stack: your apps and AWS tooling on top, the Spinifex AWS-compatible service layer, running on standard Linux on CPU and GPU hardware, deployable to Neocloud, on-premise, or the edge." width="900">
|
||||
</p>
|
||||
|
||||
- **Neocloud:** lift and shift onto partner GPU capacity (H100 / H200 / B200). Change an endpoint, keep the software.
|
||||
- **On-premise:** a multi-node HA cluster in your own data centre, so the stack, the data, and the jurisdiction stay yours.
|
||||
- **Edge:** air-gapped sites, vehicles, vessels, factories, and clinics, with compute next to the data.
|
||||
From commodity hardware up to unmodified AWS tooling, every layer is replaceable and yours to own.
|
||||
|
||||
## Install
|
||||
## Three Ways To Deploy
|
||||
|
||||
Spinifex runs on Ubuntu 26.04 or Debian 13. The full guides are at [docs.mulgadc.com](https://docs.mulgadc.com).
|
||||
The same AWS-compatible platform runs in three shapes. Pick the one that matches your hardware.
|
||||
|
||||
>*Prerequisite:* Spinifex requires a Linux bridge configured on the host for VM networking. See the [single-node install guide](https://docs.mulgadc.com/docs/install#prerequisites) for setup details.
|
||||
### Neocloud
|
||||
|
||||
**Bare metal (recommended):** download the [bootable x86 installer](https://iso.mulgadc.com/spinifex.iso) and follow the [USB install guide](https://docs.mulgadc.com/docs/install-usb).
|
||||
Lift and shift onto a partner Neocloud without rewriting workloads. GPU capacity (H100 / H200 / B200), cheaper, available now. Change an endpoint, keep the software.
|
||||
|
||||
```bash
|
||||
curl -fLO https://iso.mulgadc.com/spinifex.iso
|
||||
```
|
||||
### On-premise
|
||||
|
||||
**Single node:**
|
||||
Host Spinifex on your own hardware in your own data centre. You get a multi-node HA cluster integrated with your storage and networking, so the stack, the data, and the jurisdiction stay yours.
|
||||
|
||||
```bash
|
||||
curl -fsSL https://install.mulgadc.com | bash
|
||||
sudo /usr/local/share/spinifex/setup-ovn.sh --management
|
||||
sudo spx admin init --node node1 --nodes 1
|
||||
sudo systemctl start spinifex.target
|
||||
export AWS_PROFILE=spinifex
|
||||
aws ec2 describe-instance-types
|
||||
```
|
||||
### Edge
|
||||
|
||||
**From source:** see the [Source Install](https://docs.mulgadc.com/docs/install-source) guide.
|
||||
Cloud where the cloud can't reach. Air-gapped sites, vehicles, vessels, factories, and clinics. Compute next to the data, running through disconnection, on hardware you own. The same AWS APIs your software already uses.
|
||||
|
||||
## Web Console
|
||||
## AWS Compatibility
|
||||
|
||||
<p align="center">
|
||||
<img src=".github/assets/spinifex-ui.jpg" alt="Spinifex web console dashboard" width="900">
|
||||
</p>
|
||||
Spinifex speaks the AWS API surface. The AWS SDKs, AWS CLI, and Terraform all work unchanged, whether you deploy at the edge, on-premise, or on a partner Neocloud.
|
||||
|
||||
Spinifex ships with a built-in web console for your instances, volumes, buckets, networks, and IAM. See [Launching the Web UI](https://docs.mulgadc.com/docs/setting-up-your-cluster#7-launching-the-web-ui).
|
||||
| Service | What it is | Status |
|
||||
| --- | --- | --- |
|
||||
| **EC2** | Compute | Available |
|
||||
| **EBS** | Block Storage | Available |
|
||||
| **S3** | Object Storage | Available |
|
||||
| **VPC** | Networking | Available |
|
||||
| **IAM** | Identity & Auth | Available |
|
||||
| **ALB / NLB** | Load Balancers | Available |
|
||||
| **EKS** | Kubernetes | Available |
|
||||
| **ECR** | Container Registry | Available |
|
||||
| **ECS** | Container Service | Available |
|
||||
| **RDS** | Databases | Available |
|
||||
| **Bedrock** | AI Deployment | Q4 2026 |
|
||||
|
||||
## Architecture
|
||||
Roadmap items ship under the same AWS API surface. Code written for AWS today keeps working the moment they land. [Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
|
||||
|
||||
## Live Demo
|
||||
|
||||
Create a live sandbox in under a minute. The fastest way to experience the stack. Try Spinifex against one of your own workloads on infrastructure we run for you.
|
||||
|
||||
* Live in under a minute. No card, no install.
|
||||
* Yours for 72 hours to test and experiment.
|
||||
* Real AWS-compatible endpoint, access keys and console.
|
||||
* EC2, EBS, S3, VPC, IAM, EKS and RDS, ready to call.
|
||||
* Point your existing AWS CLI or Terraform straight at it.
|
||||
|
||||
Sign up at [https://mulgadc.com/signup](https://mulgadc.com/signup)
|
||||
|
||||
## Core Components
|
||||
|
||||
### Spinifex (Compute and Platform Services – EC2, EKS, ECS, RDS)
|
||||
|
||||
Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing an EC2 compatible API. It manages lifecycle operations like start, stop, and terminate through QEMU's QMP interface. Designed to be straightforward and scriptable, Spinifex lets you launch VMs with the AWS CLI, SDKs, or Terraform, without Kubernetes or a heavyweight orchestrator.
|
||||
|
||||
The managed services build on that same instance layer. An EKS node group, an ECS container instance, and an RDS database are Spinifex VMs on your own hardware, each driven through its own AWS API.
|
||||
|
||||
- **EC2** – VM lifecycle on bare metal, cloud-init and IMDS metadata, key pairs, tags, spot instances, capacity reservations, and GPU passthrough
|
||||
- **EKS** – Managed Kubernetes on k3s, with HA control planes, managed node groups, cluster add-ons, an OIDC provider, and IAM access entries
|
||||
- **ECS** – Task and service scheduling across container instances, with awsvpc ENIs, capacity providers, rolling deployments, and GPU-aware placement
|
||||
- **ECR** – Private container registry backed by Predastore, with content-addressed blobs deduplicated per account and lifecycle policies
|
||||
- **RDS** – Managed PostgreSQL and MariaDB, with parameter groups, automated backups, and DNS endpoints served by Northstar
|
||||
- **ALB / NLB** – Layer 7 load balancing on HAProxy and layer 4 on nginx `stream`, with health checks and ACM-issued listener certificates
|
||||
- **VPC and IAM** – OVN-backed networking and security groups, with roles, policies, and SigV4 auth applied across every service
|
||||
|
||||
### Viperblock (Block Storage – EBS Alternative)
|
||||
|
||||
[Viperblock](https://github.com/mulgadc/viperblock) is a high-performance, WAL-backed block storage service that replicates volumes across multiple nodes. It's built for reliability and speed, with support for snapshots, recovery, and direct connection to QEMU instances using NBD or virtio-blk.
|
||||
|
||||
- Fast, durable virtual disks
|
||||
- Replication for resilience
|
||||
- Exposed over NBD or embedded in VMs
|
||||
- High performance WAL logs on local NVMe drives, reducing IO traffic to S3
|
||||
- In-memory read/write block cache
|
||||
|
||||
### Predastore (Object Storage – S3-Compatible)
|
||||
|
||||
[Predastore](https://github.com/mulgadc/predastore) is a fully S3-compatible object storage system. It supports the AWS S3 API, including Signature V4 authentication, multipart uploads, and Terraform provisioning. Predastore chunks data and distributes it across nodes using Reed-Solomon erasure coding, so it survives node loss and suits large-scale or low-bandwidth sites.
|
||||
|
||||
- S3-compatible API and auth
|
||||
- Multipart uploads, streaming reads/writes
|
||||
- Data redundancy with Reed-Solomon encoding
|
||||
|
||||
### Northstar (Authoritative DNS)
|
||||
|
||||
[Northstar](https://github.com/mulgadc/northstar) is a lightweight authoritative DNS server that handles both internal service discovery and public-facing DNS for a Spinifex cluster. Zones are human-readable TOML files, served from the local filesystem or synced from Predastore, keeping DNS configuration in the same object storage as the rest of your infrastructure.
|
||||
|
||||
- Authoritative DNS over UDP, TCP, and DNS-over-TLS
|
||||
- Service discovery for cluster components via SRV records
|
||||
- Zone files stored locally or in S3-compatible storage
|
||||
|
||||
## Architecture at a Glance
|
||||
|
||||
<p align="center">
|
||||
<img src=".github/assets/architecture.svg" alt="Spinifex message-driven architecture: standard AWS clients call the AWS Gateway over HTTPS with SigV4, which publishes to a NATS bus answered by stateless ec2/ebs/s3/vpc/iam daemons over local backends." width="900">
|
||||
</p>
|
||||
|
||||
Every AWS API call is authenticated at the gateway, published to a NATS subject, and answered by whichever daemon claims it. Daemons are stateless, so you scale horizontally by starting more. Deep dive: **[`docs/DESIGN.md`](docs/DESIGN.md)**.
|
||||
Every AWS API call is authenticated at the gateway, published to a NATS subject, and answered by whichever daemon claims it. Daemons are stateless: scale horizontally by starting more. No etcd, no Kubernetes, no external control plane. Just systemd units, a NATS cluster, and your hardware. Deep dive: **[`docs/DESIGN.md`](docs/DESIGN.md)**.
|
||||
|
||||
## How We Build Spinifex
|
||||
## Self-hosted Installation
|
||||
|
||||
Spinifex is built by infrastructure engineers with deep AWS experience, including former AWS team members, drawing on decades of combined work across AWS, enterprise infrastructure, and edge computing. Engineers architect and implement it, with **Claude Code** accelerating parts of the work.
|
||||
Installation requires an Ubuntu 26.04 or Debian 13 system. See the detailed documentation at [docs.mulgadc.com](https://docs.mulgadc.com) for installing Spinifex.
|
||||
|
||||
| Claude Code assists with | Engineers own |
|
||||
| --- | --- |
|
||||
| Boilerplate AWS API structures and handlers | Architecture and scalability decisions |
|
||||
| Development guides and API documentation | Authentication, encryption, and threat modelling |
|
||||
| Test case generation and validation scenarios | Real-world performance tuning and benchmarking |
|
||||
| Large-scale restructuring and refactoring | Deployment strategy and production operations |
|
||||
### Bare Metal ISO
|
||||
|
||||
The recommended installation is a [bootable x86 installer](https://iso.mulgadc.com/spinifex.iso) for bare-metal hardware.
|
||||
|
||||
```bash
|
||||
curl -fLO https://iso.mulgadc.com/spinifex.iso
|
||||
```
|
||||
|
||||
Follow the [USB install guide](https://docs.mulgadc.com/docs/install-usb) to write the ISO to USB and install on your hardware.
|
||||
|
||||
### Single Node Install
|
||||
|
||||
>*Prerequisite:* Spinifex requires a Linux bridge configured on the host for VM networking. See the [single-node install guide](https://docs.mulgadc.com/docs/install#prerequisites) for setup details.
|
||||
|
||||
```bash
|
||||
curl -fsSL https://install.mulgadc.com | bash
|
||||
|
||||
sudo /usr/local/share/spinifex/setup-ovn.sh --management
|
||||
|
||||
sudo spx admin init --node node1 --nodes 1
|
||||
|
||||
sudo systemctl start spinifex.target
|
||||
|
||||
export AWS_PROFILE=spinifex
|
||||
|
||||
aws ec2 describe-instance-types
|
||||
```
|
||||
|
||||
### Development Setup
|
||||
|
||||
For a complete development environment see the [Source Install](https://docs.mulgadc.com/docs/install-source) documentation.
|
||||
|
||||
## Spinifex UI
|
||||
|
||||
Spinifex ships with a built-in web console, an optional alternative to the AWS CLI, SDKs, and Terraform. If you know the AWS Management Console, the Spinifex UI fills the same role: a browser-based view of your instances, volumes, buckets, VPCs, and IAM resources, without leaving your own network.
|
||||
|
||||
<p align="center">
|
||||
<img src=".github/assets/spinifex-ui.jpg" alt="Spinifex web console dashboard" width="900">
|
||||
</p>
|
||||
|
||||
Each node serves the console on port `3000` over TLS, and it comes up with `spinifex.target`:
|
||||
|
||||
- **Same API, different surface.** The UI issues the same AWS SigV4 calls the CLI does, so RBAC, audit trails, and IAM policies apply the same way.
|
||||
- **Single sign-on against your AWS credentials.** Log in with the access keys from `~/.aws/credentials` on the node where Spinifex is installed. There is no separate user database.
|
||||
- **Self-hosted, works offline.** The Spinifex binary embeds the UI and each node serves it directly, with no external CDN or analytics calls.
|
||||
|
||||
For the full walkthrough, covering first-time TLS certificate trust, login, and a feature tour, see [**Launching the Web UI**](https://docs.mulgadc.com/docs/setting-up-your-cluster#7-launching-the-web-ui) in the cluster setup guide.
|
||||
|
||||
## Development Philosophy
|
||||
|
||||
### Built by Engineers, For Engineers
|
||||
|
||||
Spinifex is developed by experienced infrastructure engineers with deep AWS expertise, including former AWS team members who understand the intricacies of building production-grade cloud services. Our team brings decades of combined experience from AWS, enterprise infrastructure, and edge computing environments.
|
||||
|
||||
### AI-Assisted Development
|
||||
|
||||
Spinifex is architected and implemented by experienced engineers, with **Claude Code** accelerating certain development tasks. The split keeps the quality and reliability standards that production infrastructure demands:
|
||||
|
||||
**How We Use Claude Code:**
|
||||
|
||||
- **Code Generation**: Boilerplate AWS API structures and handlers
|
||||
- **Documentation**: Comprehensive development guides and API documentation
|
||||
- **Testing**: Test case generation and validation scenarios
|
||||
- **Refactoring**: Large-scale code restructuring and optimization
|
||||
|
||||
**What Remains Human-Driven:**
|
||||
|
||||
- **Architecture Decisions**: Core system design and scalability choices
|
||||
- **Security Implementation**: Authentication, encryption, and threat modeling
|
||||
- **Performance Optimization**: Real-world performance tuning and benchmarking
|
||||
- **Production Operations**: Deployment strategies and operational procedures
|
||||
|
||||
## Trademarks
|
||||
|
||||
@@ -141,4 +228,4 @@ Spinifex is built by infrastructure engineers with deep AWS experience, includin
|
||||
|
||||
## License
|
||||
|
||||
Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE), with a commercial license available.
|
||||
Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE), with a commercial license available. You're free to use, modify, and deploy it anywhere you need reliable infrastructure without depending on centralized cloud platforms.
|
||||
|
||||
@@ -1,92 +0,0 @@
|
||||
package instancestate
|
||||
|
||||
// allowance is one recorded production consumer of raw instance state.
|
||||
type allowance struct {
|
||||
File, Symbol string
|
||||
Inv string // inventory item in docs/PACKAGE_BOUNDARY_MIGRATION.md
|
||||
}
|
||||
|
||||
// allowlist mirrors the ADR-0007 slice 0 inventory in
|
||||
// docs/PACKAGE_BOUNDARY_MIGRATION.md. It only shrinks: delete an entry as soon
|
||||
// as its use is gone, and never add one to make a new consumer pass.
|
||||
var allowlist = []allowance{
|
||||
{File: "spinifex/daemon/daemon.go", Symbol: "daemon.LocalStatePath", Inv: "INV-01"},
|
||||
{File: "spinifex/daemon/daemon.go", Symbol: "daemon.ReadLocalState", Inv: "INV-01"},
|
||||
{File: "spinifex/daemon/daemon.go", Symbol: "daemon.MarshalLocalState", Inv: "INV-01"},
|
||||
{File: "spinifex/daemon/daemon.go", Symbol: "daemon.WriteLocalStateBytes", Inv: "INV-01"},
|
||||
{File: "spinifex/daemon/daemon.go", Symbol: "JetStreamManager.InitKVBucket", Inv: "INV-02"},
|
||||
{File: "spinifex/daemon/daemon.go", Symbol: "JetStreamManager.InitTerminatedInstanceBucket", Inv: "INV-02"},
|
||||
{File: "spinifex/daemon/daemon.go", Symbol: "JetStreamManager.WriteNodeMarkerBestEffort", Inv: "INV-03"},
|
||||
{File: "spinifex/daemon/daemon.go", Symbol: "JetStreamManager.WriteRunningSet", Inv: "INV-03"},
|
||||
{File: "spinifex/daemon/instance_record_repair.go", Symbol: "JetStreamManager.WriteRunningSet", Inv: "INV-03"},
|
||||
{File: "spinifex/daemon/daemon.go", Symbol: "JetStreamManager.ListTerminatedInstances", Inv: "INV-04"},
|
||||
|
||||
{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.LoadState", Inv: "INV-05"},
|
||||
{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.WriteStoppedInstance", Inv: "INV-05"},
|
||||
{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.LoadStoppedInstance", Inv: "INV-05"},
|
||||
{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.DeleteStoppedInstance", Inv: "INV-05"},
|
||||
{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.UpdateStoppedInstance", Inv: "INV-05"},
|
||||
{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.ClaimStoppedInstance", Inv: "INV-05"},
|
||||
{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.ListStoppedInstances", Inv: "INV-05"},
|
||||
{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.WriteTerminatedInstance", Inv: "INV-05"},
|
||||
{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.UpdateTerminatedInstance", Inv: "INV-05"},
|
||||
{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.ListTerminatedInstances", Inv: "INV-05"},
|
||||
{File: "spinifex/daemon/vm_adapters.go", Symbol: "JetStreamManager.DeleteTerminatedInstance", Inv: "INV-05"},
|
||||
|
||||
{File: "spinifex/runtime/compute/vm/migrate.go", Symbol: "JetStreamManager.WriteStoppedInstance", Inv: "INV-06"},
|
||||
{File: "spinifex/runtime/compute/vm/migrate.go", Symbol: "JetStreamManager.WriteTerminatedInstance", Inv: "INV-06"},
|
||||
{File: "spinifex/runtime/compute/vm/shutdown.go", Symbol: "JetStreamManager.WriteTerminatedInstance", Inv: "INV-06"},
|
||||
{File: "spinifex/runtime/compute/vm/orphan_qemu_reaper.go", Symbol: "JetStreamManager.ListTerminatedInstances", Inv: "INV-06"},
|
||||
{File: "spinifex/runtime/compute/vm/teardown_reaper.go", Symbol: "JetStreamManager.ListTerminatedInstances", Inv: "INV-06"},
|
||||
{File: "spinifex/runtime/compute/vm/teardown_reaper.go", Symbol: "JetStreamManager.UpdateTerminatedInstance", Inv: "INV-06"},
|
||||
{File: "spinifex/runtime/compute/vm/teardown_reaper.go", Symbol: "JetStreamManager.DeleteTerminatedInstance", Inv: "INV-06"},
|
||||
|
||||
{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.LoadStoppedInstance", Inv: "INV-07"},
|
||||
{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.UpdateStoppedInstance", Inv: "INV-07"},
|
||||
{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.ListStoppedInstances", Inv: "INV-07"},
|
||||
{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.ListTerminatedInstances", Inv: "INV-07"},
|
||||
{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.WriteStoppedInstance", Inv: "INV-07"},
|
||||
{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.ClaimStoppedInstance", Inv: "INV-07"},
|
||||
{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.WriteTerminatedInstance", Inv: "INV-07"},
|
||||
{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.DeleteStoppedInstance", Inv: "INV-07"},
|
||||
|
||||
{File: "spinifex/daemon/daemon_handlers_image.go", Symbol: "JetStreamManager.LoadStoppedInstance", Inv: "INV-08"},
|
||||
{File: "spinifex/daemon/daemon_handlers_image.go", Symbol: "JetStreamManager.UpdateStoppedInstance", Inv: "INV-08"},
|
||||
|
||||
{File: "spinifex/daemon/instance_recovery.go", Symbol: "vm.InstanceRecord", Inv: "INV-10"},
|
||||
{File: "spinifex/daemon/instance_recovery.go", Symbol: "JetStreamManager.ListInstanceRecords", Inv: "INV-10"},
|
||||
{File: "spinifex/daemon/instance_recovery.go", Symbol: "JetStreamManager.ClaimRecoverableInstance", Inv: "INV-10"},
|
||||
{File: "spinifex/daemon/instance_recovery.go", Symbol: "JetStreamManager.ReleaseRecoveredInstance", Inv: "INV-10"},
|
||||
{File: "spinifex/daemon/instance_recovery.go", Symbol: "JetStreamManager.AbandonRecovery", Inv: "INV-10"},
|
||||
{File: "spinifex/daemon/instance_recovery.go", Symbol: "JetStreamManager.LoadInstanceRecord", Inv: "INV-10"},
|
||||
|
||||
{File: "spinifex/daemon/dns_reconcile.go", Symbol: "daemon.InstanceRecordPrefix", Inv: "INV-11"},
|
||||
{File: "spinifex/daemon/dns_reconcile.go", Symbol: "daemon.InstanceStateBucket", Inv: "INV-11"},
|
||||
{File: "spinifex/daemon/dns_reconcile.go", Symbol: "JetStreamManager.ListInstanceRecords", Inv: "INV-11"},
|
||||
|
||||
{File: "spinifex/daemon/eni_orphan_reaper.go", Symbol: "vm.InstanceRecord", Inv: "INV-12"},
|
||||
{File: "spinifex/daemon/eni_orphan_reaper.go", Symbol: "JetStreamManager.ListInstanceRecords", Inv: "INV-12"},
|
||||
{File: "spinifex/daemon/eni_orphan_reaper.go", Symbol: "JetStreamManager.ListTerminatedInstanceRecords", Inv: "INV-12"},
|
||||
|
||||
{File: "spinifex/domains/ec2/instance/state/imds_readers.go", Symbol: "daemon.LocalState", Inv: "INV-13"},
|
||||
{File: "spinifex/domains/ec2/instance/state/imds_readers.go", Symbol: "daemon.LocalStatePath", Inv: "INV-13"},
|
||||
{File: "spinifex/domains/ec2/instance/state/imds_readers.go", Symbol: "daemon.ReadLocalState", Inv: "INV-13"},
|
||||
{File: "spinifex/domains/ec2/instance/state/imds_readers.go", Symbol: "daemon.InstanceStateBucket", Inv: "INV-13"},
|
||||
{File: "spinifex/domains/ec2/instance/state/imds_readers.go", Symbol: "JetStreamManager.InitKVBucket", Inv: "INV-13"},
|
||||
|
||||
{File: "spinifex/domains/ec2/guestmetadata/instance_lookup.go", Symbol: "vm.InstanceRecord", Inv: "INV-14"},
|
||||
{File: "spinifex/domains/ec2/guestmetadata/instance_lookup.go", Symbol: "vm.VMFromRecord", Inv: "INV-14"},
|
||||
{File: "spinifex/domains/ec2/guestmetadata/instance_lookup.go", Symbol: "JetStreamManager.LoadInstanceRecord", Inv: "INV-14"},
|
||||
|
||||
{File: "spinifex/domains/network/reconcile/intent.go", Symbol: `literal "spinifex-instance-state"`, Inv: "INV-15"},
|
||||
{File: "spinifex/domains/network/reconcile/intent.go", Symbol: "vm.InstanceRecord", Inv: "INV-15"},
|
||||
|
||||
{File: "spinifex/runtime/roles/awsgw/awsgw.go", Symbol: "daemon.InstanceStateBucket", Inv: "INV-16"},
|
||||
{File: "spinifex/runtime/roles/awsgw/awsgw.go", Symbol: "daemon.InstanceRecordPrefix", Inv: "INV-16"},
|
||||
{File: "spinifex/runtime/roles/awsgw/awsgw.go", Symbol: "vm.InstanceRecord", Inv: "INV-16"},
|
||||
|
||||
{File: "spinifex/runtime/compute/cache/cache.go", Symbol: "vm.InstanceRecord", Inv: "INV-17"},
|
||||
{File: "spinifex/runtime/compute/cache/cache.go", Symbol: "vm.VMFromRecord", Inv: "INV-17"},
|
||||
|
||||
{File: "spinifex/domains/admission/quota/records.go", Symbol: "vm.InstanceRecord", Inv: "INV-18"},
|
||||
}
|
||||
@@ -1,200 +0,0 @@
|
||||
package instancestate
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type report struct {
|
||||
unrecorded []use
|
||||
stale []allowance
|
||||
malformed []string
|
||||
}
|
||||
|
||||
// check compares the scanned uses with allow.
|
||||
func check(s scan, allow []allowance) report {
|
||||
var rep report
|
||||
allowed := map[use]bool{}
|
||||
for _, a := range allow {
|
||||
u := use{File: a.File, Symbol: a.Symbol}
|
||||
switch {
|
||||
case allowed[u]:
|
||||
rep.malformed = append(rep.malformed, "duplicate entry "+u.String())
|
||||
case a.Inv == "":
|
||||
rep.malformed = append(rep.malformed, "entry "+u.String()+" cites no inventory item")
|
||||
}
|
||||
allowed[u] = true
|
||||
}
|
||||
for _, u := range s.sorted() {
|
||||
if !allowed[u] {
|
||||
rep.unrecorded = append(rep.unrecorded, u)
|
||||
}
|
||||
}
|
||||
for _, a := range allow {
|
||||
if _, ok := s[use{File: a.File, Symbol: a.Symbol}]; !ok {
|
||||
rep.stale = append(rep.stale, a)
|
||||
}
|
||||
}
|
||||
return rep
|
||||
}
|
||||
|
||||
// TestADR0007_S6_NoNewRawInstanceStateConsumers is the blocking ratchet: no
|
||||
// production file may start naming the daemon's instance-state bucket, keys,
|
||||
// local file, raw record or JetStreamManager instance methods.
|
||||
func TestADR0007_S6_NoNewRawInstanceStateConsumers(t *testing.T) {
|
||||
root, err := moduleRoot()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s, files, err := scanModule(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Guards against a walk that silently finds nothing and passes vacuously.
|
||||
if files < 500 || len(s) < 20 {
|
||||
t.Fatalf("scanned %d files and found %d uses under %s; the walk has lost the tree", files, len(s), root)
|
||||
}
|
||||
if len(allowlist) == 0 {
|
||||
t.Fatal("the allowlist is empty, so the stale check would pass vacuously")
|
||||
}
|
||||
rep := check(s, allowlist)
|
||||
|
||||
t.Run("NoUnrecordedConsumer", func(t *testing.T) {
|
||||
for _, u := range rep.unrecorded {
|
||||
t.Errorf("ADR-0007 S6 (consumers receive narrow state capabilities rather than raw stores): %s at %s; "+
|
||||
"consume an instance-domain capability instead of the daemon's raw instance state", u, s[u])
|
||||
}
|
||||
})
|
||||
t.Run("Allowlist_NoStaleEntries", func(t *testing.T) {
|
||||
for _, a := range rep.stale {
|
||||
t.Errorf("allowlist entry %s uses %s (%s) no longer matches; delete it and update the inventory",
|
||||
a.File, a.Symbol, a.Inv)
|
||||
}
|
||||
})
|
||||
t.Run("Allowlist_WellFormed", func(t *testing.T) {
|
||||
for _, m := range rep.malformed {
|
||||
t.Errorf("allowlist_test.go: %s", m)
|
||||
}
|
||||
t.Logf("%d allow-listed uses", len(allowlist))
|
||||
})
|
||||
t.Run("OwnerFilesExist", func(t *testing.T) {
|
||||
for _, f := range ownerFiles {
|
||||
if _, err := os.Stat(filepath.Join(root, f)); err != nil {
|
||||
t.Errorf("owner file %s is gone; remove it from ownerFiles and re-scan its consumers: %v", f, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
const faultSource = `package %s
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
spxd "github.com/mulgadc/spinifex/spinifex/daemon"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
|
||||
)
|
||||
|
||||
type holder struct {
|
||||
LoadInstanceRecord func() // a field name, not a use
|
||||
m *spxd.JetStreamManager
|
||||
}
|
||||
|
||||
func (h holder) WriteRunningSet() {} // a declaration, not a use
|
||||
|
||||
func use(h holder, raw []byte, jsManager *spxd.JetStreamManager, other interface{ LoadState() }) {
|
||||
_ = spxd.InstanceStateBucket
|
||||
_ = spxd.LocalStatePath("")
|
||||
var r vm.InstanceRecord
|
||||
_ = json.Unmarshal(raw, &r)
|
||||
_ = vm.VMFromRecord(&r)
|
||||
_ = (&vm.VM{}).Record()
|
||||
_, _ = h.m.LoadInstanceRecord("i-1")
|
||||
_, _, _ = jsManager.LoadState("node")
|
||||
other.LoadState()
|
||||
_ = "spinifex-instance-state"
|
||||
cluster, _ := spxd.NewJetStreamManager(nil)
|
||||
_ = cluster.InitClusterStateBucket()
|
||||
_ = cluster.WriteRunningSet("node", nil)
|
||||
}
|
||||
`
|
||||
|
||||
func parseFault(t *testing.T, rel, pkg string) scan {
|
||||
t.Helper()
|
||||
fset := token.NewFileSet()
|
||||
f, err := parser.ParseFile(fset, rel, fmt.Sprintf(faultSource, pkg), parser.SkipObjectResolution)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := scan{}
|
||||
scanFile(fset, rel, f, out)
|
||||
return out
|
||||
}
|
||||
|
||||
// TestADR0007_S6_RatchetDetectsFaults injects a new consumer, a stale entry
|
||||
// and a malformed entry, and checks the matcher's declaration exclusions.
|
||||
func TestADR0007_S6_RatchetDetectsFaults(t *testing.T) {
|
||||
const rel = "spinifex/domains/widget/w.go"
|
||||
s := parseFault(t, rel, "widget")
|
||||
|
||||
want := []string{
|
||||
"daemon.InstanceStateBucket", "daemon.LocalStatePath",
|
||||
"vm.InstanceRecord", "vm.VMFromRecord", "VM.Record()",
|
||||
"JetStreamManager.LoadInstanceRecord", "JetStreamManager.LoadState",
|
||||
"JetStreamManager.WriteRunningSet",
|
||||
`literal "spinifex-instance-state"`,
|
||||
}
|
||||
for _, sym := range want {
|
||||
if _, ok := s[use{File: rel, Symbol: sym}]; !ok {
|
||||
t.Errorf("injected use of %s was not detected; got %v", sym, s.sorted())
|
||||
}
|
||||
}
|
||||
if len(s) != len(want) {
|
||||
t.Errorf("want exactly %d uses (the manager type, its constructor, cluster-state methods, field names, method declarations and an unrelated LoadState excluded), got %v",
|
||||
len(want), s.sorted())
|
||||
}
|
||||
|
||||
stale := allowance{File: "spinifex/domains/gone/g.go", Symbol: "vm.InstanceRecord", Inv: "INV-00"}
|
||||
recorded := allowance{File: rel, Symbol: "vm.InstanceRecord", Inv: "INV-00"}
|
||||
rep := check(s, []allowance{recorded, stale, {File: rel, Symbol: "vm.VMFromRecord"}, recorded})
|
||||
if len(rep.unrecorded) != len(want)-2 {
|
||||
t.Errorf("want %d unrecorded uses, got %v", len(want)-2, rep.unrecorded)
|
||||
}
|
||||
if len(rep.stale) != 1 || rep.stale[0] != stale {
|
||||
t.Errorf("want stale entry %+v, got %+v", stale, rep.stale)
|
||||
}
|
||||
if len(rep.malformed) != 2 {
|
||||
t.Errorf("want a missing-inventory and a duplicate entry reported, got %v", rep.malformed)
|
||||
}
|
||||
}
|
||||
|
||||
// Inside the owning packages the symbols are bare identifiers, and the scan
|
||||
// must see those too.
|
||||
func TestADR0007_S6_RatchetSeesBareIdentifiersInOwnerPackages(t *testing.T) {
|
||||
src := map[string]string{
|
||||
"spinifex/daemon/new_consumer.go": "package daemon\n\nfunc f() string { return InstanceStateBucket + LocalStatePath(\"\") }\n",
|
||||
"spinifex/runtime/compute/vm/new_reader.go": "package vm\n\nfunc g(r *InstanceRecord) *VM { return VMFromRecord(r) }\n",
|
||||
}
|
||||
want := map[string][]string{
|
||||
"spinifex/daemon/new_consumer.go": {"daemon.InstanceStateBucket", "daemon.LocalStatePath"},
|
||||
"spinifex/runtime/compute/vm/new_reader.go": {"vm.InstanceRecord", "vm.VMFromRecord"},
|
||||
}
|
||||
for rel, code := range src {
|
||||
fset := token.NewFileSet()
|
||||
f, err := parser.ParseFile(fset, rel, code, parser.SkipObjectResolution)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := scan{}
|
||||
scanFile(fset, rel, f, out)
|
||||
for _, sym := range want[rel] {
|
||||
if _, ok := out[use{File: rel, Symbol: sym}]; !ok {
|
||||
t.Errorf("%s: bare use of %s not detected; got %v", rel, sym, out.sorted())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,302 +0,0 @@
|
||||
// Package instancestate freezes the production consumers of the daemon's raw
|
||||
// instance-state persistence, so ADR-0007 S6 and S7 start from a known set
|
||||
// that can only shrink.
|
||||
package instancestate
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"go/ast"
|
||||
"go/build/constraint"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
daemonPath = "github.com/mulgadc/spinifex/spinifex/daemon"
|
||||
vmPath = "github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
|
||||
daemonDir = "spinifex/daemon"
|
||||
vmDir = "spinifex/runtime/compute/vm"
|
||||
)
|
||||
|
||||
// daemonSymbols are the daemon's instance-state bucket, key and local-file
|
||||
// identifiers. The JetStreamManager type and constructor are not listed: they
|
||||
// also serve cluster state, so only the instance-state methods below count.
|
||||
var daemonSymbols = set(
|
||||
"InstanceStateBucket", "InstanceStateBucketVersion",
|
||||
"TerminatedInstanceBucket", "TerminatedInstanceBucketVersion",
|
||||
"InstanceRecordPrefix", "InstanceStatePrefix", "NodePresencePrefix",
|
||||
"StoppedInstancePrefix", "TerminatedInstancePrefix",
|
||||
"LocalState", "LocalStateSchemaVersion", "LocalStatePath", "ReadLocalState",
|
||||
"MarshalLocalState", "WriteLocalStateBytes", "LocalStateFileName",
|
||||
"LocalStateFileMode", "DefaultLocalStateDir",
|
||||
)
|
||||
|
||||
// vmSymbols are the raw persisted record and its conversion.
|
||||
var vmSymbols = set("InstanceRecord", "InstanceSpec", "InstanceStatus", "VMFromRecord")
|
||||
|
||||
// jsmMethods are JetStreamManager's instance-state methods. Matched by name, so
|
||||
// a call through an interface that mirrors them counts as a consumer too.
|
||||
var jsmMethods = set(
|
||||
"WriteInstanceRecord", "LoadInstanceRecord", "UpdateInstanceRecord",
|
||||
"DeleteInstanceRecord", "ListInstanceRecords",
|
||||
"WriteTerminatedInstanceRecord", "LoadTerminatedInstanceRecord",
|
||||
"UpdateTerminatedInstanceRecord", "DeleteTerminatedInstanceRecord",
|
||||
"ListTerminatedInstanceRecords",
|
||||
"WriteNodeMarker", "WriteNodeMarkerBestEffort", "WriteRunningSet",
|
||||
"WriteStoppedInstance", "LoadStoppedInstance", "DeleteStoppedInstance",
|
||||
"ClaimStoppedInstance", "ClaimRecoverableInstance", "ReleaseRecoveredInstance",
|
||||
"AbandonRecovery", "UpdateStoppedInstance", "ListStoppedInstances",
|
||||
"WriteTerminatedInstance", "UpdateTerminatedInstance", "ListTerminatedInstances",
|
||||
"DeleteTerminatedInstance", "LoadTerminatedInstance",
|
||||
"InitKVBucket", "InitTerminatedInstanceBucket",
|
||||
)
|
||||
|
||||
// ambiguousJSMMethods share a name with unrelated methods, so they count only
|
||||
// when called on a field or variable named like a JetStreamManager handle.
|
||||
var ambiguousJSMMethods = set("LoadState", "DeleteState")
|
||||
|
||||
// literals duplicate a bucket or file name instead of naming the owner's symbol.
|
||||
var literals = set(`"spinifex-instance-state"`, `"spinifex-terminated-instances"`, `"instance-state.json"`)
|
||||
|
||||
// ownerFiles define the symbols above. They are the implementation being
|
||||
// migrated, not consumers of it, and are listed so a deleted one is noticed.
|
||||
var ownerFiles = []string{
|
||||
"spinifex/daemon/instance_membership.go",
|
||||
"spinifex/daemon/instance_records.go",
|
||||
"spinifex/daemon/instance_records_migrate.go",
|
||||
"spinifex/daemon/instance_running_set.go",
|
||||
"spinifex/daemon/jetstream.go",
|
||||
"spinifex/daemon/local_state.go",
|
||||
"spinifex/runtime/compute/vm/record.go",
|
||||
}
|
||||
|
||||
func set(names ...string) map[string]bool {
|
||||
m := make(map[string]bool, len(names))
|
||||
for _, n := range names {
|
||||
m[n] = true
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// use is one consumer of one symbol, keyed by file so a new file is a new use.
|
||||
type use struct {
|
||||
File, Symbol string
|
||||
}
|
||||
|
||||
func (u use) String() string { return u.File + " uses " + u.Symbol }
|
||||
|
||||
// scan maps each use to the first file:line it appears at.
|
||||
type scan map[use]string
|
||||
|
||||
func moduleRoot() (string, error) {
|
||||
_, file, _, ok := runtime.Caller(0)
|
||||
if !ok {
|
||||
return "", errors.New("cannot locate test source")
|
||||
}
|
||||
return filepath.Abs(filepath.Join(filepath.Dir(file), "..", ".."))
|
||||
}
|
||||
|
||||
var excludedTopDirs = map[string]bool{"tests": true, "docs": true, "architecture": true}
|
||||
|
||||
// scanModule parses every non-test production Go file under root.
|
||||
func scanModule(root string) (scan, int, error) {
|
||||
out := scan{}
|
||||
files := 0
|
||||
fset := token.NewFileSet()
|
||||
owners := set(ownerFiles...)
|
||||
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel = filepath.ToSlash(rel)
|
||||
if d.IsDir() {
|
||||
if rel == "." {
|
||||
return nil
|
||||
}
|
||||
name := d.Name()
|
||||
if strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_") ||
|
||||
name == "testdata" || name == "vendor" || name == "node_modules" || excludedTopDirs[rel] {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(path, "go.mod")); err == nil {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
|
||||
return nil
|
||||
}
|
||||
src, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f, err := parser.ParseFile(fset, path, src, parser.ParseComments|parser.SkipObjectResolution)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ignoredByBuildTag(f.Comments, f.Package) {
|
||||
return nil
|
||||
}
|
||||
files++
|
||||
if owners[rel] {
|
||||
return nil
|
||||
}
|
||||
scanFile(fset, rel, f, out)
|
||||
return nil
|
||||
})
|
||||
return out, files, err
|
||||
}
|
||||
|
||||
// scanFile records every use in f, which lives at module-relative path rel.
|
||||
func scanFile(fset *token.FileSet, rel string, f *ast.File, out scan) {
|
||||
dir := filepath.ToSlash(filepath.Dir(rel))
|
||||
daemonName, vmName := importName(f, daemonPath, "daemon"), importName(f, vmPath, "vm")
|
||||
inDaemon, inVM := dir == daemonDir, dir == vmDir
|
||||
seesVM := inDaemon || inVM || daemonName != "" || vmName != ""
|
||||
|
||||
record := func(symbol string, pos token.Pos) {
|
||||
u := use{File: rel, Symbol: symbol}
|
||||
if _, ok := out[u]; !ok {
|
||||
out[u] = fmt.Sprintf("%s:%d", rel, fset.Position(pos).Line)
|
||||
}
|
||||
}
|
||||
|
||||
var visit func(n ast.Node) bool
|
||||
visit = func(n ast.Node) bool {
|
||||
switch n := n.(type) {
|
||||
case *ast.ImportSpec:
|
||||
return false
|
||||
case *ast.BasicLit:
|
||||
if n.Kind == token.STRING && literals[n.Value] {
|
||||
record("literal "+n.Value, n.Pos())
|
||||
}
|
||||
case *ast.CallExpr:
|
||||
if sel, ok := n.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "Record" && len(n.Args) == 0 && seesVM {
|
||||
record("VM.Record()", sel.Sel.Pos())
|
||||
}
|
||||
case *ast.SelectorExpr:
|
||||
if x, ok := n.X.(*ast.Ident); ok {
|
||||
switch {
|
||||
case daemonName != "" && x.Name == daemonName && daemonSymbols[n.Sel.Name]:
|
||||
record("daemon."+n.Sel.Name, n.Sel.Pos())
|
||||
case vmName != "" && x.Name == vmName && vmSymbols[n.Sel.Name]:
|
||||
record("vm."+n.Sel.Name, n.Sel.Pos())
|
||||
}
|
||||
}
|
||||
if jsmMethods[n.Sel.Name] || (ambiguousJSMMethods[n.Sel.Name] && jsmReceivers[receiverName(n.X)]) {
|
||||
record("JetStreamManager."+n.Sel.Name, n.Sel.Pos())
|
||||
}
|
||||
// The selected name is a field or method, never a bare package symbol.
|
||||
ast.Inspect(n.X, visit)
|
||||
return false
|
||||
case *ast.FuncDecl:
|
||||
// A method declared with one of these names is an implementation,
|
||||
// not a use; its body is still scanned.
|
||||
if n.Recv != nil {
|
||||
ast.Inspect(n.Recv, visit)
|
||||
}
|
||||
if n.Body != nil {
|
||||
ast.Inspect(n.Body, visit)
|
||||
}
|
||||
if n.Type != nil {
|
||||
ast.Inspect(n.Type, visit)
|
||||
}
|
||||
return false
|
||||
case *ast.Field:
|
||||
// Interface method and struct field names are declarations.
|
||||
if n.Type != nil {
|
||||
ast.Inspect(n.Type, visit)
|
||||
}
|
||||
if n.Tag != nil {
|
||||
ast.Inspect(n.Tag, visit)
|
||||
}
|
||||
return false
|
||||
case *ast.Ident:
|
||||
if inDaemon && daemonSymbols[n.Name] {
|
||||
record("daemon."+n.Name, n.Pos())
|
||||
}
|
||||
if inVM && vmSymbols[n.Name] {
|
||||
record("vm."+n.Name, n.Pos())
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
ast.Inspect(f, visit)
|
||||
}
|
||||
|
||||
var jsmReceivers = set("jsManager", "jsm", "js")
|
||||
|
||||
// receiverName is the last name in x: "js" for both js and a.js.
|
||||
func receiverName(x ast.Expr) string {
|
||||
switch x := x.(type) {
|
||||
case *ast.Ident:
|
||||
return x.Name
|
||||
case *ast.SelectorExpr:
|
||||
return x.Sel.Name
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// importName returns the name f refers to path by, or "" if f does not import it.
|
||||
func importName(f *ast.File, path, def string) string {
|
||||
for _, imp := range f.Imports {
|
||||
p, err := strconv.Unquote(imp.Path.Value)
|
||||
if err != nil || p != path {
|
||||
continue
|
||||
}
|
||||
if imp.Name != nil {
|
||||
if imp.Name.Name == "_" || imp.Name.Name == "." {
|
||||
return ""
|
||||
}
|
||||
return imp.Name.Name
|
||||
}
|
||||
return def
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// ignoredByBuildTag reports a `//go:build ignore` file, which is a standalone
|
||||
// program rather than part of its directory's package.
|
||||
func ignoredByBuildTag(groups []*ast.CommentGroup, pkgPos token.Pos) bool {
|
||||
for _, cg := range groups {
|
||||
if cg.Pos() >= pkgPos {
|
||||
break
|
||||
}
|
||||
for _, c := range cg.List {
|
||||
if !constraint.IsGoBuild(c.Text) {
|
||||
continue
|
||||
}
|
||||
expr, err := constraint.Parse(c.Text)
|
||||
if err == nil && expr.String() == "ignore" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s scan) sorted() []use {
|
||||
out := make([]use, 0, len(s))
|
||||
for u := range s {
|
||||
out = append(out, u)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].File != out[j].File {
|
||||
return out[i].File < out[j].File
|
||||
}
|
||||
return out[i].Symbol < out[j].Symbol
|
||||
})
|
||||
return out
|
||||
}
|
||||
@@ -1,164 +0,0 @@
|
||||
package layering
|
||||
|
||||
// allowance is one recorded forbidden import, allowed until its debt clears.
|
||||
type allowance struct {
|
||||
From, To string
|
||||
Debt string // subject of the migration record's debt entry or close-out row
|
||||
Reason string
|
||||
}
|
||||
|
||||
// allowlist mirrors the dependency-rule debt and close-out table in
|
||||
// docs/PACKAGE_BOUNDARY_MIGRATION.md. It only shrinks: add an edge only with
|
||||
// recorded debt, and delete an entry as soon as its import is gone.
|
||||
var allowlist = []allowance{
|
||||
{From: "accountteardown", To: "domains/ec2/awsapi/eip", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
|
||||
{From: "accountteardown", To: "domains/ec2/awsapi/igw", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
|
||||
{From: "accountteardown", To: "domains/ec2/awsapi/image", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
|
||||
{From: "accountteardown", To: "domains/ec2/awsapi/instance", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
|
||||
{From: "accountteardown", To: "domains/ec2/awsapi/key", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
|
||||
{From: "accountteardown", To: "domains/ec2/awsapi/launchtemplate", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
|
||||
{From: "accountteardown", To: "domains/ec2/awsapi/placementgroup", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
|
||||
{From: "accountteardown", To: "domains/ec2/awsapi/routetable", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
|
||||
{From: "accountteardown", To: "domains/ec2/awsapi/snapshot", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
|
||||
{From: "accountteardown", To: "domains/ec2/awsapi/volume", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
|
||||
{From: "accountteardown", To: "domains/ec2/awsapi/vpc", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
|
||||
|
||||
{From: "agents/ecs/agent", To: "handlers/ecs", Debt: "agents/ecs/agent", Reason: "assignment/state wire types from the ECS implementation; needs a guest/controller contract"},
|
||||
|
||||
{From: "agents/eks/tokenwebhook", To: "handlers/eks", Debt: "agents/eks/tokenwebhook", Reason: "WebhookTokenReviewResult from the EKS implementation; needs an EKS contract or projection"},
|
||||
|
||||
{From: "agents/rds/agent", To: "handlers/rds", Debt: "agents/rds/agent (dependency, narrowed)", Reason: "command protocol, wire and record types; needs a guest/controller wire contract"},
|
||||
|
||||
{From: "cmd/aws-model-coverage", To: "domains/acm/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared ECR/ACM inventories; named exception, must not construct a registration"},
|
||||
{From: "cmd/aws-model-coverage", To: "domains/ecr/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared ECR/ACM inventories; named exception, must not construct a registration"},
|
||||
|
||||
{From: "daemon", To: "domains/ec2/awsapi/instance", Debt: "Instance adapter consumers outside EC2", Reason: "daemon/eks_cp_control.go and rds_deps.go call the instance adapter; needs an EC2 capability"},
|
||||
|
||||
{From: "domains/acm", To: "handlers/iam", Debt: "ACM: the IAM AES-GCM helper import (ACM relocation residual)", Reason: "IAM AES-GCM helpers; needs a shared crypto boundary"},
|
||||
|
||||
{From: "domains/admission/quota", To: "domains/ec2/awsapi/eip", Debt: "domains/admission/quota (dependency, moved from handlers/quota unchanged)", Reason: "describe adapter used as a counter; needs a reader capability"},
|
||||
{From: "domains/admission/quota", To: "domains/ec2/awsapi/instance", Debt: "domains/admission/quota (dependency, moved from handlers/quota unchanged)", Reason: "vcpu.go counts instances through the instance adapter; needs an EC2 reader capability"},
|
||||
{From: "domains/admission/quota", To: "domains/ec2/awsapi/volume", Debt: "domains/admission/quota (dependency, moved from handlers/quota unchanged)", Reason: "describe adapter used as a counter; needs a reader capability"},
|
||||
{From: "domains/admission/quota", To: "domains/ec2/awsapi/vpc", Debt: "domains/admission/quota (dependency, moved from handlers/quota unchanged)", Reason: "describe adapter used as a counter; needs a reader capability"},
|
||||
{From: "domains/admission/quota", To: "domains/ec2/instancetypes", Debt: "domains/admission/quota → domains/ec2/instancetypes", Reason: "DefaultVCPUs sizing; needs a consumer-owned capability or EC2 projection"},
|
||||
{From: "domains/admission/quota", To: "gateway/elbv2", Debt: "domains/admission/quota (dependency, moved from handlers/quota unchanged)", Reason: "live ELBv2 counter; needs a reader capability"},
|
||||
{From: "domains/admission/quota", To: "handlers/rds", Debt: "domains/admission/quota (dependency, moved from handlers/quota unchanged)", Reason: "live RDS counter; needs a reader capability"},
|
||||
{From: "domains/admission/quota", To: "runtime/compute/vm", Debt: "domains/admission/quota (dependency, moved from handlers/quota unchanged)", Reason: "instance-record vCPU sweep; needs a reader capability"},
|
||||
|
||||
{From: "domains/dns", To: "domains/network/reconcile", Debt: "domains/dns", Reason: "takes the reconcile lock from network's implementation"},
|
||||
{From: "domains/dns", To: "runtime/compute/vm", Debt: "domains/dns", Reason: "domain-to-runtime import of VM state"},
|
||||
|
||||
{From: "domains/ec2/awsapi/instance", To: "handlers/iam", Debt: "domains/ec2/awsapi/instance (dependency)", Reason: "IAM edge needs an IAM capability"},
|
||||
{From: "domains/ec2/awsapi/instance", To: "runtime/compute/cache", Debt: "domains/ec2/awsapi/instance (dependency)", Reason: "node liveness and VM state; needs an EC2-owned projection"},
|
||||
{From: "domains/ec2/awsapi/instance", To: "runtime/compute/vm", Debt: "domains/ec2/awsapi/instance (dependency)", Reason: "node liveness and VM state; needs an EC2-owned projection"},
|
||||
|
||||
{From: "domains/ec2/awsapi/spotinstance", To: "domains/admission/quota", Debt: "domains/ec2/awsapi/spotinstance (dependency)", Reason: "RequestSpotInstances.go uses IAMService and quota.Service"},
|
||||
{From: "domains/ec2/awsapi/spotinstance", To: "handlers/iam", Debt: "domains/ec2/awsapi/spotinstance (dependency)", Reason: "RequestSpotInstances.go uses IAMService and quota.Service"},
|
||||
|
||||
{From: "domains/ec2/eip", To: "domains/network/topology", Debt: "EC2 → network implementation", Reason: "EC2 reads network implementation; clears with the EC2-owned projection"},
|
||||
|
||||
{From: "domains/ec2/guestmetadata", To: "domains/dns", Debt: "Guest metadata: the cross-domain and runtime imports above", Reason: "guest-metadata relocation residual; needs capabilities, projections or contracts"},
|
||||
{From: "domains/ec2/guestmetadata", To: "handlers/iam", Debt: "Guest metadata: the cross-domain and runtime imports above", Reason: "guest-metadata relocation residual; needs capabilities, projections or contracts"},
|
||||
{From: "domains/ec2/guestmetadata", To: "handlers/sts", Debt: "Guest metadata: the cross-domain and runtime imports above", Reason: "guest-metadata relocation residual; needs capabilities, projections or contracts"},
|
||||
{From: "domains/ec2/guestmetadata", To: "runtime/compute/vm", Debt: "Guest metadata: the cross-domain and runtime imports above", Reason: "guest-metadata relocation residual; needs capabilities, projections or contracts"},
|
||||
|
||||
{From: "domains/ec2/instance", To: "domains/dns", Debt: "domains/ec2/instance", Reason: "cross-domain implementation import of DNS"},
|
||||
{From: "domains/ec2/instance", To: "domains/network/topology", Debt: "EC2 → network implementation", Reason: "EC2 reads network implementation; clears with the EC2-owned projection"},
|
||||
{From: "domains/ec2/instance", To: "runtime/compute/gpu", Debt: "domains/ec2/instance", Reason: "VM/GPU runtime state read directly; needs a domain-owned projection"},
|
||||
{From: "domains/ec2/instance", To: "runtime/compute/vm", Debt: "domains/ec2/instance", Reason: "VM/GPU runtime state read directly; needs a domain-owned projection"},
|
||||
{From: "domains/ec2/instance/state", To: "daemon", Debt: "domains/ec2/instance/state (dependency, IMDS readers)", Reason: "wraps the daemon's local-file reader and JetStreamManager bucket open unchanged for vpcd's IMDS lookup"},
|
||||
{From: "domains/ec2/instance/state", To: "runtime/compute/vm", Debt: "domains/ec2/instance/state (dependency)", Reason: "the vpcd IMDS local reader returns vm.VM"},
|
||||
|
||||
{From: "domains/ec2/volume", To: "runtime/compute/vm", Debt: "Domain → runtime", Reason: "VM runtime state read directly; needs a domain-owned projection"},
|
||||
|
||||
{From: "domains/ec2/vpc", To: "domains/network/external", Debt: "EC2 → network implementation", Reason: "EC2 reads network implementation; clears with the EC2-owned projection"},
|
||||
{From: "domains/ec2/vpc", To: "domains/network/external/dhcp", Debt: "EC2 → network implementation", Reason: "EC2 reads network implementation; clears with the EC2-owned projection"},
|
||||
{From: "domains/ec2/vpc", To: "domains/network/identifiers", Debt: "EC2 → network implementation", Reason: "EC2 reads network implementation; clears with the EC2-owned projection"},
|
||||
{From: "domains/ec2/vpc", To: "domains/network/topology", Debt: "EC2 → network implementation", Reason: "EC2 reads network implementation; clears with the EC2-owned projection"},
|
||||
|
||||
{From: "domains/ecr/auth", To: "handlers/iam", Debt: "Cross-domain", Reason: "IAM implementation import; needs an IAM capability"},
|
||||
|
||||
{From: "domains/network/host", To: "runtime/compute/vm", Debt: "domains/network", Reason: "domain-to-runtime import"},
|
||||
{From: "domains/network/host", To: "runtime/host/command", Debt: "domains/network", Reason: "domain-to-runtime import"},
|
||||
|
||||
{From: "domains/network/reconcile", To: "domains/ec2/eip", Debt: "domains/network", Reason: "reconcile reads EC2 private records; clears with the EC2-owned projection"},
|
||||
{From: "domains/network/reconcile", To: "domains/ec2/igw", Debt: "domains/network", Reason: "reconcile reads EC2 private records; clears with the EC2-owned projection"},
|
||||
{From: "domains/network/reconcile", To: "domains/ec2/natgw", Debt: "domains/network", Reason: "reconcile reads EC2 private records; clears with the EC2-owned projection"},
|
||||
{From: "domains/network/reconcile", To: "domains/ec2/routetable", Debt: "domains/network", Reason: "reconcile reads EC2 private records; clears with the EC2-owned projection"},
|
||||
{From: "domains/network/reconcile", To: "domains/ec2/vpc", Debt: "domains/network", Reason: "reconcile reads EC2 private records; clears with the EC2-owned projection"},
|
||||
{From: "domains/network/reconcile", To: "runtime/compute/vm", Debt: "domains/network", Reason: "domain-to-runtime import"},
|
||||
|
||||
{From: "domains/ochre", To: "domains/ec2/systeminstance", Debt: "Cross-domain", Reason: "consumes another domain's implementation rather than a capability"},
|
||||
{From: "domains/ochre", To: "domains/network/systemvpc", Debt: "Cross-domain", Reason: "consumes another domain's implementation rather than a capability"},
|
||||
{From: "domains/ochre", To: "gateway/bedrock", Debt: "gateway/bedrock", Reason: "Ochre domain core still lives in gateway/bedrock; needs the recorded split"},
|
||||
{From: "domains/ochre", To: "runtime/compute/gpu", Debt: "Domain → runtime", Reason: "GPU runtime state read directly; needs a domain-owned projection"},
|
||||
|
||||
{From: "domains/ochre/vector", To: "handlers/iam", Debt: "Cross-domain", Reason: "consumes another domain's implementation rather than a capability"},
|
||||
|
||||
{From: "gateway", To: "domains/ec2/awsapi", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/account", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/capacityreservation", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/eigw", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/eip", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/idem", Debt: "gateway/ec2.go (registration; ec2_idempotency.go)", Reason: "dispatch-time idempotency wrapper; moves with the EC2 dispatch map"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/igw", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/image", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/instance", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/key", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/launchtemplate", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/natgw", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/placementgroup", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/routetable", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/snapshot", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/spotinstance", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/tags", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/volume", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/vpc", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
{From: "gateway", To: "domains/ec2/awsapi/zone", Debt: "gateway/ec2.go (registration)", Reason: "EC2 dispatch map stays in gateway until the ingress registration seam"},
|
||||
|
||||
{From: "gateway/bedrock", To: "handlers/iam", Debt: "IAM implementation consumers in STS, Ochre and ELBv2", Reason: "secret encrypt/decrypt helpers; needs a shared crypto boundary"},
|
||||
|
||||
{From: "gateway/rds", To: "domains/ec2/awsapi/instance", Debt: "Instance adapter consumers outside EC2", Reason: "enginecatalog.go calls the instance adapter; needs an EC2 capability"},
|
||||
|
||||
{From: "gateway/sts", To: "handlers/iam", Debt: "IAM implementation consumers in STS, Ochre and ELBv2", Reason: "GetCallerIdentity takes IAMService; needs an IAM capability"},
|
||||
|
||||
{From: "handlers/ecs", To: "domains/ec2/ebs/metadata", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
|
||||
{From: "handlers/ecs", To: "domains/ec2/eip", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
|
||||
{From: "handlers/ecs", To: "domains/ec2/instancetypes", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
|
||||
{From: "handlers/ecs", To: "handlers/elbv2", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
|
||||
{From: "handlers/ecs", To: "handlers/iam", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
|
||||
|
||||
{From: "handlers/eks", To: "domains/dns", Debt: "close-out: handlers/eks (ADR-0004 blocked-dependency list; inventory-eks boundary imports)", Reason: "legacy EKS calls another domain's or runtime implementation directly"},
|
||||
{From: "handlers/eks", To: "domains/ec2/ebs/metadata", Debt: "close-out: handlers/eks (ADR-0004 blocked-dependency list; inventory-eks boundary imports)", Reason: "legacy EKS calls another domain's or runtime implementation directly"},
|
||||
{From: "handlers/eks", To: "domains/ec2/instancetypes", Debt: "close-out: handlers/eks (ADR-0004 blocked-dependency list; inventory-eks boundary imports)", Reason: "legacy EKS calls another domain's or runtime implementation directly"},
|
||||
{From: "handlers/eks", To: "domains/ec2/placementgroup", Debt: "close-out: handlers/eks (ADR-0004 blocked-dependency list; inventory-eks boundary imports)", Reason: "legacy EKS calls another domain's or runtime implementation directly"},
|
||||
{From: "handlers/eks", To: "domains/ec2/systeminstance", Debt: "close-out: handlers/eks (ADR-0004 blocked-dependency list; inventory-eks boundary imports)", Reason: "legacy EKS calls another domain's or runtime implementation directly"},
|
||||
{From: "handlers/eks", To: "domains/network/systemvpc", Debt: "close-out: handlers/eks (ADR-0004 blocked-dependency list; inventory-eks boundary imports)", Reason: "legacy EKS calls another domain's or runtime implementation directly"},
|
||||
{From: "handlers/eks", To: "handlers/iam", Debt: "close-out: handlers/eks (ADR-0004 blocked-dependency list; inventory-eks boundary imports)", Reason: "legacy EKS calls another domain's or runtime implementation directly"},
|
||||
{From: "handlers/eks", To: "runtime/compute/vm", Debt: "close-out: handlers/eks (ADR-0004 blocked-dependency list; inventory-eks boundary imports)", Reason: "legacy EKS calls another domain's or runtime implementation directly"},
|
||||
|
||||
{From: "handlers/elbv2", To: "domains/acm", Debt: "handlers/elbv2 (cross-domain)", Reason: "legacy ELBv2 cross-domain import; clears when ELBv2 is extracted behind consumer-owned capabilities"},
|
||||
{From: "handlers/elbv2", To: "domains/dns", Debt: "handlers/elbv2 (cross-domain)", Reason: "legacy ELBv2 cross-domain import; clears when ELBv2 is extracted behind consumer-owned capabilities"},
|
||||
{From: "handlers/elbv2", To: "domains/ec2/systeminstance", Debt: "handlers/elbv2 (cross-domain)", Reason: "legacy ELBv2 cross-domain import; clears when ELBv2 is extracted behind consumer-owned capabilities"},
|
||||
{From: "handlers/elbv2", To: "domains/ec2/vpc", Debt: "handlers/elbv2 (cross-domain)", Reason: "legacy ELBv2 cross-domain import; clears when ELBv2 is extracted behind consumer-owned capabilities"},
|
||||
{From: "handlers/elbv2", To: "domains/network/topology", Debt: "handlers/elbv2 (cross-domain)", Reason: "legacy ELBv2 cross-domain import; clears when ELBv2 is extracted behind consumer-owned capabilities"},
|
||||
{From: "handlers/elbv2", To: "handlers/iam", Debt: "IAM implementation consumers in STS, Ochre and ELBv2", Reason: "system-instance role ensurer; needs an IAM capability"},
|
||||
{From: "handlers/elbv2", To: "lbagent", Debt: "handlers/elbv2/{agent_types,haproxy,health_checker,nginx}.go import lbagent", Reason: "health-report and cert/PID-path vocabulary; needs a guest/controller contract"},
|
||||
|
||||
{From: "handlers/rds", To: "domains/dns", Debt: "close-out: handlers/rds (ADR-0004 blocked-dependency list; inventory-rds boundary imports)", Reason: "legacy RDS calls another domain's or runtime implementation directly"},
|
||||
{From: "handlers/rds", To: "domains/ec2/instance", Debt: "close-out: handlers/rds (ADR-0004 blocked-dependency list; inventory-rds boundary imports)", Reason: "legacy RDS calls another domain's or runtime implementation directly"},
|
||||
{From: "handlers/rds", To: "domains/ec2/instancetypes", Debt: "close-out: handlers/rds (ADR-0004 blocked-dependency list; inventory-rds boundary imports)", Reason: "legacy RDS calls another domain's or runtime implementation directly"},
|
||||
{From: "handlers/rds", To: "domains/ec2/systeminstance", Debt: "close-out: handlers/rds (ADR-0004 blocked-dependency list; inventory-rds boundary imports)", Reason: "legacy RDS calls another domain's or runtime implementation directly"},
|
||||
{From: "handlers/rds", To: "domains/network/systemvpc", Debt: "close-out: handlers/rds (ADR-0004 blocked-dependency list; inventory-rds boundary imports)", Reason: "legacy RDS calls another domain's or runtime implementation directly"},
|
||||
{From: "handlers/rds", To: "handlers/iam", Debt: "close-out: handlers/rds (ADR-0004 blocked-dependency list; inventory-rds boundary imports)", Reason: "legacy RDS calls another domain's or runtime implementation directly"},
|
||||
{From: "handlers/rds", To: "runtime/compute/vm", Debt: "close-out: handlers/rds (ADR-0004 blocked-dependency list; inventory-rds boundary imports)", Reason: "legacy RDS calls another domain's or runtime implementation directly"},
|
||||
|
||||
{From: "handlers/sts", To: "handlers/iam", Debt: "IAM implementation consumers in STS, Ochre and ELBv2", Reason: "STS reads IAM service, trust-policy and OIDC-provider state; needs an IAM capability"},
|
||||
|
||||
{From: "operator/cli", To: "accountteardown", Debt: "operator/cli (dependency)", Reason: "command tree imports implementations; needs public capabilities"},
|
||||
{From: "operator/cli", To: "daemon", Debt: "operator/cli (dependency)", Reason: "command tree imports implementations; needs public capabilities"},
|
||||
{From: "operator/cli", To: "gateway", Debt: "operator/cli (dependency)", Reason: "command tree imports implementations; needs public capabilities"},
|
||||
{From: "operator/cli", To: "gateway/bedrock", Debt: "operator/cli (dependency)", Reason: "command tree imports implementations; needs public capabilities"},
|
||||
{From: "operator/cli", To: "handlers/eks", Debt: "operator/cli (dependency)", Reason: "command tree imports implementations; needs public capabilities"},
|
||||
{From: "operator/cli", To: "handlers/iam", Debt: "operator/cli (dependency)", Reason: "command tree imports implementations; needs public capabilities"},
|
||||
}
|
||||
@@ -1,163 +0,0 @@
|
||||
// Package layering enforces ADR-0001's package-boundary dependency rules over
|
||||
// the module's production import graph, ratcheted by a recorded-debt allowlist.
|
||||
package layering
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"go/ast"
|
||||
"go/build/constraint"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const modulePath = "github.com/mulgadc/spinifex"
|
||||
|
||||
// edge is one package-to-package import, both sides named as in the allowlist:
|
||||
// application packages relative to spinifex/, module-root ones (cmd/,
|
||||
// contracts/, internal/) relative to the module root.
|
||||
type edge struct {
|
||||
From, To string
|
||||
}
|
||||
|
||||
func (e edge) String() string { return e.From + " -> " + e.To }
|
||||
|
||||
// graph is the production import graph restricted to this module.
|
||||
type graph struct {
|
||||
// pkgs maps a package name to its module-relative directory.
|
||||
pkgs map[string]string
|
||||
// imports records the first file:line at which each edge appears.
|
||||
imports map[edge]string
|
||||
}
|
||||
|
||||
func newGraph() *graph {
|
||||
return &graph{pkgs: map[string]string{}, imports: map[edge]string{}}
|
||||
}
|
||||
|
||||
// shortName drops the application-root prefix so names read like the ADR and
|
||||
// the migration record; classify rejects app-root dirs that would collide.
|
||||
func shortName(full string) string {
|
||||
if rest, ok := strings.CutPrefix(full, "spinifex/"); ok {
|
||||
return rest
|
||||
}
|
||||
return full
|
||||
}
|
||||
|
||||
func moduleRoot() (string, error) {
|
||||
_, file, _, ok := runtime.Caller(0)
|
||||
if !ok {
|
||||
return "", errors.New("cannot locate test source")
|
||||
}
|
||||
// architecture/layering/graph_test.go -> module root.
|
||||
return filepath.Abs(filepath.Join(filepath.Dir(file), "..", ".."))
|
||||
}
|
||||
|
||||
// excludedTopDirs hold no production code: tests/ is test-only, docs/ only
|
||||
// embeds documentation and architecture/ is repository governance tooling.
|
||||
var excludedTopDirs = map[string]bool{"tests": true, "docs": true, "architecture": true}
|
||||
|
||||
// loadGraph parses the imports of every non-test Go file under root, so it
|
||||
// sees every build-tag variant and needs no Go toolchain or workspace.
|
||||
func loadGraph(root string) (*graph, error) {
|
||||
g := newGraph()
|
||||
fset := token.NewFileSet()
|
||||
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel = filepath.ToSlash(rel)
|
||||
if d.IsDir() {
|
||||
if rel == "." {
|
||||
return nil
|
||||
}
|
||||
name := d.Name()
|
||||
if strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_") ||
|
||||
name == "testdata" || name == "vendor" || name == "node_modules" || excludedTopDirs[rel] {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(path, "go.mod")); err == nil {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
|
||||
return nil
|
||||
}
|
||||
f, err := parser.ParseFile(fset, path, nil, parser.ImportsOnly|parser.ParseComments)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ignoredByBuildTag(f.Comments, f.Package) {
|
||||
return nil
|
||||
}
|
||||
dir := filepath.ToSlash(filepath.Dir(rel))
|
||||
if dir == "." {
|
||||
dir = ""
|
||||
}
|
||||
from := shortName(dir)
|
||||
g.pkgs[from] = dir
|
||||
for _, imp := range f.Imports {
|
||||
p, err := strconv.Unquote(imp.Path.Value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
target, ok := strings.CutPrefix(p, modulePath+"/")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
e := edge{From: from, To: shortName(target)}
|
||||
if _, seen := g.imports[e]; !seen {
|
||||
pos := fset.Position(imp.Pos())
|
||||
g.imports[e] = fmt.Sprintf("%s:%d", rel, pos.Line)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return g, err
|
||||
}
|
||||
|
||||
// ignoredByBuildTag reports a `//go:build ignore` file, which is a standalone
|
||||
// program (a generator) rather than part of its directory's package.
|
||||
func ignoredByBuildTag(groups []*ast.CommentGroup, pkgPos token.Pos) bool {
|
||||
for _, cg := range groups {
|
||||
if cg.Pos() >= pkgPos {
|
||||
break
|
||||
}
|
||||
for _, c := range cg.List {
|
||||
if !constraint.IsGoBuild(c.Text) {
|
||||
continue
|
||||
}
|
||||
expr, err := constraint.Parse(c.Text)
|
||||
if err == nil && expr.String() == "ignore" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// sortedEdges returns the graph's edges in a stable order.
|
||||
func (g *graph) sortedEdges() []edge {
|
||||
out := make([]edge, 0, len(g.imports))
|
||||
for e := range g.imports {
|
||||
out = append(out, e)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].From != out[j].From {
|
||||
return out[i].From < out[j].From
|
||||
}
|
||||
return out[i].To < out[j].To
|
||||
})
|
||||
return out
|
||||
}
|
||||
@@ -1,193 +0,0 @@
|
||||
package layering
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type violation struct {
|
||||
rule string
|
||||
edge edge
|
||||
pos string
|
||||
}
|
||||
|
||||
type report struct {
|
||||
unmapped []string
|
||||
violations []violation // every rule hit, allowlisted or not
|
||||
unrecorded []violation // hits whose edge has no allowlist entry
|
||||
stale []allowance // entries whose edge no rule rejects any more
|
||||
malformed []string
|
||||
}
|
||||
|
||||
// check classifies g, applies every rule and compares the result with allow.
|
||||
func check(g *graph, allow []allowance) report {
|
||||
var rep report
|
||||
classes := map[string]class{}
|
||||
for name, dir := range g.pkgs {
|
||||
c, ok := classify(dir)
|
||||
if !ok || c.layer == layerTests {
|
||||
rep.unmapped = append(rep.unmapped, fmt.Sprintf("%s (directory %s)", name, dir))
|
||||
continue
|
||||
}
|
||||
classes[name] = c
|
||||
}
|
||||
|
||||
allowed := map[edge]bool{}
|
||||
for _, a := range allow {
|
||||
e := edge{From: a.From, To: a.To}
|
||||
switch {
|
||||
case allowed[e]:
|
||||
rep.malformed = append(rep.malformed, "duplicate entry "+e.String())
|
||||
case a.Debt == "" || a.Reason == "":
|
||||
rep.malformed = append(rep.malformed, "entry "+e.String()+" needs both Debt and Reason")
|
||||
}
|
||||
allowed[e] = true
|
||||
}
|
||||
|
||||
hit := map[edge]bool{}
|
||||
for _, e := range g.sortedEdges() {
|
||||
from, ok := classes[e.From]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
to, ok := classify(fullPath(e.To))
|
||||
if !ok {
|
||||
rep.unmapped = append(rep.unmapped, fmt.Sprintf("%s (imported by %s at %s)", e.To, e.From, g.imports[e]))
|
||||
continue
|
||||
}
|
||||
for _, r := range rules {
|
||||
if !r.subject(from, e.From) || !r.forbids(from, to, e.From) {
|
||||
continue
|
||||
}
|
||||
v := violation{rule: r.id, edge: e, pos: g.imports[e]}
|
||||
rep.violations = append(rep.violations, v)
|
||||
hit[e] = true
|
||||
if !allowed[e] {
|
||||
rep.unrecorded = append(rep.unrecorded, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, a := range allow {
|
||||
if !hit[edge{From: a.From, To: a.To}] {
|
||||
rep.stale = append(rep.stale, a)
|
||||
}
|
||||
}
|
||||
sort.Strings(rep.unmapped)
|
||||
return rep
|
||||
}
|
||||
|
||||
func ruleByID(id string) rule {
|
||||
for _, r := range rules {
|
||||
if r.id == id {
|
||||
return r
|
||||
}
|
||||
}
|
||||
panic("unknown rule " + id)
|
||||
}
|
||||
|
||||
// TestADR0001_Layering is the blocking ratchet over the real import graph.
|
||||
func TestADR0001_Layering(t *testing.T) {
|
||||
root, err := moduleRoot()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g, err := loadGraph(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Guards against a walk that silently finds nothing and passes vacuously.
|
||||
if len(g.pkgs) < 100 || len(g.imports) < 500 {
|
||||
t.Fatalf("scanned %d packages and %d module imports under %s; the walk has lost the tree and every rule would pass vacuously",
|
||||
len(g.pkgs), len(g.imports), root)
|
||||
}
|
||||
rep := check(g, allowlist)
|
||||
|
||||
t.Run("Classification_EveryPackageMapped", func(t *testing.T) {
|
||||
for _, u := range rep.unmapped {
|
||||
t.Errorf("ADR-0001 current-to-target map: package %s has no layer; place it under an ADR-0001 home or classify it deliberately in layers_test.go", u)
|
||||
}
|
||||
})
|
||||
|
||||
for _, r := range rules {
|
||||
t.Run(r.id, func(t *testing.T) {
|
||||
subjects := 0
|
||||
for name, dir := range g.pkgs {
|
||||
if c, ok := classify(dir); ok && r.subject(c, name) {
|
||||
subjects++
|
||||
}
|
||||
}
|
||||
if subjects == 0 {
|
||||
t.Fatalf("no scanned package is governed by %s; the rule would pass vacuously", r.id)
|
||||
}
|
||||
var snippets []string
|
||||
for _, v := range rep.unrecorded {
|
||||
if v.rule != r.id {
|
||||
continue
|
||||
}
|
||||
t.Errorf("ADR-0001 %s: %s imports %s (%s); %s",
|
||||
r.clause, v.edge.From, v.edge.To, v.pos, r.hint)
|
||||
snippets = append(snippets, fmt.Sprintf("{From: %q, To: %q, Debt: \"<migration-record debt subject>\", Reason: \"<why>\"},", v.edge.From, v.edge.To))
|
||||
}
|
||||
if len(snippets) > 0 {
|
||||
t.Logf("fix the import; only debt already recorded in docs/PACKAGE_BOUNDARY_MIGRATION.md may be allow-listed:\n%s",
|
||||
strings.Join(snippets, "\n"))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("Allowlist_NoStaleEntries", func(t *testing.T) {
|
||||
for _, a := range rep.stale {
|
||||
t.Errorf("allowlist entry %s -> %s (%s) no longer matches a forbidden import; the debt is paid, so delete the entry from allowlist_test.go",
|
||||
a.From, a.To, a.Debt)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Allowlist_WellFormed", func(t *testing.T) {
|
||||
for _, m := range rep.malformed {
|
||||
t.Errorf("allowlist_test.go: %s", m)
|
||||
}
|
||||
t.Logf("%d allow-listed edges", len(allowlist))
|
||||
})
|
||||
}
|
||||
|
||||
// TestADR0001_LayeringCheckerDetectsFaults injects a forbidden edge, a stale
|
||||
// allowlist entry and an unmapped package into a synthetic graph.
|
||||
func TestADR0001_LayeringCheckerDetectsFaults(t *testing.T) {
|
||||
g := newGraph()
|
||||
for _, dir := range []string{
|
||||
"spinifex/foundation/widget", "spinifex/domains/alpha/thing",
|
||||
"spinifex/domains/beta/store", "spinifex/misc",
|
||||
} {
|
||||
g.pkgs[shortName(dir)] = dir
|
||||
}
|
||||
forbidden := edge{From: "foundation/widget", To: "domains/alpha/thing"}
|
||||
g.imports[forbidden] = "spinifex/foundation/widget/w.go:7"
|
||||
allowedEdge := edge{From: "domains/alpha/thing", To: "domains/beta/store"}
|
||||
g.imports[allowedEdge] = "spinifex/domains/alpha/thing/t.go:9"
|
||||
|
||||
staleEntry := allowance{From: "domains/beta/store", To: "domains/alpha/thing", Debt: "synthetic", Reason: "stale"}
|
||||
rep := check(g, []allowance{
|
||||
{From: allowedEdge.From, To: allowedEdge.To, Debt: "synthetic", Reason: "recorded"},
|
||||
staleEntry,
|
||||
})
|
||||
|
||||
if len(rep.unrecorded) != 1 || rep.unrecorded[0].edge != forbidden ||
|
||||
rep.unrecorded[0].rule != ruleByID("R1_FoundationImportsOnlyFoundation").id {
|
||||
t.Errorf("want only %s reported under R1, got %+v", forbidden, rep.unrecorded)
|
||||
}
|
||||
if len(rep.stale) != 1 || rep.stale[0] != staleEntry {
|
||||
t.Errorf("want stale entry %+v, got %+v", staleEntry, rep.stale)
|
||||
}
|
||||
if len(rep.unmapped) != 1 || !strings.HasPrefix(rep.unmapped[0], "misc ") {
|
||||
t.Errorf("want unmapped package misc, got %v", rep.unmapped)
|
||||
}
|
||||
var allowedHit bool
|
||||
for _, v := range rep.violations {
|
||||
allowedHit = allowedHit || v.edge == allowedEdge
|
||||
}
|
||||
if !allowedHit {
|
||||
t.Errorf("allow-listed cross-domain edge %s was not detected by R3, so the allowlist entry would read as stale", allowedEdge)
|
||||
}
|
||||
}
|
||||
@@ -1,164 +0,0 @@
|
||||
package layering
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type layer string
|
||||
|
||||
const (
|
||||
layerFoundation layer = "foundation"
|
||||
layerContracts layer = "contracts"
|
||||
layerDomain layer = "domains"
|
||||
layerIngress layer = "ingress"
|
||||
layerRuntime layer = "runtime"
|
||||
layerOperator layer = "operator"
|
||||
layerAgents layer = "agents"
|
||||
layerProviders layer = "providers"
|
||||
layerBootstrap layer = "bootstrap"
|
||||
layerCmd layer = "cmd"
|
||||
layerInternal layer = "internal"
|
||||
layerTestkit layer = "testkit"
|
||||
layerLegacy layer = "legacy"
|
||||
// layerTests is valid only as an import target; tests/ is never scanned.
|
||||
layerTests layer = "tests"
|
||||
)
|
||||
|
||||
// domainKind distinguishes the parts of a domain the rules treat differently.
|
||||
type domainKind int
|
||||
|
||||
const (
|
||||
notDomain domainKind = iota
|
||||
domainImpl // domains/<d>/... resource implementation
|
||||
domainAWSAPI // domains/<d>/awsapi/... AWS action adapters
|
||||
domainProj // domains/<d>/projection/... authorised view
|
||||
legacyImpl // handlers/<d>/... legacy domain implementation
|
||||
legacyAdapter // gateway/<d>/... legacy AWS action adapters
|
||||
)
|
||||
|
||||
type class struct {
|
||||
layer layer
|
||||
legacy string // legacy root name when layer is layerLegacy
|
||||
domain string // owning domain for domain-shaped and agent packages
|
||||
kind domainKind
|
||||
}
|
||||
|
||||
func (c class) domainShaped() bool { return c.kind != notDomain }
|
||||
|
||||
// isLegacy reports whether c sits under one of the named legacy roots.
|
||||
func (c class) isLegacy(roots ...string) bool {
|
||||
if c.layer != layerLegacy {
|
||||
return false
|
||||
}
|
||||
if len(roots) == 0 {
|
||||
return true
|
||||
}
|
||||
return slices.Contains(roots, c.legacy)
|
||||
}
|
||||
|
||||
// legacyRoots are pre-ADR-0001 application roots still standing.
|
||||
var legacyRoots = map[string]bool{
|
||||
"handlers": true, "gateway": true, "daemon": true,
|
||||
"accountteardown": true, "lbagent": true,
|
||||
}
|
||||
|
||||
// gatewayDomains names the domain each gateway/<service> adapter belongs to;
|
||||
// Bedrock is the AWS surface of the Ochre domain.
|
||||
var gatewayDomains = map[string]string{
|
||||
"bedrock": "ochre", "ecs": "ecs", "eks": "eks", "elbv2": "elbv2", "iam": "iam",
|
||||
"rds": "rds", "sts": "sts",
|
||||
}
|
||||
|
||||
// runtimeLegacy and operatorLegacy are the legacy roots whose ADR-0001 target
|
||||
// is runtime/ and operator/ respectively.
|
||||
var (
|
||||
runtimeLegacy = []string{"daemon"}
|
||||
operatorLegacy = []string{"accountteardown"}
|
||||
)
|
||||
|
||||
// classify maps a module-relative package directory to its layer. ok is false
|
||||
// for a package that has no deliberate classification.
|
||||
func classify(full string) (class, bool) {
|
||||
parts := strings.Split(full, "/")
|
||||
switch parts[0] {
|
||||
case "cmd":
|
||||
return class{layer: layerCmd}, len(parts) >= 2
|
||||
case "contracts":
|
||||
return class{layer: layerContracts}, len(parts) >= 2
|
||||
case "internal":
|
||||
if len(parts) < 2 {
|
||||
return class{}, false
|
||||
}
|
||||
if parts[1] == "testkit" || parts[1] == "tooling" {
|
||||
return class{layer: layerTestkit}, true
|
||||
}
|
||||
return class{layer: layerInternal}, true
|
||||
case "tests":
|
||||
return class{layer: layerTests}, true
|
||||
case "spinifex":
|
||||
return classifyApp(parts[1:])
|
||||
}
|
||||
return class{}, false
|
||||
}
|
||||
|
||||
func classifyApp(app []string) (class, bool) {
|
||||
if len(app) == 0 {
|
||||
return class{}, false
|
||||
}
|
||||
switch app[0] {
|
||||
case "foundation":
|
||||
return class{layer: layerFoundation}, true
|
||||
case "ingress":
|
||||
return class{layer: layerIngress}, true
|
||||
case "runtime":
|
||||
return class{layer: layerRuntime}, true
|
||||
case "operator":
|
||||
return class{layer: layerOperator}, true
|
||||
case "providers":
|
||||
return class{layer: layerProviders}, true
|
||||
case "bootstrap":
|
||||
return class{layer: layerBootstrap}, true
|
||||
case "agents":
|
||||
c := class{layer: layerAgents}
|
||||
if len(app) >= 2 {
|
||||
c.domain = app[1]
|
||||
}
|
||||
return c, true
|
||||
case "domains":
|
||||
if len(app) < 2 {
|
||||
return class{}, false
|
||||
}
|
||||
c := class{layer: layerDomain, domain: app[1], kind: domainImpl}
|
||||
if len(app) >= 3 {
|
||||
switch app[2] {
|
||||
case "awsapi":
|
||||
c.kind = domainAWSAPI
|
||||
case "projection":
|
||||
c.kind = domainProj
|
||||
}
|
||||
}
|
||||
return c, true
|
||||
}
|
||||
if !legacyRoots[app[0]] {
|
||||
return class{}, false
|
||||
}
|
||||
c := class{layer: layerLegacy, legacy: app[0]}
|
||||
switch {
|
||||
case app[0] == "handlers" && len(app) >= 2:
|
||||
c.domain, c.kind = app[1], legacyImpl
|
||||
case app[0] == "gateway" && len(app) >= 2 && gatewayDomains[app[1]] != "":
|
||||
c.domain, c.kind = gatewayDomains[app[1]], legacyAdapter
|
||||
}
|
||||
return c, true
|
||||
}
|
||||
|
||||
// fullPath reverses shortName for classification.
|
||||
func fullPath(name string) string {
|
||||
top, _, _ := strings.Cut(name, "/")
|
||||
switch top {
|
||||
case "cmd", "contracts", "internal", "tests", "docs":
|
||||
return name
|
||||
}
|
||||
return "spinifex/" + name
|
||||
}
|
||||
@@ -1,81 +0,0 @@
|
||||
package layering
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// frozenLegacyPackages is every production package still under a legacy root.
|
||||
// It only shrinks: a new entry is a new legacy package, a missing one is a
|
||||
// completed move whose line must be deleted.
|
||||
var frozenLegacyPackages = []string{
|
||||
"accountteardown",
|
||||
"daemon",
|
||||
"gateway",
|
||||
"gateway/bedrock",
|
||||
"gateway/ecs",
|
||||
"gateway/eks",
|
||||
"gateway/elbv2",
|
||||
"gateway/iam",
|
||||
"gateway/rds",
|
||||
"gateway/spx",
|
||||
"gateway/sts",
|
||||
"handlers/ecs",
|
||||
"handlers/eks",
|
||||
"handlers/elbv2",
|
||||
"handlers/iam",
|
||||
"handlers/iam/mock",
|
||||
"handlers/rds",
|
||||
"handlers/sts",
|
||||
"lbagent",
|
||||
}
|
||||
|
||||
// retiredLegacyRoots were legacy roots whose every package has moved to its
|
||||
// ADR-0001 target; recreating one would reopen a closed migration row.
|
||||
var retiredLegacyRoots = []string{"admin", "services", "utils", "vpcd"}
|
||||
|
||||
// TestADR0001_LegacyRootsFrozen: the current-to-target map gives every legacy
|
||||
// root a destination, so none may gain a package and no retired root may return.
|
||||
func TestADR0001_LegacyRootsFrozen(t *testing.T) {
|
||||
root, err := moduleRoot()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g, err := loadGraph(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
present := map[string]bool{}
|
||||
for name, dir := range g.pkgs {
|
||||
if c, ok := classify(dir); ok && c.layer == layerLegacy {
|
||||
present[name] = true
|
||||
}
|
||||
}
|
||||
if len(present) == 0 {
|
||||
t.Fatal("found no legacy packages; the freeze would pass vacuously")
|
||||
}
|
||||
frozen := map[string]bool{}
|
||||
for _, p := range frozenLegacyPackages {
|
||||
frozen[p] = true
|
||||
if !present[p] {
|
||||
t.Errorf("legacy package %s no longer exists; delete it from frozenLegacyPackages", p)
|
||||
}
|
||||
}
|
||||
for p := range present {
|
||||
if !frozen[p] {
|
||||
t.Errorf("ADR-0001 current-to-target map: %s is a new package under a legacy root; create it at its ADR-0001 target home instead", p)
|
||||
}
|
||||
}
|
||||
|
||||
for _, r := range retiredLegacyRoots {
|
||||
entries, err := os.ReadDir(filepath.Join(root, "spinifex", r))
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(entries) > 0 {
|
||||
t.Errorf("ADR-0001 current-to-target map: spinifex/%s is a retired legacy root; give the code its target home instead", r)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,170 +0,0 @@
|
||||
package layering
|
||||
|
||||
import "slices"
|
||||
|
||||
// rule is one ADR-0001 dependency rule over a classified import edge.
|
||||
type rule struct {
|
||||
id string // subtest name, led by the ADR-0001 clause it encodes
|
||||
clause string // the ADR text the rule enforces, quoted in failures
|
||||
hint string // one-line fix direction
|
||||
// subject reports whether an importer is governed by the rule; the
|
||||
// non-empty guard requires at least one governed package per rule.
|
||||
subject func(from class, fromName string) bool
|
||||
// forbids reports whether the rule rejects the edge.
|
||||
forbids func(from, to class, fromName string) bool
|
||||
}
|
||||
|
||||
// awsgwRole is the composition root ADR-0002 lets wire domain registrations.
|
||||
const awsgwRole = "runtime/roles/awsgw"
|
||||
|
||||
// qualificationBinaries are the developer/qualification mains ADR-0001 lets
|
||||
// keep reusable source in internal/testkit or internal/tooling.
|
||||
var qualificationBinaries = map[string]bool{
|
||||
"cmd/aws-model-coverage": true, "cmd/dhcptest": true, "cmd/spx-loadgen": true,
|
||||
}
|
||||
|
||||
func isLayer(c class, layers ...layer) bool {
|
||||
return slices.Contains(layers, c.layer)
|
||||
}
|
||||
|
||||
// importsDomainImplementation is any domain-owned package other than a named
|
||||
// projection, in its target or legacy home.
|
||||
func importsDomainImplementation(to class) bool {
|
||||
return to.domainShaped() && to.kind != domainProj
|
||||
}
|
||||
|
||||
var rules = []rule{
|
||||
{
|
||||
id: "R1_FoundationImportsOnlyFoundation",
|
||||
clause: "rule 1: foundation imports neither domains, ingress, runtime nor operator (enforced conservatively: foundation imports no module package above foundation)",
|
||||
hint: "move the dependency's needed primitive into foundation or invert it through a parameter",
|
||||
subject: func(f class, _ string) bool { return f.layer == layerFoundation },
|
||||
forbids: func(_, t class, _ string) bool { return t.layer != layerFoundation },
|
||||
},
|
||||
{
|
||||
id: "R2_IngressImportsNoImplementation",
|
||||
clause: "rule 2: ingress/aws imports no domain implementation; domains supply their own action registration through the ingress registration contract",
|
||||
hint: "take the capability through the registration contract wired by runtime/roles/awsgw",
|
||||
subject: func(f class, _ string) bool { return f.layer == layerIngress },
|
||||
forbids: func(_, t class, _ string) bool {
|
||||
return !isLayer(t, layerFoundation, layerContracts, layerIngress)
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "R3_DomainImportsNoOtherDomainImplementation",
|
||||
clause: "rule 3: a domain does not read another domain's private records, migrations or repository; it uses a consumer-owned capability, a named projection, or a versioned contracts/<domain>/v1 contract",
|
||||
hint: "declare a narrow capability in the consumer, or import the owner's projection or contracts/<domain>/v1",
|
||||
subject: func(f class, _ string) bool { return f.domainShaped() },
|
||||
forbids: func(f, t class, _ string) bool {
|
||||
return f.domainShaped() && importsDomainImplementation(t) && t.domain != f.domain
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "R4_ProviderImportsNoDomainImplementation",
|
||||
clause: "rule 4: a provider adapter implements a provider contract and cannot acquire AWS-visible resource authority by reading a domain's private state",
|
||||
hint: "pass the needed values into the provider contract instead of importing the domain",
|
||||
subject: func(f class, _ string) bool { return f.layer == layerProviders },
|
||||
forbids: func(_, t class, _ string) bool { return t.domainShaped() || t.isLegacy() },
|
||||
},
|
||||
{
|
||||
id: "R5_DomainImportsNoRuntimeOrOperator",
|
||||
clause: "rule 5: operator and runtime are composition consumers; they wire domain modules, so a domain does not import them",
|
||||
hint: "expose the runtime or operator state through a domain-owned projection or a capability wired at the composition root",
|
||||
subject: func(f class, _ string) bool { return f.domainShaped() },
|
||||
forbids: func(f, t class, _ string) bool {
|
||||
return f.domainShaped() && (isLayer(t, layerRuntime, layerOperator) ||
|
||||
t.isLegacy(runtimeLegacy...) || t.isLegacy(operatorLegacy...))
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "R5_AWSAPIAdaptersImportedOnlyByOwnerOrComposition",
|
||||
clause: "rule 5: operator and runtime call public use cases; a domain's awsapi action adapters are owned by that domain and wired only by the composition root",
|
||||
hint: "call an owner-provided capability instead of the AWS action adapter",
|
||||
subject: func(_ class, _ string) bool { return true },
|
||||
forbids: func(f, t class, fromName string) bool {
|
||||
if t.kind != domainAWSAPI || fromName == awsgwRole {
|
||||
return false
|
||||
}
|
||||
return !f.domainShaped() || f.domain != t.domain
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "R5_OperatorImportsNoLegacyImplementation",
|
||||
clause: "rule 5 and the cmd/spinifex map row: operator use cases reach domains through public capabilities and do not read domain-private state",
|
||||
hint: "reach the domain through its public capability; do not import a legacy implementation root",
|
||||
subject: func(f class, _ string) bool { return f.layer == layerOperator },
|
||||
forbids: func(f, t class, _ string) bool { return f.layer == layerOperator && t.isLegacy() },
|
||||
},
|
||||
{
|
||||
id: "R6_ProductionImportsNoTestkit",
|
||||
clause: "rule 6: test support is under internal/testkit; production packages do not import it",
|
||||
hint: "move the shared code to its production owner or keep the import in a _test.go file",
|
||||
subject: func(f class, n string) bool { return f.layer != layerTestkit && !qualificationBinaries[n] },
|
||||
forbids: func(f, t class, n string) bool {
|
||||
return f.layer != layerTestkit && !qualificationBinaries[n] && isLayer(t, layerTestkit, layerTests)
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "Contracts_ImportOnlyFoundationAndContracts",
|
||||
clause: "decision item 1 and the area table: contracts own cross-process vocabulary, not application interfaces, domain persistence or transport implementation",
|
||||
hint: "keep wire types self-contained; depend only on foundation or another contract",
|
||||
subject: func(f class, _ string) bool { return f.layer == layerContracts },
|
||||
forbids: func(_, t class, _ string) bool { return !isLayer(t, layerFoundation, layerContracts) },
|
||||
},
|
||||
{
|
||||
id: "Domains_ImportNoAgentsIngressOrLegacy",
|
||||
clause: "area table and domain module shape: a domain owns no other domain's implementation; only its awsapi adapters may use the generic ingress/aws machinery",
|
||||
hint: "depend on a contract or capability; leave ingress use to the domain's awsapi package",
|
||||
subject: func(f class, _ string) bool { return f.domainShaped() },
|
||||
forbids: func(f, t class, _ string) bool {
|
||||
if !f.domainShaped() {
|
||||
return false
|
||||
}
|
||||
switch {
|
||||
case t.layer == layerAgents || t.isLegacy("lbagent"):
|
||||
return true
|
||||
case t.layer == layerIngress:
|
||||
return f.kind != domainAWSAPI && f.kind != legacyAdapter
|
||||
case t.isLegacy("gateway") && !t.domainShaped():
|
||||
return true
|
||||
case t.isLegacy("handlers", "gateway"):
|
||||
return f.layer == layerDomain
|
||||
}
|
||||
return false
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "Agents_ImportNoDomainOrLegacyImplementation",
|
||||
clause: "area table: agents/<domain> own in-guest and service-role behaviour, not tenant API authority or another domain's private state",
|
||||
hint: "move the shared wire vocabulary behind a guest/controller contract under contracts/",
|
||||
subject: func(f class, _ string) bool { return f.layer == layerAgents || f.isLegacy("lbagent") },
|
||||
forbids: func(f, t class, _ string) bool {
|
||||
return (f.layer == layerAgents || f.isLegacy("lbagent")) && (t.domainShaped() || t.isLegacy())
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "Q22_STSImportsNoEKSImplementation",
|
||||
clause: "Q-22 prohibits the STS-to-EKS dependency in its present form; cut it through an identity-owned contract or an EKS projection",
|
||||
hint: "consume an EKS projection or an identity-owned contract instead of the EKS implementation",
|
||||
subject: func(f class, _ string) bool { return f.domainShaped() && f.domain == "sts" },
|
||||
forbids: func(f, t class, _ string) bool {
|
||||
return f.domainShaped() && f.domain == "sts" && t.domain == "eks" &&
|
||||
(importsDomainImplementation(t) || t.layer == layerAgents)
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "Interim_DaemonGatewayHandlersDirection",
|
||||
clause: "during migration the existing daemon -> gateway -> handlers rule remains the guardrail (Q-20, Q-29)",
|
||||
hint: "keep the dependency pointing daemon -> gateway -> handlers; invert it through a callback or capability",
|
||||
subject: func(f class, _ string) bool { return f.isLegacy("handlers", "gateway") },
|
||||
forbids: func(f, t class, _ string) bool {
|
||||
switch {
|
||||
case f.isLegacy("handlers"):
|
||||
return t.isLegacy("gateway", "daemon")
|
||||
case f.isLegacy("gateway"):
|
||||
return t.isLegacy("daemon")
|
||||
}
|
||||
return false
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
ARG VERSION=dev
|
||||
|
||||
FROM golang:1.27.3-bookworm@sha256:5c3e9b43cfd3bb95c7937462e29d87ada57b481a96fa23ad0eb61778d7a00110 AS builder
|
||||
FROM golang:1.27-bookworm AS builder
|
||||
|
||||
ARG VERSION
|
||||
|
||||
@@ -35,8 +35,8 @@ COPY bluebottle/ ./bluebottle/
|
||||
# Build spinifex binary (spx)
|
||||
RUN COMMIT=$(cd spinifex && git rev-parse --short HEAD 2>/dev/null || echo "unknown") && \
|
||||
cd spinifex && GOFIPS140=v1.0.0 go build -ldflags "-s -w \
|
||||
-X github.com/mulgadc/spinifex/spinifex/operator/cli.Version=${VERSION} \
|
||||
-X github.com/mulgadc/spinifex/spinifex/operator/cli.Commit=${COMMIT}" \
|
||||
-X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Version=${VERSION} \
|
||||
-X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Commit=${COMMIT}" \
|
||||
-o /out/spx cmd/spinifex/main.go
|
||||
|
||||
# Build nbdkit viperblock plugin (.so)
|
||||
|
||||
@@ -7,10 +7,7 @@ import (
|
||||
|
||||
"github.com/mulgadc/predastore/s3api"
|
||||
"github.com/mulgadc/spinifex/internal/awsmodel"
|
||||
acmawsapi "github.com/mulgadc/spinifex/spinifex/domains/acm/awsapi"
|
||||
awsapi "github.com/mulgadc/spinifex/spinifex/domains/ecr/awsapi"
|
||||
"github.com/mulgadc/spinifex/spinifex/gateway"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
|
||||
_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
|
||||
)
|
||||
@@ -46,15 +43,10 @@ func fail(err error) {
|
||||
}
|
||||
|
||||
func compareAll() ([]awsmodel.OperationCoverage, error) {
|
||||
// Registered services contribute their declared inventory, which needs no
|
||||
// live dependencies; nothing here serves a request.
|
||||
inventories := gateway.AWSOperationInventory(map[string]dispatch.Inventory{
|
||||
awsapi.ServiceName: awsapi.OperationInventory(),
|
||||
acmawsapi.ServiceName: acmawsapi.OperationInventory(),
|
||||
})
|
||||
dispatch := gateway.AWSOperationInventory()
|
||||
coverages := make([]awsmodel.OperationCoverage, 0, len(awsmodel.Services()))
|
||||
for _, service := range awsmodel.Services() {
|
||||
inventory, ok := inventories[string(service)]
|
||||
inventory, ok := dispatch[string(service)]
|
||||
var modelInventory awsmodel.DispatchInventory
|
||||
switch {
|
||||
case ok:
|
||||
|
||||
@@ -23,7 +23,7 @@ import (
|
||||
|
||||
"github.com/insomniacslk/dhcp/dhcpv4"
|
||||
"github.com/insomniacslk/dhcp/dhcpv4/nclient4"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/network/external/dhcp"
|
||||
"github.com/mulgadc/spinifex/spinifex/network/external/dhcp"
|
||||
|
||||
_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
|
||||
)
|
||||
|
||||
@@ -1,15 +1,175 @@
|
||||
// ecr-credential-provider is the kubelet exec credential provider for Spinifex
|
||||
// EKS worker nodes; the implementation is in agents/eks/credentialprovider.
|
||||
// ecr-credential-provider is a kubelet exec credential provider that lets a
|
||||
// Spinifex EKS worker node pull images from the internal Spinifex ECR registry.
|
||||
//
|
||||
// kubelet invokes it per-pull, feeding a CredentialProviderRequest on STDIN and
|
||||
// reading a CredentialProviderResponse on STDOUT. The provider fetches the
|
||||
// node IAM-role credentials from IMDS, SigV4-signs GetAuthorizationToken against
|
||||
// the AWS gateway's ECR endpoint, and returns the minted AWS:<jwt> as Basic auth
|
||||
// for the requested registry host.
|
||||
//
|
||||
// Static node config (gateway URL, CA, region) is read from the cloud-init env
|
||||
// file /etc/spinifex-eks/agent.env (KEY=value); real env vars override it.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/ecrauth"
|
||||
"github.com/mulgadc/spinifex/internal/guestenv"
|
||||
"github.com/mulgadc/spinifex/internal/imdscreds"
|
||||
|
||||
_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/eks/credentialprovider"
|
||||
)
|
||||
|
||||
func main() {
|
||||
os.Exit(credentialprovider.Main(os.Stdin, os.Stdout))
|
||||
const (
|
||||
credProviderAPIVersion = "credentialprovider.kubelet.k8s.io/v1"
|
||||
defaultCacheDuration = "10h"
|
||||
// expiryMargin trims the token's real TTL so kubelet re-invokes the provider
|
||||
// before the JWT actually expires mid-run.
|
||||
expiryMargin = 30 * time.Minute
|
||||
|
||||
defaultEnvFile = "/etc/spinifex-eks/agent.env"
|
||||
defaultGatewayCA = "/etc/spinifex-eks/gateway-ca.pem"
|
||||
defaultIMDSBase = "http://169.254.169.254/latest"
|
||||
)
|
||||
|
||||
// CredentialProviderRequest is the kubelet exec-provider request on STDIN.
|
||||
type CredentialProviderRequest struct {
|
||||
APIVersion string `json:"apiVersion"`
|
||||
Kind string `json:"kind"`
|
||||
Image string `json:"image"`
|
||||
}
|
||||
|
||||
// AuthConfig is a single registry credential entry.
|
||||
type AuthConfig struct {
|
||||
Username string `json:"username"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
|
||||
// CredentialProviderResponse is the kubelet exec-provider response on STDOUT.
|
||||
type CredentialProviderResponse struct {
|
||||
APIVersion string `json:"apiVersion"`
|
||||
Kind string `json:"kind"`
|
||||
CacheKeyType string `json:"cacheKeyType"`
|
||||
CacheDuration string `json:"cacheDuration"`
|
||||
Auth map[string]AuthConfig `json:"auth"`
|
||||
}
|
||||
|
||||
// config holds the static node settings the provider needs.
|
||||
type config struct {
|
||||
GatewayURL string
|
||||
GatewayCA string
|
||||
Region string
|
||||
IMDSBase string
|
||||
}
|
||||
|
||||
func main() {
|
||||
cfg := loadConfig(defaultEnvFile)
|
||||
|
||||
httpClient, err := ecrauth.GatewayHTTPClient(cfg.GatewayCA)
|
||||
if err != nil {
|
||||
slog.Error("ecr-credential-provider: build gateway client", "err", err)
|
||||
emitEmpty(os.Stdout)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err := run(os.Stdin, os.Stdout, cfg, httpClient); err != nil {
|
||||
slog.Error("ecr-credential-provider: run failed", "err", err)
|
||||
// Still emit a valid empty-auth response so kubelet has a parseable reply.
|
||||
emitEmpty(os.Stdout)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// run is the testable core: it reads a CredentialProviderRequest from in,
|
||||
// resolves credentials via IMDS + the gateway, and writes a
|
||||
// CredentialProviderResponse to out. The httpClient must trust the gateway CA.
|
||||
func run(in io.Reader, out io.Writer, cfg config, httpClient *http.Client) error {
|
||||
var req CredentialProviderRequest
|
||||
if err := json.NewDecoder(in).Decode(&req); err != nil {
|
||||
return fmt.Errorf("decode request: %w", err)
|
||||
}
|
||||
if req.Image == "" {
|
||||
return fmt.Errorf("request image is empty")
|
||||
}
|
||||
imageHost := ecrauth.HostFromImage(req.Image)
|
||||
|
||||
creds, err := imdscreds.Fetch(httpClient, cfg.IMDSBase)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fetch IMDS credentials: %w", err)
|
||||
}
|
||||
|
||||
tok, err := ecrauth.GetAuthorizationToken(cfg.Region, cfg.GatewayURL, httpClient,
|
||||
creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken)
|
||||
if err != nil {
|
||||
return fmt.Errorf("get authorization token: %w", err)
|
||||
}
|
||||
|
||||
auth := AuthConfig{Username: tok.Username, Password: tok.Password}
|
||||
authMap := map[string]AuthConfig{imageHost: auth}
|
||||
if tok.ProxyHost != "" && tok.ProxyHost != imageHost {
|
||||
authMap[tok.ProxyHost] = auth
|
||||
}
|
||||
|
||||
resp := CredentialProviderResponse{
|
||||
APIVersion: credProviderAPIVersion,
|
||||
Kind: "CredentialProviderResponse",
|
||||
CacheKeyType: "Registry",
|
||||
CacheDuration: cacheDurationFrom(tok.ExpiresAt),
|
||||
Auth: authMap,
|
||||
}
|
||||
return json.NewEncoder(out).Encode(&resp)
|
||||
}
|
||||
|
||||
// cacheDurationFrom derives a Go-duration string from the token expiry minus a
|
||||
// safety margin, falling back to the default when expiry is unknown or too near.
|
||||
func cacheDurationFrom(expiresAt time.Time) string {
|
||||
if expiresAt.IsZero() {
|
||||
return defaultCacheDuration
|
||||
}
|
||||
d := time.Until(expiresAt) - expiryMargin
|
||||
if d <= 0 {
|
||||
return defaultCacheDuration
|
||||
}
|
||||
return d.Round(time.Second).String()
|
||||
}
|
||||
|
||||
// loadConfig reads the cloud-init env file then lets real env vars override. The
|
||||
// IMDS base is fixed (not in the env file) but overridable for tests.
|
||||
func loadConfig(envFile string) config {
|
||||
get := guestenv.Load(envFile).Get
|
||||
|
||||
cfg := config{
|
||||
GatewayURL: get("EKS_GATEWAY_URL"),
|
||||
GatewayCA: get("EKS_GATEWAY_CA"),
|
||||
Region: get("EKS_REGION"),
|
||||
IMDSBase: get("EKS_IMDS_BASE"),
|
||||
}
|
||||
if cfg.GatewayCA == "" {
|
||||
cfg.GatewayCA = defaultGatewayCA
|
||||
}
|
||||
if cfg.IMDSBase == "" {
|
||||
cfg.IMDSBase = defaultIMDSBase
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// emitEmpty writes a valid CredentialProviderResponse with no auth entries.
|
||||
func emitEmpty(out io.Writer) {
|
||||
resp := CredentialProviderResponse{
|
||||
APIVersion: credProviderAPIVersion,
|
||||
Kind: "CredentialProviderResponse",
|
||||
CacheKeyType: "Registry",
|
||||
CacheDuration: defaultCacheDuration,
|
||||
Auth: map[string]AuthConfig{},
|
||||
}
|
||||
if err := json.NewEncoder(out).Encode(&resp); err != nil {
|
||||
slog.Error("ecr-credential-provider: encode empty response", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// fakeBackend serves both the IMDS role/credentials endpoints and the gateway
|
||||
// ECR GetAuthorizationToken on one TLS server, routed by request path.
|
||||
func fakeBackend(t *testing.T) *httptest.Server {
|
||||
t.Helper()
|
||||
mux := http.NewServeMux()
|
||||
|
||||
// IMDSv2 token. Real IMDS echoes the requested TTL back as a response
|
||||
// header; the SDK client parses it from there, not the body.
|
||||
mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPut {
|
||||
w.WriteHeader(http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
|
||||
_, _ = w.Write([]byte("imds-v2-token"))
|
||||
})
|
||||
// IMDS role name.
|
||||
mux.HandleFunc("/latest/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/latest/meta-data/iam/security-credentials/" {
|
||||
if r.Header.Get("X-Aws-Ec2-Metadata-Token") != "imds-v2-token" {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte("node-role"))
|
||||
return
|
||||
}
|
||||
// Role credentials.
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{
|
||||
"Code": "Success",
|
||||
"Type": "AWS-HMAC",
|
||||
"AccessKeyId": "AKIATEST",
|
||||
"SecretAccessKey": "secrettest",
|
||||
"Token": "sessiontoken",
|
||||
"AccountId": "123456789012",
|
||||
})
|
||||
})
|
||||
// Gateway ECR GetAuthorizationToken (X-Amz-Target dispatch).
|
||||
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
if !strings.Contains(r.Header.Get("X-Amz-Target"), "GetAuthorizationToken") {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
authToken := base64.StdEncoding.EncodeToString([]byte("AWS:testjwt"))
|
||||
w.Header().Set("Content-Type", "application/x-amz-json-1.1")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"authorizationData": []map[string]any{{
|
||||
"authorizationToken": authToken,
|
||||
"proxyEndpoint": "https://123456789012.dkr.ecr.us-east-1.spinifex.internal",
|
||||
}},
|
||||
})
|
||||
})
|
||||
|
||||
srv := httptest.NewTLSServer(mux)
|
||||
t.Cleanup(srv.Close)
|
||||
return srv
|
||||
}
|
||||
|
||||
func clientTrusting(srv *httptest.Server) *http.Client {
|
||||
pool := x509.NewCertPool()
|
||||
pool.AddCert(srv.Certificate())
|
||||
return &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}}
|
||||
}
|
||||
|
||||
func TestRun_ReturnsBasicAuth(t *testing.T) {
|
||||
srv := fakeBackend(t)
|
||||
client := clientTrusting(srv)
|
||||
|
||||
cfg := config{
|
||||
GatewayURL: srv.URL,
|
||||
Region: "us-east-1",
|
||||
IMDSBase: srv.URL + "/latest",
|
||||
}
|
||||
|
||||
const image = "123456789012.dkr.ecr.us-east-1.spinifex.internal/spinifex-demo:latest"
|
||||
reqBody, err := json.Marshal(CredentialProviderRequest{
|
||||
APIVersion: credProviderAPIVersion,
|
||||
Kind: "CredentialProviderRequest",
|
||||
Image: image,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal request: %v", err)
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := run(bytes.NewReader(reqBody), &out, cfg, client); err != nil {
|
||||
t.Fatalf("run: %v", err)
|
||||
}
|
||||
|
||||
var resp CredentialProviderResponse
|
||||
if err := json.Unmarshal(out.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("unmarshal response: %v\n%s", err, out.String())
|
||||
}
|
||||
|
||||
if resp.Kind != "CredentialProviderResponse" {
|
||||
t.Errorf("Kind = %q, want CredentialProviderResponse", resp.Kind)
|
||||
}
|
||||
if resp.CacheKeyType != "Registry" {
|
||||
t.Errorf("CacheKeyType = %q, want Registry", resp.CacheKeyType)
|
||||
}
|
||||
|
||||
host := "123456789012.dkr.ecr.us-east-1.spinifex.internal"
|
||||
auth, ok := resp.Auth[host]
|
||||
if !ok {
|
||||
t.Fatalf("auth map missing host %q; got %v", host, resp.Auth)
|
||||
}
|
||||
if auth.Username != "AWS" {
|
||||
t.Errorf("username = %q, want AWS", auth.Username)
|
||||
}
|
||||
if auth.Password != "testjwt" {
|
||||
t.Errorf("password = %q, want testjwt", auth.Password)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
goruntime "runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
|
||||
ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
|
||||
"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
|
||||
)
|
||||
|
||||
// version is the agent build version, reported in the register message.
|
||||
// Overridable via -ldflags "-X main.version=...".
|
||||
var version = "dev"
|
||||
|
||||
// Agent wires the ecs-agent's runtime seams: a gateway control-plane client, a
|
||||
// container runtime, and an ECR resolver. It registers the host as a container
|
||||
// instance at boot, heartbeats (re-registers) while alive, polls the gateway for
|
||||
// task assignments, and runs them through containerd, reporting state back over
|
||||
// the gateway. It never connects to NATS — the bus stays host-internal.
|
||||
type Agent struct {
|
||||
cfg config
|
||||
id identity
|
||||
cp controlPlane
|
||||
resolver ctrruntime.Resolver
|
||||
|
||||
// rt holds the container runtime, which is resolved by a loop rather than
|
||||
// once at boot. Empty until the containerd socket answers, and emptied again
|
||||
// if it stops answering.
|
||||
rt *runtimeGate
|
||||
|
||||
reg *registrar
|
||||
hb *heartbeater
|
||||
|
||||
// netns builds awsvpc task network namespaces (nil-safe; bridge/host skip it).
|
||||
netns *taskNetns
|
||||
|
||||
// cred serves task IAM role credentials at 169.254.170.2 (nil in unit tests).
|
||||
cred *credEndpoint
|
||||
|
||||
// gpu is the local pinning ledger over the host's discovered GPU UUIDs.
|
||||
// Always non-nil (newAgent seeds an empty one); New() reseeds it from
|
||||
// nvidia-smi discovery.
|
||||
gpu *gpuLedger
|
||||
|
||||
// dialRuntime connects to the container runtime. nil in unit tests, which
|
||||
// drive the gate directly.
|
||||
dialRuntime func() (ctrruntime.Runtime, error)
|
||||
|
||||
closers []func() error
|
||||
}
|
||||
|
||||
// newAgent assembles an Agent from already-built seams. Tests use this directly
|
||||
// with fakes; New builds the production seams and delegates here. rt may be nil
|
||||
// when containerd is unavailable; the assign path then reports the task STOPPED
|
||||
// rather than crashing, and the watch loop keeps dialling for it.
|
||||
func newAgent(cfg config, id identity, cp controlPlane, rt ctrruntime.Runtime, resolver ctrruntime.Resolver) *Agent {
|
||||
gate := newRuntimeGate(rt)
|
||||
a := &Agent{
|
||||
cfg: cfg,
|
||||
id: id,
|
||||
cp: cp,
|
||||
resolver: resolver,
|
||||
rt: gate,
|
||||
netns: newTaskNetns(execNetRunner{}),
|
||||
gpu: newGPULedger(id.Capacity.GPUIDs),
|
||||
}
|
||||
// Registration reports the host's capacity only while a runtime is there to
|
||||
// use it, so an agent that cannot run a container advertises none and takes
|
||||
// no placement.
|
||||
a.reg = newRegistrar(cp, id, gate.available)
|
||||
a.hb = newHeartbeater(cp, id, cfg.Heartbeat, gate.available)
|
||||
return a
|
||||
}
|
||||
|
||||
// New builds an Agent from config: it resolves the host identity from IMDS,
|
||||
// builds the gateway control-plane client, the ECR resolver and (best-effort)
|
||||
// the containerd runtime. A containerd connect failure is logged, not fatal —
|
||||
// registration and heartbeat still run so the instance is visible while the
|
||||
// runtime recovers. The ECR gateway client is built lazily on first image pull
|
||||
// (not here), so a missing or malformed gateway CA does not stop registration.
|
||||
func New(cfg config) (*Agent, error) {
|
||||
imdsClient := &http.Client{Timeout: 5 * time.Second}
|
||||
|
||||
meta, err := fetchInstanceMetadata(imdsClient, cfg.IMDSBase)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("instance metadata: %w", err)
|
||||
}
|
||||
host, _ := os.Hostname()
|
||||
id := identity{
|
||||
AccountID: meta.AccountID,
|
||||
ClusterName: cfg.ClusterName,
|
||||
InstanceID: meta.InstanceID,
|
||||
AZ: meta.AZ,
|
||||
InstanceType: meta.InstanceType,
|
||||
Hostname: host,
|
||||
Capacity: detectCapacity(discoverNvidiaGPUs(execCommandRunner)),
|
||||
AgentVersion: version,
|
||||
}
|
||||
|
||||
creds := credentials.NewIMDSProvider(imdsClient, cfg.IMDSBase)
|
||||
resolver := newLazyECRResolver(creds, cfg.Region, cfg.GatewayURL, cfg.GatewayCA)
|
||||
|
||||
// A dial that fails here is not fatal and not final: the node starts the
|
||||
// container daemon and the agent back to back, so the socket is often not
|
||||
// accepting yet. Run's watch loop keeps dialling.
|
||||
rt, perr := ctrruntime.New(cfg.ContainerdSocket)
|
||||
if perr != nil {
|
||||
slog.Warn("ecs-agent: containerd unavailable at boot, will keep retrying", "err", perr)
|
||||
}
|
||||
|
||||
cp, err := newGatewayControlPlane(cfg, creds)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build gateway control-plane: %w", err)
|
||||
}
|
||||
|
||||
a := newAgent(cfg, id, cp, rt, resolver)
|
||||
a.cred = newCredEndpoint(creds, cfg.Region, cfg.GatewayURL, cfg.GatewayCA,
|
||||
cfg.CredEndpointIP, cfg.CredEndpointPort, execNetRunner{})
|
||||
a.dialRuntime = func() (ctrruntime.Runtime, error) { return ctrruntime.New(cfg.ContainerdSocket) }
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// Run registers the instance, starts the heartbeat and assignment-poll loops,
|
||||
// and blocks until ctx is cancelled, then tears down.
|
||||
func (a *Agent) Run(ctx context.Context) error {
|
||||
if err := a.reg.Register(); err != nil {
|
||||
slog.Error("ecs-agent: initial registration failed", "err", err)
|
||||
} else {
|
||||
slog.Info("ecs-agent: registered", "cluster", a.id.ClusterName, "instance", a.id.InstanceID)
|
||||
}
|
||||
|
||||
if a.cred != nil {
|
||||
if err := a.cred.Start(); err != nil {
|
||||
slog.Error("ecs-agent: credential endpoint start failed", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
go a.hb.Run(ctx)
|
||||
|
||||
// Keeps the runtime current for the life of the process: dials while there
|
||||
// is none, and drops one that stops answering.
|
||||
if a.dialRuntime != nil {
|
||||
go a.rt.watch(ctx, a.dialRuntime, a.cfg.RuntimeRetry)
|
||||
}
|
||||
|
||||
// Re-adopt containers still running from before this restart, then poll with
|
||||
// their tasks pre-seeded so re-delivered assignments are acked, not re-run.
|
||||
go a.pollAssignments(ctx, a.reconcile(ctx))
|
||||
|
||||
<-ctx.Done()
|
||||
return a.Stop()
|
||||
}
|
||||
|
||||
// Stop closes the runtime. Safe to call once.
|
||||
func (a *Agent) Stop() error {
|
||||
var firstErr error
|
||||
if a.cred != nil {
|
||||
_ = a.cred.Stop()
|
||||
}
|
||||
for _, c := range a.closers {
|
||||
if err := c(); err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
}
|
||||
a.rt.clear()
|
||||
return firstErr
|
||||
}
|
||||
|
||||
// detectCapacity reports the host's total CPU units (1 vCPU = 1024), memory,
|
||||
// and GPU count/UUIDs from the caller's nvidia-smi discovery (nil on a
|
||||
// non-GPU host).
|
||||
func detectCapacity(gpuUUIDs []string) bus.InstanceCapacity {
|
||||
return bus.InstanceCapacity{
|
||||
CPU: goruntime.NumCPU() * 1024,
|
||||
MemoryMiB: memTotalMiB(),
|
||||
GPU: len(gpuUUIDs),
|
||||
GPUIDs: gpuUUIDs,
|
||||
}
|
||||
}
|
||||
|
||||
// memTotalMiB reads MemTotal from /proc/meminfo, returning 0 if unavailable.
|
||||
func memTotalMiB() int {
|
||||
f, err := os.Open("/proc/meminfo")
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
defer f.Close()
|
||||
scanner := bufio.NewScanner(f)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if !strings.HasPrefix(line, "MemTotal:") {
|
||||
continue
|
||||
}
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 2 {
|
||||
return 0
|
||||
}
|
||||
kb, err := strconv.Atoi(fields[1])
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return kb / 1024
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
|
||||
)
|
||||
|
||||
func TestAgent_RunRegistersThenStopsOnContext(t *testing.T) {
|
||||
cp := &fakeCP{}
|
||||
cfg := config{Heartbeat: 5 * time.Millisecond, PollInterval: 5 * time.Millisecond}
|
||||
puller := &ctrruntime.FakePuller{}
|
||||
a := newAgent(cfg, testIdentity(), cp, puller, nil)
|
||||
a.closers = append(a.closers, puller.Close)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- a.Run(ctx) }()
|
||||
|
||||
// Give it time to register + re-register (heartbeat) a few times.
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
cancel()
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatalf("Run returned error: %v", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Run did not return after cancel")
|
||||
}
|
||||
|
||||
// One boot register + at least one heartbeat re-register.
|
||||
if cp.registerCount() < 2 {
|
||||
t.Errorf("registers = %d, want >= 2 (boot + heartbeat)", cp.registerCount())
|
||||
}
|
||||
if !puller.Closed {
|
||||
t.Error("Stop did not close the puller")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectCapacity_Positive(t *testing.T) {
|
||||
c := detectCapacity(nil)
|
||||
if c.CPU <= 0 {
|
||||
t.Errorf("CPU = %d, want > 0", c.CPU)
|
||||
}
|
||||
// MemoryMiB may be 0 on platforms without /proc/meminfo; just assert non-negative.
|
||||
if c.MemoryMiB < 0 {
|
||||
t.Errorf("MemoryMiB = %d, want >= 0", c.MemoryMiB)
|
||||
}
|
||||
if c.GPU != 0 || c.GPUIDs != nil {
|
||||
t.Errorf("no GPU UUIDs given: want GPU=0/GPUIDs=nil, got %d/%v", c.GPU, c.GPUIDs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectCapacity_WithGPUUUIDs(t *testing.T) {
|
||||
c := detectCapacity([]string{"GPU-aaa", "GPU-bbb"})
|
||||
if c.GPU != 2 {
|
||||
t.Errorf("GPU = %d, want 2", c.GPU)
|
||||
}
|
||||
if len(c.GPUIDs) != 2 {
|
||||
t.Errorf("GPUIDs = %v, want 2 entries", c.GPUIDs)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/guestenv"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultEnvFile = "/etc/spinifex-ecs/agent.env"
|
||||
defaultGatewayCA = "/etc/spinifex-ecs/gateway-ca.pem"
|
||||
defaultIMDSBase = "http://169.254.169.254/latest"
|
||||
defaultContainerdSocket = "/run/containerd/containerd.sock"
|
||||
defaultHeartbeat = 30 * time.Second
|
||||
defaultPollInterval = 5 * time.Second
|
||||
)
|
||||
|
||||
// config holds the static settings the agent reads at boot. Cluster identity
|
||||
// (account ID, instance ID) is discovered at runtime from IMDS, not configured;
|
||||
// only the cluster *name* the instance was launched into is static. AccessKey /
|
||||
// SecretKey are legacy seeded creds, optional now that the agent SigV4-signs
|
||||
// gateway calls with instance-role credentials from IMDS; the agent never holds
|
||||
// a NATS token.
|
||||
type config struct {
|
||||
GatewayURL string
|
||||
GatewayCA string
|
||||
Region string
|
||||
ClusterName string
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
IMDSBase string
|
||||
ContainerdSocket string
|
||||
Heartbeat time.Duration
|
||||
PollInterval time.Duration
|
||||
RuntimeRetry time.Duration
|
||||
CredEndpointIP string
|
||||
CredEndpointPort int
|
||||
}
|
||||
|
||||
// loadConfig reads the cloud-init env file then lets real env vars override.
|
||||
func loadConfig(envFile string) config {
|
||||
get := guestenv.Load(envFile).Get
|
||||
|
||||
cfg := config{
|
||||
GatewayURL: get("ECS_GATEWAY_URL"),
|
||||
GatewayCA: get("ECS_GATEWAY_CA"),
|
||||
Region: get("ECS_REGION"),
|
||||
ClusterName: get("ECS_CLUSTER"),
|
||||
AccessKey: get("ECS_ACCESS_KEY"),
|
||||
SecretKey: get("ECS_SECRET_KEY"),
|
||||
IMDSBase: get("ECS_IMDS_BASE"),
|
||||
ContainerdSocket: get("ECS_CONTAINERD_SOCKET"),
|
||||
}
|
||||
if cfg.GatewayCA == "" {
|
||||
cfg.GatewayCA = defaultGatewayCA
|
||||
}
|
||||
if cfg.IMDSBase == "" {
|
||||
cfg.IMDSBase = defaultIMDSBase
|
||||
}
|
||||
if cfg.ContainerdSocket == "" {
|
||||
cfg.ContainerdSocket = defaultContainerdSocket
|
||||
}
|
||||
if cfg.ClusterName == "" {
|
||||
cfg.ClusterName = "default"
|
||||
}
|
||||
cfg.Heartbeat = defaultHeartbeat
|
||||
if v := get("ECS_HEARTBEAT_INTERVAL"); v != "" {
|
||||
if d, err := time.ParseDuration(v); err == nil && d > 0 {
|
||||
cfg.Heartbeat = d
|
||||
}
|
||||
}
|
||||
cfg.PollInterval = defaultPollInterval
|
||||
if v := get("ECS_POLL_INTERVAL"); v != "" {
|
||||
if d, err := time.ParseDuration(v); err == nil && d > 0 {
|
||||
cfg.PollInterval = d
|
||||
}
|
||||
}
|
||||
cfg.RuntimeRetry = defaultRuntimeRetry
|
||||
if v := get("ECS_RUNTIME_RETRY"); v != "" {
|
||||
if d, err := time.ParseDuration(v); err == nil && d > 0 {
|
||||
cfg.RuntimeRetry = d
|
||||
}
|
||||
}
|
||||
cfg.CredEndpointIP = get("ECS_CRED_ENDPOINT_IP")
|
||||
if cfg.CredEndpointIP == "" {
|
||||
cfg.CredEndpointIP = defaultCredEndpointIP
|
||||
}
|
||||
cfg.CredEndpointPort = defaultCredEndpointPort
|
||||
if v := get("ECS_CRED_ENDPOINT_PORT"); v != "" {
|
||||
if p, err := strconv.Atoi(v); err == nil && p > 0 {
|
||||
cfg.CredEndpointPort = p
|
||||
}
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestLoadConfig_Defaults(t *testing.T) {
|
||||
// Ensure env vars don't leak in from the host.
|
||||
for _, k := range []string{"ECS_GATEWAY_URL", "ECS_GATEWAY_CA", "ECS_REGION",
|
||||
"ECS_CLUSTER", "ECS_ACCESS_KEY", "ECS_SECRET_KEY", "ECS_IMDS_BASE",
|
||||
"ECS_CONTAINERD_SOCKET", "ECS_HEARTBEAT_INTERVAL", "ECS_POLL_INTERVAL"} {
|
||||
t.Setenv(k, "")
|
||||
}
|
||||
cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
|
||||
if cfg.GatewayCA != defaultGatewayCA {
|
||||
t.Errorf("GatewayCA = %q, want default", cfg.GatewayCA)
|
||||
}
|
||||
if cfg.IMDSBase != defaultIMDSBase {
|
||||
t.Errorf("IMDSBase = %q, want default", cfg.IMDSBase)
|
||||
}
|
||||
if cfg.ContainerdSocket != defaultContainerdSocket {
|
||||
t.Errorf("ContainerdSocket = %q, want default", cfg.ContainerdSocket)
|
||||
}
|
||||
if cfg.ClusterName != "default" {
|
||||
t.Errorf("ClusterName = %q, want default", cfg.ClusterName)
|
||||
}
|
||||
if cfg.Heartbeat != defaultHeartbeat {
|
||||
t.Errorf("Heartbeat = %v, want %v", cfg.Heartbeat, defaultHeartbeat)
|
||||
}
|
||||
if cfg.PollInterval != defaultPollInterval {
|
||||
t.Errorf("PollInterval = %v, want %v", cfg.PollInterval, defaultPollInterval)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfig_FileThenEnvOverride(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
envFile := filepath.Join(dir, "agent.env")
|
||||
body := "# comment\nECS_GATEWAY_URL=https://gw.file\nECS_REGION=us-west-2\nECS_CLUSTER=prod\n"
|
||||
if err := os.WriteFile(envFile, []byte(body), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("ECS_GATEWAY_URL", "https://gw.env")
|
||||
t.Setenv("ECS_ACCESS_KEY", "AKIAENV")
|
||||
|
||||
cfg := loadConfig(envFile)
|
||||
if cfg.GatewayURL != "https://gw.env" {
|
||||
t.Errorf("GatewayURL = %q, want env override", cfg.GatewayURL)
|
||||
}
|
||||
if cfg.Region != "us-west-2" {
|
||||
t.Errorf("Region = %q, want from file", cfg.Region)
|
||||
}
|
||||
if cfg.ClusterName != "prod" {
|
||||
t.Errorf("ClusterName = %q, want from file", cfg.ClusterName)
|
||||
}
|
||||
if cfg.AccessKey != "AKIAENV" {
|
||||
t.Errorf("AccessKey = %q, want env", cfg.AccessKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfig_HeartbeatOverride(t *testing.T) {
|
||||
t.Setenv("ECS_HEARTBEAT_INTERVAL", "5s")
|
||||
if cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env")); cfg.Heartbeat != 5*time.Second {
|
||||
t.Errorf("Heartbeat = %v, want 5s", cfg.Heartbeat)
|
||||
}
|
||||
// Garbage value falls back to the default, not zero.
|
||||
t.Setenv("ECS_HEARTBEAT_INTERVAL", "not-a-duration")
|
||||
if cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env")); cfg.Heartbeat != defaultHeartbeat {
|
||||
t.Errorf("Heartbeat = %v, want default on bad value", cfg.Heartbeat)
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package agent
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -9,10 +9,10 @@ import (
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ecs"
|
||||
|
||||
ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
|
||||
"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
|
||||
"github.com/mulgadc/spinifex/internal/ecsgw"
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/ecs/credentials"
|
||||
handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
|
||||
"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
|
||||
)
|
||||
|
||||
// controlPlane is the agent's channel to the AWS gateway. It replaces the direct
|
||||
@@ -21,14 +21,14 @@ import (
|
||||
type controlPlane interface {
|
||||
// Register (re-)registers the container instance. Called at boot and on every
|
||||
// heartbeat tick — the gateway's RegisterContainerInstance is idempotent and
|
||||
// refreshes LastSeen, folding liveness into registration (no ecsv1.Heartbeat).
|
||||
// refreshes LastSeen, folding liveness into registration (no bus.Heartbeat).
|
||||
Register(id identity) error
|
||||
// SubmitTaskState reports a task's RUNNING/STOPPED transition.
|
||||
SubmitTaskState(st ecsv1.TaskState) error
|
||||
SubmitTaskState(st bus.TaskState) error
|
||||
// PollAssignments drains this instance's assignment + stop inboxes, acking the
|
||||
// assign taskIDs and stop taskIDs accepted on the previous poll. Returns the
|
||||
// still-pending assignments and stop directives.
|
||||
PollAssignments(cluster, instance string, ackAssigns, ackStops []string) ([]ecsv1.Assign, []ecsv1.StopDirective, error)
|
||||
PollAssignments(cluster, instance string, ackAssigns, ackStops []string) ([]bus.Assign, []bus.StopDirective, error)
|
||||
// ReportTaskGPU reports the local ledger's pinned device UUIDs for a task's
|
||||
// containers, so the control plane can populate DescribeTasks gpuIds.
|
||||
ReportTaskGPU(cluster, task string, containers []handlers_ecs.ContainerGPUReport) error
|
||||
@@ -47,7 +47,7 @@ const pollTimeout = 30 * time.Second
|
||||
|
||||
// newGatewayControlPlane builds the SigV4 client signing with the instance-role
|
||||
// credentials from IMDS (fetched per call so rotation is transparent) + pinned CA.
|
||||
func newGatewayControlPlane(cfg Config, creds credentials.CredentialsProvider) (*gatewayControlPlane, error) {
|
||||
func newGatewayControlPlane(cfg config, creds credentials.CredentialsProvider) (*gatewayControlPlane, error) {
|
||||
client, err := ecsgw.New(cfg.GatewayURL, cfg.GatewayCA, credentialsFunc(creds), cfg.Region, pollTimeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -96,7 +96,7 @@ func (g *gatewayControlPlane) Register(id identity) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *gatewayControlPlane) SubmitTaskState(st ecsv1.TaskState) error {
|
||||
func (g *gatewayControlPlane) SubmitTaskState(st bus.TaskState) error {
|
||||
in := &ecs.SubmitTaskStateChangeInput{
|
||||
Cluster: aws.String(st.ClusterName),
|
||||
Task: aws.String(st.TaskID),
|
||||
@@ -124,7 +124,7 @@ func (g *gatewayControlPlane) SubmitTaskState(st ecsv1.TaskState) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *gatewayControlPlane) PollAssignments(cluster, instance string, ackAssigns, ackStops []string) ([]ecsv1.Assign, []ecsv1.StopDirective, error) {
|
||||
func (g *gatewayControlPlane) PollAssignments(cluster, instance string, ackAssigns, ackStops []string) ([]bus.Assign, []bus.StopDirective, error) {
|
||||
in := &handlers_ecs.PollAssignmentsInput{
|
||||
Cluster: cluster,
|
||||
ContainerInstance: instance,
|
||||
@@ -1,4 +1,4 @@
|
||||
package agent
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/ecs"
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/ecs/credentials"
|
||||
"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
|
||||
handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
|
||||
)
|
||||
|
||||
@@ -23,7 +23,7 @@ func (fixedCredsProvider) Retrieve(context.Context) (credentials.Credentials, er
|
||||
|
||||
func newTestGatewayControlPlane(t *testing.T, srv *httptest.Server) *gatewayControlPlane {
|
||||
t.Helper()
|
||||
cp, err := newGatewayControlPlane(Config{GatewayURL: srv.URL, Region: "us-east-1"}, fixedCredsProvider{})
|
||||
cp, err := newGatewayControlPlane(config{GatewayURL: srv.URL, Region: "us-east-1"}, fixedCredsProvider{})
|
||||
if err != nil {
|
||||
t.Fatalf("newGatewayControlPlane: %v", err)
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package agent
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -13,9 +13,9 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
|
||||
"github.com/mulgadc/spinifex/internal/ecrauth"
|
||||
"github.com/mulgadc/spinifex/internal/stsauth"
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/ecs/credentials"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -1,4 +1,4 @@
|
||||
package agent
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -1,20 +1,20 @@
|
||||
package agent
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
|
||||
"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
|
||||
)
|
||||
|
||||
func TestTaskCredID(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
as ecsv1.Assign
|
||||
as bus.Assign
|
||||
want string
|
||||
}{
|
||||
{"no role", ecsv1.Assign{TaskID: "t1"}, ""},
|
||||
{"role, no credID -> taskID", ecsv1.Assign{TaskID: "t1", TaskRoleARN: "arn:role"}, "t1"},
|
||||
{"role + credID", ecsv1.Assign{TaskID: "t1", CredID: "c9", TaskRoleARN: "arn:role"}, "c9"},
|
||||
{"no role", bus.Assign{TaskID: "t1"}, ""},
|
||||
{"role, no credID -> taskID", bus.Assign{TaskID: "t1", TaskRoleARN: "arn:role"}, "t1"},
|
||||
{"role + credID", bus.Assign{TaskID: "t1", CredID: "c9", TaskRoleARN: "arn:role"}, "c9"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := taskCredID(&tc.as); got != tc.want {
|
||||
@@ -1,4 +1,4 @@
|
||||
package agent
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -7,9 +7,9 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
|
||||
"github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
|
||||
"github.com/mulgadc/spinifex/internal/ecrauth"
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/ecs/credentials"
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/ecs/runtime"
|
||||
)
|
||||
|
||||
// ecrResolver implements runtime.Resolver by minting an ECR authorization token
|
||||
@@ -1,4 +1,4 @@
|
||||
package agent
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/ecs/credentials"
|
||||
"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
|
||||
)
|
||||
|
||||
// stubCreds is a credentials.CredentialsProvider returning fixed creds or error.
|
||||
@@ -1,4 +1,4 @@
|
||||
package agent
|
||||
package main
|
||||
|
||||
import (
|
||||
"os/exec"
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
package agent
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
@@ -1,4 +1,4 @@
|
||||
package agent
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
@@ -1,4 +1,4 @@
|
||||
package agent
|
||||
package main
|
||||
|
||||
import (
|
||||
"sync"
|
||||
@@ -0,0 +1,51 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
)
|
||||
|
||||
// heartbeater keeps the container instance alive by periodically re-registering
|
||||
// it through the gateway. Real ECS folds liveness into ACS; v1 folds it into an
|
||||
// idempotent RegisterContainerInstance, which refreshes the instance's LastSeen
|
||||
// so the scheduler can tell live instances from dead ones (no bus.Heartbeat).
|
||||
type heartbeater struct {
|
||||
cp controlPlane
|
||||
id identity
|
||||
interval time.Duration
|
||||
// runtimeReady reports whether a container runtime is resolved, so a beat
|
||||
// advertises capacity only while there is something to run it with. nil
|
||||
// means always ready.
|
||||
runtimeReady func() bool
|
||||
}
|
||||
|
||||
func newHeartbeater(cp controlPlane, id identity, interval time.Duration, runtimeReady func() bool) *heartbeater {
|
||||
if interval <= 0 {
|
||||
interval = defaultHeartbeat
|
||||
}
|
||||
return &heartbeater{cp: cp, id: id, interval: interval, runtimeReady: runtimeReady}
|
||||
}
|
||||
|
||||
// beat re-registers the instance once, refreshing its LastSeen.
|
||||
func (h *heartbeater) beat() error {
|
||||
ready := h.runtimeReady == nil || h.runtimeReady()
|
||||
return h.cp.Register(advertisedIdentity(h.id, ready))
|
||||
}
|
||||
|
||||
// Run beats every interval until ctx is cancelled. A failed beat is logged and
|
||||
// retried on the next tick rather than killing the loop.
|
||||
func (h *heartbeater) Run(ctx context.Context) {
|
||||
ticker := time.NewTicker(h.interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := h.beat(); err != nil {
|
||||
slog.Warn("ecs-agent: heartbeat (re-register) failed", "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+12
-12
@@ -1,6 +1,13 @@
|
||||
// Command ecs-agent runs inside a Spinifex ECS container instance; the
|
||||
// implementation is in agents/ecs/agent. This binary owns process lifecycle:
|
||||
// signal handling, startup and run failure logging, and the exit code.
|
||||
// ecs-agent runs inside a Spinifex ECS container instance (a guest VM booted from
|
||||
// the ECS-AMI, with containerd baked in). It registers the host with the ECS
|
||||
// scheduler through the AWS gateway over TLS+SigV4 (never NATS), heartbeats by
|
||||
// re-registering, polls the gateway for task assignments, runs them through
|
||||
// containerd, and reports state back over the gateway.
|
||||
//
|
||||
// Static config (gateway URL, CA, region, cluster, seeded IAM creds, containerd
|
||||
// socket) is read from the cloud-init env file /etc/spinifex-ecs/agent.env
|
||||
// (KEY=value); real env vars override it. Host identity (account, instance, AZ)
|
||||
// comes from IMDS at boot.
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -11,19 +18,12 @@ import (
|
||||
"syscall"
|
||||
|
||||
_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
|
||||
|
||||
ecsagent "github.com/mulgadc/spinifex/spinifex/agents/ecs/agent"
|
||||
)
|
||||
|
||||
// version is the agent build version, reported in the register message.
|
||||
// Overridable via -ldflags "-X main.version=...".
|
||||
var version = "dev"
|
||||
|
||||
func main() {
|
||||
cfg := ecsagent.LoadConfig(ecsagent.DefaultEnvFile)
|
||||
cfg.AgentVersion = version
|
||||
cfg := loadConfig(defaultEnvFile)
|
||||
|
||||
agent, err := ecsagent.New(cfg)
|
||||
agent, err := New(cfg)
|
||||
if err != nil {
|
||||
slog.Error("ecs-agent: startup failed", "err", err)
|
||||
os.Exit(1)
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
|
||||
"github.com/mulgadc/spinifex/internal/imdscreds"
|
||||
)
|
||||
|
||||
// instanceMetadata is the IMDS-derived identity of the host the agent runs on.
|
||||
type instanceMetadata struct {
|
||||
AccountID string
|
||||
InstanceID string
|
||||
AZ string
|
||||
InstanceType string
|
||||
}
|
||||
|
||||
// fetchInstanceMetadata reads the instance-id, AZ and account-id from IMDSv2
|
||||
// via the SDK IMDS client. base is the IMDS root, e.g.
|
||||
// "http://169.254.169.254/latest".
|
||||
func fetchInstanceMetadata(client *http.Client, base string) (instanceMetadata, error) {
|
||||
imdsClient := imdscreds.NewClient(client, base)
|
||||
ctx := context.Background()
|
||||
|
||||
instanceID, err := getMetadata(ctx, imdsClient, "instance-id")
|
||||
if err != nil {
|
||||
return instanceMetadata{}, fmt.Errorf("instance-id: %w", err)
|
||||
}
|
||||
az, err := getMetadata(ctx, imdsClient, "placement/availability-zone")
|
||||
if err != nil {
|
||||
return instanceMetadata{}, fmt.Errorf("availability-zone: %w", err)
|
||||
}
|
||||
|
||||
doc, err := imdsClient.GetInstanceIdentityDocument(ctx, &imds.GetInstanceIdentityDocumentInput{})
|
||||
if err != nil {
|
||||
return instanceMetadata{}, fmt.Errorf("identity document: %w", err)
|
||||
}
|
||||
|
||||
// The instance type comes off the identity document already in hand rather
|
||||
// than a further meta-data read; the scheduler sizes the host's awsvpc ENI
|
||||
// capacity from it.
|
||||
return instanceMetadata{
|
||||
AccountID: doc.AccountID,
|
||||
InstanceID: strings.TrimSpace(instanceID),
|
||||
AZ: strings.TrimSpace(az),
|
||||
InstanceType: strings.TrimSpace(doc.InstanceType),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// getMetadata reads a single meta-data path and returns its body as a string.
|
||||
func getMetadata(ctx context.Context, imdsClient *imds.Client, path string) (string, error) {
|
||||
out, err := imdsClient.GetMetadata(ctx, &imds.GetMetadataInput{Path: path})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer out.Content.Close()
|
||||
body, err := io.ReadAll(out.Content)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(body), nil
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func metadataStub(t *testing.T, enforceV2 bool) *httptest.Server {
|
||||
t.Helper()
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPut {
|
||||
w.WriteHeader(http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
// Real IMDS echoes the requested TTL back as a response header; the SDK
|
||||
// client parses it from there, not the body.
|
||||
w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
|
||||
_, _ = w.Write([]byte("v2-token"))
|
||||
})
|
||||
guard := func(w http.ResponseWriter, r *http.Request) bool {
|
||||
if enforceV2 && r.Header.Get("X-Aws-Ec2-Metadata-Token") != "v2-token" {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
mux.HandleFunc("/latest/meta-data/instance-id", func(w http.ResponseWriter, r *http.Request) {
|
||||
if guard(w, r) {
|
||||
_, _ = w.Write([]byte("i-0abc123"))
|
||||
}
|
||||
})
|
||||
mux.HandleFunc("/latest/meta-data/placement/availability-zone", func(w http.ResponseWriter, r *http.Request) {
|
||||
if guard(w, r) {
|
||||
_, _ = w.Write([]byte("us-east-1a"))
|
||||
}
|
||||
})
|
||||
mux.HandleFunc("/latest/dynamic/instance-identity/document", func(w http.ResponseWriter, r *http.Request) {
|
||||
if guard(w, r) {
|
||||
_, _ = w.Write([]byte(`{"accountId":"123456789012","region":"us-east-1"}`))
|
||||
}
|
||||
})
|
||||
srv := httptest.NewServer(mux)
|
||||
t.Cleanup(srv.Close)
|
||||
return srv
|
||||
}
|
||||
|
||||
func TestFetchInstanceMetadata_OK(t *testing.T) {
|
||||
srv := metadataStub(t, true)
|
||||
got, err := fetchInstanceMetadata(srv.Client(), srv.URL+"/latest")
|
||||
if err != nil {
|
||||
t.Fatalf("fetch: %v", err)
|
||||
}
|
||||
if got.InstanceID != "i-0abc123" || got.AZ != "us-east-1a" || got.AccountID != "123456789012" {
|
||||
t.Errorf("metadata mismatch: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchInstanceMetadata_TokenRequired(t *testing.T) {
|
||||
// Server returns 405 for non-PUT token; a GET-only client would 401 downstream.
|
||||
srv := metadataStub(t, true)
|
||||
if _, err := fetchInstanceMetadata(srv.Client(), srv.URL+"/latest"); err != nil {
|
||||
t.Fatalf("v2 path should succeed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchInstanceMetadata_BadDocument(t *testing.T) {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
|
||||
_, _ = w.Write([]byte("t"))
|
||||
})
|
||||
mux.HandleFunc("/latest/meta-data/instance-id", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("i-1")) })
|
||||
mux.HandleFunc("/latest/meta-data/placement/availability-zone", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("z")) })
|
||||
mux.HandleFunc("/latest/dynamic/instance-identity/document", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("not json")) })
|
||||
srv := httptest.NewServer(mux)
|
||||
t.Cleanup(srv.Close)
|
||||
if _, err := fetchInstanceMetadata(srv.Client(), srv.URL+"/latest"); err == nil {
|
||||
t.Fatal("expected parse error on bad identity document")
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package agent
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
@@ -1,4 +1,4 @@
|
||||
package agent
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
@@ -0,0 +1,115 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
|
||||
ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
|
||||
"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
|
||||
)
|
||||
|
||||
// reconcile re-syncs the gateway with what this instance is actually running
|
||||
// after an agent restart. It lists the runtime's containers and, for each task
|
||||
// in this cluster, either re-adopts it (still running: re-register IAM
|
||||
// credentials, re-attach the exit-wait, refresh RUNNING) or reports it STOPPED
|
||||
// and clears its leftovers. The returned set seeds pollAssignments so the
|
||||
// gateway's re-delivered assignments for these tasks are acked but not re-run.
|
||||
//
|
||||
// Reporting the dead ones is the load-bearing half. The exit-wait is cancelled
|
||||
// when the agent shuts down, so a container that dies while the agent is down
|
||||
// has nobody to report its exit; the gateway holds the task RUNNING forever,
|
||||
// its service never replaces it, and the target never recovers.
|
||||
//
|
||||
// It is best-effort: a nil runner or a List error returns an empty set and the
|
||||
// agent falls back to its pre-reconcile behaviour (poll + run).
|
||||
func (a *Agent) reconcile(ctx context.Context) map[string]bool {
|
||||
adopted := map[string]bool{}
|
||||
runner := a.rt.get()
|
||||
if runner == nil {
|
||||
return adopted
|
||||
}
|
||||
containers, err := runner.List(ctx)
|
||||
if err != nil {
|
||||
slog.Warn("ecs-agent: reconcile list failed", "err", err)
|
||||
return adopted
|
||||
}
|
||||
|
||||
// Group this cluster's containers back into their tasks, running or not. The
|
||||
// Assign is reconstructed from labels — enough to re-register creds and tear
|
||||
// down netns.
|
||||
type task struct {
|
||||
as *bus.Assign
|
||||
running []ctrruntime.Container
|
||||
dead []ctrruntime.Container
|
||||
}
|
||||
tasks := map[string]*task{}
|
||||
for _, c := range containers {
|
||||
if c.Labels[labelClusterName] != a.id.ClusterName {
|
||||
continue
|
||||
}
|
||||
taskID := c.Labels[labelTaskID]
|
||||
if taskID == "" {
|
||||
continue
|
||||
}
|
||||
t := tasks[taskID]
|
||||
if t == nil {
|
||||
t = &task{as: &bus.Assign{
|
||||
TaskID: taskID,
|
||||
ClusterName: a.id.ClusterName,
|
||||
CredID: c.Labels[labelCredID],
|
||||
TaskRoleARN: c.Labels[labelTaskRoleARN],
|
||||
ENIMacAddress: c.Labels[labelENIMac],
|
||||
}}
|
||||
tasks[taskID] = t
|
||||
}
|
||||
if c.Running {
|
||||
t.running = append(t.running, c)
|
||||
} else {
|
||||
t.dead = append(t.dead, c)
|
||||
}
|
||||
}
|
||||
|
||||
for taskID, t := range tasks {
|
||||
// A task with nothing running is over, whatever the gateway still believes.
|
||||
// Mixed running/dead is left to the running path: the exit-wait re-attached
|
||||
// below reports the individual container when it is reaped.
|
||||
if len(t.running) == 0 {
|
||||
a.reportStoppedTask(ctx, t.as, t.dead)
|
||||
adopted[taskID] = true
|
||||
continue
|
||||
}
|
||||
|
||||
a.registerTaskCreds(t.as)
|
||||
statuses := make([]bus.ContainerStatus, 0, len(t.running))
|
||||
for _, c := range t.running {
|
||||
name := c.Labels[labelContainerName]
|
||||
go a.waitContainer(ctx, t.as, name, c.ID)
|
||||
statuses = append(statuses, bus.ContainerStatus{Name: name, Status: bus.TaskStatusRunning, ContainerID: c.ID})
|
||||
}
|
||||
a.reportTaskState(t.as, bus.TaskStatusRunning, "", statuses)
|
||||
adopted[taskID] = true
|
||||
slog.Info("ecs-agent: re-adopted running task", "task", taskID, "containers", len(t.running))
|
||||
}
|
||||
return adopted
|
||||
}
|
||||
|
||||
// reportStoppedTask clears a task's leftover containers and tells the gateway it
|
||||
// is STOPPED, so the scheduler stops counting a task this instance is not
|
||||
// running and its service can place a replacement.
|
||||
func (a *Agent) reportStoppedTask(ctx context.Context, as *bus.Assign, dead []ctrruntime.Container) {
|
||||
runner := a.rt.get()
|
||||
statuses := make([]bus.ContainerStatus, 0, len(dead))
|
||||
for _, c := range dead {
|
||||
if runner == nil {
|
||||
break
|
||||
}
|
||||
if err := runner.Remove(ctx, c.ID); err != nil {
|
||||
slog.Warn("ecs-agent: reconcile remove failed", "task", as.TaskID, "container", c.ID, "err", err)
|
||||
}
|
||||
statuses = append(statuses, bus.ContainerStatus{
|
||||
Name: c.Labels[labelContainerName], Status: bus.TaskStatusStopped, ContainerID: c.ID,
|
||||
})
|
||||
}
|
||||
a.reportTaskState(as, bus.TaskStatusStopped, "Container exited while the agent was not running", statuses)
|
||||
slog.Info("ecs-agent: reported stopped task", "task", as.TaskID, "containers", len(dead))
|
||||
}
|
||||
@@ -0,0 +1,250 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
|
||||
"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
|
||||
)
|
||||
|
||||
const testTaskRole = "arn:aws:iam::111122223333:role/task"
|
||||
|
||||
// adoptedContainer builds a runtime.Container as the reconciler would discover it,
|
||||
// stamping only the mulga.ecs.* labels that are non-empty.
|
||||
func adoptedContainer(taskID, name, cluster, credID, role, mac string, running bool) ctrruntime.Container {
|
||||
labels := map[string]string{
|
||||
labelTaskID: taskID,
|
||||
labelContainerName: name,
|
||||
labelClusterName: cluster,
|
||||
}
|
||||
if credID != "" {
|
||||
labels[labelCredID] = credID
|
||||
}
|
||||
if role != "" {
|
||||
labels[labelTaskRoleARN] = role
|
||||
}
|
||||
if mac != "" {
|
||||
labels[labelENIMac] = mac
|
||||
}
|
||||
return ctrruntime.Container{ID: containerID(taskID, name), Labels: labels, Running: running}
|
||||
}
|
||||
|
||||
func waitedFor(t *testing.T, rt *ctrruntime.FakePuller, id string) {
|
||||
t.Helper()
|
||||
deadline := time.After(time.Second)
|
||||
for !slices.Contains(rt.Waits(), id) {
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatalf("exit-wait never re-attached for %s; waited=%v", id, rt.Waits())
|
||||
case <-time.After(2 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A running, correctly-labeled container is re-adopted: its credentials are
|
||||
// re-registered, its RUNNING state refreshed, and its exit-wait re-attached.
|
||||
func TestReconcile_AdoptsRunningLabeledContainer(t *testing.T) {
|
||||
cp := &fakeCP{}
|
||||
rt := &ctrruntime.FakePuller{
|
||||
WaitErr: errors.New("blocked"),
|
||||
Containers: []ctrruntime.Container{
|
||||
adoptedContainer("t-001", "web", "default", "t-001", testTaskRole, "52:54:00:de:ad:01", true),
|
||||
},
|
||||
}
|
||||
a := newAgent(config{}, testIdentity(), cp, rt, nil)
|
||||
a.cred = newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
|
||||
|
||||
adopted := a.reconcile(context.Background())
|
||||
|
||||
if !adopted["t-001"] {
|
||||
t.Fatalf("t-001 not adopted; got %v", adopted)
|
||||
}
|
||||
if got := a.cred.roles["t-001"]; got != testTaskRole {
|
||||
t.Errorf("cred not re-registered: roles[t-001] = %q, want %q", got, testTaskRole)
|
||||
}
|
||||
if n := countStatus(cp.taskStates(), "t-001", bus.TaskStatusRunning); n != 1 {
|
||||
t.Errorf("RUNNING reports for t-001 = %d, want 1", n)
|
||||
}
|
||||
waitedFor(t, rt, containerID("t-001", "web"))
|
||||
}
|
||||
|
||||
// A task whose container died while the agent was down must be reported STOPPED,
|
||||
// not silently skipped. The exit-wait is cancelled on agent shutdown, so nothing
|
||||
// else ever reports that exit: skipping it leaves the gateway holding the task
|
||||
// RUNNING forever, its service never places a replacement, and the load
|
||||
// balancer target stays unhealthy with no path back.
|
||||
func TestReconcile_ReportsTaskWhoseContainerDied(t *testing.T) {
|
||||
cp := &fakeCP{}
|
||||
rt := &ctrruntime.FakePuller{
|
||||
Containers: []ctrruntime.Container{
|
||||
adoptedContainer("t-dead", "web", "default", "t-dead", testTaskRole, "52:54:00:de:ad:02", false),
|
||||
},
|
||||
}
|
||||
a := newAgent(config{}, testIdentity(), cp, rt, nil)
|
||||
a.cred = newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
|
||||
|
||||
adopted := a.reconcile(context.Background())
|
||||
|
||||
if n := countStatus(cp.taskStates(), "t-dead", bus.TaskStatusStopped); n != 1 {
|
||||
t.Errorf("STOPPED reports for t-dead = %d, want 1", n)
|
||||
}
|
||||
if n := countStatus(cp.taskStates(), "t-dead", bus.TaskStatusRunning); n != 0 {
|
||||
t.Errorf("t-dead must not be reported RUNNING, got %d such reports", n)
|
||||
}
|
||||
// Seeded so a re-delivered assignment for the dead task is acked, not re-run.
|
||||
if !adopted["t-dead"] {
|
||||
t.Errorf("t-dead not seeded into the adopted set; got %v", adopted)
|
||||
}
|
||||
if want := containerID("t-dead", "web"); !slices.Contains(rt.Removed, want) {
|
||||
t.Errorf("leftover container %s not removed; removed=%v", want, rt.Removed)
|
||||
}
|
||||
}
|
||||
|
||||
// A task with one live and one dead container is still running: the live half is
|
||||
// re-adopted and the re-attached exit-wait reports the dead one when it is
|
||||
// reaped, so reconcile must not declare the whole task STOPPED.
|
||||
func TestReconcile_PartiallyDeadTaskStaysRunning(t *testing.T) {
|
||||
cp := &fakeCP{}
|
||||
rt := &ctrruntime.FakePuller{
|
||||
WaitErr: errors.New("blocked"),
|
||||
Containers: []ctrruntime.Container{
|
||||
adoptedContainer("t-mixed", "web", "default", "t-mixed", testTaskRole, "52:54:00:de:ad:03", true),
|
||||
adoptedContainer("t-mixed", "sidecar", "default", "t-mixed", testTaskRole, "52:54:00:de:ad:03", false),
|
||||
},
|
||||
}
|
||||
a := newAgent(config{}, testIdentity(), cp, rt, nil)
|
||||
a.cred = newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
|
||||
|
||||
a.reconcile(context.Background())
|
||||
|
||||
if n := countStatus(cp.taskStates(), "t-mixed", bus.TaskStatusRunning); n != 1 {
|
||||
t.Errorf("RUNNING reports for t-mixed = %d, want 1", n)
|
||||
}
|
||||
if n := countStatus(cp.taskStates(), "t-mixed", bus.TaskStatusStopped); n != 0 {
|
||||
t.Errorf("t-mixed must not be reported STOPPED while a container still runs, got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
// pollAssignments seeded with an adopted task acks its re-delivered assignment
|
||||
// but does not re-run it, while a genuinely new task still dispatches.
|
||||
func TestPollAssignments_SeededTaskNotRerun(t *testing.T) {
|
||||
mk := func(id string) bus.Assign {
|
||||
return bus.Assign{
|
||||
AccountID: "123456789012", ClusterName: "default", InstanceID: "i-abc123", TaskID: id,
|
||||
Containers: []bus.AssignContainer{{Name: "web", Image: "registry/web:1", Command: []string{"/bin/true"}}},
|
||||
}
|
||||
}
|
||||
adopted, fresh := mk("t-001"), mk("t-002")
|
||||
cp := &fakeCP{pollReplies: [][]bus.Assign{
|
||||
{adopted, fresh},
|
||||
{adopted, fresh},
|
||||
nil,
|
||||
}}
|
||||
rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
|
||||
a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, nil)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
go a.pollAssignments(ctx, map[string]bool{"t-001": true})
|
||||
|
||||
deadline := time.After(time.Second)
|
||||
for countStatus(cp.taskStates(), "t-002", bus.TaskStatusRunning) < 1 {
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatal("fresh task t-002 was never dispatched")
|
||||
case <-time.After(2 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
cancel()
|
||||
|
||||
if n := countStatus(cp.taskStates(), "t-001", bus.TaskStatusRunning); n != 0 {
|
||||
t.Errorf("adopted task t-001 was re-run %d times, want 0", n)
|
||||
}
|
||||
if n := countStatus(cp.taskStates(), "t-002", bus.TaskStatusRunning); n != 1 {
|
||||
t.Errorf("fresh task t-002 dispatched %d times, want 1", n)
|
||||
}
|
||||
var t1Acked bool
|
||||
for _, ack := range cp.acks() {
|
||||
for _, id := range ack {
|
||||
if id == "t-001" {
|
||||
t1Acked = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !t1Acked {
|
||||
t.Errorf("adopted task t-001 was never acked; acks=%v", cp.acks())
|
||||
}
|
||||
}
|
||||
|
||||
// Containers belonging to another cluster are ignored entirely: this agent
|
||||
// speaks for its own cluster only, and must neither adopt nor report them.
|
||||
//
|
||||
// A stopped container of this cluster IS handled — see
|
||||
// TestReconcile_ReportsTaskWhoseContainerDied. It lands in the adopted set so
|
||||
// its re-delivered assignment is acked rather than re-run, after reconcile has
|
||||
// reported the task STOPPED.
|
||||
func TestReconcile_IgnoresOtherClusterContainers(t *testing.T) {
|
||||
cp := &fakeCP{}
|
||||
rt := &ctrruntime.FakePuller{
|
||||
WaitErr: errors.New("blocked"),
|
||||
Containers: []ctrruntime.Container{
|
||||
adoptedContainer("t-other", "web", "other", "", "", "", true),
|
||||
adoptedContainer("t-other-dead", "web", "other", "", "", "", false),
|
||||
adoptedContainer("t-ok", "web", "default", "", "", "", true),
|
||||
},
|
||||
}
|
||||
a := newAgent(config{}, testIdentity(), cp, rt, nil)
|
||||
|
||||
adopted := a.reconcile(context.Background())
|
||||
|
||||
if len(adopted) != 1 || !adopted["t-ok"] {
|
||||
t.Fatalf("adopted = %v, want only t-ok", adopted)
|
||||
}
|
||||
for _, id := range []string{"t-other", "t-other-dead"} {
|
||||
if n := countStatus(cp.taskStates(), id, bus.TaskStatusStopped); n != 0 {
|
||||
t.Errorf("reported %d STOPPED states for another cluster's task %s, want 0", n, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A nil runner or a List error degrades to an empty set without panicking.
|
||||
func TestReconcile_DegradesGracefully(t *testing.T) {
|
||||
noRunner := newAgent(config{}, testIdentity(), &fakeCP{}, nil, nil)
|
||||
if got := noRunner.reconcile(context.Background()); len(got) != 0 {
|
||||
t.Errorf("nil runner: adopted = %v, want empty", got)
|
||||
}
|
||||
|
||||
rt := &ctrruntime.FakePuller{ListErr: errors.New("boom")}
|
||||
listErr := newAgent(config{}, testIdentity(), &fakeCP{}, rt, nil)
|
||||
if got := listErr.reconcile(context.Background()); len(got) != 0 {
|
||||
t.Errorf("list error: adopted = %v, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
// taskLabels emits the optional cred/role/MAC labels only when set.
|
||||
func TestTaskLabels_OmitsEmptyOptionalLabels(t *testing.T) {
|
||||
full := taskLabels(&bus.Assign{
|
||||
TaskID: "t1", ClusterName: "c", CredID: "x", TaskRoleARN: testTaskRole, ENIMacAddress: "mac",
|
||||
}, "web")
|
||||
for k, want := range map[string]string{
|
||||
labelCredID: "x", labelTaskRoleARN: testTaskRole, labelENIMac: "mac",
|
||||
} {
|
||||
if full[k] != want {
|
||||
t.Errorf("full labels[%q] = %q, want %q", k, full[k], want)
|
||||
}
|
||||
}
|
||||
|
||||
bare := taskLabels(&bus.Assign{TaskID: "t1", ClusterName: "c"}, "web")
|
||||
if len(bare) != 3 {
|
||||
t.Errorf("bare labels = %v, want only taskID/containerName/clusterName", bare)
|
||||
}
|
||||
for _, k := range []string{labelCredID, labelTaskRoleARN, labelENIMac} {
|
||||
if _, ok := bare[k]; ok {
|
||||
t.Errorf("bare labels unexpectedly carry %q", k)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ecs"
|
||||
handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
|
||||
"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
|
||||
)
|
||||
|
||||
// identity is the container instance's stable identity, assembled at boot from
|
||||
// IMDS (account, instance, AZ) and config (cluster). It is the join key the
|
||||
// scheduler uses to track this instance.
|
||||
type identity struct {
|
||||
AccountID string
|
||||
ClusterName string
|
||||
InstanceID string
|
||||
AZ string
|
||||
InstanceType string
|
||||
Hostname string
|
||||
Capacity bus.InstanceCapacity
|
||||
AgentVersion string
|
||||
}
|
||||
|
||||
// instanceAttributes builds the built-in container-instance attributes from the
|
||||
// host's identity. Only attributes whose value was actually discovered are
|
||||
// reported, so an empty one is absent rather than present and blank.
|
||||
func instanceAttributes(id identity) []*ecs.Attribute {
|
||||
var attrs []*ecs.Attribute
|
||||
add := func(name, value string) {
|
||||
if value == "" {
|
||||
return
|
||||
}
|
||||
attrs = append(attrs, &ecs.Attribute{Name: aws.String(name), Value: aws.String(value)})
|
||||
}
|
||||
add(handlers_ecs.AttrInstanceType, id.InstanceType)
|
||||
add(handlers_ecs.AttrAvailabilityZone, id.AZ)
|
||||
return attrs
|
||||
}
|
||||
|
||||
// advertisedIdentity is the identity to register with, given whether a
|
||||
// container runtime is available. A host with no runtime can run nothing, so it
|
||||
// advertises no capacity: the scheduler places nothing on it, and the control
|
||||
// plane reads the zero as an involuntary drain. The identity is otherwise
|
||||
// unchanged, so the instance stays visible rather than disappearing.
|
||||
func advertisedIdentity(id identity, runtimeReady bool) identity {
|
||||
if runtimeReady {
|
||||
return id
|
||||
}
|
||||
id.Capacity = bus.InstanceCapacity{}
|
||||
return id
|
||||
}
|
||||
|
||||
// registrar registers the host as a container instance through the gateway.
|
||||
type registrar struct {
|
||||
cp controlPlane
|
||||
id identity
|
||||
// runtimeReady reports whether a container runtime is resolved. nil means
|
||||
// always ready, which is what a test with no gate wants.
|
||||
runtimeReady func() bool
|
||||
}
|
||||
|
||||
func newRegistrar(cp controlPlane, id identity, runtimeReady func() bool) *registrar {
|
||||
return ®istrar{cp: cp, id: id, runtimeReady: runtimeReady}
|
||||
}
|
||||
|
||||
// ready reports the runtime state, defaulting to ready when no gate is wired.
|
||||
func (r *registrar) ready() bool { return r.runtimeReady == nil || r.runtimeReady() }
|
||||
|
||||
// Register calls the gateway's RegisterContainerInstance. The scheduler records
|
||||
// (or refreshes) the container instance on receipt.
|
||||
func (r *registrar) Register() error {
|
||||
if err := r.cp.Register(advertisedIdentity(r.id, r.ready())); err != nil {
|
||||
return fmt.Errorf("register instance %s: %w", r.id.InstanceID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"slices"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
|
||||
"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
|
||||
)
|
||||
|
||||
// fakeCP is a test double for the gateway control-plane. It records registers,
|
||||
// task-state reports and GPU reports, and replays a scripted queue of poll
|
||||
// responses.
|
||||
type fakeCP struct {
|
||||
mu sync.Mutex
|
||||
|
||||
registers int
|
||||
lastRegistered identity
|
||||
states []bus.TaskState
|
||||
gpuReports []gpuReport
|
||||
pollAcks [][]string
|
||||
pollStopAcks [][]string
|
||||
pollReplies [][]bus.Assign
|
||||
stopReplies [][]bus.StopDirective
|
||||
stopPending []bus.StopDirective
|
||||
pollCalls int
|
||||
|
||||
registerErr bool
|
||||
submitErr bool
|
||||
}
|
||||
|
||||
// gpuReport is one recorded ReportTaskGPU call.
|
||||
type gpuReport struct {
|
||||
cluster string
|
||||
task string
|
||||
containers []handlers_ecs.ContainerGPUReport
|
||||
}
|
||||
|
||||
var _ controlPlane = (*fakeCP)(nil)
|
||||
|
||||
func (f *fakeCP) Register(id identity) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.registerErr {
|
||||
return errors.New("register boom")
|
||||
}
|
||||
f.registers++
|
||||
f.lastRegistered = id
|
||||
return nil
|
||||
}
|
||||
|
||||
// registered returns the identity of the most recent Register call.
|
||||
func (f *fakeCP) registered() identity {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.lastRegistered
|
||||
}
|
||||
|
||||
func (f *fakeCP) SubmitTaskState(st bus.TaskState) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.submitErr {
|
||||
return errors.New("submit boom")
|
||||
}
|
||||
f.states = append(f.states, st)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeCP) PollAssignments(_, _ string, ackAssigns, ackStops []string) ([]bus.Assign, []bus.StopDirective, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.pollAcks = append(f.pollAcks, append([]string(nil), ackAssigns...))
|
||||
f.pollStopAcks = append(f.pollStopAcks, append([]string(nil), ackStops...))
|
||||
var out []bus.Assign
|
||||
if f.pollCalls < len(f.pollReplies) {
|
||||
out = f.pollReplies[f.pollCalls]
|
||||
}
|
||||
var stops []bus.StopDirective
|
||||
if f.pollCalls < len(f.stopReplies) {
|
||||
stops = f.stopReplies[f.pollCalls]
|
||||
}
|
||||
// stopPending is the durable inbox the gateway keeps: an acked directive is
|
||||
// deleted, and everything still in it is redelivered on every poll.
|
||||
f.stopPending = slices.DeleteFunc(f.stopPending, func(sd bus.StopDirective) bool {
|
||||
return slices.Contains(ackStops, sd.TaskID)
|
||||
})
|
||||
stops = slices.Concat(stops, f.stopPending)
|
||||
f.pollCalls++
|
||||
return out, stops, nil
|
||||
}
|
||||
|
||||
func (f *fakeCP) ReportTaskGPU(cluster, task string, containers []handlers_ecs.ContainerGPUReport) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.gpuReports = append(f.gpuReports, gpuReport{cluster: cluster, task: task, containers: containers})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeCP) gpuReportsFor(taskID string) []gpuReport {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var out []gpuReport
|
||||
for _, r := range f.gpuReports {
|
||||
if r.task == taskID {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (f *fakeCP) registerCount() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.registers
|
||||
}
|
||||
|
||||
func (f *fakeCP) taskStates() []bus.TaskState {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([]bus.TaskState(nil), f.states...)
|
||||
}
|
||||
|
||||
func (f *fakeCP) acks() [][]string {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([][]string(nil), f.pollAcks...)
|
||||
}
|
||||
|
||||
func (f *fakeCP) stopAcks() [][]string {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([][]string(nil), f.pollStopAcks...)
|
||||
}
|
||||
|
||||
func testIdentity() identity {
|
||||
return identity{
|
||||
AccountID: "123456789012",
|
||||
ClusterName: "default",
|
||||
InstanceID: "i-abc123",
|
||||
AZ: "us-east-1a",
|
||||
Hostname: "ecs-host",
|
||||
Capacity: bus.InstanceCapacity{CPU: 2048, MemoryMiB: 4096},
|
||||
AgentVersion: "test",
|
||||
}
|
||||
}
|
||||
|
||||
// testIdentityWithGPUs is testIdentity seeded with discovered GPU UUIDs, so
|
||||
// newAgent's ledger has real devices to pin from.
|
||||
func testIdentityWithGPUs(uuids ...string) identity {
|
||||
id := testIdentity()
|
||||
id.Capacity.GPU = len(uuids)
|
||||
id.Capacity.GPUIDs = uuids
|
||||
return id
|
||||
}
|
||||
|
||||
func TestRegistrar_Register(t *testing.T) {
|
||||
cp := &fakeCP{}
|
||||
if err := newRegistrar(cp, testIdentity(), nil).Register(); err != nil {
|
||||
t.Fatalf("Register: %v", err)
|
||||
}
|
||||
if cp.registerCount() != 1 {
|
||||
t.Errorf("registers = %d, want 1", cp.registerCount())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistrar_RegisterError(t *testing.T) {
|
||||
cp := &fakeCP{registerErr: true}
|
||||
if err := newRegistrar(cp, testIdentity(), nil).Register(); err == nil {
|
||||
t.Fatal("expected register error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeartbeater_Beat(t *testing.T) {
|
||||
cp := &fakeCP{}
|
||||
if err := newHeartbeater(cp, testIdentity(), time.Second, nil).beat(); err != nil {
|
||||
t.Fatalf("beat: %v", err)
|
||||
}
|
||||
if cp.registerCount() != 1 {
|
||||
t.Errorf("beat should re-register once, got %d", cp.registerCount())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeartbeater_DefaultInterval(t *testing.T) {
|
||||
h := newHeartbeater(&fakeCP{}, testIdentity(), 0, nil)
|
||||
if h.interval != defaultHeartbeat {
|
||||
t.Errorf("interval = %v, want default", h.interval)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeartbeater_RunStopsOnContext(t *testing.T) {
|
||||
cp := &fakeCP{}
|
||||
h := newHeartbeater(cp, testIdentity(), 5*time.Millisecond, nil)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() { h.Run(ctx); close(done) }()
|
||||
|
||||
time.Sleep(25 * time.Millisecond)
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Run did not stop on context cancel")
|
||||
}
|
||||
if cp.registerCount() == 0 {
|
||||
t.Error("expected at least one re-register before cancel")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRegistrar_NoRuntimeAdvertisesNoCapacity is the placement half of the boot
|
||||
// bug: an agent with no container runtime kept registering its full CPU and
|
||||
// memory, so the scheduler placed tasks it could only fail to start. Capacity is
|
||||
// advertised only while a runtime is there to use it.
|
||||
func TestRegistrar_NoRuntimeAdvertisesNoCapacity(t *testing.T) {
|
||||
cp := &fakeCP{}
|
||||
ready := false
|
||||
r := newRegistrar(cp, testIdentity(), func() bool { return ready })
|
||||
|
||||
if err := r.Register(); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
if got := cp.registered().Capacity; got.CPU != 0 || got.MemoryMiB != 0 {
|
||||
t.Fatalf("advertised capacity with no runtime: cpu=%d memory=%d", got.CPU, got.MemoryMiB)
|
||||
}
|
||||
|
||||
ready = true
|
||||
if err := r.Register(); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
want := testIdentity().Capacity
|
||||
if got := cp.registered().Capacity; got.CPU != want.CPU || got.MemoryMiB != want.MemoryMiB {
|
||||
t.Fatalf("capacity not restored once the runtime resolved: cpu=%d memory=%d", got.CPU, got.MemoryMiB)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHeartbeater_NoRuntimeAdvertisesNoCapacity is the same rule on the beat,
|
||||
// which is how a runtime lost after boot reaches the control plane.
|
||||
func TestHeartbeater_NoRuntimeAdvertisesNoCapacity(t *testing.T) {
|
||||
cp := &fakeCP{}
|
||||
h := newHeartbeater(cp, testIdentity(), time.Second, func() bool { return false })
|
||||
|
||||
if err := h.beat(); err != nil {
|
||||
t.Fatalf("beat: %v", err)
|
||||
}
|
||||
if got := cp.registered().Capacity; got.CPU != 0 || got.MemoryMiB != 0 {
|
||||
t.Fatalf("beat advertised capacity with no runtime: cpu=%d memory=%d", got.CPU, got.MemoryMiB)
|
||||
}
|
||||
// The instance still identifies itself, so it stays visible as drained rather
|
||||
// than vanishing from the cluster.
|
||||
if cp.registered().InstanceID != testIdentity().InstanceID {
|
||||
t.Error("beat dropped the instance identity")
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package agent
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
ctrruntime "github.com/mulgadc/spinifex/spinifex/agents/ecs/runtime"
|
||||
ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
|
||||
)
|
||||
|
||||
// runtimeProbeTimeout bounds the liveness probe against a runtime the gate is
|
||||
@@ -1,13 +1,11 @@
|
||||
//test:in-package — package main cannot be imported by an external test package.
|
||||
|
||||
package agent
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
ctrruntime "github.com/mulgadc/spinifex/spinifex/agents/ecs/runtime"
|
||||
ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
|
||||
)
|
||||
|
||||
// TestRuntimeGate_DialsUntilTheSocketAnswers is the boot case the agent used to
|
||||
@@ -1,4 +1,4 @@
|
||||
package agent
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -9,9 +9,9 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
|
||||
ctrruntime "github.com/mulgadc/spinifex/spinifex/agents/ecs/runtime"
|
||||
ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
|
||||
handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
|
||||
"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
|
||||
)
|
||||
|
||||
// defaultStopTimeout is applied when a container carries no explicit
|
||||
@@ -127,7 +127,7 @@ func (l *stopLedger) seen(taskID string) bool {
|
||||
// Idempotent — a task with no live containers still reports STOPPED so the
|
||||
// scheduler releases its capacity. Reports whether it got as far as saying
|
||||
// STOPPED: a caller may only ack the directive when it did.
|
||||
func (a *Agent) stopTask(ctx context.Context, sd ecsv1.StopDirective) bool {
|
||||
func (a *Agent) stopTask(ctx context.Context, sd bus.StopDirective) bool {
|
||||
runner := a.rt.get()
|
||||
if runner == nil {
|
||||
slog.Warn("ecs-agent: stop deferred, no container runtime", "task", sd.TaskID)
|
||||
@@ -138,7 +138,7 @@ func (a *Agent) stopTask(ctx context.Context, sd ecsv1.StopDirective) bool {
|
||||
slog.Warn("ecs-agent: stop list failed", "task", sd.TaskID, "err", err)
|
||||
return false
|
||||
}
|
||||
statuses := make([]ecsv1.ContainerStatus, 0)
|
||||
statuses := make([]bus.ContainerStatus, 0)
|
||||
for _, c := range containers {
|
||||
if c.Labels[labelTaskID] != sd.TaskID {
|
||||
continue
|
||||
@@ -146,15 +146,15 @@ func (a *Agent) stopTask(ctx context.Context, sd ecsv1.StopDirective) bool {
|
||||
if rerr := runner.Stop(ctx, c.ID, containerStopTimeout(c.Labels)); rerr != nil {
|
||||
slog.Warn("ecs-agent: stop failed", "task", sd.TaskID, "container", c.ID, "err", rerr)
|
||||
}
|
||||
statuses = append(statuses, ecsv1.ContainerStatus{
|
||||
Name: c.Labels[labelContainerName], Status: ecsv1.TaskStatusStopped, ContainerID: c.ID,
|
||||
statuses = append(statuses, bus.ContainerStatus{
|
||||
Name: c.Labels[labelContainerName], Status: bus.TaskStatusStopped, ContainerID: c.ID,
|
||||
})
|
||||
}
|
||||
reason := sd.Reason
|
||||
if reason == "" {
|
||||
reason = "Task stopped"
|
||||
}
|
||||
a.reportTaskState(&ecsv1.Assign{TaskID: sd.TaskID}, ecsv1.TaskStatusStopped, reason, statuses)
|
||||
a.reportTaskState(&bus.Assign{TaskID: sd.TaskID}, bus.TaskStatusStopped, reason, statuses)
|
||||
slog.Info("ecs-agent: task reaped", "task", sd.TaskID, "containers", len(statuses))
|
||||
return true
|
||||
}
|
||||
@@ -162,10 +162,10 @@ func (a *Agent) stopTask(ctx context.Context, sd ecsv1.StopDirective) bool {
|
||||
// runTask pulls each container image, starts the containers, and reports task
|
||||
// state on the bus. A missing runtime or any per-container failure reports the
|
||||
// task STOPPED with a reason; success reports RUNNING and waits for exit.
|
||||
func (a *Agent) runTask(ctx context.Context, as *ecsv1.Assign) {
|
||||
func (a *Agent) runTask(ctx context.Context, as *bus.Assign) {
|
||||
rt := a.rt.get()
|
||||
if rt == nil {
|
||||
a.reportTaskState(as, ecsv1.TaskStatusStopped, "containerd unavailable on agent", nil)
|
||||
a.reportTaskState(as, bus.TaskStatusStopped, "containerd unavailable on agent", nil)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -173,14 +173,14 @@ func (a *Agent) runTask(ctx context.Context, as *ecsv1.Assign) {
|
||||
netnsPath, err := a.setupTaskNetns(as)
|
||||
if err != nil {
|
||||
slog.Error("ecs-agent: task netns setup failed", "task", as.TaskID, "err", err)
|
||||
a.reportTaskState(as, ecsv1.TaskStatusStopped, "network setup failed: "+err.Error(), nil)
|
||||
a.reportTaskState(as, bus.TaskStatusStopped, "network setup failed: "+err.Error(), nil)
|
||||
return
|
||||
}
|
||||
|
||||
credID := a.registerTaskCreds(as)
|
||||
resolver := a.pullResolver(as)
|
||||
|
||||
statuses := make([]ecsv1.ContainerStatus, 0, len(as.Containers))
|
||||
statuses := make([]bus.ContainerStatus, 0, len(as.Containers))
|
||||
for _, c := range as.Containers {
|
||||
// startTimeout bounds pull-through-start as a unit: real ECS measures a
|
||||
// container's time-to-RUNNING from the same starting point.
|
||||
@@ -190,7 +190,7 @@ func (a *Agent) runTask(ctx context.Context, as *ecsv1.Assign) {
|
||||
cancel()
|
||||
slog.Error("ecs-agent: pull failed", "task", as.TaskID, "image", c.Image, "err", err)
|
||||
a.teardownTaskNetns(as)
|
||||
a.reportTaskState(as, ecsv1.TaskStatusStopped, "image pull failed: "+err.Error(), statuses)
|
||||
a.reportTaskState(as, bus.TaskStatusStopped, "image pull failed: "+err.Error(), statuses)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -222,22 +222,22 @@ func (a *Agent) runTask(ctx context.Context, as *ecsv1.Assign) {
|
||||
if err != nil {
|
||||
slog.Error("ecs-agent: run failed", "task", as.TaskID, "container", c.Name, "err", err)
|
||||
a.teardownTaskNetns(as)
|
||||
a.reportTaskState(as, ecsv1.TaskStatusStopped, "container start failed: "+err.Error(), statuses)
|
||||
a.reportTaskState(as, bus.TaskStatusStopped, "container start failed: "+err.Error(), statuses)
|
||||
return
|
||||
}
|
||||
statuses = append(statuses, ecsv1.ContainerStatus{
|
||||
Name: c.Name, Status: ecsv1.TaskStatusRunning, ContainerID: id, GPUIDs: gpuIDs,
|
||||
statuses = append(statuses, bus.ContainerStatus{
|
||||
Name: c.Name, Status: bus.TaskStatusRunning, ContainerID: id, GPUIDs: gpuIDs,
|
||||
})
|
||||
go a.waitContainer(ctx, as, c.Name, id)
|
||||
}
|
||||
|
||||
a.reportTaskState(as, ecsv1.TaskStatusRunning, "", statuses)
|
||||
a.reportTaskState(as, bus.TaskStatusRunning, "", statuses)
|
||||
slog.Info("ecs-agent: task running", "task", as.TaskID, "containers", len(statuses))
|
||||
}
|
||||
|
||||
// waitContainer blocks until a container exits, then reports the task STOPPED
|
||||
// with the exit code. v1 stops the whole task when its first container exits.
|
||||
func (a *Agent) waitContainer(ctx context.Context, as *ecsv1.Assign, name, containerID string) {
|
||||
func (a *Agent) waitContainer(ctx context.Context, as *bus.Assign, name, containerID string) {
|
||||
runner := a.rt.get()
|
||||
if runner == nil {
|
||||
slog.Warn("ecs-agent: cannot wait on container, no container runtime", "task", as.TaskID, "container", name)
|
||||
@@ -250,14 +250,14 @@ func (a *Agent) waitContainer(ctx context.Context, as *ecsv1.Assign, name, conta
|
||||
}
|
||||
exit := status.ExitCode
|
||||
a.teardownTaskNetns(as)
|
||||
statuses := []ecsv1.ContainerStatus{{Name: name, Status: ecsv1.TaskStatusStopped, ContainerID: containerID, ExitCode: &exit}}
|
||||
a.reportTaskState(as, ecsv1.TaskStatusStopped, fmt.Sprintf("container %s exited (%d)", name, exit), statuses)
|
||||
statuses := []bus.ContainerStatus{{Name: name, Status: bus.TaskStatusStopped, ContainerID: containerID, ExitCode: &exit}}
|
||||
a.reportTaskState(as, bus.TaskStatusStopped, fmt.Sprintf("container %s exited (%d)", name, exit), statuses)
|
||||
slog.Info("ecs-agent: task stopped", "task", as.TaskID, "container", name, "exitCode", exit)
|
||||
}
|
||||
|
||||
// setupTaskNetns builds the awsvpc task netns from the hot-plugged ENI. Bridge/
|
||||
// host tasks (no ENI MAC) and a nil controller are no-ops returning an empty path.
|
||||
func (a *Agent) setupTaskNetns(as *ecsv1.Assign) (string, error) {
|
||||
func (a *Agent) setupTaskNetns(as *bus.Assign) (string, error) {
|
||||
if as.ENIMacAddress == "" || a.netns == nil {
|
||||
return "", nil
|
||||
}
|
||||
@@ -267,7 +267,7 @@ func (a *Agent) setupTaskNetns(as *ecsv1.Assign) (string, error) {
|
||||
// teardownTaskNetns releases the task's IAM credential registration, then removes
|
||||
// the awsvpc task netns (the latter is a no-op for bridge/host tasks). It runs at
|
||||
// every task-stop path so credentials never outlive the task that owns them.
|
||||
func (a *Agent) teardownTaskNetns(as *ecsv1.Assign) {
|
||||
func (a *Agent) teardownTaskNetns(as *bus.Assign) {
|
||||
if a.cred != nil {
|
||||
a.cred.Deregister(taskCredID(as))
|
||||
}
|
||||
@@ -283,7 +283,7 @@ func (a *Agent) teardownTaskNetns(as *ecsv1.Assign) {
|
||||
// assign carries an execution role, it authorizes pulls as that role (assumed
|
||||
// over the gateway); an empty role or a nil credential endpoint (unit tests)
|
||||
// falls back to the instance-role resolver, so a pull is never worse off.
|
||||
func (a *Agent) pullResolver(as *ecsv1.Assign) ctrruntime.Resolver {
|
||||
func (a *Agent) pullResolver(as *bus.Assign) ctrruntime.Resolver {
|
||||
if as.ExecutionRoleARN == "" || a.cred == nil {
|
||||
return a.resolver
|
||||
}
|
||||
@@ -293,7 +293,7 @@ func (a *Agent) pullResolver(as *ecsv1.Assign) ctrruntime.Resolver {
|
||||
|
||||
// registerTaskCreds registers the task's credID -> role mapping with the
|
||||
// credential endpoint and returns the credID (empty when the task has no role).
|
||||
func (a *Agent) registerTaskCreds(as *ecsv1.Assign) string {
|
||||
func (a *Agent) registerTaskCreds(as *bus.Assign) string {
|
||||
credID := taskCredID(as)
|
||||
if credID == "" || a.cred == nil {
|
||||
return credID
|
||||
@@ -305,7 +305,7 @@ func (a *Agent) registerTaskCreds(as *ecsv1.Assign) string {
|
||||
// taskCredID is the credential ID a task's containers fetch credentials under:
|
||||
// the scheduler-assigned CredID, falling back to the taskID. Empty when the task
|
||||
// carries no IAM role.
|
||||
func taskCredID(as *ecsv1.Assign) string {
|
||||
func taskCredID(as *bus.Assign) string {
|
||||
if as.TaskRoleARN == "" {
|
||||
return ""
|
||||
}
|
||||
@@ -353,9 +353,9 @@ func containerStopTimeout(labels map[string]string) time.Duration {
|
||||
}
|
||||
|
||||
// toRuntimeSystemControls maps the assign's sysctl pairs to the runtime
|
||||
// package's own type, keeping agents/ecs/runtime decoupled from
|
||||
// contracts/ecs/v1.
|
||||
func toRuntimeSystemControls(in []ecsv1.SystemControl) []ctrruntime.SystemControl {
|
||||
// package's own type, keeping cmd/ecs-agent/runtime decoupled from the ecs
|
||||
// handlers' bus package.
|
||||
func toRuntimeSystemControls(in []bus.SystemControl) []ctrruntime.SystemControl {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -386,11 +386,11 @@ func (a *Agent) pinContainerGPUs(taskID, containerName string, n int) []string {
|
||||
// reportTaskState reports a task transition through the gateway's
|
||||
// SubmitTaskStateChange action, then (RUNNING with pinned GPUs) reports the
|
||||
// per-container device UUIDs, and (STOPPED) releases them back to the ledger.
|
||||
func (a *Agent) reportTaskState(as *ecsv1.Assign, status, reason string, containers []ecsv1.ContainerStatus) {
|
||||
func (a *Agent) reportTaskState(as *bus.Assign, status, reason string, containers []bus.ContainerStatus) {
|
||||
if a.cp == nil {
|
||||
return
|
||||
}
|
||||
msg := ecsv1.TaskState{
|
||||
msg := bus.TaskState{
|
||||
AccountID: a.id.AccountID,
|
||||
ClusterName: a.id.ClusterName,
|
||||
InstanceID: a.id.InstanceID,
|
||||
@@ -404,7 +404,7 @@ func (a *Agent) reportTaskState(as *ecsv1.Assign, status, reason string, contain
|
||||
slog.Error("ecs-agent: report task-state failed", "task", as.TaskID, "err", err)
|
||||
}
|
||||
a.reportTaskGPUs(as.TaskID, containers)
|
||||
if status == ecsv1.TaskStatusStopped && a.gpu != nil {
|
||||
if status == bus.TaskStatusStopped && a.gpu != nil {
|
||||
a.gpu.ReleaseTask(as.TaskID)
|
||||
}
|
||||
}
|
||||
@@ -412,7 +412,7 @@ func (a *Agent) reportTaskState(as *ecsv1.Assign, status, reason string, contain
|
||||
// reportTaskGPUs sends the control plane the pinned device UUIDs for any
|
||||
// container in the report that has them, so DescribeTasks can surface gpuIds.
|
||||
// A no-op when nothing was pinned (the common non-GPU-task case).
|
||||
func (a *Agent) reportTaskGPUs(taskID string, containers []ecsv1.ContainerStatus) {
|
||||
func (a *Agent) reportTaskGPUs(taskID string, containers []bus.ContainerStatus) {
|
||||
reports := make([]handlers_ecs.ContainerGPUReport, 0, len(containers))
|
||||
for _, c := range containers {
|
||||
if len(c.GPUIDs) > 0 {
|
||||
@@ -440,7 +440,7 @@ const (
|
||||
labelTaskID = "mulga.ecs.taskID"
|
||||
labelContainerName = "mulga.ecs.containerName"
|
||||
labelClusterName = "mulga.ecs.clusterName"
|
||||
labelCredID = "mulga.ecs.credID" //nolint:gosec // G101: label key name, not a credential value.
|
||||
labelCredID = "mulga.ecs.credID"
|
||||
labelTaskRoleARN = "mulga.ecs.taskRoleArn"
|
||||
labelENIMac = "mulga.ecs.eniMac"
|
||||
// labelStopTimeout carries a container's stopTimeout (seconds) so stopTask
|
||||
@@ -450,7 +450,7 @@ const (
|
||||
|
||||
// taskLabels are the mulga.ecs.* labels stamped on a container. The cred/role/MAC
|
||||
// labels are omitted when empty so a container carries only what it needs.
|
||||
func taskLabels(as *ecsv1.Assign, name string) map[string]string {
|
||||
func taskLabels(as *bus.Assign, name string) map[string]string {
|
||||
labels := map[string]string{
|
||||
labelTaskID: as.TaskID,
|
||||
labelContainerName: name,
|
||||
@@ -1,4 +1,4 @@
|
||||
package agent
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -7,17 +7,17 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
|
||||
ctrruntime "github.com/mulgadc/spinifex/spinifex/agents/ecs/runtime"
|
||||
ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
|
||||
"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
|
||||
)
|
||||
|
||||
func testAssign() *ecsv1.Assign {
|
||||
return &ecsv1.Assign{
|
||||
func testAssign() *bus.Assign {
|
||||
return &bus.Assign{
|
||||
AccountID: "123456789012",
|
||||
ClusterName: "default",
|
||||
InstanceID: "i-abc123",
|
||||
TaskID: "t-001",
|
||||
Containers: []ecsv1.AssignContainer{
|
||||
Containers: []bus.AssignContainer{
|
||||
{Name: "web", Image: "registry/web:1", Command: []string{"/bin/true"},
|
||||
Environment: map[string]string{"FOO": "bar"}},
|
||||
},
|
||||
@@ -25,17 +25,17 @@ func testAssign() *ecsv1.Assign {
|
||||
}
|
||||
|
||||
func newRunAgent(cp controlPlane, rt *ctrruntime.FakePuller) *Agent {
|
||||
return newAgent(Config{}, testIdentity(), cp, rt, nil)
|
||||
return newAgent(config{}, testIdentity(), cp, rt, nil)
|
||||
}
|
||||
|
||||
// runTask with no runtime reports the task STOPPED instead of crashing.
|
||||
func TestRunTask_NoRuntimeReportsStopped(t *testing.T) {
|
||||
cp := &fakeCP{}
|
||||
a := newAgent(Config{}, testIdentity(), cp, nil, nil)
|
||||
a := newAgent(config{}, testIdentity(), cp, nil, nil)
|
||||
a.runTask(context.Background(), testAssign())
|
||||
|
||||
st := cp.taskStates()
|
||||
if len(st) != 1 || st[0].LastStatus != ecsv1.TaskStatusStopped {
|
||||
if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
|
||||
t.Fatalf("want one STOPPED state, got %+v", st)
|
||||
}
|
||||
if st[0].Reason == "" {
|
||||
@@ -63,7 +63,7 @@ func TestRunTask_PullRunReportsRunning(t *testing.T) {
|
||||
}
|
||||
|
||||
st := cp.taskStates()
|
||||
if len(st) == 0 || st[len(st)-1].LastStatus != ecsv1.TaskStatusRunning {
|
||||
if len(st) == 0 || st[len(st)-1].LastStatus != bus.TaskStatusRunning {
|
||||
t.Fatalf("want RUNNING final state, got %+v", st)
|
||||
}
|
||||
if len(st[len(st)-1].Containers) != 1 || st[len(st)-1].Containers[0].ContainerID != "t-001-web" {
|
||||
@@ -87,7 +87,7 @@ func TestRunTask_RuntimeFieldsCarriedToRunSpec(t *testing.T) {
|
||||
as.Containers[0].Privileged = &priv
|
||||
as.Containers[0].PseudoTerminal = &tty
|
||||
as.Containers[0].Interactive = &interactive
|
||||
as.Containers[0].SystemControls = []ecsv1.SystemControl{{Namespace: "net.core.somaxconn", Value: "1024"}}
|
||||
as.Containers[0].SystemControls = []bus.SystemControl{{Namespace: "net.core.somaxconn", Value: "1024"}}
|
||||
as.Containers[0].CapAdd = []string{"SYS_PTRACE"}
|
||||
as.Containers[0].CapDrop = []string{"NET_RAW"}
|
||||
a.runTask(context.Background(), as)
|
||||
@@ -194,7 +194,7 @@ func TestStopTask_UsesContainerStopTimeout(t *testing.T) {
|
||||
}}}
|
||||
a := newRunAgent(cp, rt)
|
||||
|
||||
a.stopTask(context.Background(), ecsv1.StopDirective{TaskID: "t-001", Reason: "bye"})
|
||||
a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "bye"})
|
||||
|
||||
if len(rt.Stopped) != 1 || rt.Stopped[0] != "t-001-web" {
|
||||
t.Fatalf("want t-001-web stopped, got %+v", rt.Stopped)
|
||||
@@ -215,7 +215,7 @@ func TestStopTask_DefaultStopTimeoutWhenUnset(t *testing.T) {
|
||||
}}}
|
||||
a := newRunAgent(cp, rt)
|
||||
|
||||
a.stopTask(context.Background(), ecsv1.StopDirective{TaskID: "t-001", Reason: "bye"})
|
||||
a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "bye"})
|
||||
|
||||
if len(rt.StopTimeouts) != 1 || rt.StopTimeouts[0] != defaultStopTimeout {
|
||||
t.Fatalf("want default stopTimeout %v, got %+v", defaultStopTimeout, rt.StopTimeouts)
|
||||
@@ -230,7 +230,7 @@ func TestRunTask_GPUCarriedToRunSpec(t *testing.T) {
|
||||
rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
|
||||
a := newRunAgent(cp, rt)
|
||||
as := testAssign()
|
||||
as.Containers = append(as.Containers, ecsv1.AssignContainer{
|
||||
as.Containers = append(as.Containers, bus.AssignContainer{
|
||||
Name: "trainer", Image: "registry/trainer:1", GPU: 2,
|
||||
})
|
||||
a.runTask(context.Background(), as)
|
||||
@@ -252,19 +252,19 @@ func TestRunTask_GPUCarriedToRunSpec(t *testing.T) {
|
||||
func TestRunTask_GPUPinsAndReportsUUIDs(t *testing.T) {
|
||||
cp := &fakeCP{}
|
||||
rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
|
||||
a := newAgent(Config{}, testIdentityWithGPUs("GPU-aaa", "GPU-bbb"), cp, rt, nil)
|
||||
a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa", "GPU-bbb"), cp, rt, nil)
|
||||
as := testAssign()
|
||||
as.Containers = append(as.Containers, ecsv1.AssignContainer{
|
||||
as.Containers = append(as.Containers, bus.AssignContainer{
|
||||
Name: "trainer", Image: "registry/trainer:1", GPU: 1,
|
||||
})
|
||||
a.runTask(context.Background(), as)
|
||||
|
||||
st := cp.taskStates()
|
||||
if len(st) == 0 || st[len(st)-1].LastStatus != ecsv1.TaskStatusRunning {
|
||||
if len(st) == 0 || st[len(st)-1].LastStatus != bus.TaskStatusRunning {
|
||||
t.Fatalf("want RUNNING final state, got %+v", st)
|
||||
}
|
||||
last := st[len(st)-1]
|
||||
var trainer *ecsv1.ContainerStatus
|
||||
var trainer *bus.ContainerStatus
|
||||
for i := range last.Containers {
|
||||
if last.Containers[i].Name == "trainer" {
|
||||
trainer = &last.Containers[i]
|
||||
@@ -296,9 +296,9 @@ func TestRunTask_GPUPinsAndReportsUUIDs(t *testing.T) {
|
||||
func TestRunTask_GPUIDsReachRunSpec(t *testing.T) {
|
||||
cp := &fakeCP{}
|
||||
rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
|
||||
a := newAgent(Config{}, testIdentityWithGPUs("GPU-aaa", "GPU-bbb"), cp, rt, nil)
|
||||
a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa", "GPU-bbb"), cp, rt, nil)
|
||||
as := testAssign()
|
||||
as.Containers = append(as.Containers, ecsv1.AssignContainer{
|
||||
as.Containers = append(as.Containers, bus.AssignContainer{
|
||||
Name: "trainer", Image: "registry/trainer:1", GPU: 1,
|
||||
})
|
||||
a.runTask(context.Background(), as)
|
||||
@@ -339,7 +339,7 @@ func TestRunTask_GPUShortLedgerStillRuns(t *testing.T) {
|
||||
rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
|
||||
a := newRunAgent(cp, rt) // testIdentity() has no discovered GPU UUIDs
|
||||
as := testAssign()
|
||||
as.Containers = append(as.Containers, ecsv1.AssignContainer{
|
||||
as.Containers = append(as.Containers, bus.AssignContainer{
|
||||
Name: "trainer", Image: "registry/trainer:1", GPU: 2,
|
||||
})
|
||||
a.runTask(context.Background(), as)
|
||||
@@ -361,7 +361,7 @@ func TestStopTask_ReleasesGPUsBackToLedger(t *testing.T) {
|
||||
"mulga.ecs.taskID": "t-001", "mulga.ecs.containerName": "trainer",
|
||||
}},
|
||||
}}
|
||||
a := newAgent(Config{}, testIdentityWithGPUs("GPU-aaa"), cp, rt, nil)
|
||||
a := newAgent(config{}, testIdentityWithGPUs("GPU-aaa"), cp, rt, nil)
|
||||
if _, err := a.gpu.Pin(gpuKey("t-001", "trainer"), 1); err != nil {
|
||||
t.Fatalf("seed pin: %v", err)
|
||||
}
|
||||
@@ -369,7 +369,7 @@ func TestStopTask_ReleasesGPUsBackToLedger(t *testing.T) {
|
||||
t.Fatalf("setup: want ledger fully pinned, free = %v", a.gpu.free)
|
||||
}
|
||||
|
||||
a.stopTask(context.Background(), ecsv1.StopDirective{TaskID: "t-001", Reason: "bye"})
|
||||
a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "bye"})
|
||||
|
||||
if len(a.gpu.free) != 1 {
|
||||
t.Errorf("want the GPU released back to the ledger, free = %v", a.gpu.free)
|
||||
@@ -387,7 +387,7 @@ func TestRunTask_PullFailureReportsStopped(t *testing.T) {
|
||||
t.Errorf("container should not run after pull failure, got %+v", rt.Runs)
|
||||
}
|
||||
st := cp.taskStates()
|
||||
if len(st) != 1 || st[0].LastStatus != ecsv1.TaskStatusStopped {
|
||||
if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
|
||||
t.Fatalf("want STOPPED, got %+v", st)
|
||||
}
|
||||
}
|
||||
@@ -400,7 +400,7 @@ func TestRunTask_RunFailureReportsStopped(t *testing.T) {
|
||||
a.runTask(context.Background(), testAssign())
|
||||
|
||||
st := cp.taskStates()
|
||||
if len(st) != 1 || st[0].LastStatus != ecsv1.TaskStatusStopped {
|
||||
if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
|
||||
t.Fatalf("want STOPPED, got %+v", st)
|
||||
}
|
||||
}
|
||||
@@ -415,7 +415,7 @@ func TestRunTask_ExitReportsStoppedWithCode(t *testing.T) {
|
||||
deadline := time.After(time.Second)
|
||||
for {
|
||||
st := cp.taskStates()
|
||||
if len(st) > 0 && st[len(st)-1].LastStatus == ecsv1.TaskStatusStopped {
|
||||
if len(st) > 0 && st[len(st)-1].LastStatus == bus.TaskStatusStopped {
|
||||
last := st[len(st)-1]
|
||||
if len(last.Containers) != 1 || last.Containers[0].ExitCode == nil || *last.Containers[0].ExitCode != 7 {
|
||||
t.Fatalf("want exit code 7, got %+v", last.Containers)
|
||||
@@ -433,13 +433,13 @@ func TestRunTask_ExitReportsStoppedWithCode(t *testing.T) {
|
||||
// pollAssignments dispatches each assign once and acks it on the next poll.
|
||||
func TestPollAssignments_DispatchesOnceAndAcks(t *testing.T) {
|
||||
as := testAssign()
|
||||
cp := &fakeCP{pollReplies: [][]ecsv1.Assign{
|
||||
cp := &fakeCP{pollReplies: [][]bus.Assign{
|
||||
{*as}, // poll 1: one pending assign
|
||||
{*as}, // poll 2: still present (not yet acked when poll 2 was issued)
|
||||
nil, // poll 3: gateway dropped it after the ack
|
||||
}}
|
||||
rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
|
||||
a := newAgent(Config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, nil)
|
||||
a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, nil)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
go a.pollAssignments(ctx, map[string]bool{})
|
||||
@@ -449,7 +449,7 @@ func TestPollAssignments_DispatchesOnceAndAcks(t *testing.T) {
|
||||
// exactly one RUNNING report.
|
||||
deadline := time.After(time.Second)
|
||||
for {
|
||||
if running := countStatus(cp.taskStates(), as.TaskID, ecsv1.TaskStatusRunning); running >= 1 {
|
||||
if running := countStatus(cp.taskStates(), as.TaskID, bus.TaskStatusRunning); running >= 1 {
|
||||
break
|
||||
}
|
||||
select {
|
||||
@@ -461,7 +461,7 @@ func TestPollAssignments_DispatchesOnceAndAcks(t *testing.T) {
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
cancel()
|
||||
|
||||
if running := countStatus(cp.taskStates(), as.TaskID, ecsv1.TaskStatusRunning); running != 1 {
|
||||
if running := countStatus(cp.taskStates(), as.TaskID, bus.TaskStatusRunning); running != 1 {
|
||||
t.Errorf("task dispatched %d times, want exactly 1 (dedup on taskID)", running)
|
||||
}
|
||||
// The taskID must appear in a later poll's ack list.
|
||||
@@ -492,19 +492,19 @@ func TestStopTask_ReapsLabeledContainersAndReportsStopped(t *testing.T) {
|
||||
}}
|
||||
a := newRunAgent(cp, rt)
|
||||
|
||||
a.stopTask(context.Background(), ecsv1.StopDirective{TaskID: "t-001", Reason: "bye"})
|
||||
a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "bye"})
|
||||
|
||||
if len(rt.Stopped) != 1 || rt.Stopped[0] != "t-001-web" {
|
||||
t.Fatalf("want only t-001-web stopped, got %+v", rt.Stopped)
|
||||
}
|
||||
st := cp.taskStates()
|
||||
if len(st) != 1 || st[0].LastStatus != ecsv1.TaskStatusStopped {
|
||||
if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
|
||||
t.Fatalf("want one STOPPED report, got %+v", st)
|
||||
}
|
||||
if st[0].Reason != "bye" {
|
||||
t.Errorf("want reason %q, got %q", "bye", st[0].Reason)
|
||||
}
|
||||
if len(st[0].Containers) != 1 || st[0].Containers[0].Status != ecsv1.TaskStatusStopped {
|
||||
if len(st[0].Containers) != 1 || st[0].Containers[0].Status != bus.TaskStatusStopped {
|
||||
t.Errorf("want one STOPPED container, got %+v", st[0].Containers)
|
||||
}
|
||||
}
|
||||
@@ -516,13 +516,13 @@ func TestStopTask_NoContainersStillReportsStopped(t *testing.T) {
|
||||
rt := &ctrruntime.FakePuller{}
|
||||
a := newRunAgent(cp, rt)
|
||||
|
||||
a.stopTask(context.Background(), ecsv1.StopDirective{TaskID: "t-001", Reason: "gone"})
|
||||
a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "gone"})
|
||||
|
||||
if len(rt.Removed) != 0 {
|
||||
t.Errorf("nothing to remove, got %+v", rt.Removed)
|
||||
}
|
||||
st := cp.taskStates()
|
||||
if len(st) != 1 || st[0].LastStatus != ecsv1.TaskStatusStopped {
|
||||
if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
|
||||
t.Fatalf("want one STOPPED report, got %+v", st)
|
||||
}
|
||||
}
|
||||
@@ -532,20 +532,20 @@ func TestStopTask_NoContainersStillReportsStopped(t *testing.T) {
|
||||
func TestPollAssignments_StopReapsAndSuppressesAssign(t *testing.T) {
|
||||
as := testAssign()
|
||||
cp := &fakeCP{
|
||||
pollReplies: [][]ecsv1.Assign{{*as}, {*as}, nil},
|
||||
stopPending: []ecsv1.StopDirective{{TaskID: as.TaskID, Reason: "bye"}},
|
||||
pollReplies: [][]bus.Assign{{*as}, {*as}, nil},
|
||||
stopPending: []bus.StopDirective{{TaskID: as.TaskID, Reason: "bye"}},
|
||||
}
|
||||
rt := &ctrruntime.FakePuller{Containers: []ctrruntime.Container{
|
||||
{ID: "t-001-web", Running: true, Labels: map[string]string{"mulga.ecs.taskID": "t-001"}},
|
||||
}}
|
||||
a := newAgent(Config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, nil)
|
||||
a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, nil)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
go a.pollAssignments(ctx, map[string]bool{})
|
||||
|
||||
deadline := time.After(time.Second)
|
||||
for {
|
||||
if stopped := countStatus(cp.taskStates(), as.TaskID, ecsv1.TaskStatusStopped); stopped >= 1 {
|
||||
if stopped := countStatus(cp.taskStates(), as.TaskID, bus.TaskStatusStopped); stopped >= 1 {
|
||||
break
|
||||
}
|
||||
select {
|
||||
@@ -561,7 +561,7 @@ func TestPollAssignments_StopReapsAndSuppressesAssign(t *testing.T) {
|
||||
if len(rt.Runs) != 0 {
|
||||
t.Errorf("assign for a stopping task should not run, got %+v", rt.Runs)
|
||||
}
|
||||
if running := countStatus(cp.taskStates(), as.TaskID, ecsv1.TaskStatusRunning); running != 0 {
|
||||
if running := countStatus(cp.taskStates(), as.TaskID, bus.TaskStatusRunning); running != 0 {
|
||||
t.Errorf("stopping task should never report RUNNING, got %d", running)
|
||||
}
|
||||
acked := false
|
||||
@@ -591,9 +591,9 @@ func countStopAcks(acks [][]string, taskID string) int {
|
||||
// A reap that cannot run is never acked, so the directive stays in the gateway's
|
||||
// inbox and is redelivered and retried until the runtime comes back.
|
||||
func TestPollAssignments_UnperformableStopIsNotAckedAndRetried(t *testing.T) {
|
||||
cp := &fakeCP{stopPending: []ecsv1.StopDirective{{TaskID: "t-001", Reason: "bye"}}}
|
||||
cp := &fakeCP{stopPending: []bus.StopDirective{{TaskID: "t-001", Reason: "bye"}}}
|
||||
rt := &ctrruntime.FakePuller{ListErr: errors.New("containerd is down")}
|
||||
a := newAgent(Config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, nil)
|
||||
a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, nil)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
@@ -612,7 +612,7 @@ func TestPollAssignments_UnperformableStopIsNotAckedAndRetried(t *testing.T) {
|
||||
if n := countStopAcks(cp.stopAcks(), "t-001"); n != 0 {
|
||||
t.Errorf("a reap that never ran was acked %d times; stopAcks=%v", n, cp.stopAcks())
|
||||
}
|
||||
if stopped := countStatus(cp.taskStates(), "t-001", ecsv1.TaskStatusStopped); stopped != 0 {
|
||||
if stopped := countStatus(cp.taskStates(), "t-001", bus.TaskStatusStopped); stopped != 0 {
|
||||
t.Errorf("a reap that never ran reported STOPPED %d times", stopped)
|
||||
}
|
||||
}
|
||||
@@ -620,7 +620,7 @@ func TestPollAssignments_UnperformableStopIsNotAckedAndRetried(t *testing.T) {
|
||||
// Once the runtime recovers, the redelivered directive is reaped, reported
|
||||
// STOPPED and acked exactly once, which deletes it from the inbox.
|
||||
func TestPollAssignments_RecoveredStopIsAckedExactlyOnce(t *testing.T) {
|
||||
cp := &fakeCP{stopPending: []ecsv1.StopDirective{{TaskID: "t-001", Reason: "bye"}}}
|
||||
cp := &fakeCP{stopPending: []bus.StopDirective{{TaskID: "t-001", Reason: "bye"}}}
|
||||
rt := &ctrruntime.FakePuller{
|
||||
ListErr: errors.New("containerd is down"),
|
||||
ListErrFor: 1,
|
||||
@@ -628,7 +628,7 @@ func TestPollAssignments_RecoveredStopIsAckedExactlyOnce(t *testing.T) {
|
||||
{ID: "t-001-web", Running: true, Labels: map[string]string{"mulga.ecs.taskID": "t-001"}},
|
||||
},
|
||||
}
|
||||
a := newAgent(Config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, nil)
|
||||
a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, nil)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
@@ -648,7 +648,7 @@ func TestPollAssignments_RecoveredStopIsAckedExactlyOnce(t *testing.T) {
|
||||
if n := countStopAcks(cp.stopAcks(), "t-001"); n != 1 {
|
||||
t.Errorf("stop acked %d times, want exactly 1; stopAcks=%v", n, cp.stopAcks())
|
||||
}
|
||||
if stopped := countStatus(cp.taskStates(), "t-001", ecsv1.TaskStatusStopped); stopped != 1 {
|
||||
if stopped := countStatus(cp.taskStates(), "t-001", bus.TaskStatusStopped); stopped != 1 {
|
||||
t.Errorf("task reported STOPPED %d times, want exactly 1", stopped)
|
||||
}
|
||||
if reaped := rt.Reaped(); len(reaped) != 1 || reaped[0] != "t-001-web" {
|
||||
@@ -661,13 +661,13 @@ func TestPollAssignments_RecoveredStopIsAckedExactlyOnce(t *testing.T) {
|
||||
func TestPollAssignments_StoppedTaskIsNotRerunByRedeliveredAssign(t *testing.T) {
|
||||
as := testAssign()
|
||||
cp := &fakeCP{
|
||||
pollReplies: [][]ecsv1.Assign{{*as}, {*as}, {*as}, {*as}, {*as}, {*as}},
|
||||
stopPending: []ecsv1.StopDirective{{TaskID: as.TaskID, Reason: "bye"}},
|
||||
pollReplies: [][]bus.Assign{{*as}, {*as}, {*as}, {*as}, {*as}, {*as}},
|
||||
stopPending: []bus.StopDirective{{TaskID: as.TaskID, Reason: "bye"}},
|
||||
}
|
||||
rt := &ctrruntime.FakePuller{Containers: []ctrruntime.Container{
|
||||
{ID: "t-001-web", Running: true, Labels: map[string]string{"mulga.ecs.taskID": "t-001"}},
|
||||
}}
|
||||
a := newAgent(Config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, nil)
|
||||
a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, nil)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
@@ -688,7 +688,7 @@ func TestPollAssignments_StoppedTaskIsNotRerunByRedeliveredAssign(t *testing.T)
|
||||
if runs := rt.RunCalls(); len(runs) != 0 {
|
||||
t.Errorf("stopped task was started by a redelivered assign: %+v", runs)
|
||||
}
|
||||
if running := countStatus(cp.taskStates(), as.TaskID, ecsv1.TaskStatusRunning); running != 0 {
|
||||
if running := countStatus(cp.taskStates(), as.TaskID, bus.TaskStatusRunning); running != 0 {
|
||||
t.Errorf("stopped task reported RUNNING %d times", running)
|
||||
}
|
||||
}
|
||||
@@ -696,8 +696,8 @@ func TestPollAssignments_StoppedTaskIsNotRerunByRedeliveredAssign(t *testing.T)
|
||||
// stopTask with no runtime is a no-op: nothing to reap, no state reported.
|
||||
func TestStopTask_NilRunnerNoOp(t *testing.T) {
|
||||
cp := &fakeCP{}
|
||||
a := newAgent(Config{}, testIdentity(), cp, nil, nil)
|
||||
a.stopTask(context.Background(), ecsv1.StopDirective{TaskID: "t-001", Reason: "x"})
|
||||
a := newAgent(config{}, testIdentity(), cp, nil, nil)
|
||||
a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "x"})
|
||||
if len(cp.taskStates()) != 0 {
|
||||
t.Fatalf("nil runner should report no state, got %+v", cp.taskStates())
|
||||
}
|
||||
@@ -708,7 +708,7 @@ func TestStopTask_ListErrorNoReport(t *testing.T) {
|
||||
cp := &fakeCP{}
|
||||
rt := &ctrruntime.FakePuller{ListErr: errors.New("list boom")}
|
||||
a := newRunAgent(cp, rt)
|
||||
a.stopTask(context.Background(), ecsv1.StopDirective{TaskID: "t-001", Reason: "x"})
|
||||
a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001", Reason: "x"})
|
||||
if len(cp.taskStates()) != 0 {
|
||||
t.Fatalf("list error should report no state, got %+v", cp.taskStates())
|
||||
}
|
||||
@@ -725,13 +725,13 @@ func TestStopTask_EmptyReasonReapsWithDefault(t *testing.T) {
|
||||
},
|
||||
}}}
|
||||
a := newRunAgent(cp, rt)
|
||||
a.stopTask(context.Background(), ecsv1.StopDirective{TaskID: "t-001"})
|
||||
a.stopTask(context.Background(), bus.StopDirective{TaskID: "t-001"})
|
||||
|
||||
if len(rt.Stopped) != 1 || rt.Stopped[0] != "t-001-web" {
|
||||
t.Fatalf("want container t-001-web reaped, got %+v", rt.Stopped)
|
||||
}
|
||||
st := cp.taskStates()
|
||||
if len(st) != 1 || st[0].LastStatus != ecsv1.TaskStatusStopped {
|
||||
if len(st) != 1 || st[0].LastStatus != bus.TaskStatusStopped {
|
||||
t.Fatalf("want one STOPPED report, got %+v", st)
|
||||
}
|
||||
if st[0].Reason != "Task stopped" {
|
||||
@@ -739,7 +739,7 @@ func TestStopTask_EmptyReasonReapsWithDefault(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func countStatus(states []ecsv1.TaskState, taskID, status string) int {
|
||||
func countStatus(states []bus.TaskState, taskID, status string) int {
|
||||
n := 0
|
||||
for _, st := range states {
|
||||
if st.TaskID == taskID && st.LastStatus == status {
|
||||
@@ -807,12 +807,12 @@ func (sentinelResolver) Authorize(context.Context, string) (string, string, stri
|
||||
// exec-role resolver is built.
|
||||
func TestPullResolver(t *testing.T) {
|
||||
fake := sentinelResolver{}
|
||||
a := newAgent(Config{Region: "us-east-1"}, testIdentity(), &fakeCP{}, nil, fake)
|
||||
a := newAgent(config{Region: "us-east-1"}, testIdentity(), &fakeCP{}, nil, fake)
|
||||
|
||||
if got := a.pullResolver(&ecsv1.Assign{TaskID: "t1"}); got != fake {
|
||||
if got := a.pullResolver(&bus.Assign{TaskID: "t1"}); got != fake {
|
||||
t.Fatalf("no execution role: want instance resolver fallback")
|
||||
}
|
||||
execAssign := &ecsv1.Assign{TaskID: "t1", ExecutionRoleARN: "arn:aws:iam::1:role/e"}
|
||||
execAssign := &bus.Assign{TaskID: "t1", ExecutionRoleARN: "arn:aws:iam::1:role/e"}
|
||||
if got := a.pullResolver(execAssign); got != fake {
|
||||
t.Fatalf("nil cred endpoint: want instance resolver fallback")
|
||||
}
|
||||
+170
-17
@@ -1,43 +1,196 @@
|
||||
// eks-gateway-fetch runs inside the EKS K3s control-plane VM and fetches a
|
||||
// control-plane resource from the host through the AWS gateway, writing it to
|
||||
// stdout as TSV; the implementation is in agents/eks/gatewayfetch.
|
||||
// control-plane resource from the host through the AWS gateway, instead of
|
||||
// dialing core NATS directly. It is the read-side companion to
|
||||
// eks-gateway-publish.
|
||||
//
|
||||
// It SigV4-signs (service "eks") an HTTPS GET to the gateway, retrying with
|
||||
// backoff until the gateway returns 2xx or the attempt budget is exhausted (so
|
||||
// a degraded link surfaces as a non-zero exit), then emits the staged add-ons
|
||||
// as tab-separated lines the on-VM addon-sync shell agent consumes without a
|
||||
// JSON parser:
|
||||
//
|
||||
// <addonName>\t<addonVersion>\t<serviceAccountRoleArn>\t<base64(configurationValues)>
|
||||
//
|
||||
// The configuration values are base64-encoded so embedded tabs/newlines cannot
|
||||
// break the line framing.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// eks-gateway-fetch -resource addons # GET /clusters/{cluster}/internal-addons/{accountId}
|
||||
// eks-gateway-fetch -resource recovery -instance-id i-… # GET /clusters/{cluster}/internal-recovery/{accountId}/{instanceId}
|
||||
//
|
||||
// The recovery resource returns a single epoch\taction\tsnapshot line the on-VM
|
||||
// k3s-recovery agent applies before k3s starts (etcd cluster-reset / wipe-rejoin).
|
||||
//
|
||||
// Flags default to environment variables seeded by cloud-init:
|
||||
// EKS_GATEWAY_URL, EKS_GATEWAY_CA, EKS_ACCESS_KEY, EKS_SECRET_KEY, EKS_REGION,
|
||||
// EKS_ACCOUNT_ID, EKS_CLUSTER_NAME, EKS_INSTANCE_ID.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"bufio"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/eksgw"
|
||||
|
||||
_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
|
||||
)
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/eks/gatewayfetch"
|
||||
// stagedAddon mirrors the gateway's StagedAddonManifest wire shape; duplicated
|
||||
// here to keep this tiny VM binary free of the handler package's dependency
|
||||
// graph.
|
||||
type stagedAddon struct {
|
||||
AddonName string `json:"addonName"`
|
||||
AddonVersion string `json:"addonVersion"`
|
||||
ServiceAccountRoleArn string `json:"serviceAccountRoleArn"`
|
||||
ConfigurationValues string `json:"configurationValues"`
|
||||
}
|
||||
|
||||
type internalAddonsResponse struct {
|
||||
Addons []stagedAddon `json:"addons"`
|
||||
}
|
||||
|
||||
const (
|
||||
maxAttempts = 30
|
||||
retryDelay = 5 * time.Second
|
||||
)
|
||||
|
||||
func main() {
|
||||
cfg := gatewayfetch.Config{Out: os.Stdout}
|
||||
flag.StringVar(&cfg.GatewayURL, "gateway", os.Getenv("EKS_GATEWAY_URL"), "Gateway URL (e.g. https://10.15.8.1:9999)")
|
||||
flag.StringVar(&cfg.GatewayCA, "gateway-ca", os.Getenv("EKS_GATEWAY_CA"), "Path to gateway TLS CA PEM (optional; falls back to system trust)")
|
||||
flag.StringVar(&cfg.AccessKey, "access-key", os.Getenv("EKS_ACCESS_KEY"), "AWS access key ID")
|
||||
flag.StringVar(&cfg.SecretKey, "secret-key", os.Getenv("EKS_SECRET_KEY"), "AWS secret access key")
|
||||
flag.StringVar(&cfg.Region, "region", os.Getenv("EKS_REGION"), "AWS region for SigV4 signing")
|
||||
flag.StringVar(&cfg.AccountID, "account-id", os.Getenv("EKS_ACCOUNT_ID"), "Cluster account ID")
|
||||
flag.StringVar(&cfg.ClusterName, "cluster", os.Getenv("EKS_CLUSTER_NAME"), "Cluster name")
|
||||
flag.StringVar(&cfg.InstanceID, "instance-id", os.Getenv("EKS_INSTANCE_ID"), "This member's EC2 instance ID (recovery resource)")
|
||||
flag.StringVar(&cfg.Resource, "resource", "addons", "Resource to fetch: addons | recovery")
|
||||
var (
|
||||
gatewayURL string
|
||||
gatewayCA string
|
||||
accessKey string
|
||||
secretKey string
|
||||
region string
|
||||
accountID string
|
||||
clusterName string
|
||||
resource string
|
||||
instanceID string
|
||||
)
|
||||
|
||||
flag.StringVar(&gatewayURL, "gateway", os.Getenv("EKS_GATEWAY_URL"), "Gateway URL (e.g. https://10.15.8.1:9999)")
|
||||
flag.StringVar(&gatewayCA, "gateway-ca", os.Getenv("EKS_GATEWAY_CA"), "Path to gateway TLS CA PEM (optional; falls back to system trust)")
|
||||
flag.StringVar(&accessKey, "access-key", os.Getenv("EKS_ACCESS_KEY"), "AWS access key ID")
|
||||
flag.StringVar(&secretKey, "secret-key", os.Getenv("EKS_SECRET_KEY"), "AWS secret access key")
|
||||
flag.StringVar(®ion, "region", os.Getenv("EKS_REGION"), "AWS region for SigV4 signing")
|
||||
flag.StringVar(&accountID, "account-id", os.Getenv("EKS_ACCOUNT_ID"), "Cluster account ID")
|
||||
flag.StringVar(&clusterName, "cluster", os.Getenv("EKS_CLUSTER_NAME"), "Cluster name")
|
||||
flag.StringVar(&instanceID, "instance-id", os.Getenv("EKS_INSTANCE_ID"), "This member's EC2 instance ID (recovery resource)")
|
||||
flag.StringVar(&resource, "resource", "addons", "Resource to fetch: addons | recovery")
|
||||
flag.Parse()
|
||||
|
||||
if err := gatewayfetch.Run(context.Background(), cfg); err != nil {
|
||||
slog.Error("eks-gateway-fetch: " + err.Error())
|
||||
os.Exit(1)
|
||||
switch {
|
||||
case accountID == "":
|
||||
fatal("--account-id is required (or set EKS_ACCOUNT_ID)")
|
||||
case clusterName == "":
|
||||
fatal("--cluster is required (or set EKS_CLUSTER_NAME)")
|
||||
case resource != "addons" && resource != "recovery":
|
||||
fatal("--resource must be addons or recovery")
|
||||
case resource == "recovery" && instanceID == "":
|
||||
fatal("--instance-id is required for --resource recovery (or set EKS_INSTANCE_ID)")
|
||||
}
|
||||
|
||||
client, err := eksgw.New(gatewayURL, gatewayCA, accessKey, secretKey, region)
|
||||
if err != nil {
|
||||
fatal(fmt.Sprintf("build gateway client: %v", err))
|
||||
}
|
||||
|
||||
var (
|
||||
path string
|
||||
emit func(io.Writer, []byte) error
|
||||
)
|
||||
switch resource {
|
||||
case "recovery":
|
||||
path = "/clusters/" + clusterName + "/internal-recovery/" + accountID + "/" + instanceID
|
||||
emit = emitRecoveryTSV
|
||||
default:
|
||||
path = "/clusters/" + clusterName + "/internal-addons/" + accountID
|
||||
emit = emitAddonsTSV
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= maxAttempts; attempt++ {
|
||||
body, err := client.Get(path)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
slog.Warn("eks-gateway-fetch: attempt failed", "resource", resource, "attempt", attempt, "err", err)
|
||||
if attempt < maxAttempts {
|
||||
time.Sleep(retryDelay)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if werr := emit(os.Stdout, body); werr != nil {
|
||||
fatal(fmt.Sprintf("render response: %v", werr))
|
||||
}
|
||||
return
|
||||
}
|
||||
fatal(fmt.Sprintf("fetch failed after %d attempts: %v", maxAttempts, lastErr))
|
||||
}
|
||||
|
||||
// emitAddonsTSV decodes the internal-addons response and writes one TSV line per
|
||||
// staged add-on to w. An empty add-on set writes nothing (the agent then GCs
|
||||
// every locally-rendered manifest), which is the correct steady state for a
|
||||
// cluster with no managed add-ons.
|
||||
func emitAddonsTSV(out io.Writer, body []byte) error {
|
||||
var resp internalAddonsResponse
|
||||
if err := json.Unmarshal(body, &resp); err != nil {
|
||||
return fmt.Errorf("unmarshal internal-addons response: %w", err)
|
||||
}
|
||||
w := bufio.NewWriter(out)
|
||||
for _, a := range resp.Addons {
|
||||
cfg := base64.StdEncoding.EncodeToString([]byte(a.ConfigurationValues))
|
||||
if _, err := fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", a.AddonName, a.AddonVersion, a.ServiceAccountRoleArn, cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return w.Flush()
|
||||
}
|
||||
|
||||
// recoveryDirective mirrors the gateway's RecoveryDirective wire shape; duplicated
|
||||
// here to keep this tiny VM binary free of the handler package's dependency graph.
|
||||
type recoveryDirective struct {
|
||||
Epoch int64 `json:"epoch"`
|
||||
Action string `json:"action"`
|
||||
Snapshot string `json:"snapshot"`
|
||||
SnapshotRequired bool `json:"snapshotRequired"`
|
||||
}
|
||||
|
||||
type internalRecoveryResponse struct {
|
||||
Directive recoveryDirective `json:"directive"`
|
||||
}
|
||||
|
||||
// emitRecoveryTSV decodes the recovery directive and writes a single
|
||||
// epoch\taction\tsnapshot\tsnapshotRequired line the on-VM k3s-recovery agent
|
||||
// consumes without a JSON parser. An unset action defaults to "none" (steady
|
||||
// state); snapshotRequired is 1 when the snapshot MUST restore (fresh DR seed
|
||||
// aborts boot on fetch failure) else 0.
|
||||
func emitRecoveryTSV(out io.Writer, body []byte) error {
|
||||
var resp internalRecoveryResponse
|
||||
if err := json.Unmarshal(body, &resp); err != nil {
|
||||
return fmt.Errorf("unmarshal internal-recovery response: %w", err)
|
||||
}
|
||||
d := resp.Directive
|
||||
if d.Action == "" {
|
||||
d.Action = "none"
|
||||
}
|
||||
required := "0"
|
||||
if d.SnapshotRequired {
|
||||
required = "1"
|
||||
}
|
||||
if _, err := fmt.Fprintf(out, "%d\t%s\t%s\t%s\n", d.Epoch, d.Action, d.Snapshot, required); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func fatal(msg string) {
|
||||
slog.Error("eks-gateway-fetch: " + msg)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestEmitAddonsTSV(t *testing.T) {
|
||||
body := []byte(`{"addons":[
|
||||
{"addonName":"spinifex-noop","addonVersion":"0.1.0"},
|
||||
{"addonName":"aws-load-balancer-controller","addonVersion":"2.8.1",
|
||||
"serviceAccountRoleArn":"arn:aws:iam::111122223333:role/alb",
|
||||
"configurationValues":"{\"replicaCount\":2}"}
|
||||
]}`)
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := emitAddonsTSV(&buf, body); err != nil {
|
||||
t.Fatalf("emitAddonsTSV: %v", err)
|
||||
}
|
||||
cfg := base64.StdEncoding.EncodeToString([]byte(`{"replicaCount":2}`))
|
||||
want := "spinifex-noop\t0.1.0\t\t\n" +
|
||||
"aws-load-balancer-controller\t2.8.1\tarn:aws:iam::111122223333:role/alb\t" + cfg + "\n"
|
||||
if buf.String() != want {
|
||||
t.Fatalf("TSV mismatch:\n got: %q\nwant: %q", buf.String(), want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmitAddonsTSV_Empty(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
if err := emitAddonsTSV(&buf, []byte(`{"addons":[]}`)); err != nil {
|
||||
t.Fatalf("emitAddonsTSV: %v", err)
|
||||
}
|
||||
if buf.Len() != 0 {
|
||||
t.Fatalf("empty addon set must emit nothing, got %q", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmitAddonsTSV_BadJSON(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
if err := emitAddonsTSV(&buf, []byte(`{not-json`)); err == nil {
|
||||
t.Fatal("expected error on malformed JSON")
|
||||
}
|
||||
}
|
||||
+109
-18
@@ -1,6 +1,13 @@
|
||||
// eks-gateway-publish runs inside the EKS K3s control-plane VM and relays a
|
||||
// single JSON publication from stdin to the host through the AWS gateway; the
|
||||
// implementation is in agents/eks/gatewaypublish.
|
||||
// single publication to the host through the AWS gateway, instead of dialing
|
||||
// core NATS directly.
|
||||
//
|
||||
// It reads a JSON payload on stdin, wraps it as
|
||||
// {accountId, channel, kind, payload}, SigV4-signs (service "eks") an HTTPS
|
||||
// POST to {gateway}/clusters/{cluster}/internal-publish, and retries with
|
||||
// backoff until the gateway returns 2xx or the attempt budget is exhausted —
|
||||
// so a degraded link surfaces as a non-zero exit rather than a silently
|
||||
// dropped message (the failure mode of fire-and-forget `nats pub`).
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
@@ -13,31 +20,115 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/eksgw"
|
||||
|
||||
_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/eks/gatewaypublish"
|
||||
)
|
||||
|
||||
const (
|
||||
// Retry budget: the control plane reaches readiness before this runs, so a
|
||||
// failing POST means a degraded link, not a cold start. Bounded so a stuck
|
||||
// boot still terminates the OpenRC service.
|
||||
maxAttempts = 30
|
||||
retryDelay = 5 * time.Second
|
||||
)
|
||||
|
||||
type publishBody struct {
|
||||
AccountID string `json:"accountId"`
|
||||
Channel string `json:"channel"`
|
||||
Kind string `json:"kind"`
|
||||
Payload json.RawMessage `json:"payload"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
cfg := gatewaypublish.Config{Payload: os.Stdin}
|
||||
flag.StringVar(&cfg.GatewayURL, "gateway", os.Getenv("EKS_GATEWAY_URL"), "Gateway URL (e.g. https://10.15.8.1:9999)")
|
||||
flag.StringVar(&cfg.GatewayCA, "gateway-ca", os.Getenv("EKS_GATEWAY_CA"), "Path to gateway TLS CA PEM (optional; falls back to system trust)")
|
||||
flag.StringVar(&cfg.AccessKey, "access-key", os.Getenv("EKS_ACCESS_KEY"), "AWS access key ID")
|
||||
flag.StringVar(&cfg.SecretKey, "secret-key", os.Getenv("EKS_SECRET_KEY"), "AWS secret access key")
|
||||
flag.StringVar(&cfg.Region, "region", os.Getenv("EKS_REGION"), "AWS region for SigV4 signing")
|
||||
flag.StringVar(&cfg.AccountID, "account-id", os.Getenv("EKS_ACCOUNT_ID"), "Cluster account ID")
|
||||
flag.StringVar(&cfg.ClusterName, "cluster", os.Getenv("EKS_CLUSTER_NAME"), "Cluster name")
|
||||
flag.StringVar(&cfg.Channel, "channel", "", "Publish channel: bootstrap|state|addon")
|
||||
flag.StringVar(&cfg.Kind, "kind", "", "Bootstrap subject kind (bootstrap channel only)")
|
||||
var (
|
||||
gatewayURL string
|
||||
gatewayCA string
|
||||
accessKey string
|
||||
secretKey string
|
||||
region string
|
||||
accountID string
|
||||
clusterName string
|
||||
channel string
|
||||
kind string
|
||||
)
|
||||
|
||||
flag.StringVar(&gatewayURL, "gateway", os.Getenv("EKS_GATEWAY_URL"), "Gateway URL (e.g. https://10.15.8.1:9999)")
|
||||
flag.StringVar(&gatewayCA, "gateway-ca", os.Getenv("EKS_GATEWAY_CA"), "Path to gateway TLS CA PEM (optional; falls back to system trust)")
|
||||
flag.StringVar(&accessKey, "access-key", os.Getenv("EKS_ACCESS_KEY"), "AWS access key ID")
|
||||
flag.StringVar(&secretKey, "secret-key", os.Getenv("EKS_SECRET_KEY"), "AWS secret access key")
|
||||
flag.StringVar(®ion, "region", os.Getenv("EKS_REGION"), "AWS region for SigV4 signing")
|
||||
flag.StringVar(&accountID, "account-id", os.Getenv("EKS_ACCOUNT_ID"), "Cluster account ID")
|
||||
flag.StringVar(&clusterName, "cluster", os.Getenv("EKS_CLUSTER_NAME"), "Cluster name")
|
||||
flag.StringVar(&channel, "channel", "", "Publish channel: bootstrap|state|addon")
|
||||
flag.StringVar(&kind, "kind", "", "Bootstrap subject kind (bootstrap channel only)")
|
||||
flag.Parse()
|
||||
|
||||
if err := gatewaypublish.Run(context.Background(), cfg); err != nil {
|
||||
slog.Error("eks-gateway-publish: " + err.Error())
|
||||
os.Exit(1)
|
||||
switch {
|
||||
case accountID == "":
|
||||
fatal("--account-id is required (or set EKS_ACCOUNT_ID)")
|
||||
case clusterName == "":
|
||||
fatal("--cluster is required (or set EKS_CLUSTER_NAME)")
|
||||
case channel != "bootstrap" && channel != "state" && channel != "addon":
|
||||
fatal("--channel must be bootstrap, state or addon")
|
||||
case channel == "bootstrap" && kind == "":
|
||||
fatal("--kind is required for the bootstrap channel")
|
||||
}
|
||||
|
||||
payload, err := io.ReadAll(os.Stdin)
|
||||
if err != nil {
|
||||
fatal(fmt.Sprintf("read stdin payload: %v", err))
|
||||
}
|
||||
if len(bytes.TrimSpace(payload)) == 0 {
|
||||
fatal("empty stdin payload")
|
||||
}
|
||||
if !json.Valid(payload) {
|
||||
fatal("stdin payload is not valid JSON")
|
||||
}
|
||||
|
||||
body, err := json.Marshal(publishBody{
|
||||
AccountID: accountID,
|
||||
Channel: channel,
|
||||
Kind: kind,
|
||||
Payload: json.RawMessage(payload),
|
||||
})
|
||||
if err != nil {
|
||||
fatal(fmt.Sprintf("marshal request body: %v", err))
|
||||
}
|
||||
|
||||
client, err := eksgw.New(gatewayURL, gatewayCA, accessKey, secretKey, region)
|
||||
if err != nil {
|
||||
fatal(fmt.Sprintf("build gateway client: %v", err))
|
||||
}
|
||||
path := "/clusters/" + clusterName + "/internal-publish"
|
||||
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= maxAttempts; attempt++ {
|
||||
if _, err := client.Post(path, body); err != nil {
|
||||
lastErr = err
|
||||
slog.Warn("eks-gateway-publish: attempt failed",
|
||||
"channel", channel, "kind", kind, "attempt", attempt, "err", err)
|
||||
if attempt < maxAttempts {
|
||||
time.Sleep(retryDelay)
|
||||
}
|
||||
continue
|
||||
}
|
||||
slog.Info("eks-gateway-publish: published", "channel", channel, "kind", kind)
|
||||
return
|
||||
}
|
||||
fatal(fmt.Sprintf("publish failed after %d attempts: %v", maxAttempts, lastErr))
|
||||
}
|
||||
|
||||
func fatal(msg string) {
|
||||
slog.Error("eks-gateway-publish: " + msg)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,16 @@
|
||||
// Command eks-konnectivity-cert mints, then reuses, a K3s-CA-signed serving
|
||||
// certificate for the on-node konnectivity-server; the implementation is in
|
||||
// agents/eks/konnectivitycert.
|
||||
// Command eks-konnectivity-cert mints, then reuses, an ECDSA-P256 serving
|
||||
// certificate for the on-node konnectivity-server, signed by the K3s cluster CA.
|
||||
//
|
||||
// The konnectivity-agent (DaemonSet on workers) verifies the server it dials
|
||||
// using the in-pod ServiceAccount CA (/var/run/secrets/.../ca.crt), which in
|
||||
// K3s is the same server-ca that signs the apiserver. Signing the konnectivity
|
||||
// serving cert with that CA is therefore what lets the agent validate the
|
||||
// reverse-tunnel endpoint with no separate CA distribution.
|
||||
//
|
||||
// The eks-node AMI carries no openssl, so this is the Go equivalent: load the
|
||||
// K3s server CA cert+key, issue a leaf (ServerAuth) cert carrying the cluster's
|
||||
// konnectivity SANs (the NLB private-endpoint IP, public endpoint IP, NLB DNS),
|
||||
// and cache it under -dir so the OpenRC service is byte-stable across restarts.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
@@ -13,27 +23,225 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/pem"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math/big"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/eks/konnectivitycert"
|
||||
)
|
||||
|
||||
func main() {
|
||||
cfg := konnectivitycert.Config{Out: os.Stdout}
|
||||
flag.StringVar(&cfg.Dir, "dir", "", "directory to cache tls.crt/tls.key in")
|
||||
flag.StringVar(&cfg.CACert, "ca-cert", "", "PEM path of the signing (K3s server) CA cert")
|
||||
flag.StringVar(&cfg.CAKey, "ca-key", "", "PEM path of the signing CA private key")
|
||||
flag.StringVar(&cfg.CN, "cn", "konnectivity-server", "certificate common name")
|
||||
flag.StringVar(&cfg.SANs, "sans", "", "comma-separated SANs (IP or DNS)")
|
||||
dir := flag.String("dir", "", "directory to cache tls.crt/tls.key in")
|
||||
caCert := flag.String("ca-cert", "", "PEM path of the signing (K3s server) CA cert")
|
||||
caKey := flag.String("ca-key", "", "PEM path of the signing CA private key")
|
||||
cn := flag.String("cn", "konnectivity-server", "certificate common name")
|
||||
sansCSV := flag.String("sans", "", "comma-separated SANs (IP or DNS)")
|
||||
flag.Parse()
|
||||
|
||||
if err := konnectivitycert.Run(context.Background(), cfg); err != nil {
|
||||
slog.Error("eks-konnectivity-cert: " + err.Error())
|
||||
os.Exit(1)
|
||||
if *dir == "" || *caCert == "" || *caKey == "" || *sansCSV == "" {
|
||||
fatal("eks-konnectivity-cert: -dir, -ca-cert, -ca-key and -sans are required")
|
||||
}
|
||||
dns, ips := splitSANs(*sansCSV)
|
||||
if len(dns) == 0 && len(ips) == 0 {
|
||||
fatal("eks-konnectivity-cert: -sans lists no usable names")
|
||||
}
|
||||
|
||||
certPath, keyPath, err := ensureCert(*dir, *caCert, *caKey, *cn, dns, ips)
|
||||
if err != nil {
|
||||
fatal("eks-konnectivity-cert: " + err.Error())
|
||||
}
|
||||
fmt.Printf("%s\t%s\n", certPath, keyPath)
|
||||
}
|
||||
|
||||
// ensureCert returns the cached cert/key paths under dir, minting and persisting
|
||||
// a fresh CA-signed pair when none is present, readable, and still SAN-current.
|
||||
func ensureCert(dir, caCertPath, caKeyPath, cn string, dns []string, ips []net.IP) (string, string, error) {
|
||||
certPath := filepath.Join(dir, "tls.crt")
|
||||
keyPath := filepath.Join(dir, "tls.key")
|
||||
|
||||
if certCoversSANs(certPath, keyPath, dns, ips) {
|
||||
return certPath, keyPath, nil
|
||||
}
|
||||
|
||||
caCert, caKey, err := loadCA(caCertPath, caKeyPath)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
certPEM, keyPEM, err := issueLeaf(caCert, caKey, cn, dns, ips)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if err = os.MkdirAll(dir, 0o700); err != nil {
|
||||
return "", "", fmt.Errorf("mkdir %s: %w", dir, err)
|
||||
}
|
||||
if err = os.WriteFile(certPath, certPEM, 0o644); err != nil {
|
||||
return "", "", fmt.Errorf("write %s: %w", certPath, err)
|
||||
}
|
||||
if err = os.WriteFile(keyPath, keyPEM, 0o600); err != nil {
|
||||
return "", "", fmt.Errorf("write %s: %w", keyPath, err)
|
||||
}
|
||||
return certPath, keyPath, nil
|
||||
}
|
||||
|
||||
// certCoversSANs reports whether the cached cert exists, parses, has a key, and
|
||||
// already carries every requested SAN — so a cert minted before the endpoint
|
||||
// set changed is re-issued rather than served stale.
|
||||
func certCoversSANs(certPath, keyPath string, dns []string, ips []net.IP) bool {
|
||||
c, err := os.ReadFile(certPath)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if k, kerr := os.ReadFile(keyPath); kerr != nil || len(k) == 0 {
|
||||
return false
|
||||
}
|
||||
der := decodeFirst(c)
|
||||
if der == nil {
|
||||
return false
|
||||
}
|
||||
crt, err := x509.ParseCertificate(der)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
have := make(map[string]struct{}, len(crt.DNSNames)+len(crt.IPAddresses))
|
||||
for _, n := range crt.DNSNames {
|
||||
have[n] = struct{}{}
|
||||
}
|
||||
for _, ip := range crt.IPAddresses {
|
||||
have[ip.String()] = struct{}{}
|
||||
}
|
||||
for _, n := range dns {
|
||||
if _, ok := have[n]; !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
for _, ip := range ips {
|
||||
if _, ok := have[ip.String()]; !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// loadCA parses the PEM CA cert + EC/PKCS8 private key used to sign the leaf.
|
||||
func loadCA(certPath, keyPath string) (*x509.Certificate, *ecdsa.PrivateKey, error) {
|
||||
certPEM, err := os.ReadFile(certPath)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("read ca cert %s: %w", certPath, err)
|
||||
}
|
||||
der := decodeFirst(certPEM)
|
||||
if der == nil {
|
||||
return nil, nil, fmt.Errorf("ca cert %s: no PEM block", certPath)
|
||||
}
|
||||
caCert, err := x509.ParseCertificate(der)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("parse ca cert: %w", err)
|
||||
}
|
||||
|
||||
keyPEM, err := os.ReadFile(keyPath)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("read ca key %s: %w", keyPath, err)
|
||||
}
|
||||
keyDER := decodeFirst(keyPEM)
|
||||
if keyDER == nil {
|
||||
return nil, nil, fmt.Errorf("ca key %s: no PEM block", keyPath)
|
||||
}
|
||||
caKey, err := parseECKey(keyDER)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return caCert, caKey, nil
|
||||
}
|
||||
|
||||
// parseECKey accepts the EC SEC1 or PKCS8 encodings K3s may write for the CA key.
|
||||
func parseECKey(der []byte) (*ecdsa.PrivateKey, error) {
|
||||
if k, err := x509.ParseECPrivateKey(der); err == nil {
|
||||
return k, nil
|
||||
}
|
||||
k8, err := x509.ParsePKCS8PrivateKey(der)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse ca key (sec1 and pkcs8 failed): %w", err)
|
||||
}
|
||||
ec, ok := k8.(*ecdsa.PrivateKey)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("ca key is %T, want ecdsa", k8)
|
||||
}
|
||||
return ec, nil
|
||||
}
|
||||
|
||||
// issueLeaf mints an ECDSA-P256 ServerAuth leaf for cn + SANs, signed by caCert/caKey.
|
||||
func issueLeaf(caCert *x509.Certificate, caKey *ecdsa.PrivateKey, cn string, dns []string, ips []net.IP) (certPEM, keyPEM []byte, err error) {
|
||||
priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("generate key: %w", err)
|
||||
}
|
||||
serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("serial: %w", err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
tmpl := x509.Certificate{
|
||||
SerialNumber: serial,
|
||||
Subject: pkix.Name{CommonName: cn},
|
||||
NotBefore: now.Add(-time.Hour),
|
||||
NotAfter: now.AddDate(10, 0, 0),
|
||||
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
BasicConstraintsValid: true,
|
||||
DNSNames: dns,
|
||||
IPAddresses: ips,
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, &tmpl, caCert, &priv.PublicKey, caKey)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create cert: %w", err)
|
||||
}
|
||||
keyDER, err := x509.MarshalECPrivateKey(priv)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("marshal key: %w", err)
|
||||
}
|
||||
certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
|
||||
keyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
|
||||
return certPEM, keyPEM, nil
|
||||
}
|
||||
|
||||
// splitSANs partitions the CSV into DNS names and IPs (an entry that parses as an
|
||||
// IP goes to IPAddresses, else DNSNames), matching how SAN matching is typed.
|
||||
func splitSANs(s string) (dns []string, ips []net.IP) {
|
||||
for p := range strings.SplitSeq(s, ",") {
|
||||
p = strings.TrimSpace(p)
|
||||
if p == "" {
|
||||
continue
|
||||
}
|
||||
if ip := net.ParseIP(p); ip != nil {
|
||||
ips = append(ips, ip)
|
||||
continue
|
||||
}
|
||||
dns = append(dns, p)
|
||||
}
|
||||
return dns, ips
|
||||
}
|
||||
|
||||
// decodeFirst returns the DER bytes of the first PEM block, or nil.
|
||||
func decodeFirst(pemBytes []byte) []byte {
|
||||
blk, _ := pem.Decode(pemBytes)
|
||||
if blk == nil {
|
||||
return nil
|
||||
}
|
||||
return blk.Bytes
|
||||
}
|
||||
|
||||
func fatal(msg string) {
|
||||
slog.Error(msg)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/pem"
|
||||
"math/big"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// writeTestCA mints a self-signed EC CA (cert + SEC1 key PEM) into dir, returning
|
||||
// the two paths — a stand-in for K3s' /var/lib/rancher/k3s/server/tls/server-ca.*.
|
||||
func writeTestCA(t *testing.T, dir string) (certPath, keyPath string) {
|
||||
t.Helper()
|
||||
priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatalf("gen ca key: %v", err)
|
||||
}
|
||||
tmpl := x509.Certificate{
|
||||
SerialNumber: big.NewInt(1),
|
||||
Subject: pkix.Name{CommonName: "k3s-server-ca"},
|
||||
NotBefore: time.Now().Add(-time.Hour),
|
||||
NotAfter: time.Now().AddDate(10, 0, 0),
|
||||
KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature,
|
||||
BasicConstraintsValid: true,
|
||||
IsCA: true,
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv)
|
||||
if err != nil {
|
||||
t.Fatalf("create ca cert: %v", err)
|
||||
}
|
||||
keyDER, err := x509.MarshalECPrivateKey(priv)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal ca key: %v", err)
|
||||
}
|
||||
certPath = filepath.Join(dir, "server-ca.crt")
|
||||
keyPath = filepath.Join(dir, "server-ca.key")
|
||||
if err := os.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644); err != nil {
|
||||
t.Fatalf("write ca cert: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o600); err != nil {
|
||||
t.Fatalf("write ca key: %v", err)
|
||||
}
|
||||
return certPath, keyPath
|
||||
}
|
||||
|
||||
func leafFrom(t *testing.T, certPath string) *x509.Certificate {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(certPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read leaf: %v", err)
|
||||
}
|
||||
crt, err := x509.ParseCertificate(decodeFirst(b))
|
||||
if err != nil {
|
||||
t.Fatalf("parse leaf: %v", err)
|
||||
}
|
||||
return crt
|
||||
}
|
||||
|
||||
func TestEnsureCertSignsLeafWithSANsThatChainsToCA(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
caCert, caKey := writeTestCA(t, dir)
|
||||
|
||||
dns, ips := splitSANs("10.32.100.4,203.0.113.9,eks-toc.example")
|
||||
certPath, keyPath, err := ensureCert(dir, caCert, caKey, "konnectivity-server", dns, ips)
|
||||
if err != nil {
|
||||
t.Fatalf("ensureCert: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(keyPath); err != nil {
|
||||
t.Fatalf("key not written: %v", err)
|
||||
}
|
||||
|
||||
leaf := leafFrom(t, certPath)
|
||||
if leaf.IsCA {
|
||||
t.Error("konnectivity serving cert must be a leaf, not a CA")
|
||||
}
|
||||
// The agent dials the NLB private-endpoint IP and verifies via the in-pod CA,
|
||||
// so the IP must be a SAN or the tunnel TLS handshake fails.
|
||||
if err := leaf.VerifyHostname("10.32.100.4"); err != nil {
|
||||
t.Errorf("IP SAN 10.32.100.4 missing: %v", err)
|
||||
}
|
||||
|
||||
// The leaf must chain to the K3s server CA — that CA is the in-pod ca.crt the
|
||||
// agent trusts, so no separate CA distribution is needed.
|
||||
caPEM, _ := os.ReadFile(caCert)
|
||||
roots := x509.NewCertPool()
|
||||
if !roots.AppendCertsFromPEM(caPEM) {
|
||||
t.Fatal("append CA failed")
|
||||
}
|
||||
if _, err := leaf.Verify(x509.VerifyOptions{
|
||||
Roots: roots,
|
||||
DNSName: "eks-toc.example",
|
||||
KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
}); err != nil {
|
||||
t.Errorf("leaf does not chain to the K3s server CA: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureCertReusesThenReissuesOnSANChange(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
caCert, caKey := writeTestCA(t, dir)
|
||||
|
||||
dns, ips := splitSANs("10.32.100.4,eks-toc.example")
|
||||
c1, _, err := ensureCert(dir, caCert, caKey, "konnectivity-server", dns, ips)
|
||||
if err != nil {
|
||||
t.Fatalf("first ensureCert: %v", err)
|
||||
}
|
||||
first, _ := os.ReadFile(c1)
|
||||
|
||||
// Same SANs → cached pair reused byte-for-byte (a churning cert would bounce
|
||||
// the konnectivity-server on every restart).
|
||||
if _, _, err := ensureCert(dir, caCert, caKey, "konnectivity-server", dns, ips); err != nil {
|
||||
t.Fatalf("second ensureCert: %v", err)
|
||||
}
|
||||
if second, _ := os.ReadFile(c1); string(first) != string(second) {
|
||||
t.Error("ensureCert re-minted instead of reusing the SAN-current cached cert")
|
||||
}
|
||||
|
||||
// A new endpoint in the SAN set must force a re-issue.
|
||||
dns2, ips2 := splitSANs("10.32.100.4,eks-toc.example,10.32.100.99")
|
||||
if _, _, err := ensureCert(dir, caCert, caKey, "konnectivity-server", dns2, ips2); err != nil {
|
||||
t.Fatalf("third ensureCert: %v", err)
|
||||
}
|
||||
if third, _ := os.ReadFile(c1); string(first) == string(third) {
|
||||
t.Error("ensureCert served a stale cert that does not cover the new SAN")
|
||||
}
|
||||
if err := leafFrom(t, c1).VerifyHostname("10.32.100.99"); err != nil {
|
||||
t.Errorf("re-issued cert missing new SAN: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitSANsPartitionsIPsAndDNS(t *testing.T) {
|
||||
dns, ips := splitSANs(" 10.0.0.1 , host.example , , 2001:db8::1 ")
|
||||
if len(dns) != 1 || dns[0] != "host.example" {
|
||||
t.Errorf("dns = %v, want [host.example]", dns)
|
||||
}
|
||||
wantIPs := []net.IP{net.ParseIP("10.0.0.1"), net.ParseIP("2001:db8::1")}
|
||||
if len(ips) != len(wantIPs) {
|
||||
t.Fatalf("ips = %v, want %v", ips, wantIPs)
|
||||
}
|
||||
for i, ip := range ips {
|
||||
if !ip.Equal(wantIPs[i]) {
|
||||
t.Errorf("ips[%d] = %v, want %v", i, ip, wantIPs[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/base64"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"net"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/bluebottle/pkg/tlsconfig"
|
||||
)
|
||||
|
||||
// serverTLSConfig is the webhook's loopback serving TLS config.
|
||||
func serverTLSConfig(cert tls.Certificate) *tls.Config {
|
||||
return &tls.Config{
|
||||
Certificates: []tls.Certificate{cert},
|
||||
MinVersion: tls.VersionTLS13,
|
||||
CurvePreferences: tlsconfig.Curves,
|
||||
}
|
||||
}
|
||||
|
||||
// ensureServingCert loads the persisted self-signed serving cert/key, or mints
|
||||
// a fresh ECDSA-P256 pair (SAN 127.0.0.1 / localhost) and persists it. Reusing
|
||||
// across restarts keeps the CA the apiserver loaded into its webhook kubeconfig
|
||||
// valid even if the webhook process is restarted. Returns the parsed cert and
|
||||
// the certificate PEM (for embedding as the kubeconfig CA bundle).
|
||||
func ensureServingCert(certPath, keyPath string) (tls.Certificate, []byte, error) {
|
||||
// Fall through to regenerate on a missing/corrupt/incompatible persisted
|
||||
// pair. The cert PEM is re-read because tls.Certificate keeps only DER.
|
||||
if cert, err := tls.LoadX509KeyPair(certPath, keyPath); err == nil {
|
||||
if certPEM, err := os.ReadFile(certPath); err == nil {
|
||||
return cert, certPEM, nil
|
||||
}
|
||||
}
|
||||
|
||||
certPEM, keyPEM, err := generateSelfSigned()
|
||||
if err != nil {
|
||||
return tls.Certificate{}, nil, err
|
||||
}
|
||||
if err := os.WriteFile(certPath, certPEM, 0o644); err != nil {
|
||||
return tls.Certificate{}, nil, fmt.Errorf("write cert %s: %w", certPath, err)
|
||||
}
|
||||
if err := os.WriteFile(keyPath, keyPEM, 0o600); err != nil {
|
||||
return tls.Certificate{}, nil, fmt.Errorf("write key %s: %w", keyPath, err)
|
||||
}
|
||||
cert, err := tls.X509KeyPair(certPEM, keyPEM)
|
||||
if err != nil {
|
||||
return tls.Certificate{}, nil, fmt.Errorf("parse minted keypair: %w", err)
|
||||
}
|
||||
return cert, certPEM, nil
|
||||
}
|
||||
|
||||
func generateSelfSigned() (certPEM, keyPEM []byte, err error) {
|
||||
priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("generate key: %w", err)
|
||||
}
|
||||
serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("serial: %w", err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
tmpl := x509.Certificate{
|
||||
SerialNumber: serial,
|
||||
Subject: pkix.Name{CommonName: "eks-token-webhook"},
|
||||
NotBefore: now.Add(-time.Hour),
|
||||
NotAfter: now.AddDate(10, 0, 0),
|
||||
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
BasicConstraintsValid: true,
|
||||
IsCA: true,
|
||||
IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback},
|
||||
DNSNames: []string{"localhost"},
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create cert: %w", err)
|
||||
}
|
||||
keyDER, err := x509.MarshalECPrivateKey(priv)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("marshal key: %w", err)
|
||||
}
|
||||
certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
|
||||
keyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
|
||||
return certPEM, keyPEM, nil
|
||||
}
|
||||
|
||||
// writeAPIServerKubeconfig writes the kubeconfig kube-apiserver loads via
|
||||
// --authentication-token-webhook-config-file. It points the apiserver at the
|
||||
// loopback webhook and trusts its self-signed serving cert as the CA bundle.
|
||||
func writeAPIServerKubeconfig(path, addr string, certPEM []byte) error {
|
||||
caB64 := base64.StdEncoding.EncodeToString(certPEM)
|
||||
kubeconfig := fmt.Sprintf(`apiVersion: v1
|
||||
kind: Config
|
||||
clusters:
|
||||
- name: mulga-eks-token-webhook
|
||||
cluster:
|
||||
certificate-authority-data: %s
|
||||
server: https://%s/authenticate
|
||||
users:
|
||||
- name: kube-apiserver
|
||||
contexts:
|
||||
- name: webhook
|
||||
context:
|
||||
cluster: mulga-eks-token-webhook
|
||||
user: kube-apiserver
|
||||
current-context: webhook
|
||||
`, caB64, addr)
|
||||
if err := os.WriteFile(path, []byte(kubeconfig), 0o600); err != nil {
|
||||
return fmt.Errorf("write %s: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+179
-24
@@ -1,45 +1,76 @@
|
||||
// Command eks-token-webhook is the K8s TokenReview webhook authenticator baked
|
||||
// into the eks-server AMI; the implementation is in agents/eks/tokenwebhook.
|
||||
// It reads its configuration from the first-boot env cloud-init seeds onto the
|
||||
// VM and binds loopback only.
|
||||
// eks-token-webhook is the K8s TokenReview webhook authenticator baked into
|
||||
// the eks-server AMI. Per kubectl call the kube-apiserver POSTs the bearer
|
||||
// token (produced by `aws eks get-token`); the webhook relays it through the
|
||||
// AWS gateway broker (SigV4 HTTPS POST to /clusters/{name}/token-review), which
|
||||
// host-side runs the STS verify (cross-cluster x-k8s-aws-id pin) + AccessEntry
|
||||
// KV lookup and returns the resolved identity. The webhook never speaks core
|
||||
// NATS — same gateway-broker model as ELBv2's lb-agent and eks-gateway-publish.
|
||||
//
|
||||
// It binds 127.0.0.1 only; the kube-apiserver is the sole client (loopback).
|
||||
// On startup it writes the apiserver webhook kubeconfig (with its self-signed
|
||||
// serving CA) so k3s can be pointed at it via
|
||||
// --authentication-token-webhook-config-file.
|
||||
//
|
||||
// SigV4 creds come from the AWS SDK chain (IMDS instance-role) when static
|
||||
// EKS_ACCESS_KEY/EKS_SECRET_KEY are absent, matching the sibling helpers.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/eksgw"
|
||||
handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
|
||||
|
||||
_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/eks/tokenwebhook"
|
||||
)
|
||||
|
||||
func loadConfig() (tokenwebhook.Config, error) {
|
||||
c := tokenwebhook.Config{
|
||||
Addr: flagAddr,
|
||||
GatewayURL: os.Getenv("EKS_GATEWAY_URL"),
|
||||
GatewayCA: os.Getenv("EKS_GATEWAY_CA"),
|
||||
AccessKey: os.Getenv("EKS_ACCESS_KEY"),
|
||||
SecretKey: os.Getenv("EKS_SECRET_KEY"),
|
||||
Region: os.Getenv("EKS_REGION"),
|
||||
AccountID: os.Getenv("EKS_ACCOUNT_ID"),
|
||||
ClusterName: os.Getenv("EKS_CLUSTER_NAME"),
|
||||
CertPath: envOr("EKS_WEBHOOK_CERT", "/etc/spinifex-eks/token-webhook.crt"),
|
||||
KeyPath: envOr("EKS_WEBHOOK_KEY", "/etc/spinifex-eks/token-webhook.key"),
|
||||
Kubeconfig: envOr("EKS_WEBHOOK_KUBECONFIG", "/etc/spinifex-eks/token-webhook.kubeconfig"),
|
||||
// config is the webhook's runtime configuration, sourced from the first-boot
|
||||
// env (/etc/spinifex-eks/first-boot.env) cloud-init seeds onto the VM.
|
||||
type config struct {
|
||||
addr string
|
||||
gatewayURL string
|
||||
gatewayCA string
|
||||
accessKey string
|
||||
secretKey string
|
||||
region string
|
||||
accountID string
|
||||
clusterName string
|
||||
certPath string
|
||||
keyPath string
|
||||
kubeconfig string
|
||||
}
|
||||
|
||||
func loadConfig() (config, error) {
|
||||
c := config{
|
||||
addr: flagAddr,
|
||||
gatewayURL: os.Getenv("EKS_GATEWAY_URL"),
|
||||
gatewayCA: os.Getenv("EKS_GATEWAY_CA"),
|
||||
accessKey: os.Getenv("EKS_ACCESS_KEY"),
|
||||
secretKey: os.Getenv("EKS_SECRET_KEY"),
|
||||
region: os.Getenv("EKS_REGION"),
|
||||
accountID: os.Getenv("EKS_ACCOUNT_ID"),
|
||||
clusterName: os.Getenv("EKS_CLUSTER_NAME"),
|
||||
certPath: envOr("EKS_WEBHOOK_CERT", "/etc/spinifex-eks/token-webhook.crt"),
|
||||
keyPath: envOr("EKS_WEBHOOK_KEY", "/etc/spinifex-eks/token-webhook.key"),
|
||||
kubeconfig: envOr("EKS_WEBHOOK_KUBECONFIG", "/etc/spinifex-eks/token-webhook.kubeconfig"),
|
||||
}
|
||||
// Static SigV4 creds are optional: when absent, eksgw.New signs with the
|
||||
// AWS SDK chain (IMDS instance-role creds), the same path the CP VM's
|
||||
// sibling helpers use. The CP VM launches on an instance profile, so
|
||||
// buildK3sUserData omits EKS_ACCESS_KEY/EKS_SECRET_KEY by design.
|
||||
switch {
|
||||
case c.GatewayURL == "":
|
||||
case c.gatewayURL == "":
|
||||
return c, errors.New("EKS_GATEWAY_URL not set")
|
||||
case c.AccountID == "":
|
||||
case c.accountID == "":
|
||||
return c, errors.New("EKS_ACCOUNT_ID not set")
|
||||
case c.ClusterName == "":
|
||||
case c.clusterName == "":
|
||||
return c, errors.New("EKS_CLUSTER_NAME not set")
|
||||
}
|
||||
return c, nil
|
||||
@@ -66,8 +97,132 @@ func main() {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err := tokenwebhook.Run(context.Background(), cfg); err != nil {
|
||||
if err := run(cfg); err != nil {
|
||||
slog.Error("eks-token-webhook fatal", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(cfg config) error {
|
||||
// Serving cert + apiserver kubeconfig. Persisted and reused across restarts
|
||||
// so a webhook crash does not invalidate the CA the apiserver already loaded.
|
||||
tlsCert, certPEM, err := ensureServingCert(cfg.certPath, cfg.keyPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("serving cert: %w", err)
|
||||
}
|
||||
if err := writeAPIServerKubeconfig(cfg.kubeconfig, cfg.addr, certPEM); err != nil {
|
||||
return fmt.Errorf("write apiserver kubeconfig: %w", err)
|
||||
}
|
||||
|
||||
client, err := eksgw.New(cfg.gatewayURL, cfg.gatewayCA, cfg.accessKey, cfg.secretKey, cfg.region)
|
||||
if err != nil {
|
||||
return fmt.Errorf("build gateway client: %w", err)
|
||||
}
|
||||
|
||||
authr := &authenticator{
|
||||
accountID: cfg.accountID,
|
||||
clusterName: cfg.clusterName,
|
||||
review: gatewayReviewer(client, cfg.clusterName, cfg.accountID),
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("ok"))
|
||||
})
|
||||
mux.HandleFunc("/authenticate", authr.handle)
|
||||
|
||||
server := &http.Server{
|
||||
Addr: cfg.addr,
|
||||
Handler: mux,
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
TLSConfig: serverTLSConfig(tlsCert),
|
||||
}
|
||||
slog.Info("eks-token-webhook listening", "addr", cfg.addr, "cluster", cfg.clusterName)
|
||||
if err := server.ListenAndServeTLS("", ""); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
return fmt.Errorf("serve: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// tokenReviewRequest is the body POSTed to /clusters/{name}/token-review. The
|
||||
// webhook holds system SigV4 creds (system account, not the cluster account),
|
||||
// so accountId names the cluster account explicitly — same as PublishInternal.
|
||||
type tokenReviewRequest struct {
|
||||
AccountID string `json:"accountId"`
|
||||
Token string `json:"token"`
|
||||
}
|
||||
|
||||
// gatewayReviewer returns the per-call relay: SigV4-POST {accountId,token} to
|
||||
// the gateway and decode the resolved identity. A non-2xx / transport error is
|
||||
// surfaced (handle() turns it into a 5xx so the apiserver retries) rather than a
|
||||
// silent deny, distinguishing "webhook broker down" from "token rejected".
|
||||
func gatewayReviewer(client *eksgw.Client, clusterName, accountID string) func(string) (handlers_eks.WebhookTokenReviewResult, error) {
|
||||
path := "/clusters/" + clusterName + "/token-review"
|
||||
return func(token string) (handlers_eks.WebhookTokenReviewResult, error) {
|
||||
body, err := json.Marshal(tokenReviewRequest{AccountID: accountID, Token: token})
|
||||
if err != nil {
|
||||
return handlers_eks.WebhookTokenReviewResult{}, fmt.Errorf("marshal request: %w", err)
|
||||
}
|
||||
respBody, err := client.Post(path, body)
|
||||
if err != nil {
|
||||
return handlers_eks.WebhookTokenReviewResult{}, err
|
||||
}
|
||||
var res handlers_eks.WebhookTokenReviewResult
|
||||
if err := json.Unmarshal(respBody, &res); err != nil {
|
||||
return handlers_eks.WebhookTokenReviewResult{}, fmt.Errorf("decode response: %w", err)
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
}
|
||||
|
||||
// authenticator resolves a TokenReview by relaying to the gateway broker.
|
||||
type authenticator struct {
|
||||
accountID string
|
||||
clusterName string
|
||||
review func(token string) (handlers_eks.WebhookTokenReviewResult, error)
|
||||
}
|
||||
|
||||
func (a *authenticator) handle(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
var review tokenReview
|
||||
if err := json.NewDecoder(r.Body).Decode(&review); err != nil {
|
||||
http.Error(w, "bad request", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
res, err := a.review(review.Spec.Token)
|
||||
if err != nil {
|
||||
// Broker fault (gateway down, transport error): a 5xx tells the apiserver
|
||||
// to retry rather than treating a transient outage as a hard deny.
|
||||
slog.Error("TokenReview relay failed", "cluster", a.clusterName, "err", err)
|
||||
http.Error(w, "token review unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
status := tokenReviewStatus{Authenticated: res.Authenticated}
|
||||
if res.Authenticated {
|
||||
status.User = userInfo{
|
||||
Username: res.Username,
|
||||
UID: res.UID,
|
||||
Groups: res.Groups,
|
||||
}
|
||||
// Confirm the token for the audiences the apiserver asked about; an empty
|
||||
// status.audiences is rejected when --api-audiences is configured.
|
||||
status.Audiences = review.Spec.Audiences
|
||||
}
|
||||
slog.Info("TokenReview decision", "authenticated", status.Authenticated, "username", status.User.Username, "audiences", status.Audiences)
|
||||
|
||||
resp := tokenReview{
|
||||
APIVersion: "authentication.k8s.io/v1",
|
||||
Kind: "TokenReview",
|
||||
Status: status,
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(resp); err != nil {
|
||||
slog.Error("encode TokenReview response", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,12 +1,127 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// postReview drives authenticator.handle with a TokenReview carrying token and
|
||||
// audiences, returning the decoded response status and HTTP code.
|
||||
func postReview(t *testing.T, a *authenticator, token string, audiences []string) (tokenReviewStatus, int) {
|
||||
t.Helper()
|
||||
body, err := json.Marshal(tokenReview{
|
||||
APIVersion: "authentication.k8s.io/v1",
|
||||
Kind: "TokenReview",
|
||||
Spec: tokenReviewSpec{Token: token, Audiences: audiences},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/authenticate", strings.NewReader(string(body)))
|
||||
w := httptest.NewRecorder()
|
||||
a.handle(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
return tokenReviewStatus{}, w.Code
|
||||
}
|
||||
var resp tokenReview
|
||||
require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp))
|
||||
return resp.Status, w.Code
|
||||
}
|
||||
|
||||
func TestHandle_GrantsAndStampsAudiences(t *testing.T) {
|
||||
a := &authenticator{
|
||||
accountID: "111122223333",
|
||||
clusterName: "alpha",
|
||||
review: func(string) (handlers_eks.WebhookTokenReviewResult, error) {
|
||||
return handlers_eks.WebhookTokenReviewResult{
|
||||
Authenticated: true,
|
||||
Username: "arn:aws:iam::111122223333:role/admin",
|
||||
UID: "AROAEXAMPLE:session",
|
||||
Groups: []string{"system:masters"},
|
||||
}, nil
|
||||
},
|
||||
}
|
||||
|
||||
status, code := postReview(t, a, "tok", []string{"https://kubernetes.default.svc"})
|
||||
require.Equal(t, http.StatusOK, code)
|
||||
require.True(t, status.Authenticated)
|
||||
assert.Equal(t, "arn:aws:iam::111122223333:role/admin", status.User.Username)
|
||||
assert.Equal(t, "AROAEXAMPLE:session", status.User.UID)
|
||||
assert.Equal(t, []string{"system:masters"}, status.User.Groups)
|
||||
assert.Equal(t, []string{"https://kubernetes.default.svc"}, status.Audiences)
|
||||
}
|
||||
|
||||
func TestHandle_DeniesWithoutStampingAudiences(t *testing.T) {
|
||||
a := &authenticator{
|
||||
clusterName: "alpha",
|
||||
review: func(string) (handlers_eks.WebhookTokenReviewResult, error) {
|
||||
return handlers_eks.WebhookTokenReviewResult{Authenticated: false}, nil
|
||||
},
|
||||
}
|
||||
|
||||
status, code := postReview(t, a, "tok", []string{"https://kubernetes.default.svc"})
|
||||
require.Equal(t, http.StatusOK, code)
|
||||
assert.False(t, status.Authenticated)
|
||||
assert.Empty(t, status.User.Username)
|
||||
assert.Empty(t, status.Audiences, "unauthenticated review must not stamp audiences")
|
||||
}
|
||||
|
||||
// A broker fault is a 5xx (retryable), not a silent deny — a transient gateway
|
||||
// outage must not look identical to a rejected token.
|
||||
func TestHandle_BrokerFaultReturns5xx(t *testing.T) {
|
||||
a := &authenticator{
|
||||
clusterName: "alpha",
|
||||
review: func(string) (handlers_eks.WebhookTokenReviewResult, error) {
|
||||
return handlers_eks.WebhookTokenReviewResult{}, errors.New("gateway returned 503")
|
||||
},
|
||||
}
|
||||
|
||||
_, code := postReview(t, a, "tok", nil)
|
||||
assert.Equal(t, http.StatusServiceUnavailable, code)
|
||||
}
|
||||
|
||||
func TestEnsureServingCert_PersistsAndReuses(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
certPath := filepath.Join(dir, "wh.crt")
|
||||
keyPath := filepath.Join(dir, "wh.key")
|
||||
|
||||
c1, pem1, err := ensureServingCert(certPath, keyPath)
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, pem1)
|
||||
require.NotNil(t, serverTLSConfig(c1))
|
||||
|
||||
// Second call reuses the persisted pair (same cert bytes).
|
||||
_, pem2, err := ensureServingCert(certPath, keyPath)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, pem1, pem2)
|
||||
}
|
||||
|
||||
func TestWriteAPIServerKubeconfig(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "wh.kubeconfig")
|
||||
certPEM := []byte("-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----\n")
|
||||
|
||||
require.NoError(t, writeAPIServerKubeconfig(path, "127.0.0.1:8443", certPEM))
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
require.NoError(t, err)
|
||||
body := string(data)
|
||||
assert.Contains(t, body, "server: https://127.0.0.1:8443/authenticate")
|
||||
assert.Contains(t, body, base64.StdEncoding.EncodeToString(certPEM))
|
||||
assert.Contains(t, body, "current-context: webhook")
|
||||
}
|
||||
|
||||
// The CP VM launches on an instance profile, so cloud-init omits the static
|
||||
// EKS_ACCESS_KEY/EKS_SECRET_KEY and eksgw.New signs with IMDS creds. loadConfig
|
||||
// must accept absent static keys; gating on them aborts every IMDS-mode boot,
|
||||
@@ -33,8 +148,8 @@ func TestLoadConfig_AcceptsAbsentStaticCreds(t *testing.T) {
|
||||
})
|
||||
cfg, err := loadConfig()
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, cfg.AccessKey)
|
||||
assert.Empty(t, cfg.SecretKey)
|
||||
assert.Empty(t, cfg.accessKey)
|
||||
assert.Empty(t, cfg.secretKey)
|
||||
|
||||
// Genuinely-required vars are still enforced.
|
||||
setEnv(t, map[string]string{
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
package tokenwebhook
|
||||
package main
|
||||
|
||||
// Minimal authentication.k8s.io/v1 TokenReview wire types. Hand-rolled to avoid
|
||||
// pulling k8s.io/api into the spinifex module for these few fields; the shapes
|
||||
+127
-13
@@ -1,6 +1,12 @@
|
||||
// Command eks-webhook-cert mints, then reuses, a self-signed serving certificate
|
||||
// for an in-cluster admission webhook and prints its base64 PEM; the
|
||||
// implementation is in agents/eks/webhookcert.
|
||||
// Command eks-webhook-cert mints, then reuses, a self-signed ECDSA-P256 serving
|
||||
// certificate for an in-cluster admission webhook and prints its base64 PEM so
|
||||
// the addon renderer can embed it as the webhook Secret + each caBundle.
|
||||
//
|
||||
// The eks-node AMI carries no openssl, so this is the Go equivalent of the
|
||||
// upstream Helm chart's install-time genSelfSignedCert: one cert per cluster,
|
||||
// cached on disk under -dir, stable across addon-sync ticks. The single cert is
|
||||
// its own CA (IsCA + ServerAuth), so the same PEM serves as both the TLS leaf
|
||||
// and the caBundle the apiserver trusts.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
@@ -11,25 +17,133 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/base64"
|
||||
"encoding/pem"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math/big"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/eks/webhookcert"
|
||||
)
|
||||
|
||||
func main() {
|
||||
cfg := webhookcert.Config{Out: os.Stdout}
|
||||
flag.StringVar(&cfg.Dir, "dir", "", "directory to cache tls.crt/tls.key in")
|
||||
flag.StringVar(&cfg.CN, "cn", "", "certificate common name")
|
||||
flag.StringVar(&cfg.DNS, "dns", "", "comma-separated DNS SANs")
|
||||
dir := flag.String("dir", "", "directory to cache tls.crt/tls.key in")
|
||||
cn := flag.String("cn", "", "certificate common name")
|
||||
dnsCSV := flag.String("dns", "", "comma-separated DNS SANs")
|
||||
flag.Parse()
|
||||
|
||||
if err := webhookcert.Run(context.Background(), cfg); err != nil {
|
||||
slog.Error("eks-webhook-cert: " + err.Error())
|
||||
os.Exit(1)
|
||||
if *dir == "" || *cn == "" || *dnsCSV == "" {
|
||||
fatal("eks-webhook-cert: -dir, -cn and -dns are required")
|
||||
}
|
||||
dnsNames := splitCSV(*dnsCSV)
|
||||
if len(dnsNames) == 0 {
|
||||
fatal("eks-webhook-cert: -dns lists no names")
|
||||
}
|
||||
|
||||
certPEM, keyPEM, err := ensureCert(*dir, *cn, dnsNames)
|
||||
if err != nil {
|
||||
fatal("eks-webhook-cert: " + err.Error())
|
||||
}
|
||||
b := base64.StdEncoding.EncodeToString
|
||||
fmt.Printf("%s\t%s\t%s\n", b(certPEM), b(certPEM), b(keyPEM))
|
||||
}
|
||||
|
||||
// ensureCert returns the cached cert/key under dir, minting and persisting a
|
||||
// fresh self-signed pair (for cn + dnsNames) when none is present or readable.
|
||||
func ensureCert(dir, cn string, dnsNames []string) (certPEM, keyPEM []byte, err error) {
|
||||
certPath := filepath.Join(dir, "tls.crt")
|
||||
keyPath := filepath.Join(dir, "tls.key")
|
||||
|
||||
if c, cerr := os.ReadFile(certPath); cerr == nil {
|
||||
if k, kerr := os.ReadFile(keyPath); kerr == nil {
|
||||
if _, perr := x509.ParseCertificate(decodeFirst(c)); perr == nil && len(k) > 0 {
|
||||
return c, k, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
certPEM, keyPEM, err = generateSelfSigned(cn, dnsNames)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if err = os.MkdirAll(dir, 0o700); err != nil {
|
||||
return nil, nil, fmt.Errorf("mkdir %s: %w", dir, err)
|
||||
}
|
||||
if err = os.WriteFile(certPath, certPEM, 0o644); err != nil {
|
||||
return nil, nil, fmt.Errorf("write %s: %w", certPath, err)
|
||||
}
|
||||
if err = os.WriteFile(keyPath, keyPEM, 0o600); err != nil {
|
||||
return nil, nil, fmt.Errorf("write %s: %w", keyPath, err)
|
||||
}
|
||||
return certPEM, keyPEM, nil
|
||||
}
|
||||
|
||||
// generateSelfSigned mints an ECDSA-P256 self-signed cert valid as both a CA and
|
||||
// a server cert, with the given common name and DNS SANs.
|
||||
func generateSelfSigned(cn string, dnsNames []string) (certPEM, keyPEM []byte, err error) {
|
||||
priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("generate key: %w", err)
|
||||
}
|
||||
serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("serial: %w", err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
tmpl := x509.Certificate{
|
||||
SerialNumber: serial,
|
||||
Subject: pkix.Name{CommonName: cn},
|
||||
NotBefore: now.Add(-time.Hour),
|
||||
NotAfter: now.AddDate(10, 0, 0),
|
||||
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
BasicConstraintsValid: true,
|
||||
IsCA: true,
|
||||
DNSNames: dnsNames,
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create cert: %w", err)
|
||||
}
|
||||
keyDER, err := x509.MarshalECPrivateKey(priv)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("marshal key: %w", err)
|
||||
}
|
||||
certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
|
||||
keyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
|
||||
return certPEM, keyPEM, nil
|
||||
}
|
||||
|
||||
// decodeFirst returns the DER bytes of the first PEM block, or nil.
|
||||
func decodeFirst(pemBytes []byte) []byte {
|
||||
blk, _ := pem.Decode(pemBytes)
|
||||
if blk == nil {
|
||||
return nil
|
||||
}
|
||||
return blk.Bytes
|
||||
}
|
||||
|
||||
func splitCSV(s string) []string {
|
||||
var out []string
|
||||
for p := range strings.SplitSeq(s, ",") {
|
||||
if p = strings.TrimSpace(p); p != "" {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func fatal(msg string) {
|
||||
slog.Error(msg)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/x509"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const (
|
||||
testCN = "aws-load-balancer-webhook-service.kube-system.svc"
|
||||
testDNS = "aws-load-balancer-webhook-service.kube-system.svc.cluster.local"
|
||||
)
|
||||
|
||||
func parseLeaf(t *testing.T, certPEM []byte) *x509.Certificate {
|
||||
t.Helper()
|
||||
leaf, err := x509.ParseCertificate(decodeFirst(certPEM))
|
||||
if err != nil {
|
||||
t.Fatalf("parse cert: %v", err)
|
||||
}
|
||||
return leaf
|
||||
}
|
||||
|
||||
func TestEnsureCertMintsServingCert(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
certPEM, keyPEM, err := ensureCert(dir, testCN, []string{testCN, testDNS})
|
||||
if err != nil {
|
||||
t.Fatalf("ensureCert: %v", err)
|
||||
}
|
||||
if len(keyPEM) == 0 {
|
||||
t.Fatal("empty key PEM")
|
||||
}
|
||||
leaf := parseLeaf(t, certPEM)
|
||||
|
||||
// Both webhook service DNS names must be present, or the apiserver's TLS
|
||||
// dial to the webhook fails SAN verification.
|
||||
for _, want := range []string{testCN, testDNS} {
|
||||
if err := leaf.VerifyHostname(want); err != nil {
|
||||
t.Errorf("SAN %q missing: %v", want, err)
|
||||
}
|
||||
}
|
||||
if !leaf.IsCA {
|
||||
t.Error("cert must be a CA so the same PEM serves as the webhook caBundle")
|
||||
}
|
||||
|
||||
// The single self-signed cert must verify against itself as a root — this is
|
||||
// exactly what the apiserver does with caBundle == the serving cert.
|
||||
roots := x509.NewCertPool()
|
||||
roots.AddCert(leaf)
|
||||
if _, err := leaf.Verify(x509.VerifyOptions{
|
||||
Roots: roots,
|
||||
DNSName: testCN,
|
||||
KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
}); err != nil {
|
||||
t.Errorf("self-signed cert does not verify as its own CA: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureCertReusesCachedPair(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cert1, key1, err := ensureCert(dir, testCN, []string{testCN})
|
||||
if err != nil {
|
||||
t.Fatalf("first ensureCert: %v", err)
|
||||
}
|
||||
cert2, key2, err := ensureCert(dir, testCN, []string{testCN})
|
||||
if err != nil {
|
||||
t.Fatalf("second ensureCert: %v", err)
|
||||
}
|
||||
if string(cert1) != string(cert2) || string(key1) != string(key2) {
|
||||
t.Error("ensureCert regenerated instead of reusing the cached pair; a churning cert re-applies the addon every tick")
|
||||
}
|
||||
|
||||
// The private key must not be world-readable on disk.
|
||||
info, err := os.Stat(filepath.Join(dir, "tls.key"))
|
||||
if err != nil {
|
||||
t.Fatalf("stat key: %v", err)
|
||||
}
|
||||
if info.Mode().Perm()&0o077 != 0 {
|
||||
t.Errorf("tls.key mode %o is group/other-accessible", info.Mode().Perm())
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -17,7 +17,7 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/operator/installer/install"
|
||||
"github.com/mulgadc/spinifex/cmd/installer/install"
|
||||
)
|
||||
|
||||
// listDisks is the block-device scan, indirected so tests can drive selection
|
||||
+1
-1
@@ -10,7 +10,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/operator/installer/install"
|
||||
"github.com/mulgadc/spinifex/cmd/installer/install"
|
||||
)
|
||||
|
||||
const gib = 1 << 30
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user