Compare commits
239 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 784e1f909a | |||
| 60cf4ca314 | |||
| fab4036972 | |||
| 2eb0f5ec83 | |||
| 561582b971 | |||
| 68c9a9123a | |||
| 0fd85d117b | |||
| 51d7ac91f5 | |||
| 707dbefa13 | |||
| 4050b63ada | |||
| 955714f557 | |||
| 2e770e8859 | |||
| f3ee105ed4 | |||
| 00070b1ecd | |||
| 9405d2c233 | |||
| a431fd3385 | |||
| 3c285b2f37 | |||
| da3e4e5b73 | |||
| 23be7c201a | |||
| e5b9fef0ae | |||
| 5271b16d2a | |||
| e06baa3aef | |||
| ded44feedb | |||
| cf4a6abafb | |||
| 06f93802b8 | |||
| 0fc93e975a | |||
| d066786faa | |||
| 55054ad0ac | |||
| 27304bbfd6 | |||
| a52b078678 | |||
| 26d53602a3 | |||
| 060204514e | |||
| 12cea4f6f7 | |||
| 099a69b23b | |||
| 6462ccc41f | |||
| ae8ee1e5fe | |||
| 488f1c7bf9 | |||
| 3617e4a045 | |||
| 24281cb860 | |||
| e58838bc67 | |||
| f0e4ffe70c | |||
| 18adea9486 | |||
| f06def6894 | |||
| ef7d30578f | |||
| b76679e571 | |||
| d7d922e963 | |||
| 64e3711893 | |||
| ad9568a6c6 | |||
| dedb3d8d75 | |||
| cc1186fa48 | |||
| dbdb827a69 | |||
| c94b0f22f6 | |||
| 71567d0998 | |||
| 11e8b68ddc | |||
| 8f4754da13 | |||
| 8b965d24ce | |||
| ac2620df58 | |||
| 2feb264863 | |||
| ddadfa2812 | |||
| 5923143062 | |||
| 46ca3dfba1 | |||
| 596b5b1733 | |||
| 1c5eaaf852 | |||
| 49117cf7b0 | |||
| 3a9985c79d | |||
| c42a352064 | |||
| 61686f2530 | |||
| d5f4d48956 | |||
| acfa078586 | |||
| 17dd3f325b | |||
| d2b02414b5 | |||
| af3fe610f9 | |||
| c8092975c2 | |||
| 49a4a07e00 | |||
| 02fd49be7e | |||
| 2aac488fa7 | |||
| 11ef3b63b8 | |||
| d5fbb7b628 | |||
| d953d0a973 | |||
| 43e84874df | |||
| 3f7664ddf9 | |||
| 7528c063cb | |||
| 772e1b4c37 | |||
| 05256f1cfc | |||
| 920f0487a3 | |||
| 919d083bbf | |||
| 816b3168b7 | |||
| a33fbcc696 | |||
| 2989a55309 | |||
| f12a0d8e98 | |||
| 7dde151766 | |||
| e7a3be3b4e | |||
| f92d73d6af | |||
| c02cfb806b | |||
| e1916c99f8 | |||
| 8b589c99c7 | |||
| 5216b18f88 | |||
| 7aaf6bf5e7 | |||
| 6f9875abb9 | |||
| 3503215bae | |||
| e5509443b3 | |||
| ee4f571687 | |||
| 49aea1b365 | |||
| d9f19be941 | |||
| 598e3aec8d | |||
| 7957186998 | |||
| 3a778d579e | |||
| fffa2e4a47 | |||
| 31b5356109 | |||
| 6eac30217f | |||
| 2511cc4cc3 | |||
| a6b6bb73a9 | |||
| 744a73ef1e | |||
| 2df87981f2 | |||
| 7fc79d95bd | |||
| 0c440a6717 | |||
| 00e38f9e76 | |||
| 57ea08be2a | |||
| 10fc0af2c0 | |||
| 524c20db8b | |||
| 2ed2a1c9dc | |||
| ac403f57bc | |||
| 6a41d269b9 | |||
| 1bb470517c | |||
| c545aa6220 | |||
| e006837e1c | |||
| f9dbe9526d | |||
| 738f1e6d4e | |||
| 42ca999e0b | |||
| f35853c1d3 | |||
| b586b7a1f9 | |||
| 24e2572ebd | |||
| 0ac0a106c0 | |||
| b2b7822f6a | |||
| fbf76cbedc | |||
| b557386c4b | |||
| e04a551185 | |||
| 0e81918345 | |||
| 8de934767d | |||
| 8b3b23deff | |||
| 8a4ab4438c | |||
| 6daa340044 | |||
| b4d4fa65f8 | |||
| 2e2465dd25 | |||
| 5d2c3e749f | |||
| fb0f8e9291 | |||
| bdfa01459c | |||
| f707df92e7 | |||
| f1ebb74acc | |||
| bde54ffd2f | |||
| d01e9b400d | |||
| 77b46481f1 | |||
| a16b667cbb | |||
| 7f634496b6 | |||
| 3a849c4c18 | |||
| 2f609efa14 | |||
| eccbf48273 | |||
| 84df82c26f | |||
| 29770dc798 | |||
| 09392d18a1 | |||
| 142ca6eec3 | |||
| 36949804e8 | |||
| a43febc187 | |||
| e7cb02dedf | |||
| be84b73abe | |||
| 3dbe352d1b | |||
| fcfcc1dcff | |||
| cd4363ba53 | |||
| 90739c889b | |||
| a4e5494e66 | |||
| efdd255435 | |||
| ed8a6f2fb2 | |||
| 0fb5c83a26 | |||
| 07bda82f1e | |||
| 49b94444ba | |||
| 1c21123c68 | |||
| 15c9308e41 | |||
| 3da9d9e60b | |||
| 04644bfbc9 | |||
| 71301983bd | |||
| 359eb7a386 | |||
| ef782a2758 | |||
| a7a774a7fc | |||
| bd14e2d67d | |||
| b5879f44ff | |||
| 0f025e8332 | |||
| 4c4ca253b4 | |||
| 3a3807f08e | |||
| 26f12e7107 | |||
| 5865fc418e | |||
| 02c02f6632 | |||
| 9075b13fe8 | |||
| 600fb41012 | |||
| 236dd2e87d | |||
| 96bcb4746d | |||
| 316a1b7c03 | |||
| 7fdabe1207 | |||
| 9498481161 | |||
| f3bebacbe5 | |||
| 2a19384fe1 | |||
| b6e50c488c | |||
| 5ad2346b3b | |||
| 4981828497 | |||
| 838de354c2 | |||
| 29aef59dae | |||
| 6644e6dc9f | |||
| a0b4d840e2 | |||
| bec3a78167 | |||
| 966a28f546 | |||
| c861642d44 | |||
| 72d234e95a | |||
| c09f0b5f11 | |||
| bc51816ad8 | |||
| ebc36d6f0c | |||
| 261b695e53 | |||
| e4d90b30d3 | |||
| bf3b830152 | |||
| b087d5d469 | |||
| fd9d69f05b | |||
| 75471404b6 | |||
| 848a662aac | |||
| ab45ea84ad | |||
| f50630ed4b | |||
| abb3da8af2 | |||
| 7e008c4689 | |||
| d91312d2f9 | |||
| c5d96ee56c | |||
| 788483807f | |||
| 429c5f13e5 | |||
| 49d3990252 | |||
| dbcda95b3c | |||
| 562e19aeca | |||
| 19c35326a3 | |||
| 68b1e18c72 | |||
| 9f9b7be9f7 | |||
| 68d3018b3e | |||
| 1a24217b2c | |||
| c09f15b50f | |||
| de4be25d80 |
@@ -0,0 +1,3 @@
|
||||
# The committed bundle carries the AWS SDK's template strings verbatim,
|
||||
# including trailing spaces that are part of their values.
|
||||
spinifex/runtime/roles/spinifexui/frontend/dist/** -whitespace
|
||||
@@ -6,10 +6,10 @@
|
||||
# Set the matching branch protection rule on `dev` and `main` to
|
||||
# "Require review from Code Owners" or this file is advisory only.
|
||||
|
||||
# Network layer is governed by ADR-0006 invariant tests.
|
||||
# Network layer is governed by its invariant tests.
|
||||
# Changes here must respect those clauses
|
||||
# see spinifex/network/invariants/
|
||||
# docs/development/proposals/az-local-architecture/0006-spinifex-network-layer-contract.md
|
||||
/spinifex/network/ @joshsiv-mulga
|
||||
/spinifex/vpcd/ @joshsiv-mulga
|
||||
/spinifex/network/invariants/ @joshsiv-mulga
|
||||
# see spinifex/domains/network/invariants/
|
||||
# mulga docs/development/proposals/region-local-architecture/0006-spinifex-network-layer-contract.md (proposal, not accepted)
|
||||
/spinifex/domains/network/ @joshsiv-mulga
|
||||
/spinifex/runtime/roles/vpcd/ @joshsiv-mulga
|
||||
/spinifex/domains/network/invariants/ @joshsiv-mulga
|
||||
|
||||
@@ -8,7 +8,7 @@ Actions job summary and emits one `::error` annotation per failed test case.
|
||||
```yaml
|
||||
- name: Publish test summary
|
||||
if: always()
|
||||
uses: ./spinifex/.github/actions/junit-report # or your own checkout path
|
||||
uses: $/.github/actions/junit-report
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
title: "Go E2E results"
|
||||
|
||||
@@ -27,7 +27,7 @@
|
||||
<line x1="510.0" y1="212.0" x2="510.0" y2="220.0" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
|
||||
<rect x="28.0" y="224.0" width="964.0" height="94.0" rx="13" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
|
||||
<text x="48.0" y="251.0" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#EE4B12" font-weight="700" text-anchor="start" letter-spacing="1.5">AWS GATEWAY</text>
|
||||
<text x="48.0" y="272.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">spinifex/services/awsgw · TLS termination · SigV4 verification · IAM policy eval · NATS routing</text>
|
||||
<text x="48.0" y="272.0" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">spinifex/runtime/roles/awsgw · TLS termination · SigV4 verification · IAM policy eval · NATS routing</text>
|
||||
<rect x="46.0" y="287.0" width="68.2" height="22.0" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="80.1" y="301.9" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle" fill-opacity="0.9">TLS 1.3</text>
|
||||
<rect x="123.2" y="287.0" width="88.0" height="22.0" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
|
||||
|
Before Width: | Height: | Size: 20 KiB After Width: | Height: | Size: 20 KiB |
@@ -51,7 +51,7 @@
|
||||
<!-- ═══ AWS GATEWAY ════════════════════════════════════ -->
|
||||
<rect x="16" y="154" width="868" height="104" rx="8" fill="url(#gw-grad)" stroke="#DC2626" stroke-width="1.3" stroke-opacity="0.75"/>
|
||||
<text x="32" y="176" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#F87171" font-weight="700" letter-spacing="1">AWS GATEWAY · port 9999</text>
|
||||
<text x="876" y="176" font-family="'SF Mono', ui-monospace, monospace" font-size="9" fill="#71717a" text-anchor="end">spinifex/services/awsgw/awsgw.go</text>
|
||||
<text x="876" y="176" font-family="'SF Mono', ui-monospace, monospace" font-size="9" fill="#71717a" text-anchor="end">spinifex/runtime/roles/awsgw/awsgw.go</text>
|
||||
|
||||
<!-- Two-column feature bullets -->
|
||||
<g font-family="'Segoe UI', sans-serif" font-size="10.5" fill="#f5f5f5">
|
||||
|
||||
|
Before Width: | Height: | Size: 12 KiB After Width: | Height: | Size: 12 KiB |
@@ -19,8 +19,8 @@
|
||||
</defs>
|
||||
<rect width="900" height="480" fill="url(#bg)" rx="12"/>
|
||||
|
||||
<text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">NGINX + ALB · VPC WITH PUBLIC AND PRIVATE SUBNETS</text>
|
||||
<text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">10.20.0.0/16 · multi-AZ</text>
|
||||
<text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">NGINX + ALB · PUBLIC SUBNETS · SPREAD ACROSS NODES</text>
|
||||
<text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">10.20.0.0/16 · server_count = 3</text>
|
||||
|
||||
<!-- Internet -->
|
||||
<rect x="350" y="54" width="200" height="40" rx="8" fill="url(#inet-grad)" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.55"/>
|
||||
@@ -41,58 +41,45 @@
|
||||
<line x1="230" y1="184" x2="230" y2="202" stroke="#DC2626" stroke-width="1" stroke-dasharray="3,3" opacity="0.6"/>
|
||||
<line x1="670" y1="184" x2="670" y2="202" stroke="#DC2626" stroke-width="1" stroke-dasharray="3,3" opacity="0.6"/>
|
||||
|
||||
<!-- public_a -->
|
||||
<g transform="translate(108, 202)">
|
||||
<rect width="244" height="104" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
|
||||
<rect width="244" height="150" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
|
||||
<text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">public_a</text>
|
||||
<text x="224" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">az-a</text>
|
||||
<text x="224" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">10.20.1.0/24</text>
|
||||
<line x1="16" y1="32" x2="228" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
|
||||
|
||||
<rect x="20" y="42" width="200" height="22" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="120" y="57" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle" font-weight="600">ALB ENI</text>
|
||||
|
||||
<rect x="20" y="70" width="200" height="22" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="120" y="85" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle" font-weight="600">NAT GW</text>
|
||||
<rect x="22" y="42" width="200" height="22" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="122" y="57" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle" font-weight="600">ALB ENI</text>
|
||||
<rect x="22" y="70" width="200" height="36" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="122" y="86" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle" font-weight="600">nginx-alb-1</text>
|
||||
<text x="122" y="100" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">public IP · Ubuntu · cloud-init</text>
|
||||
<rect x="22" y="110" width="200" height="36" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="122" y="126" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle" font-weight="600">nginx-alb-3</text>
|
||||
<text x="122" y="140" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">public IP · Ubuntu · cloud-init</text>
|
||||
</g>
|
||||
|
||||
<!-- public_b -->
|
||||
<g transform="translate(548, 202)">
|
||||
<rect width="244" height="104" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
|
||||
<rect width="244" height="150" rx="8" fill="url(#pub-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.55"/>
|
||||
<text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#3fb950" font-weight="700">public_b</text>
|
||||
<text x="224" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">az-b</text>
|
||||
<text x="224" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">10.20.2.0/24</text>
|
||||
<line x1="16" y1="32" x2="228" y2="32" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.5"/>
|
||||
|
||||
<rect x="20" y="56" width="200" height="22" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="120" y="71" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle" font-weight="600">ALB ENI</text>
|
||||
<rect x="22" y="42" width="200" height="22" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="122" y="57" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle" font-weight="600">ALB ENI</text>
|
||||
<rect x="22" y="70" width="200" height="36" rx="4" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="122" y="86" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle" font-weight="600">nginx-alb-2</text>
|
||||
<text x="122" y="100" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">public IP · Ubuntu · cloud-init</text>
|
||||
</g>
|
||||
|
||||
<!-- SNAT arrow from public_a NAT → private row -->
|
||||
<line x1="230" y1="306" x2="230" y2="340" stroke="#F87171" stroke-width="1.2" stroke-dasharray="3,3" opacity="0.65"/>
|
||||
<polygon points="226,336 234,336 230,344" fill="#F87171" opacity="0.8"/>
|
||||
<rect x="256" y="314" width="240" height="20" rx="10" fill="#0a0a0a" stroke="#F87171" stroke-width="0.6" stroke-opacity="0.55"/>
|
||||
<text x="376" y="328" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#FCA5A5" text-anchor="middle">SNAT for private subnets (egress)</text>
|
||||
|
||||
<!-- private_a -->
|
||||
<g transform="translate(108, 350)">
|
||||
<rect width="244" height="104" rx="8" fill="url(#priv-grad)" stroke="#a855f7" stroke-width="1.2" stroke-opacity="0.55"/>
|
||||
<text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#c084fc" font-weight="700">private_a</text>
|
||||
<text x="224" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#d8b4fe" text-anchor="end">az-a</text>
|
||||
<line x1="16" y1="32" x2="228" y2="32" stroke="#a855f7" stroke-opacity="0.3" stroke-width="0.5"/>
|
||||
|
||||
<rect x="20" y="44" width="200" height="48" rx="5" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="120" y="64" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#d8b4fe" text-anchor="middle" font-weight="700">nginx_1</text>
|
||||
<text x="120" y="80" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">Debian 12 · cloud-init</text>
|
||||
</g>
|
||||
|
||||
<!-- private_b -->
|
||||
<g transform="translate(548, 350)">
|
||||
<rect width="244" height="104" rx="8" fill="url(#priv-grad)" stroke="#a855f7" stroke-width="1.2" stroke-opacity="0.55"/>
|
||||
<text x="20" y="24" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#c084fc" font-weight="700">private_b</text>
|
||||
<text x="224" y="24" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#d8b4fe" text-anchor="end">az-b</text>
|
||||
<line x1="16" y1="32" x2="228" y2="32" stroke="#a855f7" stroke-opacity="0.3" stroke-width="0.5"/>
|
||||
|
||||
<rect x="20" y="44" width="200" height="48" rx="5" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="120" y="64" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#d8b4fe" text-anchor="middle" font-weight="700">nginx_2</text>
|
||||
<text x="120" y="80" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">Debian 12 · cloud-init</text>
|
||||
</g>
|
||||
<!-- spread placement group -->
|
||||
<rect x="108" y="372" width="684" height="88" rx="8" fill="url(#priv-grad)" stroke="#a855f7" stroke-width="1.2" stroke-opacity="0.55"/>
|
||||
<text x="128" y="394" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#c084fc" font-weight="700">nginx-alb-spread</text>
|
||||
<text x="772" y="394" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#d8b4fe" text-anchor="end">placement group · strategy spread · one server per node</text>
|
||||
<line x1="124" y1="402" x2="776" y2="402" stroke="#a855f7" stroke-opacity="0.3" stroke-width="0.5"/>
|
||||
<rect x="128" y="412" width="204" height="36" rx="5" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="230" y="428" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#d8b4fe" text-anchor="middle" font-weight="700">node1</text>
|
||||
<text x="230" y="442" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">one web server</text>
|
||||
<rect x="348" y="412" width="204" height="36" rx="5" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="450" y="428" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#d8b4fe" text-anchor="middle" font-weight="700">node2</text>
|
||||
<text x="450" y="442" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">one web server</text>
|
||||
<rect x="568" y="412" width="204" height="36" rx="5" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="670" y="428" font-family="'SF Mono', ui-monospace, monospace" font-size="11" fill="#d8b4fe" text-anchor="middle" font-weight="700">node3</text>
|
||||
<text x="670" y="442" font-family="'Segoe UI', sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">one web server</text>
|
||||
</svg>
|
||||
|
||||
|
Before Width: | Height: | Size: 7.1 KiB After Width: | Height: | Size: 7.5 KiB |
@@ -0,0 +1,110 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="900" height="580" viewBox="0 0 900 580">
|
||||
<defs>
|
||||
<linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
|
||||
<stop offset="0%" style="stop-color:#0a0a0a"/>
|
||||
<stop offset="100%" style="stop-color:#0d1117"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="vpc-grad" x1="0%" y1="0%" x2="0%" y2="100%">
|
||||
<stop offset="0%" style="stop-color:#58a6ff14"/>
|
||||
<stop offset="100%" style="stop-color:#58a6ff05"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="node-grad" x1="0%" y1="0%" x2="0%" y2="100%">
|
||||
<stop offset="0%" style="stop-color:#3fb9501f"/>
|
||||
<stop offset="100%" style="stop-color:#3fb95008"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="svc-grad" x1="0%" y1="0%" x2="0%" y2="100%">
|
||||
<stop offset="0%" style="stop-color:#a855f71f"/>
|
||||
<stop offset="100%" style="stop-color:#a855f708"/>
|
||||
</linearGradient>
|
||||
<marker id="arrow" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse">
|
||||
<path d="M0,0 L10,5 L0,10 z" fill="#71717a"/>
|
||||
</marker>
|
||||
</defs>
|
||||
<rect width="900" height="580" fill="url(#bg)" rx="12"/>
|
||||
<text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">SPINIFEX ON VULTR · ONE NODE OR THREE</text>
|
||||
<text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">--nodes 1 or 3 · vx1-g-16c-64g or bare metal</text>
|
||||
<rect x="30" y="52" width="250" height="56" rx="8" fill="#e4e4e70d" stroke="#e4e4e7" stroke-width="1.1" stroke-opacity="0.6"/>
|
||||
<text x="155.0" y="76" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#e4e4e7" text-anchor="middle" font-weight="700">Your machine</text>
|
||||
<text x="155.0" y="94" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">deploy-vultr.sh · OpenTofu or Terraform</text>
|
||||
<rect x="325" y="52" width="250" height="56" rx="8" fill="#3fb9500d" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.6"/>
|
||||
<text x="450.0" y="76" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#3fb950" text-anchor="middle" font-weight="700">Internet</text>
|
||||
<text x="450.0" y="94" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#7ee787" text-anchor="middle">customers reach your EC2 and load balancers</text>
|
||||
<rect x="620" y="52" width="250" height="56" rx="8" fill="#58a6ff12" stroke="#58a6ff" stroke-width="1.1" stroke-opacity="0.6"/>
|
||||
<text x="745.0" y="76" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#79c0ff" text-anchor="middle" font-weight="700">Vultr API</text>
|
||||
<text x="745.0" y="94" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#a5d6ff" text-anchor="middle">VPC · servers · reserved IPs</text>
|
||||
<rect x="30" y="160" width="840" height="400" rx="10" fill="url(#vpc-grad)" stroke="#58a6ff" stroke-width="1.2" stroke-opacity="0.55" stroke-dasharray="6,4"/>
|
||||
<text x="50" y="184" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#79c0ff" font-weight="700">Vultr VPC</text>
|
||||
<text x="850" y="184" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#58a6ff" text-anchor="end">private network 10.10.8.0/24</text>
|
||||
<line x1="155" y1="108" x2="155" y2="158" stroke="#71717a" stroke-width="1.2" opacity="0.8" marker-end="url(#arrow)"/>
|
||||
<text x="165" y="138" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#a1a1aa" text-anchor="start">Terraform builds · SSH installs</text>
|
||||
<line x1="450" y1="108" x2="450" y2="158" stroke="#71717a" stroke-width="1.2" opacity="0.8" marker-end="url(#arrow)"/>
|
||||
<text x="460" y="138" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#a1a1aa" text-anchor="start">reserved IPs</text>
|
||||
<line x1="745" y1="158" x2="745" y2="110" stroke="#71717a" stroke-width="1.2" opacity="0.8" marker-end="url(#arrow)"/>
|
||||
<text x="735" y="138" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#a1a1aa" text-anchor="end">Spinifex calls the API</text>
|
||||
<rect x="60" y="205" width="250" height="118" rx="8" fill="url(#node-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.6"/>
|
||||
<text x="78" y="229" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#3fb950" font-weight="700">node1</text>
|
||||
<text x="292" y="229" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">Ubuntu 24.04 · Spinifex</text>
|
||||
<line x1="74" y1="237" x2="296" y2="237" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.6"/>
|
||||
<rect x="76" y="250" width="74" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="113.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">AWS API</text>
|
||||
<rect x="160" y="250" width="60" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="190.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">EBS</text>
|
||||
<rect x="230" y="250" width="60" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="260.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">S3</text>
|
||||
<text x="185" y="300" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">VPC networking · your instances</text>
|
||||
<rect x="325" y="205" width="250" height="118" rx="8" fill="url(#node-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.6" stroke-dasharray="5,3"/>
|
||||
<text x="343" y="229" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#3fb950" font-weight="700">node2</text>
|
||||
<text x="557" y="229" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">with --nodes 3</text>
|
||||
<line x1="339" y1="237" x2="561" y2="237" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.6"/>
|
||||
<rect x="341" y="250" width="74" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="378.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">AWS API</text>
|
||||
<rect x="425" y="250" width="60" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="455.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">EBS</text>
|
||||
<rect x="495" y="250" width="60" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="525.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">S3</text>
|
||||
<text x="450" y="300" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">VPC networking · your instances</text>
|
||||
<rect x="590" y="205" width="250" height="118" rx="8" fill="url(#node-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.6" stroke-dasharray="5,3"/>
|
||||
<text x="608" y="229" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#3fb950" font-weight="700">node3</text>
|
||||
<text x="822" y="229" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">with --nodes 3</text>
|
||||
<line x1="604" y1="237" x2="826" y2="237" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.6"/>
|
||||
<rect x="606" y="250" width="74" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="643.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">AWS API</text>
|
||||
<rect x="690" y="250" width="60" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="720.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">EBS</text>
|
||||
<rect x="760" y="250" width="60" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="790.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">S3</text>
|
||||
<text x="715" y="300" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">VPC networking · your instances</text>
|
||||
<line x1="310" y1="264" x2="325" y2="264" stroke="#58a6ff" stroke-width="1.6" stroke-dasharray="3,3" opacity="0.9"/>
|
||||
<line x1="575" y1="264" x2="590" y2="264" stroke="#58a6ff" stroke-width="1.6" stroke-dasharray="3,3" opacity="0.9"/>
|
||||
<text x="450" y="348" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#79c0ff" text-anchor="middle">nodes cluster over the private network: control plane, EBS replicas and S3 shards</text>
|
||||
<rect x="60" y="370" width="780" height="166" rx="8" fill="url(#svc-grad)" stroke="#a855f7" stroke-width="1.2" stroke-opacity="0.55"/>
|
||||
<text x="80" y="394" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#c084fc" font-weight="700">AWS services across the fleet</text>
|
||||
<text x="820" y="394" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#d8b4fe" text-anchor="end">spread across the nodes; with three, moved to a survivor if one fails</text>
|
||||
<line x1="76" y1="402" x2="824" y2="402" stroke="#a855f7" stroke-opacity="0.3" stroke-width="0.6"/>
|
||||
<rect x="76" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="119.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">EC2</text>
|
||||
<text x="119.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">instances</text>
|
||||
<rect x="169" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="212.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">ALB / NLB</text>
|
||||
<text x="212.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">load balancers</text>
|
||||
<rect x="262" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="305.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">EKS</text>
|
||||
<text x="305.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">Kubernetes</text>
|
||||
<rect x="355" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="398.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">ECS</text>
|
||||
<text x="398.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">containers</text>
|
||||
<rect x="448" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="491.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">RDS</text>
|
||||
<text x="491.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">databases</text>
|
||||
<rect x="541" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="584.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">S3</text>
|
||||
<text x="584.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">object storage</text>
|
||||
<rect x="634" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="677.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">EBS</text>
|
||||
<text x="677.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">volumes</text>
|
||||
<rect x="727" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="770.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">IAM</text>
|
||||
<text x="770.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">access</text>
|
||||
<text x="450" y="500" font-family="'Segoe UI', system-ui, sans-serif" font-size="10.5" fill="#a1a1aa" text-anchor="middle">Use the AWS CLI, SDKs and Terraform against your own endpoint, unmodified.</text>
|
||||
<text x="450" y="520" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#71717a" text-anchor="middle">Public addresses are Vultr reserved IPs, attached to the node each resource runs on.</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 13 KiB |
|
After Width: | Height: | Size: 725 KiB |
|
After Width: | Height: | Size: 743 KiB |
@@ -23,11 +23,6 @@ updates:
|
||||
timezone: Australia/Sydney
|
||||
cooldown:
|
||||
default-days: 3
|
||||
ignore:
|
||||
- dependency-name: "*"
|
||||
update-types:
|
||||
- "version-update:semver-minor"
|
||||
- "version-update:semver-patch"
|
||||
groups:
|
||||
github-actions:
|
||||
patterns:
|
||||
@@ -35,7 +30,7 @@ updates:
|
||||
|
||||
- package-ecosystem: npm
|
||||
target-branch: dev
|
||||
directory: /spinifex/services/spinifexui/frontend
|
||||
directory: /spinifex/runtime/roles/spinifexui/frontend
|
||||
schedule:
|
||||
interval: weekly
|
||||
day: sunday
|
||||
@@ -50,3 +45,28 @@ updates:
|
||||
npm-dependencies:
|
||||
patterns:
|
||||
- "*"
|
||||
|
||||
# Base images are pinned by digest, so this is what moves them. The golang
|
||||
# version is deliberate and changes with go.mod; only its digest moves here.
|
||||
- package-ecosystem: docker
|
||||
target-branch: dev
|
||||
directories:
|
||||
- /build
|
||||
- /scripts
|
||||
- /docs/terraform-workbooks/demo-app
|
||||
schedule:
|
||||
interval: weekly
|
||||
day: sunday
|
||||
timezone: Australia/Sydney
|
||||
cooldown:
|
||||
default-days: 3
|
||||
ignore:
|
||||
- dependency-name: golang
|
||||
update-types:
|
||||
- "version-update:semver-major"
|
||||
- "version-update:semver-minor"
|
||||
- "version-update:semver-patch"
|
||||
groups:
|
||||
docker-images:
|
||||
patterns:
|
||||
- "*"
|
||||
|
||||
@@ -165,7 +165,7 @@
|
||||
"runner": "ci-multi",
|
||||
"extra": "",
|
||||
"suite": "instancerecovery",
|
||||
"timeout_minutes": 105
|
||||
"timeout_minutes": 125
|
||||
},
|
||||
{
|
||||
"cell": 32,
|
||||
@@ -190,5 +190,60 @@
|
||||
"extra": "",
|
||||
"suite": "kvquorum",
|
||||
"timeout_minutes": 55
|
||||
},
|
||||
{
|
||||
"cell": 34,
|
||||
"install": "binary",
|
||||
"topo": "single",
|
||||
"net": "dhcp",
|
||||
"host": "oracle-10",
|
||||
"arch": "x86_64",
|
||||
"runner": "ci-single",
|
||||
"extra": "",
|
||||
"suite": ""
|
||||
},
|
||||
{
|
||||
"cell": 35,
|
||||
"install": "binary",
|
||||
"topo": "single",
|
||||
"net": "static",
|
||||
"host": "oracle-10",
|
||||
"arch": "x86_64",
|
||||
"runner": "ci-single",
|
||||
"extra": "reboot-single",
|
||||
"suite": ""
|
||||
},
|
||||
{
|
||||
"cell": 36,
|
||||
"install": "binary",
|
||||
"topo": "single",
|
||||
"net": "nat",
|
||||
"host": "oracle-10",
|
||||
"arch": "x86_64",
|
||||
"runner": "ci-single",
|
||||
"extra": "nat-single",
|
||||
"suite": ""
|
||||
},
|
||||
{
|
||||
"cell": 37,
|
||||
"install": "binary",
|
||||
"topo": "real-multi",
|
||||
"net": "dhcp",
|
||||
"host": "oracle-10",
|
||||
"arch": "x86_64",
|
||||
"runner": "ci-multi",
|
||||
"extra": "",
|
||||
"suite": ""
|
||||
},
|
||||
{
|
||||
"cell": 38,
|
||||
"install": "binary",
|
||||
"topo": "real-multi",
|
||||
"net": "nat",
|
||||
"host": "oracle-10",
|
||||
"arch": "x86_64",
|
||||
"runner": "ci-multi",
|
||||
"extra": "nat-multi",
|
||||
"suite": ""
|
||||
}
|
||||
]
|
||||
|
||||
@@ -5,3 +5,21 @@
|
||||
## Changes
|
||||
|
||||
<!-- The notable changes, as bullets. -->
|
||||
|
||||
## Account-scoped lifecycle impact
|
||||
|
||||
<!-- Complete this section only when the PR changes account-scoped resource lifecycle, account termination, resource cleanup, or a lifecycle boundary. Otherwise write: N/A. -->
|
||||
|
||||
- Lifecycle owner and resource family:
|
||||
- Owner-local ordering, recovery, retained-state, or force impact:
|
||||
- Cross-domain dependency or ownership impact:
|
||||
- Evidence added or rechecked:
|
||||
- ADR-0008 lifecycle register / ADR-0001 migration-record / bead updates:
|
||||
|
||||
## Mulga design
|
||||
|
||||
<!-- One line, so a reviewer knows whether the design records in the private mulga repository need to change. Write either "Updated: <mulga PR or bead id>" or "Not needed: <reason>". This is a reminder, not a gate. -->
|
||||
|
||||
## Community Contribution License
|
||||
|
||||
- [ ] All community contributions in this pull request are licensed to Mulga under the terms of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). By creating this pull request I represent that I have the right to license these contributions to Mulga under the Apache License 2.0.
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
#!/usr/bin/env bash
|
||||
# Resolve the tag of each sibling repo (viperblock, predastore, northstar,
|
||||
# bluebottle) a release build checks out. A release ships what go.mod requires,
|
||||
# so each sibling is built at the version go.mod names, never a branch tip. A
|
||||
# pseudo-version has no tag to check out, so it fails the release rather than
|
||||
# silently building something else.
|
||||
#
|
||||
# Takes the path to spinifex's go.mod and writes viperblock=<tag>,
|
||||
# predastore=<tag>, northstar=<tag> and bluebottle=<tag> to $GITHUB_OUTPUT for
|
||||
# the checkout steps to consume.
|
||||
set -euo pipefail
|
||||
|
||||
gomod="${1:?usage: $0 <path to spinifex go.mod>}"
|
||||
: "${GITHUB_OUTPUT:?GITHUB_OUTPUT must be set}"
|
||||
|
||||
for repo in viperblock predastore northstar bluebottle; do
|
||||
module="github.com/mulgadc/${repo}"
|
||||
version=$(awk -v m="$module" '$1 == m { print $2; exit }' "$gomod")
|
||||
if [[ -z "$version" ]]; then
|
||||
echo "::error::${gomod} does not require ${module}"
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! "$version" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ || "$version" =~ [0-9]{14}-[0-9a-f]{12}$ ]]; then
|
||||
echo "::error::${module} is required at ${version}, which is not a release tag; tag ${repo} and bump go.mod before releasing"
|
||||
exit 1
|
||||
fi
|
||||
echo "${repo}: building against ${version}"
|
||||
echo "${repo}=${version}" >>"$GITHUB_OUTPUT"
|
||||
done
|
||||
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Warn when a PR body has no filled "Mulga design" line. Reads PR_BODY; never fails the job."""
|
||||
|
||||
import os
|
||||
import re
|
||||
|
||||
COMMENT_RE = re.compile(r"<!--.*?-->", re.S)
|
||||
SECTION_RE = re.compile(r"^#{1,6}\s*Mulga design\s*$", re.I)
|
||||
LINE_RE = re.compile(r"^\s*(Updated|Not needed)\s*:\s*(.*?)\s*$", re.I)
|
||||
PLACEHOLDER_RE = re.compile(r"<[^<>]*>")
|
||||
|
||||
|
||||
def filled(body):
|
||||
inside = False
|
||||
for line in COMMENT_RE.sub("", body or "").splitlines():
|
||||
if SECTION_RE.match(line):
|
||||
inside = True
|
||||
elif inside and line.lstrip().startswith("#"):
|
||||
return False
|
||||
elif inside:
|
||||
m = LINE_RE.match(line)
|
||||
if m and PLACEHOLDER_RE.sub("", m.group(2)).strip():
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
if filled(os.environ.get("PR_BODY", "")):
|
||||
print("The Mulga design line is filled in.")
|
||||
else:
|
||||
print('::warning title=Mulga design::Add a line under "## Mulga design": "Updated: <mulga PR or bead id>" or "Not needed: <reason>".')
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -28,6 +28,9 @@
|
||||
# that runs identically on either, and because a quota left switched on would
|
||||
# cap every suite sharing the environment — so it must be proved to restore
|
||||
# itself wherever it runs, not only on one node.
|
||||
# route53 belongs to both topologies because every node serves every hosted
|
||||
# zone from one shared bucket: on one node it proves the zone loads at all, on
|
||||
# three that each node's sync reaches the same set.
|
||||
# storagefault runs last in both lists and belongs to both topologies, for
|
||||
# different reasons. On one node it is RS(1,0), so freezing the only predastore
|
||||
# is a total outage and the guest-survival assertion is unambiguous. On three it
|
||||
@@ -37,7 +40,7 @@
|
||||
# received. It is last because it SIGSTOPs a cluster-wide
|
||||
# service: anything sharing the environment would see the same outage and report
|
||||
# a failure that says nothing about itself.
|
||||
E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota storagefault"
|
||||
E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota route53 storagefault"
|
||||
|
||||
# Suites runnable against a multi-node environment.
|
||||
#
|
||||
@@ -57,7 +60,7 @@ E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota
|
||||
# replicated rather than sitting on one node, so it runs before the pair that
|
||||
# take a node away — a cluster whose buckets are already wrong should report
|
||||
# that, not report a recovery failing.
|
||||
E2E_SUITES_MULTI="multinode lb cert quota kvquorum lbrecovery instancerecovery storagefault"
|
||||
E2E_SUITES_MULTI="multinode lb cert quota route53 kvquorum lbrecovery instancerecovery storagefault"
|
||||
|
||||
# grep -xE alternation form, for narrowing a requested set to the eligible one.
|
||||
E2E_SUITES_SINGLE_RE="$(printf '%s' "${E2E_SUITES_SINGLE}" | tr ' ' '|')"
|
||||
@@ -114,8 +117,9 @@ e2e_suite_timeout() {
|
||||
# storage case waits for a pause measured at 4m47s and then holds four
|
||||
# minutes to prove nothing moved. Three tests, each taking a node away and
|
||||
# bringing it back, and the partition case adds a lease validity, a lease
|
||||
# TTL and a boot on top of everything the other two wait for.
|
||||
instancerecovery) echo "90m" ;;
|
||||
# TTL and a boot on top of everything the other two wait for. The data
|
||||
# test lets its guest age past the pending watchdog twice, about 20 more.
|
||||
instancerecovery) echo "110m" ;;
|
||||
# lbrecovery pays for a load balancer twice: once to stand one up with two
|
||||
# backends placed around it, and once for the recovery itself — a stale
|
||||
# owner, a lease, a boot, HAProxy's own start, and its health checks against
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
import unittest
|
||||
|
||||
from mulga_design_check import filled
|
||||
|
||||
|
||||
class FilledTest(unittest.TestCase):
|
||||
def test_accepts_either_form(self):
|
||||
self.assertTrue(filled("## Mulga design\nUpdated: mulga#275"))
|
||||
self.assertTrue(filled("## Mulga design\n\nnot needed: internal refactor\n## Next"))
|
||||
|
||||
def test_rejects_unanswered(self):
|
||||
self.assertFalse(filled(""))
|
||||
self.assertFalse(filled("## Mulga design\n<!-- Updated: x -->"))
|
||||
self.assertFalse(filled("## Mulga design\nUpdated: <mulga PR or bead id>"))
|
||||
self.assertFalse(filled("## Mulga design\nUpdated:"))
|
||||
|
||||
def test_scoped_to_the_section(self):
|
||||
self.assertFalse(filled("Updated: mulga#1"))
|
||||
self.assertFalse(filled("## Mulga design\n## Other\nUpdated: mulga#1"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,145 @@
|
||||
# This file is machine-generated by `gh actions-lock`.
|
||||
# Do not edit by hand; run `gh actions-lock` to update.
|
||||
# Docs: https://gh.io/actions-lockfile
|
||||
version: 'v0.0.3'
|
||||
workflows:
|
||||
'.github/workflows/apt-packages.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
'.github/workflows/dev-release.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/setup-go@v7.0.0'
|
||||
- 'docker/setup-buildx-action@v4.4.1'
|
||||
'.github/workflows/e2e-baremetal.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/create-github-app-token@v3.3.0'
|
||||
- 'actions/upload-artifact@v7.0.2'
|
||||
'.github/workflows/e2e-cloudvendor-nightly.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/create-github-app-token@v3.3.0'
|
||||
- 'actions/upload-artifact@v7.0.2'
|
||||
- 'hashicorp/setup-terraform@v4.0.1'
|
||||
'.github/workflows/e2e-cloudvendor-vultr.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/create-github-app-token@v3.3.0'
|
||||
- 'actions/upload-artifact@v7.0.2'
|
||||
- 'hashicorp/setup-terraform@v4.0.1'
|
||||
'.github/workflows/e2e-ddil.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/create-github-app-token@v3.3.0'
|
||||
- 'actions/upload-artifact@v7.0.2'
|
||||
'.github/workflows/e2e-nightly.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/create-github-app-token@v3.3.0'
|
||||
- 'actions/download-artifact@v8.0.2'
|
||||
- 'actions/upload-artifact@v7.0.2'
|
||||
'.github/workflows/e2e-suites.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/create-github-app-token@v3.3.0'
|
||||
- 'actions/setup-go@v7.0.0'
|
||||
- 'actions/upload-artifact@v7.0.2'
|
||||
'.github/workflows/e2e.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/create-github-app-token@v3.3.0'
|
||||
- 'actions/download-artifact@v8.0.2'
|
||||
- 'actions/setup-go@v7.0.0'
|
||||
- 'actions/upload-artifact@v7.0.2'
|
||||
'.github/workflows/gpu-e2e.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/setup-go@v7.0.0'
|
||||
- 'actions/upload-artifact@v7.0.2'
|
||||
'.github/workflows/image-scripts.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/setup-go@v7.0.0'
|
||||
'.github/workflows/mulga-design.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
'.github/workflows/perf-3node.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/create-github-app-token@v3.3.0'
|
||||
- 'actions/setup-go@v7.0.0'
|
||||
- 'actions/upload-artifact@v7.0.2'
|
||||
'.github/workflows/publish-system-ami.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/setup-go@v7.0.0'
|
||||
'.github/workflows/release.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/create-github-app-token@v3.3.0'
|
||||
- 'actions/download-artifact@v8.0.2'
|
||||
- 'actions/setup-go@v7.0.0'
|
||||
- 'actions/upload-artifact@v7.0.2'
|
||||
- 'docker/setup-buildx-action@v4.4.1'
|
||||
- 'softprops/action-gh-release@v3.0.3'
|
||||
'.github/workflows/segscan-oracle.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/create-github-app-token@v3.3.0'
|
||||
- 'actions/setup-go@v7.0.0'
|
||||
'.github/workflows/spinifex.yml':
|
||||
- 'actions/cache@v6.1.0'
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/setup-go@v7.0.0'
|
||||
- 'golangci/golangci-lint-action@v9.3.0'
|
||||
'.github/workflows/ui.yml':
|
||||
- 'actions/checkout@v7.0.1'
|
||||
- 'actions/setup-node@v7.1.0'
|
||||
- 'actions/upload-artifact@v7.0.2'
|
||||
- 'pnpm/action-setup@v6.1.0'
|
||||
dependencies:
|
||||
'actions/cache@v6.1.0':
|
||||
ref: 'v6.1.0'
|
||||
commit: 'sha1-55cc8345863c7cc4c66a329aec7e433d2d1c52a9'
|
||||
owner_id: 44036562
|
||||
repo_id: 215566462
|
||||
'actions/checkout@v7.0.1':
|
||||
ref: 'v7.0.1'
|
||||
commit: 'sha1-3d3c42e5aac5ba805825da76410c181273ba90b1'
|
||||
owner_id: 44036562
|
||||
repo_id: 197814629
|
||||
'actions/create-github-app-token@v3.3.0':
|
||||
ref: 'v3.3.0'
|
||||
commit: 'sha1-64ab729a6294000f5aa4a4bc0b73d1f32bc45510'
|
||||
owner_id: 44036562
|
||||
repo_id: 642580244
|
||||
'actions/download-artifact@v8.0.2':
|
||||
ref: 'v8.0.2'
|
||||
commit: 'sha1-9000827ccba6bdab643e8b6fd33ac0654aef8333'
|
||||
owner_id: 44036562
|
||||
repo_id: 192626254
|
||||
'actions/setup-go@v7.0.0':
|
||||
ref: 'v7.0.0'
|
||||
commit: 'sha1-b7ad1dad31e06c5925ef5d2fc7ad053ef454303e'
|
||||
owner_id: 44036562
|
||||
repo_id: 192624594
|
||||
'actions/setup-node@v7.1.0':
|
||||
ref: 'v7.1.0'
|
||||
commit: 'sha1-949feb2413d6458794dcd2491c4babbbce0c15c1'
|
||||
owner_id: 44036562
|
||||
repo_id: 189476904
|
||||
'actions/upload-artifact@v7.0.2':
|
||||
ref: 'v7.0.2'
|
||||
commit: 'sha1-cf430e030ddbb5b0abf93d22962f4752f3646cd9'
|
||||
owner_id: 44036562
|
||||
repo_id: 192625955
|
||||
'docker/setup-buildx-action@v4.4.1':
|
||||
ref: 'v4.4.1'
|
||||
commit: 'sha1-f87e5991a6d7451dcb8d9637bfbc97413f497069'
|
||||
owner_id: 5429470
|
||||
repo_id: 288485773
|
||||
'golangci/golangci-lint-action@v9.3.0':
|
||||
ref: 'v9.3.0'
|
||||
commit: 'sha1-ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a'
|
||||
owner_id: 35628013
|
||||
repo_id: 232539879
|
||||
'hashicorp/setup-terraform@v4.0.1':
|
||||
ref: 'v4.0.1'
|
||||
commit: 'sha1-dfe3c3f87815947d99a8997f908cb6525fc44e9e'
|
||||
owner_id: 761456
|
||||
repo_id: 258271226
|
||||
'pnpm/action-setup@v6.1.0':
|
||||
ref: 'v6.1.0'
|
||||
commit: 'sha1-ea17c68df8912ef543352723c149a84f56e3d413'
|
||||
owner_id: 21320719
|
||||
repo_id: 262289747
|
||||
'softprops/action-gh-release@v3.0.3':
|
||||
ref: 'v3.0.3'
|
||||
commit: 'sha1-efb35369e0ad2afab669f228072c1b0d510eae64'
|
||||
owner_id: 2242
|
||||
repo_id: 204253808
|
||||
@@ -2,7 +2,8 @@ name: Apt Packages
|
||||
|
||||
# Path-filtered like image-scripts.yml: this resolves setup.sh's runtime package
|
||||
# list against each supported suite's real indices, so a renamed, dropped, or
|
||||
# ambiguously virtual package fails here rather than on a user's machine.
|
||||
# ambiguously virtual package fails here rather than on a user's machine. The
|
||||
# dnf job does the same for Oracle Linux, through setup.sh's own repo setup.
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
@@ -11,6 +12,7 @@ on:
|
||||
paths:
|
||||
- scripts/setup.sh
|
||||
- scripts/lint-apt-packages.sh
|
||||
- scripts/lint-dnf-packages.sh
|
||||
- .github/workflows/apt-packages.yml
|
||||
pull_request:
|
||||
branches:
|
||||
@@ -19,6 +21,7 @@ on:
|
||||
paths:
|
||||
- scripts/setup.sh
|
||||
- scripts/lint-apt-packages.sh
|
||||
- scripts/lint-dnf-packages.sh
|
||||
- .github/workflows/apt-packages.yml
|
||||
workflow_dispatch:
|
||||
|
||||
@@ -40,7 +43,18 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
|
||||
- name: Resolve runtime package list
|
||||
run: ./scripts/lint-apt-packages.sh ${{ matrix.image }}
|
||||
|
||||
dnf_packages:
|
||||
name: Resolve dnf list (oraclelinux:10)
|
||||
runs-on: ubuntu-26.04
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v7.0.1
|
||||
|
||||
- name: Resolve runtime package list
|
||||
run: ./scripts/lint-dnf-packages.sh oraclelinux:10
|
||||
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
# Full history and tags, because the tag name is git describe and a shallow
|
||||
# clone has no tag to describe against.
|
||||
- name: Checkout Spinifex
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
path: spinifex
|
||||
fetch-depth: 0
|
||||
@@ -42,40 +42,41 @@ jobs:
|
||||
# dev, not each repository's default branch, which is what release.yml takes.
|
||||
# A dev build of spinifex against main of the others is a pairing nobody runs.
|
||||
- name: Checkout Predastore
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
ref: dev
|
||||
path: predastore
|
||||
|
||||
- name: Checkout Viperblock
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/viperblock
|
||||
ref: dev
|
||||
path: viperblock
|
||||
|
||||
- name: Checkout Northstar
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/northstar
|
||||
ref: dev
|
||||
path: northstar
|
||||
|
||||
- name: Checkout Bluebottle
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/bluebottle
|
||||
ref: dev
|
||||
path: bluebottle
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v4.4.1
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v7.0.0
|
||||
with:
|
||||
go-version-file: spinifex/go.mod
|
||||
cache: false
|
||||
|
||||
# The tag carries the describe string so the commit is recoverable from the
|
||||
# version alone, and the -dev suffix is what release.yml's own convention
|
||||
@@ -101,9 +102,9 @@ jobs:
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
|
||||
- name: Build microVM artifacts (lb-agent + kernel + initramfs)
|
||||
- name: Build microVM artifacts (lb-agent + kernel + initramfs + qboot + SQS store and worker images)
|
||||
if: steps.version.outputs.exists != 'true'
|
||||
run: cd spinifex && make build-microvm-image
|
||||
run: cd spinifex && make build-microvm-image build-sqs-store-microvm-image build-sqs-worker-microvm-image
|
||||
|
||||
- name: Build amd64 Tarball
|
||||
if: steps.version.outputs.exists != 'true'
|
||||
@@ -143,6 +144,7 @@ jobs:
|
||||
--notes-file "$RUNNER_TEMP/notes.md" \
|
||||
"dist/spinifex-$TAG-linux-amd64.tar.gz" \
|
||||
"dist/spinifex-$TAG-linux-amd64.tar.gz.sha256" \
|
||||
dist/qboot-*.tar.gz \
|
||||
"scripts/setup.sh"
|
||||
|
||||
# The assertion that matters. A prerelease must never become the latest
|
||||
@@ -167,11 +169,7 @@ jobs:
|
||||
|
||||
# Kept rather than unbounded: the Worker reads the first page of releases to
|
||||
# resolve the channel, so enough prereleases would eventually push the real
|
||||
# ones off it, and each build carries two assets.
|
||||
#
|
||||
# Twenty, not five. A pinned dev tag is how an external evaluator installs a
|
||||
# specific build, and five is about a week of them -- short enough that the tag
|
||||
# someone was given stops resolving while they are still using it.
|
||||
# ones off it. Five leaves a few recent builds to roll back to.
|
||||
- name: Prune old dev prereleases
|
||||
if: steps.version.outputs.exists != 'true'
|
||||
env:
|
||||
@@ -179,7 +177,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
gh api "repos/$GITHUB_REPOSITORY/releases?per_page=100" \
|
||||
--jq 'map(select(.prerelease and (.tag_name | endswith("-dev")))) | .[20:] | .[].tag_name' \
|
||||
--jq 'map(select(.prerelease and (.tag_name | endswith("-dev")))) | .[5:] | .[].tag_name' \
|
||||
> stale.txt
|
||||
while read -r tag; do
|
||||
[ -n "$tag" ] || continue
|
||||
|
||||
@@ -77,18 +77,18 @@ jobs:
|
||||
if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi
|
||||
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
|
||||
- uses: actions/create-github-app-token@v3
|
||||
- uses: actions/create-github-app-token@v3.3.0
|
||||
id: app-token
|
||||
with:
|
||||
client-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
repositories: mulga
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/mulga
|
||||
token: ${{ steps.app-token.outputs.token }}
|
||||
@@ -151,7 +151,8 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Install go-junit-report on runner
|
||||
run: go install github.com/jstemmer/go-junit-report/v2@latest
|
||||
# Version from go.mod's tool directive, checked against go.sum.
|
||||
run: GOWORK=off go install github.com/jstemmer/go-junit-report/v2
|
||||
|
||||
# The eks suite's CreateCluster boots a control-plane VM from the eks-node
|
||||
# AMI. Built here because libguestfs needs source and network, which the
|
||||
@@ -221,20 +222,20 @@ jobs:
|
||||
- name: Ship test results to Elasticsearch
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
uses: ./.github/actions/junit-to-es
|
||||
uses: $/.github/actions/junit-to-es
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
cell: ${{ env.BM_CELL }}
|
||||
|
||||
- name: Analyze e2e failures
|
||||
if: always()
|
||||
uses: ./.github/actions/e2e-analyze
|
||||
uses: $/.github/actions/e2e-analyze
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
log-dir: ${{ env.ARTIFACT_DIR }}
|
||||
title: "E2E failure analysis (cell ${{ env.BM_CELL }})"
|
||||
|
||||
- uses: actions/upload-artifact@v7
|
||||
- uses: actions/upload-artifact@v7.0.2
|
||||
if: always()
|
||||
with:
|
||||
name: e2e-artifacts-cell-${{ env.BM_CELL }}-${{ github.run_id }}
|
||||
|
||||
@@ -76,11 +76,11 @@ jobs:
|
||||
- name: Put the Terraform state outside the checkout
|
||||
run: echo "OCI_STATE_ROOT=$RUNNER_WORKSPACE/oci-state" >> "$GITHUB_ENV"
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
- uses: actions/create-github-app-token@v3
|
||||
- uses: actions/create-github-app-token@v3.3.0
|
||||
id: app-token
|
||||
with:
|
||||
client-id: ${{ secrets.APP_ID }}
|
||||
@@ -92,14 +92,14 @@ jobs:
|
||||
|
||||
# build-install-artifacts.sh lives in mulga. The workbooks and their driver
|
||||
# are in spinifex, so only the one directory is needed here.
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/mulga
|
||||
path: mulga
|
||||
token: ${{ steps.app-token.outputs.token }}
|
||||
sparse-checkout: scripts/tofu-cluster
|
||||
|
||||
- uses: hashicorp/setup-terraform@v4
|
||||
- uses: hashicorp/setup-terraform@v4.0.1
|
||||
with:
|
||||
terraform_wrapper: false
|
||||
|
||||
@@ -273,7 +273,7 @@ jobs:
|
||||
# collect step above chose, so terraform.tfstate cannot reach an artifact.
|
||||
- name: Upload the run
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
with:
|
||||
name: e2e-oci-${{ github.run_id }}
|
||||
path: |
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
name: E2E Cloud Vendor Vultr
|
||||
|
||||
# On demand only. Deploys a ref to Vultr with deploy-vultr.sh, runs the same
|
||||
# published Terraform workbooks as the hypervisor nightly and OCI, and destroys
|
||||
# the deployment, failing if anything is left billing.
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
nodes:
|
||||
description: "1 or 3 nodes"
|
||||
required: false
|
||||
default: "1"
|
||||
type: choice
|
||||
options: ["1", "3"]
|
||||
plan:
|
||||
description: "Vultr plan: vx1-g-16c-64g (VM) or vbm-8c-64gb-amd (bare metal)"
|
||||
required: false
|
||||
default: vx1-g-16c-64g
|
||||
region:
|
||||
description: "Vultr region"
|
||||
required: false
|
||||
default: ord
|
||||
workbooks:
|
||||
description: "Space-separated workbooks (blank = the driver's full list)"
|
||||
required: false
|
||||
source:
|
||||
description: "tree (build this ref) or release (the published installer)"
|
||||
required: false
|
||||
default: tree
|
||||
type: choice
|
||||
options: [tree, release]
|
||||
channel:
|
||||
description: "With source=release: latest, or dev for the newest prerelease"
|
||||
required: false
|
||||
default: latest
|
||||
keep_on_fail:
|
||||
description: "Leave a failed deployment up to inspect. It keeps billing."
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
runner:
|
||||
description: "Runner label (blank = the VULTR_E2E_RUNNER variable, else ubuntu-26.04)"
|
||||
required: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# Deployments are named per run, but the account's reserved-IP quota is shared,
|
||||
# so a second dispatch queues rather than competing with the first.
|
||||
concurrency:
|
||||
group: e2e-vultr
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
vultr:
|
||||
# Hosted by default: the job only drives Terraform and SSH. The API key's
|
||||
# access control must admit this runner, so a hosted runner needs a key
|
||||
# whose list holds 0.0.0.0/0; a self-hosted runner can use a fixed address.
|
||||
runs-on: ${{ inputs.runner || vars.VULTR_E2E_RUNNER || 'ubuntu-26.04' }}
|
||||
timeout-minutes: 240
|
||||
env:
|
||||
VULTR_NAME: e2e-${{ github.run_id }}
|
||||
VULTR_NODES: ${{ inputs.nodes }}
|
||||
VULTR_PLAN: ${{ inputs.plan }}
|
||||
VULTR_REGION: ${{ inputs.region }}
|
||||
VULTR_SOURCE: ${{ inputs.source }}
|
||||
VULTR_CHANNEL: ${{ inputs.channel }}
|
||||
VULTR_ARTIFACT_DIR: ${{ github.workspace }}/e2e-vultr
|
||||
SPX_REF: ${{ github.ref_name }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
path: spinifex
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/create-github-app-token@v3.3.0
|
||||
id: app-token
|
||||
with:
|
||||
client-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
repositories: mulga
|
||||
|
||||
# build-install-artifacts.sh lives in mulga; the workbooks and their driver
|
||||
# are in spinifex, so only the one directory is needed here.
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/mulga
|
||||
path: mulga
|
||||
token: ${{ steps.app-token.outputs.token }}
|
||||
sparse-checkout: scripts/tofu-cluster
|
||||
|
||||
- uses: hashicorp/setup-terraform@v4.0.1
|
||||
with:
|
||||
terraform_wrapper: false
|
||||
|
||||
- name: Run Vultr
|
||||
working-directory: spinifex/scripts/terraform/vultr-spx
|
||||
env:
|
||||
VULTR_API_KEY: ${{ secrets.VULTR_API_KEY }}
|
||||
MULGA_ROOT: ${{ github.workspace }}/mulga
|
||||
# Through the environment, never interpolated into the script: a
|
||||
# dispatch input is untrusted text to the shell.
|
||||
WORKBOOKS_INPUT: ${{ inputs.workbooks }}
|
||||
VULTR_KEEP_ON_FAIL: ${{ inputs.keep_on_fail && '1' || '0' }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
if [ -n "$WORKBOOKS_INPUT" ]; then export WORKBOOKS="$WORKBOOKS_INPUT"; fi
|
||||
./e2e-vultr.sh 2>&1 | tee "$RUNNER_TEMP/e2e-vultr.log"
|
||||
exit "${PIPESTATUS[0]}"
|
||||
|
||||
# always(): a cancelled or timed-out job is the one that would leave
|
||||
# servers and reserved IPs billing. destroy works from this run's state.
|
||||
- name: Sweep anything left behind
|
||||
if: always() && !inputs.keep_on_fail
|
||||
working-directory: spinifex/scripts/terraform/vultr-spx
|
||||
env:
|
||||
VULTR_API_KEY: ${{ secrets.VULTR_API_KEY }}
|
||||
run: ./e2e-vultr.sh --sweep-only
|
||||
|
||||
- name: Publish the verdict
|
||||
if: always()
|
||||
run: |
|
||||
{
|
||||
echo "### Vultr: ${VULTR_NODES} x ${VULTR_PLAN} in ${VULTR_REGION}, ${VULTR_SOURCE} build of ${SPX_REF}"
|
||||
echo '```'
|
||||
cat "$VULTR_ARTIFACT_DIR/verdict.txt" 2>/dev/null || echo "no verdict written"
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# Logs and the verdict only. The throwaway SSH key and the Terraform state
|
||||
# under ~/.spinifex never reach an artifact.
|
||||
- name: Upload the run
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
with:
|
||||
name: e2e-vultr-${{ github.run_id }}
|
||||
path: |
|
||||
${{ runner.temp }}/e2e-vultr.log
|
||||
${{ github.workspace }}/e2e-vultr/*.log
|
||||
${{ github.workspace }}/e2e-vultr/verdict.txt
|
||||
${{ github.workspace }}/e2e-vultr/node*/*.log
|
||||
retention-days: 14
|
||||
@@ -38,19 +38,19 @@ jobs:
|
||||
- name: Clean Workspace
|
||||
run: sudo rm -rf "$GITHUB_WORKSPACE"/*
|
||||
|
||||
- uses: actions/create-github-app-token@v3
|
||||
- uses: actions/create-github-app-token@v3.3.0
|
||||
id: app-token
|
||||
with:
|
||||
client-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
repositories: mulga
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
name: Checkout Spinifex
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
name: Checkout Mulga (tofu-cluster)
|
||||
with:
|
||||
repository: mulgadc/mulga
|
||||
@@ -81,7 +81,7 @@ jobs:
|
||||
# Adds ci.job_id alongside ci.cell, before Bootstrap bakes the attrs into
|
||||
# the VMs, so a telemetry doc points back at this job's log directly.
|
||||
- name: Resolve CI job id
|
||||
uses: ./spinifex/.github/actions/otel-job-id
|
||||
uses: $/.github/actions/otel-job-id
|
||||
with:
|
||||
job-name: DDIL Real Multi-Node
|
||||
|
||||
@@ -175,7 +175,7 @@ jobs:
|
||||
|
||||
- name: Publish test summary
|
||||
if: always()
|
||||
uses: ./spinifex/.github/actions/junit-report
|
||||
uses: $/.github/actions/junit-report
|
||||
with:
|
||||
junit-glob: ${{ runner.temp }}/ddil-results/junit-*.xml
|
||||
title: "DDIL scenario results"
|
||||
@@ -185,7 +185,7 @@ jobs:
|
||||
- name: Ship test results to Elasticsearch
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
uses: ./spinifex/.github/actions/junit-to-es
|
||||
uses: $/.github/actions/junit-to-es
|
||||
with:
|
||||
junit-glob: ${{ runner.temp }}/ddil-results/junit-*.xml
|
||||
cell: ddil
|
||||
@@ -214,7 +214,7 @@ jobs:
|
||||
|
||||
- name: Upload DDIL Results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
with:
|
||||
name: ddil-results-${{ github.run_id }}
|
||||
path: ${{ runner.temp }}/ddil-results
|
||||
@@ -234,7 +234,7 @@ jobs:
|
||||
|
||||
- name: Upload tofu log
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
with:
|
||||
name: tofu-log-ddil-${{ github.run_id }}
|
||||
path: ${{ env.TOFU_LOG }}
|
||||
|
||||
@@ -10,7 +10,7 @@ on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
cells:
|
||||
description: "Comma-separated cell numbers to run, no spaces (blank = all). 1-3, 8-10, 17-19 tofu permutation matrix, 20 baremetal, 21-22 ISO, 23 predastore throughput, 24 predastore fault injection, 25-27 service suites (eks, ecs, rds), 28 guest disk performance, 29 storage fault injection, 30 routed NAT on three nodes, 31 instance recovery on three nodes, 32 load balancer recovery on three nodes, 33 KV bucket quorum under node loss on three nodes"
|
||||
description: "Comma-separated cell numbers to run, no spaces (blank = all). 1-3, 8-10, 17-19 tofu permutation matrix, 34-36 Oracle Linux single node (basic, reboot, routed NAT), 37-38 Oracle Linux on three nodes (multi-node subset with IPsec, routed NAT), 20 baremetal, 21-22 ISO, 23 predastore throughput, 24 predastore fault injection, 25-27 service suites (eks, ecs, rds), 28 guest disk performance, 29 storage fault injection, 30 routed NAT on three nodes, 31 instance recovery on three nodes, 32 load balancer recovery on three nodes, 33 KV bucket quorum under node loss on three nodes"
|
||||
required: false
|
||||
|
||||
permissions:
|
||||
@@ -33,6 +33,7 @@ concurrency:
|
||||
env:
|
||||
GIT_BRANCH: ${{ github.ref_name }}
|
||||
UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-26.04.qcow2
|
||||
ORACLE_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-oracle-10.qcow2
|
||||
SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
|
||||
SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
|
||||
|
||||
@@ -43,7 +44,7 @@ jobs:
|
||||
cells: ${{ steps.pick.outputs.cells }}
|
||||
any: ${{ steps.pick.outputs.any }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
sparse-checkout: |
|
||||
.github/e2e-cells.json
|
||||
@@ -128,21 +129,21 @@ jobs:
|
||||
if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi
|
||||
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
|
||||
|
||||
- uses: actions/create-github-app-token@v3
|
||||
- uses: actions/create-github-app-token@v3.3.0
|
||||
id: app-token
|
||||
with:
|
||||
client-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
repositories: mulga
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/mulga
|
||||
token: ${{ steps.app-token.outputs.token }}
|
||||
sparse-checkout: scripts/tofu-cluster
|
||||
path: mulga
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
@@ -151,7 +152,7 @@ jobs:
|
||||
# links each cell's telemetry to the job log that produced it. Runs
|
||||
# before Bootstrap bakes SPINIFEX_OTEL_ATTRS into the VM's telemetry.env.
|
||||
- name: Resolve CI job id
|
||||
uses: ./spinifex/.github/actions/otel-job-id
|
||||
uses: $/.github/actions/otel-job-id
|
||||
with:
|
||||
job-name: "cell-${{ matrix.cell }} (${{ matrix.topo }}/${{ matrix.net }}/${{ matrix.host }}${{ matrix.extra && format('/{0}', matrix.extra) || '' }})"
|
||||
|
||||
@@ -195,7 +196,8 @@ jobs:
|
||||
# back onto the debian gold.
|
||||
TOFU_VAR_ARGS=""
|
||||
case "$MATRIX_HOST" in
|
||||
ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}" ;;
|
||||
ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE} -var=host_os=ubuntu" ;;
|
||||
oracle-*) TOFU_VAR_ARGS="-var=host_image_override=${ORACLE_GOLD_IMAGE} -var=host_os=oracle" ;;
|
||||
esac
|
||||
|
||||
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
|
||||
@@ -687,7 +689,7 @@ jobs:
|
||||
'echo "host: $(hostname)"; echo "loadavg: $(cat /proc/loadavg)";
|
||||
grep -E "^(MemTotal|MemAvailable|SwapFree):" /proc/meminfo;
|
||||
echo "cpu(steal is field 8): $(grep "^cpu " /proc/stat)";
|
||||
echo "running qemu: $(pgrep -c qemu-system-x86 || echo 0)"' 2>&1
|
||||
echo "running qemu: $(pgrep -c "qemu-system-x86|qemu-kvm" || echo 0)"' 2>&1
|
||||
} >> "${ARTIFACT_DIR}/node-load.txt"
|
||||
exit 0
|
||||
|
||||
@@ -795,7 +797,8 @@ jobs:
|
||||
- name: Install go-junit-report
|
||||
if: always()
|
||||
run: |
|
||||
go install github.com/jstemmer/go-junit-report/v2@latest
|
||||
# Version from go.mod's tool directive, checked against go.sum.
|
||||
GOWORK=off go -C spinifex install github.com/jstemmer/go-junit-report/v2
|
||||
echo "$HOME/go/bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Convert test logs to JUnit XML
|
||||
@@ -812,20 +815,20 @@ jobs:
|
||||
- name: Ship test results to Elasticsearch
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
uses: ./spinifex/.github/actions/junit-to-es
|
||||
uses: $/.github/actions/junit-to-es
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
cell: ${{ matrix.cell }}
|
||||
|
||||
- name: Analyze e2e failures
|
||||
if: always()
|
||||
uses: ./spinifex/.github/actions/e2e-analyze
|
||||
uses: $/.github/actions/e2e-analyze
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
log-dir: ${{ env.ARTIFACT_DIR }}
|
||||
title: "E2E failure analysis (cell ${{ matrix.cell }})"
|
||||
|
||||
- uses: actions/upload-artifact@v7
|
||||
- uses: actions/upload-artifact@v7.0.2
|
||||
if: always()
|
||||
with:
|
||||
name: e2e-artifacts-cell-${{ matrix.cell }}-${{ github.run_id }}
|
||||
@@ -837,7 +840,7 @@ jobs:
|
||||
# pulling nineteen journals it has no use for.
|
||||
- name: Upload junit only
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
with:
|
||||
name: junit-cell-${{ matrix.cell }}-${{ github.run_id }}
|
||||
path: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
@@ -857,7 +860,8 @@ jobs:
|
||||
# different image path than apply did.
|
||||
TOFU_VAR_ARGS=""
|
||||
case "$MATRIX_HOST" in
|
||||
ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}" ;;
|
||||
ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE} -var=host_os=ubuntu" ;;
|
||||
oracle-*) TOFU_VAR_ARGS="-var=host_image_override=${ORACLE_GOLD_IMAGE} -var=host_os=oracle" ;;
|
||||
esac
|
||||
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
|
||||
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
|
||||
@@ -868,7 +872,7 @@ jobs:
|
||||
|
||||
- name: Upload tofu log
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
with:
|
||||
name: tofu-log-cell-${{ matrix.cell }}-${{ github.run_id }}
|
||||
path: ${{ env.TOFU_LOG }}
|
||||
@@ -973,20 +977,20 @@ jobs:
|
||||
{ sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
|
||||
echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
|
||||
|
||||
- uses: actions/create-github-app-token@v3
|
||||
- uses: actions/create-github-app-token@v3.3.0
|
||||
id: app-token
|
||||
with:
|
||||
client-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
repositories: mulga
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/mulga
|
||||
token: ${{ steps.app-token.outputs.token }}
|
||||
path: mulga
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
ref: ${{ env.GIT_BRANCH }}
|
||||
path: mulga/spinifex
|
||||
@@ -998,25 +1002,25 @@ jobs:
|
||||
run: bash .github/resolve-sibling-refs.sh
|
||||
working-directory: mulga/spinifex
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/viperblock
|
||||
ref: ${{ steps.siblings.outputs.viperblock }}
|
||||
path: mulga/viperblock
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
ref: ${{ steps.siblings.outputs.predastore }}
|
||||
path: mulga/predastore
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/northstar
|
||||
ref: ${{ steps.siblings.outputs.northstar }}
|
||||
path: mulga/northstar
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/bluebottle
|
||||
ref: ${{ steps.siblings.outputs.bluebottle }}
|
||||
@@ -1151,7 +1155,7 @@ jobs:
|
||||
/tmp/smoke-$GITHUB_RUN_ID.log | tail -1)
|
||||
[ -n "$qemu" ] && cp "$qemu" /tmp/cell21-artifacts-$GITHUB_RUN_ID/qemu.log || true
|
||||
|
||||
- uses: actions/upload-artifact@v7
|
||||
- uses: actions/upload-artifact@v7.0.2
|
||||
if: always()
|
||||
with:
|
||||
name: e2e-artifacts-cell-21-${{ github.run_id }}
|
||||
@@ -1220,20 +1224,20 @@ jobs:
|
||||
# disks, which is what the tmpfs assertion below is about.
|
||||
echo "WORK_DIR=$E2E_ROOT/work/cell22-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
|
||||
|
||||
- uses: actions/create-github-app-token@v3
|
||||
- uses: actions/create-github-app-token@v3.3.0
|
||||
id: app-token
|
||||
with:
|
||||
client-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
repositories: mulga
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/mulga
|
||||
token: ${{ steps.app-token.outputs.token }}
|
||||
path: mulga
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
ref: ${{ env.GIT_BRANCH }}
|
||||
path: mulga/spinifex
|
||||
@@ -1245,25 +1249,25 @@ jobs:
|
||||
run: bash .github/resolve-sibling-refs.sh
|
||||
working-directory: mulga/spinifex
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/viperblock
|
||||
ref: ${{ steps.siblings.outputs.viperblock }}
|
||||
path: mulga/viperblock
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
ref: ${{ steps.siblings.outputs.predastore }}
|
||||
path: mulga/predastore
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/northstar
|
||||
ref: ${{ steps.siblings.outputs.northstar }}
|
||||
path: mulga/northstar
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/bluebottle
|
||||
ref: ${{ steps.siblings.outputs.bluebottle }}
|
||||
@@ -1317,7 +1321,7 @@ jobs:
|
||||
cp "$WORK_DIR"/*-serial.log /tmp/cell22-artifacts-$GITHUB_RUN_ID/ 2>/dev/null ||
|
||||
echo "no per-node serial logs — the run passed and cleaned up after itself"
|
||||
|
||||
- uses: actions/upload-artifact@v7
|
||||
- uses: actions/upload-artifact@v7.0.2
|
||||
if: always()
|
||||
with:
|
||||
name: e2e-artifacts-cell-22-${{ github.run_id }}
|
||||
@@ -1369,7 +1373,7 @@ jobs:
|
||||
echo "PERF_RESULTS_ROOT=$E2E_ROOT/predastore-perf-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Checkout Spinifex
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
@@ -1384,7 +1388,7 @@ jobs:
|
||||
working-directory: spinifex
|
||||
|
||||
- name: Checkout Predastore
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
ref: ${{ steps.siblings.outputs.predastore }}
|
||||
@@ -1437,7 +1441,7 @@ jobs:
|
||||
# Report-only. The baseline comparison and the -10% gate are mulga-wpjsx;
|
||||
# until the run-to-run spread is known from real nights, a threshold would
|
||||
# be a guess, and a gate that cries wolf gets muted.
|
||||
- uses: actions/upload-artifact@v7
|
||||
- uses: actions/upload-artifact@v7.0.2
|
||||
if: always()
|
||||
with:
|
||||
name: predastore-perf-${{ github.run_id }}
|
||||
@@ -1496,7 +1500,7 @@ jobs:
|
||||
echo "STRESS_RESULTS_ROOT=$E2E_ROOT/predastore-stress-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Checkout Spinifex
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
@@ -1508,7 +1512,7 @@ jobs:
|
||||
working-directory: spinifex
|
||||
|
||||
- name: Checkout Predastore
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
ref: ${{ steps.siblings.outputs.predastore }}
|
||||
@@ -1563,7 +1567,7 @@ jobs:
|
||||
mkdir -p "$STRESS_RESULTS_ROOT"
|
||||
make e2e-stress 2>&1 | tee "$STRESS_RESULTS_ROOT/e2e-stress.log"
|
||||
|
||||
- uses: actions/upload-artifact@v7
|
||||
- uses: actions/upload-artifact@v7.0.2
|
||||
if: always()
|
||||
with:
|
||||
name: predastore-stress-${{ github.run_id }}
|
||||
@@ -1600,13 +1604,13 @@ jobs:
|
||||
if: always()
|
||||
runs-on: ubuntu-26.04
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
sparse-checkout: .github/scripts/e2e-run-summary.py
|
||||
sparse-checkout-cone-mode: false
|
||||
|
||||
- name: Collect junit
|
||||
uses: actions/download-artifact@v8
|
||||
uses: actions/download-artifact@v8.0.2
|
||||
with:
|
||||
pattern: junit-cell-*-${{ github.run_id }}
|
||||
path: junit
|
||||
|
||||
@@ -103,25 +103,25 @@ jobs:
|
||||
| sort -r | xargs -r -I{} sudo umount -l -R {} || true
|
||||
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
|
||||
|
||||
- uses: actions/create-github-app-token@v3
|
||||
- uses: actions/create-github-app-token@v3.3.0
|
||||
id: app-token
|
||||
with:
|
||||
client-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
repositories: mulga
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/mulga
|
||||
token: ${{ steps.app-token.outputs.token }}
|
||||
sparse-checkout: scripts/tofu-cluster
|
||||
path: mulga
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
- uses: actions/setup-go@v7.0.0
|
||||
with:
|
||||
go-version-file: spinifex/go.mod
|
||||
cache: false
|
||||
@@ -157,7 +157,7 @@ jobs:
|
||||
# the env: the suite legs share this env and its baked telemetry.env, so
|
||||
# the provisioner is the only job the daemon telemetry can belong to.
|
||||
- name: Resolve CI job id
|
||||
uses: ./spinifex/.github/actions/otel-job-id
|
||||
uses: $/.github/actions/otel-job-id
|
||||
with:
|
||||
job-name: Single env · provision
|
||||
|
||||
@@ -300,7 +300,7 @@ jobs:
|
||||
|
||||
- name: Upload tofu log
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
with:
|
||||
name: tofu-log-single-${{ github.run_id }}
|
||||
path: ${{ env.TOFU_LOG }}
|
||||
@@ -323,18 +323,19 @@ jobs:
|
||||
SSH_KEY: ${{ needs.single_provision.outputs.ssh_key }}
|
||||
ARTIFACT_DIR: /tmp/spinifex-e2e-single-${{ github.run_id }}-${{ matrix.suite }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
- uses: actions/setup-go@v7.0.0
|
||||
with:
|
||||
go-version-file: spinifex/go.mod
|
||||
cache: false
|
||||
|
||||
- name: Install go-junit-report
|
||||
run: |
|
||||
go install github.com/jstemmer/go-junit-report/v2@latest
|
||||
# Version from go.mod's tool directive, checked against go.sum.
|
||||
GOWORK=off go -C spinifex install github.com/jstemmer/go-junit-report/v2
|
||||
echo "$HOME/go/bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Run ${{ matrix.suite }}
|
||||
@@ -390,7 +391,7 @@ jobs:
|
||||
|
||||
- name: Publish test summary
|
||||
if: always()
|
||||
uses: ./spinifex/.github/actions/junit-report
|
||||
uses: $/.github/actions/junit-report
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
title: "E2E ${{ matrix.suite }} results"
|
||||
@@ -400,20 +401,20 @@ jobs:
|
||||
- name: Ship test results to Elasticsearch
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
uses: ./spinifex/.github/actions/junit-to-es
|
||||
uses: $/.github/actions/junit-to-es
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
cell: single
|
||||
|
||||
- name: Analyze e2e failures
|
||||
if: always()
|
||||
uses: ./spinifex/.github/actions/e2e-analyze
|
||||
uses: $/.github/actions/e2e-analyze
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
log-dir: ${{ env.ARTIFACT_DIR }}
|
||||
title: "E2E ${{ matrix.suite }} failure analysis"
|
||||
|
||||
- uses: actions/upload-artifact@v7
|
||||
- uses: actions/upload-artifact@v7.0.2
|
||||
if: always()
|
||||
with:
|
||||
name: e2e-single-${{ matrix.suite }}-${{ github.run_id }}
|
||||
@@ -444,7 +445,7 @@ jobs:
|
||||
# Reuse the persisted mulga/ tofu-cluster workspace from single_provision.
|
||||
# Re-checkout only spinifex (for ci-fmt + diag scripts); checking out mulga
|
||||
# again would wipe the local tofu state and orphan the VM.
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
@@ -467,7 +468,7 @@ jobs:
|
||||
| tar -C "${ARTIFACT_DIR}/lb-console" -xf - 2>/dev/null
|
||||
true
|
||||
|
||||
- uses: actions/upload-artifact@v7
|
||||
- uses: actions/upload-artifact@v7.0.2
|
||||
if: always()
|
||||
with:
|
||||
name: e2e-single-diag-${{ github.run_id }}
|
||||
@@ -523,7 +524,7 @@ jobs:
|
||||
|
||||
- name: Upload tofu log
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
with:
|
||||
name: tofu-log-single-td-${{ github.run_id }}
|
||||
path: ${{ env.TOFU_LOG }}
|
||||
@@ -571,25 +572,25 @@ jobs:
|
||||
| sort -r | xargs -r -I{} sudo umount -l -R {} || true
|
||||
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
|
||||
|
||||
- uses: actions/create-github-app-token@v3
|
||||
- uses: actions/create-github-app-token@v3.3.0
|
||||
id: app-token
|
||||
with:
|
||||
client-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
repositories: mulga
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/mulga
|
||||
token: ${{ steps.app-token.outputs.token }}
|
||||
sparse-checkout: scripts/tofu-cluster
|
||||
path: mulga
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
- uses: actions/setup-go@v7.0.0
|
||||
with:
|
||||
go-version-file: spinifex/go.mod
|
||||
cache: false
|
||||
@@ -606,7 +607,7 @@ jobs:
|
||||
|
||||
# See the matching note on single_provision.
|
||||
- name: Resolve CI job id
|
||||
uses: ./spinifex/.github/actions/otel-job-id
|
||||
uses: $/.github/actions/otel-job-id
|
||||
with:
|
||||
job-name: Multi env · provision
|
||||
|
||||
@@ -714,7 +715,7 @@ jobs:
|
||||
|
||||
- name: Upload tofu log
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
with:
|
||||
name: tofu-log-multi-${{ github.run_id }}
|
||||
path: ${{ env.TOFU_LOG }}
|
||||
@@ -737,18 +738,19 @@ jobs:
|
||||
NODES_CSV: ${{ needs.multi_provision.outputs.nodes_csv }}
|
||||
ARTIFACT_DIR: /tmp/spinifex-e2e-multi-${{ github.run_id }}-${{ matrix.suite }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
- uses: actions/setup-go@v7.0.0
|
||||
with:
|
||||
go-version-file: spinifex/go.mod
|
||||
cache: false
|
||||
|
||||
- name: Install go-junit-report
|
||||
run: |
|
||||
go install github.com/jstemmer/go-junit-report/v2@latest
|
||||
# Version from go.mod's tool directive, checked against go.sum.
|
||||
GOWORK=off go -C spinifex install github.com/jstemmer/go-junit-report/v2
|
||||
echo "$HOME/go/bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Run ${{ matrix.suite }}
|
||||
@@ -804,7 +806,7 @@ jobs:
|
||||
|
||||
- name: Publish test summary
|
||||
if: always()
|
||||
uses: ./spinifex/.github/actions/junit-report
|
||||
uses: $/.github/actions/junit-report
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
title: "Multi-Node E2E ${{ matrix.suite }} results"
|
||||
@@ -814,20 +816,20 @@ jobs:
|
||||
- name: Ship test results to Elasticsearch
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
uses: ./spinifex/.github/actions/junit-to-es
|
||||
uses: $/.github/actions/junit-to-es
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
cell: multi
|
||||
|
||||
- name: Analyze e2e failures
|
||||
if: always()
|
||||
uses: ./spinifex/.github/actions/e2e-analyze
|
||||
uses: $/.github/actions/e2e-analyze
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
log-dir: ${{ env.ARTIFACT_DIR }}
|
||||
title: "Multi-Node E2E ${{ matrix.suite }} failure analysis"
|
||||
|
||||
- uses: actions/upload-artifact@v7
|
||||
- uses: actions/upload-artifact@v7.0.2
|
||||
if: always()
|
||||
with:
|
||||
name: e2e-multi-${{ matrix.suite }}-${{ github.run_id }}
|
||||
@@ -854,7 +856,7 @@ jobs:
|
||||
SSH_KEY: ${{ needs.multi_provision.outputs.ssh_key }}
|
||||
NODES_CSV: ${{ needs.multi_provision.outputs.nodes_csv }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
@@ -908,7 +910,7 @@ jobs:
|
||||
|
||||
- name: Upload tofu log
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
with:
|
||||
name: tofu-log-multi-td-${{ github.run_id }}
|
||||
path: ${{ env.TOFU_LOG }}
|
||||
|
||||
@@ -54,7 +54,7 @@ jobs:
|
||||
single_timeout: ${{ steps.budgets.outputs.single_timeout }}
|
||||
multi_timeout: ${{ steps.budgets.outputs.multi_timeout }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
path: spinifex
|
||||
sparse-checkout: .github/scripts
|
||||
@@ -142,25 +142,25 @@ jobs:
|
||||
# stale suite-status.txt / analysis dirs can't leak into this attempt.
|
||||
rm -rf "${ARTIFACT_DIR}"
|
||||
|
||||
- uses: actions/create-github-app-token@v3
|
||||
- uses: actions/create-github-app-token@v3.3.0
|
||||
id: app-token
|
||||
with:
|
||||
client-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
repositories: mulga
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/mulga
|
||||
token: ${{ steps.app-token.outputs.token }}
|
||||
sparse-checkout: scripts/tofu-cluster
|
||||
path: mulga
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
- uses: actions/setup-go@v7.0.0
|
||||
with:
|
||||
go-version-file: spinifex/go.mod
|
||||
cache: false
|
||||
@@ -178,7 +178,7 @@ jobs:
|
||||
# Adds ci.job_id alongside ci.cell, before Bootstrap bakes the attrs into
|
||||
# the VM, so a telemetry doc points back at this job's log directly.
|
||||
- name: Resolve CI job id
|
||||
uses: ./spinifex/.github/actions/otel-job-id
|
||||
uses: $/.github/actions/otel-job-id
|
||||
with:
|
||||
job-name: Single-Node E2E
|
||||
|
||||
@@ -372,7 +372,8 @@ jobs:
|
||||
|
||||
- name: Install go-junit-report on runner
|
||||
run: |
|
||||
go install github.com/jstemmer/go-junit-report/v2@latest
|
||||
# Version from go.mod's tool directive, checked against go.sum.
|
||||
GOWORK=off go -C spinifex install github.com/jstemmer/go-junit-report/v2
|
||||
echo "$HOME/go/bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Ship test binaries to VM
|
||||
@@ -512,7 +513,7 @@ jobs:
|
||||
|
||||
- name: Publish test summary
|
||||
if: always()
|
||||
uses: ./spinifex/.github/actions/junit-report
|
||||
uses: $/.github/actions/junit-report
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
title: "E2E results"
|
||||
@@ -522,20 +523,27 @@ jobs:
|
||||
- name: Ship test results to Elasticsearch
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
uses: ./spinifex/.github/actions/junit-to-es
|
||||
uses: $/.github/actions/junit-to-es
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
cell: single
|
||||
|
||||
- name: Analyze e2e failures
|
||||
if: always()
|
||||
uses: ./spinifex/.github/actions/e2e-analyze
|
||||
uses: $/.github/actions/e2e-analyze
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
log-dir: ${{ env.ARTIFACT_DIR }}
|
||||
title: "E2E failure analysis"
|
||||
|
||||
- uses: actions/upload-artifact@v7
|
||||
# The WAN capture is most of the artifact and only matters when a run
|
||||
# failed. Anything short of a clean pass (suite rc, a failed setup or
|
||||
# collect step, a missing rc) keeps it.
|
||||
- name: Drop the packet capture from a passing single-node run
|
||||
if: always() && steps.e2e.outputs.rc == '0' && job.status == 'success'
|
||||
run: rm -f "${ARTIFACT_DIR}"/net-diag/wan.pcap
|
||||
|
||||
- uses: actions/upload-artifact@v7.0.2
|
||||
if: always()
|
||||
with:
|
||||
name: e2e-artifacts-${{ github.run_id }}
|
||||
@@ -579,7 +587,7 @@ jobs:
|
||||
|
||||
- name: Upload tofu log
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
with:
|
||||
name: tofu-log-single-${{ github.run_id }}
|
||||
path: ${{ env.TOFU_LOG }}
|
||||
@@ -642,25 +650,25 @@ jobs:
|
||||
# stale suite-status.txt / analysis dirs can't leak into this attempt.
|
||||
rm -rf "${ARTIFACT_DIR}"
|
||||
|
||||
- uses: actions/create-github-app-token@v3
|
||||
- uses: actions/create-github-app-token@v3.3.0
|
||||
id: app-token
|
||||
with:
|
||||
client-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
repositories: mulga
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/mulga
|
||||
token: ${{ steps.app-token.outputs.token }}
|
||||
sparse-checkout: scripts/tofu-cluster
|
||||
path: mulga
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
- uses: actions/setup-go@v7.0.0
|
||||
with:
|
||||
go-version-file: spinifex/go.mod
|
||||
cache: false
|
||||
@@ -677,7 +685,7 @@ jobs:
|
||||
|
||||
# See the matching note on e2e_single.
|
||||
- name: Resolve CI job id
|
||||
uses: ./spinifex/.github/actions/otel-job-id
|
||||
uses: $/.github/actions/otel-job-id
|
||||
with:
|
||||
job-name: Multi-Node E2E
|
||||
|
||||
@@ -758,7 +766,8 @@ jobs:
|
||||
|
||||
- name: Install go-junit-report on runner
|
||||
run: |
|
||||
go install github.com/jstemmer/go-junit-report/v2@latest
|
||||
# Version from go.mod's tool directive, checked against go.sum.
|
||||
GOWORK=off go -C spinifex install github.com/jstemmer/go-junit-report/v2
|
||||
echo "$HOME/go/bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Ship test binaries to node1
|
||||
@@ -965,7 +974,7 @@ jobs:
|
||||
|
||||
- name: Publish test summary
|
||||
if: always()
|
||||
uses: ./spinifex/.github/actions/junit-report
|
||||
uses: $/.github/actions/junit-report
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
title: "Multi-Node E2E results"
|
||||
@@ -975,20 +984,27 @@ jobs:
|
||||
- name: Ship test results to Elasticsearch
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
uses: ./spinifex/.github/actions/junit-to-es
|
||||
uses: $/.github/actions/junit-to-es
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
cell: multi
|
||||
|
||||
- name: Analyze e2e failures
|
||||
if: always()
|
||||
uses: ./spinifex/.github/actions/e2e-analyze
|
||||
uses: $/.github/actions/e2e-analyze
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
log-dir: ${{ env.ARTIFACT_DIR }}
|
||||
title: "Multi-Node E2E failure analysis"
|
||||
|
||||
- uses: actions/upload-artifact@v7
|
||||
# The WAN captures are 90% of the artifact and only matter when a run
|
||||
# failed. Anything short of a clean pass (suite rc, a failed setup or
|
||||
# collect step, a missing rc) keeps them.
|
||||
- name: Drop the packet captures from a passing multi-node run
|
||||
if: always() && steps.e2e.outputs.rc == '0' && job.status == 'success'
|
||||
run: rm -f "${ARTIFACT_DIR}"/net-diag-node*/wan.pcap
|
||||
|
||||
- uses: actions/upload-artifact@v7.0.2
|
||||
if: always()
|
||||
with:
|
||||
name: e2e-multi-artifacts-${{ github.run_id }}
|
||||
@@ -1035,7 +1051,7 @@ jobs:
|
||||
|
||||
- name: Upload tofu log
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
with:
|
||||
name: tofu-log-multi-${{ github.run_id }}
|
||||
path: ${{ env.TOFU_LOG }}
|
||||
@@ -1105,14 +1121,14 @@ jobs:
|
||||
# the aggregation script below treats an absent suite-status.txt as
|
||||
# "no failures from that leg".
|
||||
- name: Download single-node artifacts
|
||||
uses: actions/download-artifact@v8
|
||||
uses: actions/download-artifact@v8.0.2
|
||||
continue-on-error: true
|
||||
with:
|
||||
name: e2e-artifacts-${{ github.run_id }}
|
||||
path: ${{ github.workspace }}/es-summary-single
|
||||
|
||||
- name: Download multi-node artifacts
|
||||
uses: actions/download-artifact@v8
|
||||
uses: actions/download-artifact@v8.0.2
|
||||
continue-on-error: true
|
||||
with:
|
||||
name: e2e-multi-artifacts-${{ github.run_id }}
|
||||
|
||||
@@ -95,10 +95,10 @@ jobs:
|
||||
if: steps.gate.outputs.paused != 'true'
|
||||
run: sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
if: steps.gate.outputs.paused != 'true'
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
- uses: actions/setup-go@v7.0.0
|
||||
if: steps.gate.outputs.paused != 'true'
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
@@ -187,7 +187,8 @@ jobs:
|
||||
- name: Install go-junit-report on runner
|
||||
if: steps.gate.outputs.paused != 'true'
|
||||
run: |
|
||||
go install github.com/jstemmer/go-junit-report/v2@latest
|
||||
# Version from go.mod's tool directive, checked against go.sum.
|
||||
GOWORK=off go install github.com/jstemmer/go-junit-report/v2
|
||||
echo "$HOME/go/bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Ship GPU e2e test binary to wattle
|
||||
@@ -243,7 +244,7 @@ jobs:
|
||||
|
||||
- name: Publish test summary
|
||||
if: always() && steps.gate.outputs.paused != 'true'
|
||||
uses: ./.github/actions/junit-report
|
||||
uses: $/.github/actions/junit-report
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
title: "GPU E2E results"
|
||||
@@ -251,20 +252,20 @@ jobs:
|
||||
- name: Ship test results to Elasticsearch
|
||||
if: always() && steps.gate.outputs.paused != 'true'
|
||||
continue-on-error: true
|
||||
uses: ./.github/actions/junit-to-es
|
||||
uses: $/.github/actions/junit-to-es
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
cell: gpu
|
||||
|
||||
- name: Analyze e2e failures
|
||||
if: always() && steps.gate.outputs.paused != 'true'
|
||||
uses: ./.github/actions/e2e-analyze
|
||||
uses: $/.github/actions/e2e-analyze
|
||||
with:
|
||||
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
||||
log-dir: ${{ env.ARTIFACT_DIR }}
|
||||
title: "GPU E2E failure analysis"
|
||||
|
||||
- uses: actions/upload-artifact@v7
|
||||
- uses: actions/upload-artifact@v7.0.2
|
||||
if: always() && steps.gate.outputs.paused != 'true'
|
||||
with:
|
||||
name: gpu-e2e-artifacts-${{ github.run_id }}
|
||||
|
||||
@@ -29,13 +29,16 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
|
||||
- name: Run image script tests + shellcheck
|
||||
run: make test-images
|
||||
|
||||
eks_mkosi_image:
|
||||
name: Build mkosi EKS image
|
||||
# A fork PR's code must never run on a persistent self-hosted runner: it
|
||||
# shares disk, caches and credentials with the release jobs.
|
||||
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
|
||||
runs-on: [self-hosted, ci-single]
|
||||
concurrency:
|
||||
group: eks-mkosi-image-${{ github.ref }}
|
||||
@@ -43,10 +46,10 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v7.0.0
|
||||
with:
|
||||
go-version-file: "go.mod"
|
||||
# Self-hosted: the module + build cache already persists on local
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
name: Mulga design
|
||||
|
||||
# Reminds the author to say whether the Mulga design records need to change.
|
||||
# It reads only the pull request body, uses no secrets, and warns without ever
|
||||
# failing: the check is a prompt, not a gate.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- dev
|
||||
types: [opened, edited, reopened]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
mulga_design:
|
||||
name: Mulga design line
|
||||
if: ${{ !endsWith(github.actor, '[bot]') }}
|
||||
runs-on: ubuntu-26.04
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
sparse-checkout: .github/scripts
|
||||
|
||||
- name: Check the Mulga design line
|
||||
env:
|
||||
PR_BODY: ${{ github.event.pull_request.body }}
|
||||
working-directory: .github/scripts
|
||||
run: |
|
||||
python3 -m unittest -q test_mulga_design_check
|
||||
python3 mulga_design_check.py
|
||||
@@ -80,25 +80,25 @@ jobs:
|
||||
echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
|
||||
echo "PERF_RESULTS_ROOT=$E2E_ROOT/predastore-perf-3node-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
|
||||
|
||||
- uses: actions/create-github-app-token@v3
|
||||
- uses: actions/create-github-app-token@v3.3.0
|
||||
id: app-token
|
||||
with:
|
||||
client-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
repositories: mulga
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/mulga
|
||||
token: ${{ steps.app-token.outputs.token }}
|
||||
sparse-checkout: scripts/tofu-cluster
|
||||
path: mulga
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
- uses: actions/setup-go@v7.0.0
|
||||
with:
|
||||
go-version-file: spinifex/go.mod
|
||||
cache: false
|
||||
@@ -116,7 +116,7 @@ jobs:
|
||||
run: bash .github/resolve-sibling-refs.sh
|
||||
working-directory: spinifex
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
id: predastore
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
@@ -366,7 +366,7 @@ jobs:
|
||||
# Report-only, matching cell-23: the baseline comparison and any
|
||||
# regression gate are future work once run-to-run spread is known from
|
||||
# real bare-metal runs.
|
||||
- uses: actions/upload-artifact@v7
|
||||
- uses: actions/upload-artifact@v7.0.2
|
||||
if: always()
|
||||
with:
|
||||
name: predastore-perf-3node-${{ github.run_id }}
|
||||
@@ -407,7 +407,7 @@ jobs:
|
||||
|
||||
- name: Upload tofu log
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
with:
|
||||
name: tofu-log-perf3node-${{ github.run_id }}
|
||||
path: ${{ env.TOFU_LOG }}
|
||||
|
||||
@@ -17,28 +17,12 @@ name: Publish System AMIs
|
||||
# spx admin images import --name spinifex-rds-postgres
|
||||
# spx admin images import --name spinifex-rds-mariadb
|
||||
#
|
||||
# Publishing overwrites the mirror artifact, so it is a deliberate action, not
|
||||
# something every push should trigger: it runs manually, or from release.yml
|
||||
# once a tag has passed its gates. The `variant` input scopes which images
|
||||
# build — the Ubuntu GPU legs are 16G and much slower (NVIDIA driver download +
|
||||
# DKMS build), so publishing only the CPU AMIs stays cheap.
|
||||
# Publishing overwrites the mirror artifact, so it runs only from release.yml
|
||||
# once a tag has passed its gates, with siblings at the tags go.mod requires.
|
||||
# The `variant` input scopes which images build — the Ubuntu GPU legs are 16G
|
||||
# and much slower (NVIDIA driver download + DKMS build).
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
variant:
|
||||
description: "Which images to build"
|
||||
type: choice
|
||||
default: all
|
||||
options:
|
||||
- all
|
||||
- cpu
|
||||
- gpu
|
||||
- rds
|
||||
dry_run:
|
||||
description: "Build + convert only; skip the R2 upload"
|
||||
type: boolean
|
||||
default: false
|
||||
workflow_call:
|
||||
inputs:
|
||||
variant:
|
||||
@@ -94,40 +78,44 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout Spinifex
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
- name: Resolve sibling release tags from go.mod
|
||||
id: siblings
|
||||
run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
|
||||
|
||||
- name: Checkout Viperblock
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/viperblock
|
||||
ref: dev
|
||||
ref: ${{ steps.siblings.outputs.viperblock }}
|
||||
path: viperblock
|
||||
|
||||
- name: Checkout Predastore
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
ref: dev
|
||||
ref: ${{ steps.siblings.outputs.predastore }}
|
||||
path: predastore
|
||||
|
||||
- name: Checkout Northstar
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/northstar
|
||||
ref: dev
|
||||
ref: ${{ steps.siblings.outputs.northstar }}
|
||||
path: northstar
|
||||
|
||||
- name: Checkout Bluebottle
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/bluebottle
|
||||
ref: dev
|
||||
ref: ${{ steps.siblings.outputs.bluebottle }}
|
||||
path: bluebottle
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v7.0.0
|
||||
with:
|
||||
go-version-file: "spinifex/go.mod"
|
||||
# Self-hosted runner: the Go module + build cache already persists on
|
||||
|
||||
@@ -5,51 +5,60 @@ on:
|
||||
tags: [ "v*" ]
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
distro-amd64:
|
||||
runs-on: ubuntu-26.04
|
||||
steps:
|
||||
- name: Checkout Spinifex
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
- name: Resolve sibling release tags from go.mod
|
||||
id: siblings
|
||||
run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
|
||||
|
||||
- name: Checkout Predastore
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
ref: ${{ steps.siblings.outputs.predastore }}
|
||||
path: predastore
|
||||
|
||||
- name: Checkout Viperblock
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/viperblock
|
||||
ref: ${{ steps.siblings.outputs.viperblock }}
|
||||
path: viperblock
|
||||
|
||||
- name: Checkout Northstar
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/northstar
|
||||
ref: ${{ steps.siblings.outputs.northstar }}
|
||||
path: northstar
|
||||
|
||||
- name: Checkout Bluebottle
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/bluebottle
|
||||
ref: ${{ steps.siblings.outputs.bluebottle }}
|
||||
path: bluebottle
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v4.4.1
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v7.0.0
|
||||
with:
|
||||
go-version-file: spinifex/go.mod
|
||||
cache: false
|
||||
|
||||
- name: Build microVM artifacts (lb-agent + kernel + initramfs)
|
||||
run: cd spinifex && make build-microvm-image
|
||||
- name: Build microVM artifacts (lb-agent + kernel + initramfs + qboot + SQS store and worker images)
|
||||
run: cd spinifex && make build-microvm-image build-sqs-store-microvm-image build-sqs-worker-microvm-image
|
||||
|
||||
- name: Build amd64 Tarball
|
||||
env:
|
||||
@@ -57,48 +66,57 @@ jobs:
|
||||
run: cd spinifex && make distro-amd64 VERSION="$VERSION"
|
||||
|
||||
- name: Upload amd64 Artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
with:
|
||||
name: distro-amd64
|
||||
path: |
|
||||
spinifex/dist/spinifex-${{ github.ref_name }}-linux-amd64.tar.gz
|
||||
spinifex/dist/spinifex-${{ github.ref_name }}-linux-amd64.tar.gz.sha256
|
||||
spinifex/dist/qboot-*.tar.gz
|
||||
spinifex/scripts/setup.sh
|
||||
|
||||
distro-arm64:
|
||||
runs-on: ubuntu-26.04-arm
|
||||
steps:
|
||||
- name: Checkout Spinifex
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
- name: Resolve sibling release tags from go.mod
|
||||
id: siblings
|
||||
run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
|
||||
|
||||
- name: Checkout Predastore
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
ref: ${{ steps.siblings.outputs.predastore }}
|
||||
path: predastore
|
||||
|
||||
- name: Checkout Viperblock
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/viperblock
|
||||
ref: ${{ steps.siblings.outputs.viperblock }}
|
||||
path: viperblock
|
||||
|
||||
- name: Checkout Northstar
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/northstar
|
||||
ref: ${{ steps.siblings.outputs.northstar }}
|
||||
path: northstar
|
||||
|
||||
- name: Checkout Bluebottle
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/bluebottle
|
||||
ref: ${{ steps.siblings.outputs.bluebottle }}
|
||||
path: bluebottle
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v4.4.1
|
||||
|
||||
- name: Build arm64 Tarball
|
||||
env:
|
||||
@@ -106,7 +124,7 @@ jobs:
|
||||
run: cd spinifex && make distro-arm64 VERSION="$VERSION"
|
||||
|
||||
- name: Upload arm64 Artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
with:
|
||||
name: distro-arm64
|
||||
path: |
|
||||
@@ -116,21 +134,23 @@ jobs:
|
||||
release:
|
||||
needs: [ distro-amd64, distro-arm64 ]
|
||||
runs-on: ubuntu-26.04
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Download amd64 Artifacts
|
||||
uses: actions/download-artifact@v8
|
||||
uses: actions/download-artifact@v8.0.2
|
||||
with:
|
||||
name: distro-amd64
|
||||
path: artifacts
|
||||
|
||||
- name: Download arm64 Artifacts
|
||||
uses: actions/download-artifact@v8
|
||||
uses: actions/download-artifact@v8.0.2
|
||||
with:
|
||||
name: distro-arm64
|
||||
path: artifacts
|
||||
|
||||
- name: Create Draft GitHub Release
|
||||
uses: softprops/action-gh-release@v3
|
||||
uses: softprops/action-gh-release@v3.0.3
|
||||
with:
|
||||
draft: true
|
||||
files: |
|
||||
@@ -140,7 +160,7 @@ jobs:
|
||||
generate_release_notes: true
|
||||
|
||||
- name: Upload Tarball for Verify Job
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
with:
|
||||
name: release-assets
|
||||
path: artifacts/
|
||||
@@ -167,14 +187,14 @@ jobs:
|
||||
| sort -r | xargs -r -I{} sudo umount -l -R {} || true
|
||||
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
|
||||
|
||||
- uses: actions/create-github-app-token@v3
|
||||
- uses: actions/create-github-app-token@v3.3.0
|
||||
id: app-token
|
||||
with:
|
||||
client-id: ${{ secrets.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
repositories: mulga
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/mulga
|
||||
token: ${{ steps.app-token.outputs.token }}
|
||||
@@ -183,13 +203,13 @@ jobs:
|
||||
|
||||
# Only the shared shell helpers: this job builds nothing from source, but
|
||||
# it drives the same libvirt provider the e2e cells retry around.
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
sparse-checkout: .github/scripts
|
||||
path: spinifex
|
||||
|
||||
- name: Download Release Assets
|
||||
uses: actions/download-artifact@v8
|
||||
uses: actions/download-artifact@v8.0.2
|
||||
with:
|
||||
name: release-assets
|
||||
path: release-assets
|
||||
@@ -231,6 +251,8 @@ jobs:
|
||||
promote:
|
||||
needs: [ verify, baremetal ]
|
||||
runs-on: ubuntu-26.04
|
||||
permissions:
|
||||
contents: write
|
||||
env:
|
||||
TAG: ${{ github.ref_name }}
|
||||
REPO: ${{ github.repository }}
|
||||
@@ -263,6 +285,8 @@ jobs:
|
||||
needs: [ verify, baremetal ]
|
||||
if: failure()
|
||||
runs-on: ubuntu-26.04
|
||||
permissions:
|
||||
contents: write
|
||||
env:
|
||||
TAG: ${{ github.ref_name }}
|
||||
REPO: ${{ github.repository }}
|
||||
@@ -280,6 +304,8 @@ jobs:
|
||||
iso:
|
||||
needs: release
|
||||
runs-on: [ self-hosted, ci-single ]
|
||||
permissions:
|
||||
contents: write
|
||||
# ISO build (~60-90m) overlaps the parallel baremetal run (~120m); the
|
||||
# upload gate then waits for baremetal to finish, so the job must outlast it.
|
||||
timeout-minutes: 180
|
||||
@@ -294,7 +320,7 @@ jobs:
|
||||
sudo umount -l "${ROOTFS}/proc" 2>/dev/null || true
|
||||
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
|
||||
|
||||
- uses: actions/create-github-app-token@v3
|
||||
- uses: actions/create-github-app-token@v3.3.0
|
||||
id: app-token
|
||||
with:
|
||||
client-id: ${{ secrets.APP_ID }}
|
||||
@@ -302,37 +328,45 @@ jobs:
|
||||
repositories: mulga
|
||||
|
||||
- name: Checkout Spinifex
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
path: spinifex
|
||||
ref: ${{ github.ref }}
|
||||
|
||||
- name: Resolve sibling release tags from go.mod
|
||||
id: siblings
|
||||
run: bash spinifex/.github/resolve-release-sibling-refs.sh spinifex/go.mod
|
||||
|
||||
- name: Checkout Predastore
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
ref: ${{ steps.siblings.outputs.predastore }}
|
||||
path: predastore
|
||||
|
||||
- name: Checkout Viperblock
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/viperblock
|
||||
ref: ${{ steps.siblings.outputs.viperblock }}
|
||||
path: viperblock
|
||||
|
||||
- name: Checkout Northstar
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/northstar
|
||||
ref: ${{ steps.siblings.outputs.northstar }}
|
||||
path: northstar
|
||||
|
||||
- name: Checkout Bluebottle
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/bluebottle
|
||||
ref: ${{ steps.siblings.outputs.bluebottle }}
|
||||
path: bluebottle
|
||||
|
||||
- name: Checkout Mulga iso-builder
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/mulga
|
||||
token: ${{ steps.app-token.outputs.token }}
|
||||
@@ -343,7 +377,7 @@ jobs:
|
||||
run: sudo make -C mulga/scripts/iso-builder clean
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v7.0.0
|
||||
with:
|
||||
go-version-file: spinifex/go.mod
|
||||
cache: false
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
#
|
||||
# tests/integration/segscan_oracle_test.go needs the mulga umbrella repo's
|
||||
# scripts/segscan checked out alongside spinifex (it is a separate Go module,
|
||||
# `package main`, and cannot be imported — see spinifex/testutil/segscanoracle).
|
||||
# `package main`, and cannot be imported — see internal/testkit/segscanoracle).
|
||||
# spinifex.yml's jobs check out only spinifex/viperblock/predastore and run on
|
||||
# every PR; adding a second-repo checkout there would make every spinifex PR
|
||||
# depend on mulga main being healthy and pay that checkout cost on every run,
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout Spinifex
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
path: spinifex
|
||||
fetch-depth: 2
|
||||
@@ -38,34 +38,34 @@ jobs:
|
||||
working-directory: spinifex
|
||||
|
||||
- name: Checkout Viperblock
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/viperblock
|
||||
ref: ${{ steps.siblings.outputs.viperblock }}
|
||||
path: viperblock
|
||||
|
||||
- name: Checkout Predastore
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
ref: ${{ steps.siblings.outputs.predastore }}
|
||||
path: predastore
|
||||
|
||||
- name: Checkout Northstar
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/northstar
|
||||
ref: ${{ steps.siblings.outputs.northstar }}
|
||||
path: northstar
|
||||
|
||||
- name: Checkout Bluebottle
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/bluebottle
|
||||
ref: ${{ steps.siblings.outputs.bluebottle }}
|
||||
path: bluebottle
|
||||
|
||||
- uses: actions/create-github-app-token@v3
|
||||
- uses: actions/create-github-app-token@v3.3.0
|
||||
id: app-token
|
||||
with:
|
||||
client-id: ${{ secrets.APP_ID }}
|
||||
@@ -73,7 +73,7 @@ jobs:
|
||||
repositories: mulga
|
||||
|
||||
- name: Checkout Mulga (scripts/segscan only)
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/mulga
|
||||
token: ${{ steps.app-token.outputs.token }}
|
||||
@@ -81,7 +81,7 @@ jobs:
|
||||
path: mulga
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v7.0.0
|
||||
with:
|
||||
cache-dependency-path: "spinifex/go.sum"
|
||||
go-version-file: "spinifex/go.mod"
|
||||
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout Spinifex
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
path: spinifex
|
||||
fetch-depth: 2
|
||||
@@ -40,28 +40,28 @@ jobs:
|
||||
working-directory: spinifex
|
||||
|
||||
- name: Checkout Viperblock
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/viperblock
|
||||
ref: ${{ steps.siblings.outputs.viperblock }}
|
||||
path: viperblock
|
||||
|
||||
- name: Checkout Predastore
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
ref: ${{ steps.siblings.outputs.predastore }}
|
||||
path: predastore
|
||||
|
||||
- name: Checkout Northstar
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/northstar
|
||||
ref: ${{ steps.siblings.outputs.northstar }}
|
||||
path: northstar
|
||||
|
||||
- name: Checkout Bluebottle
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/bluebottle
|
||||
ref: ${{ steps.siblings.outputs.bluebottle }}
|
||||
@@ -71,7 +71,7 @@ jobs:
|
||||
# the last go.sum change and never refreshed it. Both caches are managed
|
||||
# below instead.
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v7.0.0
|
||||
with:
|
||||
go-version-file: "spinifex/go.mod"
|
||||
cache: false
|
||||
@@ -80,7 +80,7 @@ jobs:
|
||||
# setup-go hashed spinifex/go.sum alone, which left the sibling modules'
|
||||
# dependencies out of the key and re-downloading on every run.
|
||||
- name: Cache Go modules
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v6.1.0
|
||||
with:
|
||||
path: ~/go/pkg/mod
|
||||
key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
|
||||
@@ -93,7 +93,7 @@ jobs:
|
||||
# a PR reads the base branch's entry without spending a minute uploading
|
||||
# one only that PR would ever read.
|
||||
- name: Restore Go build cache
|
||||
uses: actions/cache/restore@v6
|
||||
uses: actions/cache/restore@v6.1.0
|
||||
with:
|
||||
path: ~/.cache/go-build
|
||||
key: gobuild-${{ github.job }}-${{ github.run_id }}
|
||||
@@ -119,9 +119,6 @@ jobs:
|
||||
- name: Build Spinifex
|
||||
run: make -C spinifex build
|
||||
|
||||
- name: Manifest check + drift lint
|
||||
run: make -C spinifex manifest-check manifest-lint
|
||||
|
||||
- name: Run Tests with Coverage
|
||||
env:
|
||||
SPINIFEX_REQUIRE_CONFORMANCE_TOOLS: "1"
|
||||
@@ -150,7 +147,7 @@ jobs:
|
||||
|
||||
- name: Save Go build cache
|
||||
if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
|
||||
uses: actions/cache/save@v6
|
||||
uses: actions/cache/save@v6.1.0
|
||||
with:
|
||||
path: ~/.cache/go-build
|
||||
key: gobuild-${{ github.job }}-${{ github.run_id }}
|
||||
@@ -161,7 +158,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout Spinifex
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
path: spinifex
|
||||
|
||||
@@ -173,28 +170,28 @@ jobs:
|
||||
working-directory: spinifex
|
||||
|
||||
- name: Checkout Viperblock
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/viperblock
|
||||
ref: ${{ steps.siblings.outputs.viperblock }}
|
||||
path: viperblock
|
||||
|
||||
- name: Checkout Predastore
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
ref: ${{ steps.siblings.outputs.predastore }}
|
||||
path: predastore
|
||||
|
||||
- name: Checkout Northstar
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/northstar
|
||||
ref: ${{ steps.siblings.outputs.northstar }}
|
||||
path: northstar
|
||||
|
||||
- name: Checkout Bluebottle
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/bluebottle
|
||||
ref: ${{ steps.siblings.outputs.bluebottle }}
|
||||
@@ -204,7 +201,7 @@ jobs:
|
||||
# the last go.sum change and never refreshed it. Both caches are managed
|
||||
# below instead.
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v7.0.0
|
||||
with:
|
||||
go-version-file: "spinifex/go.mod"
|
||||
cache: false
|
||||
@@ -213,7 +210,7 @@ jobs:
|
||||
# setup-go hashed spinifex/go.sum alone, which left the sibling modules'
|
||||
# dependencies out of the key and re-downloading on every run.
|
||||
- name: Cache Go modules
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v6.1.0
|
||||
with:
|
||||
path: ~/go/pkg/mod
|
||||
key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
|
||||
@@ -226,7 +223,7 @@ jobs:
|
||||
# a PR reads the base branch's entry without spending a minute uploading
|
||||
# one only that PR would ever read.
|
||||
- name: Restore Go build cache
|
||||
uses: actions/cache/restore@v6
|
||||
uses: actions/cache/restore@v6.1.0
|
||||
with:
|
||||
path: ~/.cache/go-build
|
||||
key: gobuild-${{ github.job }}-${{ github.run_id }}
|
||||
@@ -251,7 +248,7 @@ jobs:
|
||||
|
||||
- name: Save Go build cache
|
||||
if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
|
||||
uses: actions/cache/save@v6
|
||||
uses: actions/cache/save@v6.1.0
|
||||
with:
|
||||
path: ~/.cache/go-build
|
||||
key: gobuild-${{ github.job }}-${{ github.run_id }}
|
||||
@@ -262,7 +259,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout Spinifex
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
path: spinifex
|
||||
fetch-depth: 2
|
||||
@@ -275,28 +272,28 @@ jobs:
|
||||
working-directory: spinifex
|
||||
|
||||
- name: Checkout Viperblock
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/viperblock
|
||||
ref: ${{ steps.siblings.outputs.viperblock }}
|
||||
path: viperblock
|
||||
|
||||
- name: Checkout Predastore
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/predastore
|
||||
ref: ${{ steps.siblings.outputs.predastore }}
|
||||
path: predastore
|
||||
|
||||
- name: Checkout Northstar
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/northstar
|
||||
ref: ${{ steps.siblings.outputs.northstar }}
|
||||
path: northstar
|
||||
|
||||
- name: Checkout Bluebottle
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
repository: mulgadc/bluebottle
|
||||
ref: ${{ steps.siblings.outputs.bluebottle }}
|
||||
@@ -306,7 +303,7 @@ jobs:
|
||||
# the last go.sum change and never refreshed it. Both caches are managed
|
||||
# below instead.
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v7.0.0
|
||||
with:
|
||||
go-version-file: "spinifex/go.mod"
|
||||
cache: false
|
||||
@@ -315,7 +312,7 @@ jobs:
|
||||
# setup-go hashed spinifex/go.sum alone, which left the sibling modules'
|
||||
# dependencies out of the key and re-downloading on every run.
|
||||
- name: Cache Go modules
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v6.1.0
|
||||
with:
|
||||
path: ~/go/pkg/mod
|
||||
key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
|
||||
@@ -328,7 +325,7 @@ jobs:
|
||||
# a PR reads the base branch's entry without spending a minute uploading
|
||||
# one only that PR would ever read.
|
||||
- name: Restore Go build cache
|
||||
uses: actions/cache/restore@v6
|
||||
uses: actions/cache/restore@v6.1.0
|
||||
with:
|
||||
path: ~/.cache/go-build
|
||||
key: gobuild-${{ github.job }}-${{ github.run_id }}
|
||||
@@ -338,9 +335,9 @@ jobs:
|
||||
run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
|
||||
|
||||
- name: Lint
|
||||
uses: golangci/golangci-lint-action@v9
|
||||
uses: golangci/golangci-lint-action@v9.3.0
|
||||
with:
|
||||
version: v2.13.0
|
||||
version: v2.14.0
|
||||
working-directory: spinifex
|
||||
|
||||
- name: Govulncheck
|
||||
@@ -363,7 +360,7 @@ jobs:
|
||||
|
||||
- name: Save Go build cache
|
||||
if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
|
||||
uses: actions/cache/save@v6
|
||||
uses: actions/cache/save@v6.1.0
|
||||
with:
|
||||
path: ~/.cache/go-build
|
||||
key: gobuild-${{ github.job }}-${{ github.run_id }}
|
||||
|
||||
@@ -6,14 +6,24 @@ on:
|
||||
- main
|
||||
- dev
|
||||
paths:
|
||||
- spinifex/services/spinifexui/frontend/**
|
||||
- spinifex/runtime/roles/spinifexui/frontend/**
|
||||
- spinifex/runtime/roles/spinifexui/*.go
|
||||
- spinifex/operator/admin/instance_metrics*.go
|
||||
- spinifex/operator/admin/live_streams*.go
|
||||
- spinifex/runtime/roles/qmpcollector/**
|
||||
- contracts/telemetry/**
|
||||
- .github/workflows/ui.yml
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- dev
|
||||
paths:
|
||||
- spinifex/services/spinifexui/frontend/**
|
||||
- spinifex/runtime/roles/spinifexui/frontend/**
|
||||
- spinifex/runtime/roles/spinifexui/*.go
|
||||
- spinifex/operator/admin/instance_metrics*.go
|
||||
- spinifex/operator/admin/live_streams*.go
|
||||
- spinifex/runtime/roles/qmpcollector/**
|
||||
- contracts/telemetry/**
|
||||
- .github/workflows/ui.yml
|
||||
workflow_dispatch:
|
||||
|
||||
@@ -26,25 +36,25 @@ jobs:
|
||||
runs-on: ubuntu-26.04
|
||||
defaults:
|
||||
run:
|
||||
working-directory: spinifex/services/spinifexui/frontend
|
||||
working-directory: spinifex/runtime/roles/spinifexui/frontend
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
sparse-checkout: spinifex/services/spinifexui/frontend
|
||||
sparse-checkout: spinifex/runtime/roles/spinifexui/frontend
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v6
|
||||
uses: pnpm/action-setup@v6.1.0
|
||||
with:
|
||||
package_json_file: spinifex/services/spinifexui/frontend/package.json
|
||||
package_json_file: spinifex/runtime/roles/spinifexui/frontend/package.json
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v7
|
||||
uses: actions/setup-node@v7.1.0
|
||||
with:
|
||||
cache: "pnpm"
|
||||
cache-dependency-path: spinifex/services/spinifexui/frontend/pnpm-lock.yaml
|
||||
node-version-file: spinifex/services/spinifexui/frontend/.nvmrc
|
||||
cache-dependency-path: spinifex/runtime/roles/spinifexui/frontend/pnpm-lock.yaml
|
||||
node-version-file: spinifex/runtime/roles/spinifexui/frontend/.nvmrc
|
||||
|
||||
- name: Install Dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
@@ -58,5 +68,96 @@ jobs:
|
||||
- name: Build Frontend
|
||||
run: pnpm build
|
||||
|
||||
- name: Check for generated route tree drift
|
||||
run: git diff --exit-code -- src/routeTree.gen.ts
|
||||
# spx embeds the committed dist/, so a bundle that differs from a clean
|
||||
# build of this source would ship JavaScript CI never built. Porcelain
|
||||
# rather than git diff, so added and deleted chunks count too.
|
||||
- name: Check for generated file drift
|
||||
run: |
|
||||
drift=$(git status --porcelain -- src/routeTree.gen.ts src/lib/cn-tables.gen.ts dist)
|
||||
if [ -n "$drift" ]; then
|
||||
echo "::error::Generated files differ from a clean build. Run pnpm build and commit the result."
|
||||
echo "$drift"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The pinned image lives in one file that scripts/ui-playwright.sh also reads,
|
||||
# so make ui-baselines and the job below draw screenshots in the same image.
|
||||
ui_e2e_image:
|
||||
name: UI End-to-End Image
|
||||
runs-on: ubuntu-26.04
|
||||
outputs:
|
||||
image: ${{ steps.read.outputs.image }}
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
sparse-checkout: spinifex/runtime/roles/spinifexui/frontend/playwright.image
|
||||
sparse-checkout-cone-mode: false
|
||||
|
||||
- name: Read Playwright Image
|
||||
id: read
|
||||
run: echo "image=$(tr -d '[:space:]' < spinifex/runtime/roles/spinifexui/frontend/playwright.image)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# The browser suite: the console against a faked API on Vite dev, plus the
|
||||
# built dist under the production CSP. It runs in Playwright's own image, which
|
||||
# carries the browsers, so screenshots rasterise as they did for the baselines.
|
||||
ui_e2e:
|
||||
name: UI End-to-End
|
||||
needs: ui_e2e_image
|
||||
runs-on: ubuntu-26.04
|
||||
timeout-minutes: 20
|
||||
container:
|
||||
image: ${{ needs.ui_e2e_image.outputs.image }}
|
||||
options: --ipc=host
|
||||
defaults:
|
||||
run:
|
||||
working-directory: spinifex/runtime/roles/spinifexui/frontend
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v7.0.1
|
||||
with:
|
||||
sparse-checkout: spinifex/runtime/roles/spinifexui/frontend
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v6.1.0
|
||||
with:
|
||||
package_json_file: spinifex/runtime/roles/spinifexui/frontend/package.json
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v7.1.0
|
||||
with:
|
||||
cache: "pnpm"
|
||||
cache-dependency-path: spinifex/runtime/roles/spinifexui/frontend/pnpm-lock.yaml
|
||||
node-version-file: spinifex/runtime/roles/spinifexui/frontend/.nvmrc
|
||||
|
||||
- name: Install Dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
# The image's browsers belong to one Playwright release, so a lockfile
|
||||
# bump without an image bump would otherwise fail on a missing executable.
|
||||
- name: Check Playwright Matches the Image
|
||||
env:
|
||||
IMAGE: ${{ needs.ui_e2e_image.outputs.image }}
|
||||
run: |
|
||||
version=$(pnpm exec playwright --version | awk '{print $2}')
|
||||
case "$IMAGE" in
|
||||
*":v$version-"*) ;;
|
||||
*) echo "::error::Lockfile has Playwright $version but the image is $IMAGE. Update playwright.image."; exit 1 ;;
|
||||
esac
|
||||
|
||||
- name: Build Frontend
|
||||
run: pnpm build
|
||||
|
||||
- name: Run End-to-End Suite
|
||||
run: pnpm e2e
|
||||
|
||||
- name: Upload Screenshot Diffs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7.0.2
|
||||
with:
|
||||
name: ui-e2e-test-results
|
||||
path: spinifex/runtime/roles/spinifexui/frontend/test-results/
|
||||
if-no-files-found: ignore
|
||||
retention-days: 14
|
||||
|
||||
@@ -19,7 +19,16 @@ bin/
|
||||
dist/
|
||||
build/microvm/vmlinuz
|
||||
build/microvm/initramfs.cpio.gz
|
||||
!spinifex/services/spinifexui/frontend/dist/
|
||||
build/microvm/qboot.rom
|
||||
build/microvm/qboot.LICENSE
|
||||
build/microvm/sqs-store/vmlinuz
|
||||
build/microvm/sqs-store/initramfs.cpio.gz
|
||||
build/microvm/sqs-store/.stage.*/
|
||||
build/microvm/sqs-worker/vmlinuz
|
||||
build/microvm/sqs-worker/initramfs.cpio.gz
|
||||
build/microvm/sqs-worker/.stage.*/
|
||||
build/qboot/
|
||||
!spinifex/runtime/roles/spinifexui/frontend/dist/
|
||||
tmp/
|
||||
*.test
|
||||
*.out
|
||||
|
||||
@@ -71,17 +71,18 @@ linters:
|
||||
- pkg: math/rand$
|
||||
desc: Use math/rand/v2 instead, see https://go.dev/blog/randv2
|
||||
|
||||
# The EC2 control plane reaches block storage through ebsprovider, not
|
||||
# The control plane reaches block storage through the EBS provider, not
|
||||
# by importing the engine. viperblock.New starts a chunk uploader, so
|
||||
# every construction is a writer against a volume viperblockd serves.
|
||||
# Nothing under handlers/ may import viperblock, in any form, tests
|
||||
# No domain or legacy handler may import viperblock, in any form, tests
|
||||
# included. A types-only import counts: one became an engine import.
|
||||
ec2-control-plane:
|
||||
files:
|
||||
- '**/spinifex/domains/**'
|
||||
- '**/spinifex/handlers/**'
|
||||
deny:
|
||||
- pkg: github.com/mulgadc/viperblock
|
||||
desc: Use the ebsprovider.EBSProvider interface; the control plane must not construct or name viperblock types.
|
||||
desc: Use the providers/ebs EBSProvider interface; the control plane must not construct or name viperblock types.
|
||||
|
||||
embeddedstructfieldcheck:
|
||||
forbid-mutex: true
|
||||
|
||||
@@ -16,6 +16,7 @@ Every AWS call enters through a SigV4-authenticated gateway, is published to NAT
|
||||
| [`docs/COMMANDS.md`](docs/COMMANDS.md) | The `spx admin` CLI and the systemd units it manages. |
|
||||
| [`docs/coverage/`](docs/coverage/README.md) | Which AWS API operations each service implements. |
|
||||
| [`scripts/README.md`](scripts/README.md) | Install, dev-environment, and image-build scripts. |
|
||||
| [`docs/PACKAGE_BOUNDARY_MIGRATION.md`](docs/PACKAGE_BOUNDARY_MIGRATION.md) | ADR-0001 implementation inventory and incremental source moves. |
|
||||
|
||||
User-facing docs are published at [docs.mulgadc.com](https://docs.mulgadc.com).
|
||||
|
||||
@@ -53,14 +54,14 @@ AWS_PROFILE=spinifex aws ec2 describe-instance-types
|
||||
|
||||
A list of instance types means the cluster is up.
|
||||
|
||||
`make preflight` also needs `shellcheck` and `golangci-lint` (CI pins v2.13.0):
|
||||
`make preflight` also needs `shellcheck` and `golangci-lint`
|
||||
|
||||
```bash
|
||||
sudo apt-get install -y shellcheck
|
||||
curl -sSfL https://golangci-lint.run/install.sh | sh -s -- -b "$(go env GOPATH)/bin" v2.13.0
|
||||
curl -sSfL https://golangci-lint.run/install.sh | sh -s -- -b "$(go env GOPATH)/bin" v2.14.0
|
||||
```
|
||||
|
||||
The web console additionally needs Node.js 24 and `pnpm`.
|
||||
The web console additionally needs `pnpm` and the Node.js version pinned in `spinifex/runtime/roles/spinifexui/frontend/.nvmrc`.
|
||||
|
||||
## Development Workflow
|
||||
|
||||
@@ -75,7 +76,7 @@ The web console additionally needs Node.js 24 and `pnpm`.
|
||||
|
||||
Follow the services with `journalctl -u 'spinifex-*' -f`.
|
||||
|
||||
The web console lives in `spinifex/services/spinifexui/frontend/` (`pnpm install`, `pnpm dev`, `pnpm lint`, `pnpm test`). `spx` embeds its checked-in `dist/`, so run `make build-ui` and commit the rebuilt `dist/` with your change.
|
||||
The web console lives in `spinifex/runtime/roles/spinifexui/frontend/` (`pnpm install`, `pnpm dev`, `pnpm lint`, `pnpm test`). `spx` embeds its checked-in `dist/`, so run `make build-ui` and commit the rebuilt `dist/` with your change.
|
||||
|
||||
## Creating a Pull Request
|
||||
|
||||
|
||||
@@ -54,12 +54,25 @@ build: go_build build-installer build-lb-agent generate-aws-model-coverage
|
||||
# Build spinifex-ui frontend (requires pnpm)
|
||||
build-ui:
|
||||
@echo -e "\n....Building spinifex-ui frontend...."
|
||||
cd spinifex/services/spinifexui/frontend && pnpm build
|
||||
cd spinifex/runtime/roles/spinifexui/frontend && pnpm build
|
||||
|
||||
# Playwright suite for the console, against Vite dev and the built dist under
|
||||
# the console's CSP. Not part of preflight; needs `pnpm exec playwright install chromium`.
|
||||
test-ui-e2e: build-ui
|
||||
cd spinifex/runtime/roles/spinifexui/frontend && pnpm e2e
|
||||
|
||||
# The same suite inside the pinned Playwright image CI uses (podman or docker),
|
||||
# so screenshots compare as they will on the runner. ui-baselines redraws them.
|
||||
ui-e2e-container:
|
||||
scripts/ui-playwright.sh compare
|
||||
|
||||
ui-baselines:
|
||||
scripts/ui-playwright.sh baselines
|
||||
|
||||
# GO commands
|
||||
VERSION ?= $(shell git describe --tags --always --dirty --exclude '*-dev')
|
||||
COMMIT ?= $(shell git rev-parse --short HEAD)
|
||||
LDFLAGS := -s -w -X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Version=$(VERSION) -X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Commit=$(COMMIT)
|
||||
LDFLAGS := -s -w -X github.com/mulgadc/spinifex/spinifex/operator/cli.Version=$(VERSION) -X github.com/mulgadc/spinifex/spinifex/operator/cli.Commit=$(COMMIT)
|
||||
|
||||
go_build:
|
||||
@echo -e "\n....Building $(GO_PROJECT_NAME)"
|
||||
@@ -140,20 +153,74 @@ $(MICROVM_ARTIFACTS) &: $(MICROVM_INPUTS)
|
||||
bin/lb-agent:
|
||||
$(MAKE) build-lb-agent
|
||||
|
||||
build-microvm-image: $(MICROVM_ARTIFACTS) ## Build microVM kernel + initramfs (incremental — skips when up to date)
|
||||
# The SQS store microVM image: a separate artifact from the LB image above, not
|
||||
# installed or released by any target yet.
|
||||
SQS_STORE_MICROVM_OUT_DIR := $(MICROVM_OUT_DIR)/sqs-store
|
||||
SQS_STORE_MICROVM_ARTIFACTS := $(SQS_STORE_MICROVM_OUT_DIR)/vmlinuz $(SQS_STORE_MICROVM_OUT_DIR)/initramfs.cpio.gz
|
||||
SQS_STORE_MICROVM_INPUTS := scripts/build-sqs-store-microvm-image.sh scripts/lib/sqs-store-microvm-publish.sh $(SQS_STORE_MICROVM_OUT_DIR)/init.sh $(SQS_STORE_MICROVM_OUT_DIR)/inittab $(addprefix $(SQS_STORE_MICROVM_OUT_DIR)/,sqs-store-lib.sh sqs-store-prepare sqs-store-run sqs-store-stop) bin/nats-server
|
||||
|
||||
install-microvm: $(MICROVM_ARTIFACTS) ## Install microVM artifacts to /usr/share/spinifex/microvm/
|
||||
$(SQS_STORE_MICROVM_ARTIFACTS) &: $(SQS_STORE_MICROVM_INPUTS)
|
||||
./scripts/build-sqs-store-microvm-image.sh
|
||||
|
||||
# The dedicated SQS store server, built from the go.mod-pinned module so the
|
||||
# image ships exactly the reviewed version. Static for the musl guest.
|
||||
bin/nats-server: go.mod go.sum
|
||||
@mkdir -p bin
|
||||
GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags '-s -w' -o $@ github.com/nats-io/nats-server/v2
|
||||
|
||||
build-sqs-store-microvm-image: $(SQS_STORE_MICROVM_ARTIFACTS) ## Build the SQS store microVM kernel and initramfs
|
||||
|
||||
# The shared SQS worker microVM image: the store image's /init and kernel with
|
||||
# the worker binary instead of nats-server. Not installed or released yet.
|
||||
SQS_WORKER_MICROVM_OUT_DIR := $(MICROVM_OUT_DIR)/sqs-worker
|
||||
SQS_WORKER_MICROVM_ARTIFACTS := $(SQS_WORKER_MICROVM_OUT_DIR)/vmlinuz $(SQS_WORKER_MICROVM_OUT_DIR)/initramfs.cpio.gz
|
||||
SQS_WORKER_MICROVM_INPUTS := scripts/build-sqs-store-microvm-image.sh scripts/lib/sqs-store-microvm-publish.sh $(SQS_STORE_MICROVM_OUT_DIR)/init.sh $(SQS_WORKER_MICROVM_OUT_DIR)/inittab $(addprefix $(SQS_WORKER_MICROVM_OUT_DIR)/,sqs-worker-lib.sh sqs-worker-prepare sqs-worker-ready sqs-worker-stop) bin/sqs-worker
|
||||
|
||||
$(SQS_WORKER_MICROVM_ARTIFACTS) &: $(SQS_WORKER_MICROVM_INPUTS)
|
||||
SQS_MICROVM_PROFILE=worker ./scripts/build-sqs-store-microvm-image.sh
|
||||
|
||||
bin/sqs-worker: go.mod go.sum $(shell find cmd/sqs-worker spinifex/agents/sqs -name '*.go' ! -name '*_test.go')
|
||||
@mkdir -p bin
|
||||
GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags '-s -w' -o $@ ./cmd/sqs-worker
|
||||
|
||||
build-sqs-worker-microvm-image: $(SQS_WORKER_MICROVM_ARTIFACTS) ## Build the shared SQS worker microVM kernel and initramfs
|
||||
|
||||
# qboot is the microvm firmware. The source tarball ships with each release
|
||||
# because qboot is GPL-2.0.
|
||||
QBOOT_COMMIT := $(shell sed -n 's/^QBOOT_COMMIT="\(.*\)"$$/\1/p' scripts/build-qboot.sh)
|
||||
QBOOT_SOURCE := build/qboot/qboot-$(QBOOT_COMMIT).tar.gz
|
||||
QBOOT_ARTIFACTS := $(MICROVM_OUT_DIR)/qboot.rom $(MICROVM_OUT_DIR)/qboot.LICENSE $(QBOOT_SOURCE)
|
||||
|
||||
$(QBOOT_ARTIFACTS) &: scripts/build-qboot.sh
|
||||
./scripts/build-qboot.sh
|
||||
|
||||
build-microvm-image: $(MICROVM_ARTIFACTS) $(QBOOT_ARTIFACTS) ## Build microVM kernel, initramfs and qboot firmware (incremental — skips when up to date)
|
||||
|
||||
install-sqs-store-microvm: $(SQS_STORE_MICROVM_ARTIFACTS) ## Install the SQS store microVM image to /usr/share/spinifex/microvm/sqs-store/
|
||||
sudo install -d /usr/share/spinifex/microvm/sqs-store
|
||||
sudo rm -f /usr/share/spinifex/microvm/sqs-store/initramfs.cpio.gz
|
||||
sudo install -m 0644 $(SQS_STORE_MICROVM_OUT_DIR)/vmlinuz /usr/share/spinifex/microvm/sqs-store/vmlinuz
|
||||
sudo install -m 0644 $(SQS_STORE_MICROVM_OUT_DIR)/initramfs.cpio.gz /usr/share/spinifex/microvm/sqs-store/initramfs.cpio.gz
|
||||
|
||||
install-sqs-worker-microvm: $(SQS_WORKER_MICROVM_ARTIFACTS) ## Install the SQS worker microVM image to /usr/share/spinifex/microvm/sqs-worker/
|
||||
sudo install -d /usr/share/spinifex/microvm/sqs-worker
|
||||
sudo rm -f /usr/share/spinifex/microvm/sqs-worker/initramfs.cpio.gz
|
||||
sudo install -m 0644 $(SQS_WORKER_MICROVM_OUT_DIR)/vmlinuz /usr/share/spinifex/microvm/sqs-worker/vmlinuz
|
||||
sudo install -m 0644 $(SQS_WORKER_MICROVM_OUT_DIR)/initramfs.cpio.gz /usr/share/spinifex/microvm/sqs-worker/initramfs.cpio.gz
|
||||
|
||||
install-microvm: $(MICROVM_ARTIFACTS) $(QBOOT_ARTIFACTS) ## Install microVM artifacts to /usr/share/spinifex/microvm/
|
||||
sudo install -d /usr/share/spinifex/microvm
|
||||
sudo install -m 0644 $(MICROVM_OUT_DIR)/vmlinuz /usr/share/spinifex/microvm/vmlinuz
|
||||
sudo install -m 0644 $(MICROVM_OUT_DIR)/initramfs.cpio.gz /usr/share/spinifex/microvm/initramfs.cpio.gz
|
||||
sudo install -m 0644 $(MICROVM_OUT_DIR)/qboot.rom /usr/share/spinifex/microvm/qboot.rom
|
||||
sudo install -m 0644 $(MICROVM_OUT_DIR)/qboot.LICENSE /usr/share/spinifex/microvm/qboot.LICENSE
|
||||
|
||||
# the pre-commit gate: manifest checks, lint, vuln, and the unit and e2e-harness tiers.
|
||||
# the pre-commit gate: lint, vuln, and the unit and e2e-harness tiers.
|
||||
# integration and race tests skipped to keep quick, they run in CI.
|
||||
#
|
||||
# -j2 overlaps the only two heavy gates, lint and test-cover: alone each leaves
|
||||
# most of the machine idle, so pairing them cuts a cold run by about a quarter.
|
||||
# -Otarget keeps each gate's output in one block instead of interleaving them.
|
||||
# manifest-check manifest-lint test-package-check removed, can add back later
|
||||
preflight:
|
||||
@$(MAKE) --no-print-directory -j2 -Otarget QUIET=1 lint govulncheck test-cover diff-coverage test-harness test-build-scripts
|
||||
@echo -e "\n ✅ Preflight passed — safe to commit."
|
||||
@@ -167,7 +234,7 @@ preflight:
|
||||
# reason: between them they install the software, wire OVN, erase the state and
|
||||
# remove the install, so a shellcheck warning in one is an outage or a wrong
|
||||
# deletion, not a style note.
|
||||
NODE_SCRIPTS := scripts/setup.sh scripts/setup-ovn.sh scripts/install-node.sh scripts/smoke-test.sh scripts/node-reset.sh scripts/uninstall-spx.sh
|
||||
NODE_SCRIPTS := scripts/setup.sh scripts/setup-ovn.sh scripts/install-node.sh scripts/smoke-test.sh scripts/node-reset.sh scripts/uninstall-spx.sh scripts/lib/distro.sh scripts/lib/stage-sqs-store-microvm.sh
|
||||
|
||||
test-build-scripts:
|
||||
@echo -e "\n....Running build/scripts/**/*_test.sh...."
|
||||
@@ -175,13 +242,14 @@ test-build-scripts:
|
||||
echo "-- $$t"; \
|
||||
sh "$$t" || exit 1; \
|
||||
done
|
||||
@echo -e "\n....Running scripts/*_test.sh...."
|
||||
@for t in $$(find scripts -maxdepth 1 -name '*_test.sh' | sort); do \
|
||||
@echo -e "\n....Running scripts/*_test.sh, scripts/lib/*_test.sh and scripts/appliance/*_test.sh...."
|
||||
@for t in $$( (find scripts -maxdepth 1 -name '*_test.sh'; find scripts/lib scripts/appliance -name '*_test.sh') | sort); do \
|
||||
echo "-- $$t"; \
|
||||
sh "$$t" || exit 1; \
|
||||
done
|
||||
@echo -e "\n....Running shellcheck over build/scripts/**/*.sh + the node scripts...."
|
||||
shellcheck -S warning $$(find build/scripts -name '*.sh' | sort) $(NODE_SCRIPTS)
|
||||
shellcheck -S warning -x scripts/build-oracle-appliance.sh $$(find scripts/appliance -name '*.sh' | sort)
|
||||
@echo " test-build-scripts ok"
|
||||
|
||||
# E2E harness unit tests. Build-tagged `e2e` so they're skipped by the
|
||||
@@ -220,7 +288,7 @@ aws-model-coverage: generate-aws-model-coverage
|
||||
@go run ./cmd/aws-model-coverage
|
||||
|
||||
# Segscan storage oracle: needs the mulga umbrella repo's scripts/segscan
|
||||
# checked out alongside spinifex (see spinifex/testutil/segscanoracle), which
|
||||
# checked out alongside spinifex (see internal/testkit/segscanoracle), which
|
||||
# is not the default local or CI layout, so this is a separate target from
|
||||
# test-integration rather than folded into it. Skips itself when segscan's
|
||||
# source isn't found.
|
||||
@@ -228,27 +296,10 @@ test-segscan-oracle:
|
||||
@echo -e "\n....Running segscan storage oracle test...."
|
||||
$(_Q)LOG_IGNORE=1 go test -tags=integration,segscanoracle -timeout 120s ./tests/integration/... -run TestSegscanOracle $(_RACEQ)
|
||||
|
||||
# Validate docs/service-interfaces.yaml. Schema check + cross-reference
|
||||
# of services/suites/fixtures + on-disk path existence.
|
||||
manifest-check:
|
||||
@echo -e "\n....Checking service-interfaces.yaml...."
|
||||
@go run ./tests/e2e/manifest-check/cmd/manifest-check -repo-root . -manifest docs/service-interfaces.yaml
|
||||
|
||||
# Drift guards: direct-create fixture lint + NATS subject lint,
|
||||
# ratcheted against tests/e2e/manifest-lint/baseline.txt. Fails only on NEW
|
||||
# drift beyond the baseline.
|
||||
manifest-lint:
|
||||
@echo -e "\n....Linting manifest drift (fixtures + subjects)...."
|
||||
@go run ./tests/e2e/manifest-lint/cmd/manifest-lint -repo-root .
|
||||
|
||||
# Accept current drift into the baseline. Run after an intentional change.
|
||||
manifest-lint-update:
|
||||
@go run ./tests/e2e/manifest-lint/cmd/manifest-lint -repo-root . -update
|
||||
|
||||
# Run unit tests
|
||||
test:
|
||||
@echo -e "\n....Running tests for $(GO_PROJECT_NAME)...."
|
||||
LOG_IGNORE=1 go test -timeout 180s ./spinifex/... ./cmd/... ./internal/...
|
||||
LOG_IGNORE=1 go test -timeout 180s ./spinifex/... ./cmd/... ./internal/... ./contracts/... ./architecture/...
|
||||
|
||||
# Empty locally, where reusing a cached result between runs is the point.
|
||||
# CI passes -count=1 so a green run means the suite executed against that commit
|
||||
@@ -259,24 +310,13 @@ GOTESTFLAGS ?=
|
||||
COVERPROFILE ?= coverage.out
|
||||
test-cover:
|
||||
@echo -e "\n....Running tests with coverage for $(GO_PROJECT_NAME)...."
|
||||
$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -timeout 180s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... ./cmd/... ./internal/... $(_COVQ)
|
||||
$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -timeout 180s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... ./cmd/... ./internal/... ./contracts/... ./architecture/... $(_COVQ)
|
||||
@scripts/check-coverage.sh $(COVERPROFILE) $(QUIET)
|
||||
|
||||
# Refresh the README coverage badge. Measured as awesome-go documents
|
||||
# (-coverpkg=./... over ./...), with cmd/ entrypoints left out of the total.
|
||||
coverage-badge:
|
||||
@echo -e "\n....Measuring whole-module coverage for the README badge...."
|
||||
LOG_IGNORE=1 go test $(GOTESTFLAGS) -timeout 600s -covermode=atomic -coverpkg=./... -coverprofile=coverage-badge.out ./...
|
||||
@grep -v '/cmd/' coverage-badge.out > coverage-badge-nocmd.out
|
||||
@pct=$$(go tool cover -func=coverage-badge-nocmd.out | tail -1 | awk '{print $$NF}' | tr -d '%'); \
|
||||
color=$$(awk -v p="$$pct" 'BEGIN { print (p >= 80) ? "brightgreen" : (p >= 70) ? "yellow" : "red" }'); \
|
||||
sed -i -E "s|badge/coverage-[0-9.]+%25-[a-z]+|badge/coverage-$${pct}%25-$${color}|" README.md; \
|
||||
echo "Coverage $${pct}% ($${color}); README badge updated"
|
||||
|
||||
# Run unit tests with race detector
|
||||
test-race:
|
||||
@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
|
||||
$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... $(_RACEQ)
|
||||
$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... ./contracts/... ./architecture/... $(_RACEQ)
|
||||
|
||||
# Unit tests for in-repo GitHub Actions (e.g. .github/actions/e2e-analyze).
|
||||
# Kept out of `test-cover` so coverage % isn't diluted by CI-only tooling.
|
||||
@@ -344,14 +384,20 @@ fuzz:
|
||||
# Microvm artifacts are reinstalled when they already exist on disk — the rule's
|
||||
# input timestamps drive a rebuild only if anything actually changed. On a fresh
|
||||
# checkout (no build/microvm/vmlinuz yet) the install step is skipped; run
|
||||
# `make install-microvm` explicitly the first time.
|
||||
# `make install-microvm` explicitly the first time. They go in before the
|
||||
# preflight, which checks the microvm firmware.
|
||||
deploy: build
|
||||
sudo install -m 755 bin/spx /usr/local/bin/spx
|
||||
@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ]; then \
|
||||
$(MAKE) install-microvm; \
|
||||
else \
|
||||
echo "[deploy] microvm artifacts absent — run 'make install-microvm' for first-time setup"; \
|
||||
fi
|
||||
@if [ "$${SKIP_ASSET_PREFLIGHT:-}" != "1" ]; then \
|
||||
if ! sudo /usr/local/bin/spx admin preflight; then \
|
||||
echo ""; \
|
||||
echo "[deploy] host-asset preflight failed: this node's helper scripts or sudoers"; \
|
||||
echo " grants don't match the binary just installed. Remediate with:"; \
|
||||
echo "[deploy] host-asset preflight failed: this node's helper scripts, sudoers"; \
|
||||
echo " grants, QEMU or microvm firmware don't match the binary just installed. Remediate with:"; \
|
||||
echo " scripts/update-nodes.sh (from the mulga root)"; \
|
||||
echo " or:"; \
|
||||
echo " make reinstall"; \
|
||||
@@ -361,11 +407,6 @@ deploy: build
|
||||
else \
|
||||
echo "[deploy] SKIP_ASSET_PREFLIGHT=1 — skipping host-asset preflight"; \
|
||||
fi
|
||||
@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ]; then \
|
||||
$(MAKE) install-microvm; \
|
||||
else \
|
||||
echo "[deploy] microvm artifacts absent — run 'make install-microvm' for first-time setup"; \
|
||||
fi
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl restart spinifex.target
|
||||
|
||||
@@ -376,23 +417,32 @@ reinstall:
|
||||
|
||||
clean:
|
||||
rm -f ./bin/$(GO_PROJECT_NAME)
|
||||
rm -rf spinifex/services/spinifexui/frontend/dist
|
||||
rm -rf spinifex/runtime/roles/spinifexui/frontend/dist
|
||||
|
||||
# On dnf hosts the runtime packages and repositories come from setup.sh, so the
|
||||
# two cannot drift; nbdkit-devel, for building the plugin, is only in CodeReady.
|
||||
install-system:
|
||||
@echo -e "\n....Installing system dependencies for $(ARCH)...."
|
||||
sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
||||
-o Dpkg::Options::="--force-confdef" \
|
||||
-o Dpkg::Options::="--force-confold" \
|
||||
nbdkit nbdkit-plugin-dev pkg-config qemu-system-x86 qemu-system-arm qemu-utils \
|
||||
ovmf qemu-efi-aarch64 \
|
||||
libvirt-daemon-system libvirt-clients libvirt-dev make gcc jq curl \
|
||||
iproute2 netcat-openbsd openssh-client wget git unzip sudo xz-utils file \
|
||||
ovn-central ovn-host openvswitch-switch dhcpcd-base
|
||||
@pm=$$(INSTALL_SPINIFEX_LIB_ONLY=1 bash -c '. scripts/setup.sh && distro_pkg_manager') || exit 1; \
|
||||
if [ "$$pm" = dnf ]; then \
|
||||
INSTALL_SPINIFEX_LIB_ONLY=1 bash -c '. scripts/setup.sh && setup_sudo && detect_os && install_dnf_deps' && \
|
||||
sudo dnf config-manager --set-enabled ol10_codeready_builder && \
|
||||
sudo dnf install -y nbdkit-devel pkgconf-pkg-config make gcc openssh-clients git sudo; \
|
||||
else \
|
||||
sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
||||
-o Dpkg::Options::="--force-confdef" \
|
||||
-o Dpkg::Options::="--force-confold" \
|
||||
nbdkit nbdkit-plugin-dev pkg-config qemu-system-x86 qemu-system-arm qemu-utils \
|
||||
ovmf qemu-efi-aarch64 \
|
||||
libvirt-daemon-system libvirt-clients libvirt-dev make gcc jq curl \
|
||||
iproute2 netcat-openbsd openssh-client wget git unzip sudo xz-utils file \
|
||||
ovn-central ovn-host openvswitch-switch dhcpcd-base; \
|
||||
fi
|
||||
|
||||
install-go:
|
||||
@echo -e "\n....Installing Go 1.27.0 for $(ARCH) ($(GO_ARCH))...."
|
||||
@echo -e "\n....Installing Go 1.27.3 for $(ARCH) ($(GO_ARCH))...."
|
||||
@if [ ! -d "/usr/local/go" ]; then \
|
||||
curl -L https://go.dev/dl/go1.27.0.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
|
||||
curl -L https://go.dev/dl/go1.27.3.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
|
||||
else \
|
||||
echo "Go already installed in /usr/local/go"; \
|
||||
fi
|
||||
@@ -449,14 +499,17 @@ distro-amd64:
|
||||
-f build/Dockerfile.distro \
|
||||
--output type=local,dest=dist/amd64/ \
|
||||
../
|
||||
@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ] && [ -f $(MICROVM_OUT_DIR)/initramfs.cpio.gz ]; then \
|
||||
@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ] && [ -f $(MICROVM_OUT_DIR)/initramfs.cpio.gz ] && [ -f $(MICROVM_OUT_DIR)/qboot.rom ] && [ -f $(QBOOT_SOURCE) ]; then \
|
||||
echo "[distro-amd64] staging microvm artifacts into tarball"; \
|
||||
mkdir -p dist/amd64/microvm; \
|
||||
cp $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz dist/amd64/microvm/; \
|
||||
cp $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz $(MICROVM_OUT_DIR)/qboot.rom $(MICROVM_OUT_DIR)/qboot.LICENSE dist/amd64/microvm/; \
|
||||
cp $(QBOOT_SOURCE) dist/; \
|
||||
else \
|
||||
echo "[distro-amd64] WARNING: microvm artifacts missing — tarball will not include them"; \
|
||||
echo "[distro-amd64] run 'make build-microvm-image' before 'make distro-amd64'"; \
|
||||
fi
|
||||
./scripts/lib/stage-sqs-store-microvm.sh $(SQS_STORE_MICROVM_OUT_DIR) dist/amd64
|
||||
./scripts/lib/stage-sqs-store-microvm.sh $(SQS_WORKER_MICROVM_OUT_DIR) dist/amd64 sqs-worker
|
||||
tar -czf dist/spinifex-$(VERSION)-linux-amd64.tar.gz -C dist/amd64 .
|
||||
sha256sum dist/spinifex-$(VERSION)-linux-amd64.tar.gz > dist/spinifex-$(VERSION)-linux-amd64.tar.gz.sha256
|
||||
|
||||
@@ -475,7 +528,7 @@ distro-arm64:
|
||||
distro-clean:
|
||||
rm -rf dist/
|
||||
|
||||
.PHONY: test-package-check build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image install-microvm go_build preflight test test-cover coverage-badge test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
|
||||
.PHONY: test-package-check build build-ui test-ui-e2e ui-e2e-container ui-baselines build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-store-microvm-image build-sqs-worker-microvm-image install-microvm install-sqs-store-microvm install-sqs-worker-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
|
||||
deploy reinstall clean \
|
||||
install-system install-go install-aws quickinstall \
|
||||
lint fix govulncheck nilaway \
|
||||
|
||||
@@ -4,225 +4,138 @@
|
||||
|
||||
<p align="center">
|
||||
<a href="https://go.dev"><img src="https://img.shields.io/badge/Go-1.27+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
|
||||
<a href="https://pkg.go.dev/github.com/mulgadc/spinifex"><img src="https://img.shields.io/badge/pkg.go.dev-reference-007D9C?style=flat-square&logo=go&logoColor=white" alt="Go Reference"></a>
|
||||
<a href="https://github.com/mulgadc/spinifex/blob/main/Makefile"><img src="https://img.shields.io/badge/coverage-82.9%25-brightgreen?style=flat-square" alt="Coverage"></a>
|
||||
<a href="https://pkg.go.dev/github.com/mulgadc/spinifex"><img src="https://img.shields.io/badge/pkg.go.dev-Reference-007D9C?style=flat-square&logo=go&logoColor=white" alt="Go Reference"></a>
|
||||
<a href="LICENSE"><img src="https://img.shields.io/badge/License-AGPL--3.0-3fb950?style=flat-square" alt="License"></a>
|
||||
<a href="https://mulgadc.com"><img src="https://img.shields.io/badge/home-mulga-orange?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTE2LjcxOCA4Ljg5MWMtMS4yODUgMS4zNy0zLjE3OCAyLjMxNy00LjY5NSAzLjQ0LS44NTQuNjMtMy4wODkgMi4yNy0xLjYxIDMuMzQ0IDEuNzg4IDEuMjk4IDYuMjQzLjE2NCA3Ljc4NS0xLjI2Ljg0LS43NzUuODE0LTEuODIyLS4zMjgtMi4yNTktMS4yMzUtLjQ3Mi0yLjY2LS4xMTEtMy45MTMuMDg3LS4wNDIuMDA3LS4xMzEuMDU1LS4xMjMtLjAyLjUyMy0uMzM2Ljk5My0uNzUgMS41MDMtMS4xMDIuNDkyLS4zNCAxLjA5Ny0uODI3IDEuNy0uODM4IDEuODcxLS4wMzQgMy43OTkuODkgNC4yODcgMi44MTUuODExIDMuMjAzLTMuMDA2IDUuNzE1LTUuNzg0IDUuOTQybDEuNjE0LS43NDdjLjYwNS0uMzYgMS4yMTctLjczNCAxLjc1Mi0xLjE5Ni4xMzMtLjExNS4yMy0uMjYzLjM0Mi0uMzczLjAyNy0uMDI2LjMwMi0uMTQ0LjE0NC0uMTUtMS40NDEuOTQ1LTMuMTI3IDEuNTcyLTQuODEzIDEuOTMyLS41OC4xMjMtMS4xOTUuMjQyLTEuNzg1LjI1Ny4wMTUuMDguMDg3LjA2NC4xNC4wOC40NzYuMTU0IDEuMDIuMjQ1IDEuNTE2LjMyLjA2OC4wNTItLjAwNi4wNzQtLjA2LjA4MS0uNDQ4LjA1OC0uOTIzLjE0My0xLjM3LjE2Ni0xLjI3LjA2NC0yLjU1LS4wNjgtMy43NzctLjM4bC0uNjktMS45NTdjLS4wNi4wMDYtLjA2My4wNi0uMDc5LjEwMy0uMDYuMTctLjMwNSAxLjQxNy0uMzg3IDEuNDM1LS42Ni0uMi0xLjI1MS0uNjQ3LTEuNzM2LTEuMTMybDEuMTUtMi4wNi0xLjgzNSAxLjAxMWMtLjI1OC0uNTgzLS4zMDUtMS4yNDMtLjIxOS0xLjg3bDIuMzc1LTEuMThjLS43MzktLjA0OS0xLjQ4Ni4wNjgtMi4yMi4xNDUuMTI1LS4zNi4yNzctLjcxOC40NjItMS4wNTEuMDY3LS4xMjIuNDI2LS43MjEuNTItLjcyNC44Ni4xNjQgMS43ODYuMjQxIDIuNjQ4LjA3NGwtMS44Ny0uNzcxLS4wNjktLjA5M2MuMzgyLS4zNi43Ny0uNzE3IDEuMTk5LTEuMDI0LjgwNC4yMzQgMS42My40MjIgMi40NzMuNDMzTDkuMzUgOS4zNDJhNyA3IDAgMCAxIC41NjctLjQwMWMuMTYtLjEwMy42MDktLjQwMi43NzItLjM4LjY1LjIyMSAxLjMyNC4zNzggMi4wMS40MzMtLjMxMy0uMzI1LS45MDktLjU2NC0xLjIxLS44NjgtLjAzMy0uMDM0LS4wNTgtLjAzNi0uMDQzLS4wOTguNzg3LS40NTIgMS42NjItLjkwMSAyLjM0LTEuNTE3LjY5LS42MjkgMS40MjEtMS42NTYuOTI0LTIuNjA1LjU3My4xODIgMS4wNjcuOTA2IDEuMDU0IDEuNTEyLS4wMzQgMS42NzYtMS44MjIgMy4xNy0yLjk0MyA0LjIyMiAxLjczMi0uNzI4IDMuNzE0LTIuMjMgMy43MS00LjMwNS0uMDAzLTEuNjg4LTEuNTQtMi4zNjUtMi45OTMtMi41MThhMy4yIDMuMiAwIDAgMS0uMzg1IDEuMDg3Yy0uNDI4LjcxOS0xLjMwMiAxLjE2OC0xLjc1OCAxLjkxNS0uMzExLjUxLS4zNyAxLjE5NS0xLjAzMSAxLjM5LS4yNy4wOC0uNjE3LjA5My0uODk3LjA5NGwuNjE1LS4zMjVjLjY4OS0uNTA2LjY3Ny0xLjQyIDEuMDk5LTIuMS0uMDUtLjA3LS41NDUtLjItLjY2LS4yMjktLjUxLS4xMjUtMS4zNzUtLjI4NS0xLjg4Ni0uMjUtLjE1Ny4wMS0uODY5LjEyMS0uOTMuMjQyLS4wODguMTc3LjM0MS45My40NjggMS4xMDEuMDI1LjAzNS4wNzcuMDI2LjA4LjAzLjAyNC4wMzEuMDA3LjEwNS0uMDYuMDhhNSA1IDAgMCAxLS40MDQtLjI4M2MtLjUwMy0uMzg4LTEuMDc4LS45NzQtMS40OTktMS40NDctLjA2OC0uMTY2LS4xNi0uMzUuMDAyLS40ODcuODg4LS41NSAxLjc2OC0xLjEyNiAyLjY3LTEuNjUxIDIuNjcxLTEuNTU4IDUuNzExLTMuMzE4IDguMjY3LS40NjUgMi4xMTkgMi4zNjYgMS41MTQgNS4yMTItLjUxMiA3LjM3MnptLTguMjI0LTUuMzhjLjY5OS0uMTIyIDIuMDE4LjU3MiAyLjIzNS0uNDU2LjAyMS0uMS0uMDM2LS4xNTcuMDItLjI1NS4wNDMtLjA3My4yODYtLjI1LjM3LS4zMTguMjctLjIxNy41NzctLjM4Ny44NDItLjYxMi0xLjAyOS4xNjYtMi4wNjUuNzU2LTIuOTY0IDEuMjc3LS4xNDkuMDg2LS4zMS4xNzYtLjQ1MS4yNzMtLjAzNy4wMjUtLjA3NS0uMDA2LS4wNTMuMDltLTQuODMgMTAuODQ0Yy0xLjcxNSAxLjg4MS0xLjMyNSA0LjU3LjQ5NCA2LjIxNyAxLjgxMSAxLjY0MSA0LjY2IDIuMjIzIDcuMDQ4IDIuMTA4IDIuMTUyLS4xMDMgNC4zMzctLjgxMiA2LjQ2LS4wOTEuODI1LjI4IDEuNTQ2LjgwNSAyLjE2IDEuNDExLS4wODMtLjQtLjMyNC0uODE5LS41NTgtMS4xNTktMi4xMy0zLjA5NS02LjI3LTEuOTM1LTkuNDI2LTIuNTUzLTEuNzExLS4zMzUtMy40OTEtMS4xMTgtNC41MzMtMi41NjctLjkwMS0xLjI1My0xLjA0Mi0yLjczLS41OTUtNC4xOTQtLjA5MS0uMDktLjk1Mi43MjEtMS4wNS44MjhtNi4zNjYtOC42NjdjLS4zODIuMzM5LS43ODcuNjYtMS4yMTIuOTQ4bC0xLjIwNy41OWMxLjAyMi4wMzkgMi4wOC0uNTQ2IDIuNDItMS41MzciLz48L3N2Zz4=" alt="mulgadc.com"></a>
|
||||
<a href="https://docs.mulgadc.com"><img src="https://img.shields.io/badge/docs-spinifex-0969da?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTYuNSAySDIwYTEgMSAwIDAgMSAxIDF2MTQuNWExIDEgMCAwIDEtMSAxSDYuNWExLjUgMS41IDAgMCAwIDAgM0gyMWExIDEgMCAwIDEgMCAySDYuNUEzLjUgMy41IDAgMCAxIDMgMjBWNS41QTMuNSAzLjUgMCAwIDEgNi41IDJ6bTEgNGExIDEgMCAwIDAgMCAyaDlhMSAxIDAgMCAwIDAtMmgtOXptMCA0YTEgMSAwIDAgMCAwIDJoOWExIDEgMCAwIDAgMC0yaC05eiIvPjwvc3ZnPg==" alt="docs.mulgadc.com"></a>
|
||||
<a href="https://mulgadc.com"><img src="https://img.shields.io/badge/Site-mulga-orange?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTE2LjcxOCA4Ljg5MWMtMS4yODUgMS4zNy0zLjE3OCAyLjMxNy00LjY5NSAzLjQ0LS44NTQuNjMtMy4wODkgMi4yNy0xLjYxIDMuMzQ0IDEuNzg4IDEuMjk4IDYuMjQzLjE2NCA3Ljc4NS0xLjI2Ljg0LS43NzUuODE0LTEuODIyLS4zMjgtMi4yNTktMS4yMzUtLjQ3Mi0yLjY2LS4xMTEtMy45MTMuMDg3LS4wNDIuMDA3LS4xMzEuMDU1LS4xMjMtLjAyLjUyMy0uMzM2Ljk5My0uNzUgMS41MDMtMS4xMDIuNDkyLS4zNCAxLjA5Ny0uODI3IDEuNy0uODM4IDEuODcxLS4wMzQgMy43OTkuODkgNC4yODcgMi44MTUuODExIDMuMjAzLTMuMDA2IDUuNzE1LTUuNzg0IDUuOTQybDEuNjE0LS43NDdjLjYwNS0uMzYgMS4yMTctLjczNCAxLjc1Mi0xLjE5Ni4xMzMtLjExNS4yMy0uMjYzLjM0Mi0uMzczLjAyNy0uMDI2LjMwMi0uMTQ0LjE0NC0uMTUtMS40NDEuOTQ1LTMuMTI3IDEuNTcyLTQuODEzIDEuOTMyLS41OC4xMjMtMS4xOTUuMjQyLTEuNzg1LjI1Ny4wMTUuMDguMDg3LjA2NC4xNC4wOC40NzYuMTU0IDEuMDIuMjQ1IDEuNTE2LjMyLjA2OC4wNTItLjAwNi4wNzQtLjA2LjA4MS0uNDQ4LjA1OC0uOTIzLjE0My0xLjM3LjE2Ni0xLjI3LjA2NC0yLjU1LS4wNjgtMy43NzctLjM4bC0uNjktMS45NTdjLS4wNi4wMDYtLjA2My4wNi0uMDc5LjEwMy0uMDYuMTctLjMwNSAxLjQxNy0uMzg3IDEuNDM1LS42Ni0uMi0xLjI1MS0uNjQ3LTEuNzM2LTEuMTMybDEuMTUtMi4wNi0xLjgzNSAxLjAxMWMtLjI1OC0uNTgzLS4zMDUtMS4yNDMtLjIxOS0xLjg3bDIuMzc1LTEuMThjLS43MzktLjA0OS0xLjQ4Ni4wNjgtMi4yMi4xNDUuMTI1LS4zNi4yNzctLjcxOC40NjItMS4wNTEuMDY3LS4xMjIuNDI2LS43MjEuNTItLjcyNC44Ni4xNjQgMS43ODYuMjQxIDIuNjQ4LjA3NGwtMS44Ny0uNzcxLS4wNjktLjA5M2MuMzgyLS4zNi43Ny0uNzE3IDEuMTk5LTEuMDI0LjgwNC4yMzQgMS42My40MjIgMi40NzMuNDMzTDkuMzUgOS4zNDJhNyA3IDAgMCAxIC41NjctLjQwMWMuMTYtLjEwMy42MDktLjQwMi43NzItLjM4LjY1LjIyMSAxLjMyNC4zNzggMi4wMS40MzMtLjMxMy0uMzI1LS45MDktLjU2NC0xLjIxLS44NjgtLjAzMy0uMDM0LS4wNTgtLjAzNi0uMDQzLS4wOTguNzg3LS40NTIgMS42NjItLjkwMSAyLjM0LTEuNTE3LjY5LS42MjkgMS40MjEtMS42NTYuOTI0LTIuNjA1LjU3My4xODIgMS4wNjcuOTA2IDEuMDU0IDEuNTEyLS4wMzQgMS42NzYtMS44MjIgMy4xNy0yLjk0MyA0LjIyMiAxLjczMi0uNzI4IDMuNzE0LTIuMjMgMy43MS00LjMwNS0uMDAzLTEuNjg4LTEuNTQtMi4zNjUtMi45OTMtMi41MThhMy4yIDMuMiAwIDAgMS0uMzg1IDEuMDg3Yy0uNDI4LjcxOS0xLjMwMiAxLjE2OC0xLjc1OCAxLjkxNS0uMzExLjUxLS4zNyAxLjE5NS0xLjAzMSAxLjM5LS4yNy4wOC0uNjE3LjA5My0uODk3LjA5NGwuNjE1LS4zMjVjLjY4OS0uNTA2LjY3Ny0xLjQyIDEuMDk5LTIuMS0uMDUtLjA3LS41NDUtLjItLjY2LS4yMjktLjUxLS4xMjUtMS4zNzUtLjI4NS0xLjg4Ni0uMjUtLjE1Ny4wMS0uODY5LjEyMS0uOTMuMjQyLS4wODguMTc3LjM0MS45My40NjggMS4xMDEuMDI1LjAzNS4wNzcuMDI2LjA4LjAzLjAyNC4wMzEuMDA3LjEwNS0uMDYuMDhhNSA1IDAgMCAxLS40MDQtLjI4M2MtLjUwMy0uMzg4LTEuMDc4LS45NzQtMS40OTktMS40NDctLjA2OC0uMTY2LS4xNi0uMzUuMDAyLS40ODcuODg4LS41NSAxLjc2OC0xLjEyNiAyLjY3LTEuNjUxIDIuNjcxLTEuNTU4IDUuNzExLTMuMzE4IDguMjY3LS40NjUgMi4xMTkgMi4zNjYgMS41MTQgNS4yMTItLjUxMiA3LjM3MnptLTguMjI0LTUuMzhjLjY5OS0uMTIyIDIuMDE4LjU3MiAyLjIzNS0uNDU2LjAyMS0uMS0uMDM2LS4xNTcuMDItLjI1NS4wNDMtLjA3My4yODYtLjI1LjM3LS4zMTguMjctLjIxNy41NzctLjM4Ny44NDItLjYxMi0xLjAyOS4xNjYtMi4wNjUuNzU2LTIuOTY0IDEuMjc3LS4xNDkuMDg2LS4zMS4xNzYtLjQ1MS4yNzMtLjAzNy4wMjUtLjA3NS0uMDA2LS4wNTMuMDltLTQuODMgMTAuODQ0Yy0xLjcxNSAxLjg4MS0xLjMyNSA0LjU3LjQ5NCA2LjIxNyAxLjgxMSAxLjY0MSA0LjY2IDIuMjIzIDcuMDQ4IDIuMTA4IDIuMTUyLS4xMDMgNC4zMzctLjgxMiA2LjQ2LS4wOTEuODI1LjI4IDEuNTQ2LjgwNSAyLjE2IDEuNDExLS4wODMtLjQtLjMyNC0uODE5LS41NTgtMS4xNTktMi4xMy0zLjA5NS02LjI3LTEuOTM1LTkuNDI2LTIuNTUzLTEuNzExLS4zMzUtMy40OTEtMS4xMTgtNC41MzMtMi41NjctLjkwMS0xLjI1My0xLjA0Mi0yLjczLS41OTUtNC4xOTQtLjA5MS0uMDktLjk1Mi43MjEtMS4wNS44MjhtNi4zNjYtOC42NjdjLS4zODIuMzM5LS43ODcuNjYtMS4yMTIuOTQ4bC0xLjIwNy41OWMxLjAyMi4wMzkgMi4wOC0uNTQ2IDIuNDItMS41MzciLz48L3N2Zz4=" alt="mulgadc.com"></a>
|
||||
<a href="https://docs.mulgadc.com"><img src="https://img.shields.io/badge/Docs-spinifex-0969da?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTYuNSAySDIwYTEgMSAwIDAgMSAxIDF2MTQuNWExIDEgMCAwIDEtMSAxSDYuNWExLjUgMS41IDAgMCAwIDAgM0gyMWExIDEgMCAwIDEgMCAySDYuNUEzLjUgMy41IDAgMCAxIDMgMjBWNS41QTMuNSAzLjUgMCAwIDEgNi41IDJ6bTEgNGExIDEgMCAwIDAgMCAyaDlhMSAxIDAgMCAwIDAtMmgtOXptMCA0YTEgMSAwIDAgMCAwIDJoOWExIDEgMCAwIDAgMC0yaC05eiIvPjwvc3ZnPg==" alt="docs.mulgadc.com"></a>
|
||||
</p>
|
||||
|
||||
|
||||
<p align="center">
|
||||
<a href="#what-is-spinifex">What is Spinifex?</a> ·
|
||||
<a href="#the-platform">Platform</a> ·
|
||||
<a href="#three-ways-to-deploy">Deploy</a> ·
|
||||
<a href="#aws-compatibility">AWS Compatibility</a> ·
|
||||
<a href="#live-demo">Live Demo</a> ·
|
||||
<a href="#core-components">Components</a> ·
|
||||
<a href="#architecture-at-a-glance">Architecture</a> ·
|
||||
<a href="#self-hosted-installation">Installation</a> ·
|
||||
<a href="https://docs.mulgadc.com">Docs</a>
|
||||
<a href="#try-it-live">Live Demo</a> ·
|
||||
<a href="#services">Services</a> ·
|
||||
<a href="#deploy-anywhere">Deploy</a> ·
|
||||
<a href="#install">Install</a> ·
|
||||
<a href="#architecture">Architecture</a>
|
||||
</p>
|
||||
|
||||
---
|
||||
|
||||
# Spinifex: the open, AWS-compatible cloud you run yourself
|
||||
|
||||
Spinifex, built by [Mulga](https://mulgadc.com), recreates the AWS services your software already uses (EC2, EBS, S3, VPC, IAM) on infrastructure you control. Move workloads off the hyperscalers, run AI at a fraction of the cost, or take the cloud to the edge, without rewriting a line. The exact build you ship to AWS, Spinifex serves on your own hardware: same APIs, same Terraform, same SDKs.
|
||||
Spinifex, built by [Mulga](https://mulgadc.com), recreates the AWS services your software already uses on infrastructure you control. Move workloads off the hyperscalers, run AI at a fraction of the cost, or take the cloud to the edge, without rewriting a line.
|
||||
|
||||
## What is Spinifex?
|
||||
|
||||
Spinifex is an open-source infrastructure platform that recreates the AWS service surface (EC2, EBS, S3, VPC, IAM) on bare-metal, edge, and on-premises environments. Run cloud-native software without a hyperscaler.
|
||||
|
||||
Built for teams that need:
|
||||
|
||||
- The AWS tooling you already use (AWS CLI, SDKs, Terraform), with nothing to rewrite
|
||||
- Full control of the stack: your hardware, your network, your data, your keys
|
||||
- A cloud that keeps running offline, through disconnection, with no external control plane
|
||||
- An open core (AGPL-3.0), auditable and yours to keep, with no lock-in
|
||||
- **Your AWS tooling, unchanged.** The AWS CLI, SDKs, and Terraform work as they are.
|
||||
- **Full control of the stack.** Your hardware, data, and keys, with no external control plane, running through disconnection.
|
||||
- **Open core under AGPL-3.0.** Auditable, yours to keep, with no lock-in.
|
||||
|
||||
You change *where* your software runs, not *what* it is.
|
||||
|
||||
## The Platform
|
||||
## Try It Live
|
||||
|
||||
Get a real Spinifex endpoint in under a minute, with no card and no install. The sandbox is yours for 72 hours, with access keys, the web console, and EC2, EBS, S3, VPC, IAM, EKS, and RDS ready to call. Point your existing AWS CLI or Terraform straight at it.
|
||||
|
||||
**[Create your sandbox →](https://mulgadc.com/signup)**
|
||||
|
||||
## Same Commands, Your Cloud
|
||||
|
||||
```bash
|
||||
export AWS_PROFILE=spinifex
|
||||
aws ec2 describe-images
|
||||
aws ec2 run-instances --image-id ami-xxxxxxxx --instance-type t3.micro
|
||||
aws s3 mb s3://my-bucket
|
||||
```
|
||||
|
||||
For Terraform, start from the [example workbooks](https://docs.mulgadc.com/docs/nginx-webserver).
|
||||
|
||||
## Why Not LocalStack or Floci?
|
||||
|
||||
[LocalStack](https://github.com/localstack/localstack) and [Floci](https://github.com/floci-io/floci) are emulators for local development and CI testing. An emulator returns a plausible response; Spinifex boots the machine. A call launches a real KVM guest, attaches a replicated Viperblock volume, or writes an erasure-coded object into Predastore, so Spinifex can run your production workloads on hardware you own. Test against an emulator, then run in production on Spinifex.
|
||||
|
||||
## Services
|
||||
|
||||
The platform serves **405 AWS API operations** across the services below. See exactly which ones in the [coverage report](https://docs.mulgadc.com/coverage).
|
||||
|
||||
| Service | What it is | Built on | Status |
|
||||
| --- | --- | --- | --- |
|
||||
| **EC2** | Compute | QEMU/KVM, with spot instances and GPU passthrough | Available |
|
||||
| **EBS** | Block storage | [Viperblock](https://github.com/mulgadc/viperblock): replicated, with a WAL on local NVMe | Available |
|
||||
| **S3** | Object storage | [Predastore](https://github.com/mulgadc/predastore): Reed-Solomon erasure coding | Available |
|
||||
| **VPC** | Networking | OVN, with security groups | Available |
|
||||
| **IAM** | Identity & auth | SigV4 across every service | Available |
|
||||
| **ALB / NLB** | Load balancers | HAProxy (L7), nginx `stream` (L4), with ACM certificates | Available |
|
||||
| **EKS** | Kubernetes | k3s, with HA control planes and managed node groups | Available |
|
||||
| **ECR** | Container registry | Predastore, deduplicated per account | Available |
|
||||
| **ECS** | Container service | Spinifex instances | Available |
|
||||
| **RDS** | Databases | PostgreSQL and MariaDB, with automated backups | Available |
|
||||
| **Route 53** | DNS | [Northstar](https://github.com/mulgadc/northstar): authoritative DNS with hosted zones | Available |
|
||||
| **Bedrock** | AI deployment | vLLM for text generation and Hugging Face TEI for embeddings and reranking | Q4 2026 |
|
||||
| **Lambda** | Serverless functions | | Q1 2027 |
|
||||
| **SQS** | Message queues | | Q1 2027 |
|
||||
|
||||
Roadmap services ship under the same AWS API surface, so code written for AWS today works the moment they land. [Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
|
||||
|
||||
## Deploy Anywhere
|
||||
|
||||
<p align="center">
|
||||
<img src=".github/assets/platform.svg" alt="The Spinifex stack: your apps and AWS tooling on top, the Spinifex AWS-compatible service layer, running on standard Linux on CPU and GPU hardware, deployable to Neocloud, on-premise, or the edge." width="900">
|
||||
</p>
|
||||
|
||||
From commodity hardware up to unmodified AWS tooling, every layer is replaceable and yours to own.
|
||||
- **Neocloud:** lift and shift onto partner GPU capacity (H100 / H200 / B200). Change an endpoint, keep the software.
|
||||
- **On-premise:** a multi-node HA cluster in your own data centre, so the stack, the data, and the jurisdiction stay yours.
|
||||
- **Edge:** air-gapped sites, vehicles, vessels, factories, and clinics, with compute next to the data.
|
||||
|
||||
## Three Ways To Deploy
|
||||
## Install
|
||||
|
||||
The same AWS-compatible platform runs in three shapes. Pick the one that matches your hardware.
|
||||
Spinifex runs on Ubuntu 26.04 or Debian 13. The full guides are at [docs.mulgadc.com](https://docs.mulgadc.com).
|
||||
|
||||
### Neocloud
|
||||
>*Prerequisite:* Spinifex requires a Linux bridge configured on the host for VM networking. See the [single-node install guide](https://docs.mulgadc.com/docs/install#prerequisites) for setup details.
|
||||
|
||||
Lift and shift onto a partner Neocloud without rewriting workloads. GPU capacity (H100 / H200 / B200), cheaper, available now. Change an endpoint, keep the software.
|
||||
|
||||
### On-premise
|
||||
|
||||
Host Spinifex on your own hardware in your own data centre. You get a multi-node HA cluster integrated with your storage and networking, so the stack, the data, and the jurisdiction stay yours.
|
||||
|
||||
### Edge
|
||||
|
||||
Cloud where the cloud can't reach. Air-gapped sites, vehicles, vessels, factories, and clinics. Compute next to the data, running through disconnection, on hardware you own. The same AWS APIs your software already uses.
|
||||
|
||||
## AWS Compatibility
|
||||
|
||||
Spinifex speaks the AWS API surface. The AWS SDKs, AWS CLI, and Terraform all work unchanged, whether you deploy at the edge, on-premise, or on a partner Neocloud.
|
||||
|
||||
| Service | What it is | Status |
|
||||
| --- | --- | --- |
|
||||
| **EC2** | Compute | Available |
|
||||
| **EBS** | Block Storage | Available |
|
||||
| **S3** | Object Storage | Available |
|
||||
| **VPC** | Networking | Available |
|
||||
| **IAM** | Identity & Auth | Available |
|
||||
| **ALB / NLB** | Load Balancers | Available |
|
||||
| **EKS** | Kubernetes | Available |
|
||||
| **ECR** | Container Registry | Available |
|
||||
| **ECS** | Container Service | Available |
|
||||
| **RDS** | Databases | Available |
|
||||
| **Bedrock** | AI Deployment | Q4 2026 |
|
||||
|
||||
Roadmap items ship under the same AWS API surface. Code written for AWS today keeps working the moment they land. [Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
|
||||
|
||||
## Live Demo
|
||||
|
||||
Create a live sandbox in under a minute. The fastest way to experience the stack. Try Spinifex against one of your own workloads on infrastructure we run for you.
|
||||
|
||||
* Live in under a minute. No card, no install.
|
||||
* Yours for 72 hours to test and experiment.
|
||||
* Real AWS-compatible endpoint, access keys and console.
|
||||
* EC2, EBS, S3, VPC, IAM, EKS and RDS, ready to call.
|
||||
* Point your existing AWS CLI or Terraform straight at it.
|
||||
|
||||
Sign up at [https://mulgadc.com/signup](https://mulgadc.com/signup)
|
||||
|
||||
## Core Components
|
||||
|
||||
### Spinifex (Compute and Platform Services – EC2, EKS, ECS, RDS)
|
||||
|
||||
Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing an EC2 compatible API. It manages lifecycle operations like start, stop, and terminate through QEMU's QMP interface. Designed to be straightforward and scriptable, Spinifex lets you launch VMs with the AWS CLI, SDKs, or Terraform, without Kubernetes or a heavyweight orchestrator.
|
||||
|
||||
The managed services build on that same instance layer. An EKS node group, an ECS container instance, and an RDS database are Spinifex VMs on your own hardware, each driven through its own AWS API.
|
||||
|
||||
- **EC2** – VM lifecycle on bare metal, cloud-init and IMDS metadata, key pairs, tags, spot instances, capacity reservations, and GPU passthrough
|
||||
- **EKS** – Managed Kubernetes on k3s, with HA control planes, managed node groups, cluster add-ons, an OIDC provider, and IAM access entries
|
||||
- **ECS** – Task and service scheduling across container instances, with awsvpc ENIs, capacity providers, rolling deployments, and GPU-aware placement
|
||||
- **ECR** – Private container registry backed by Predastore, with content-addressed blobs deduplicated per account and lifecycle policies
|
||||
- **RDS** – Managed PostgreSQL and MariaDB, with parameter groups, automated backups, and DNS endpoints served by Northstar
|
||||
- **ALB / NLB** – Layer 7 load balancing on HAProxy and layer 4 on nginx `stream`, with health checks and ACM-issued listener certificates
|
||||
- **VPC and IAM** – OVN-backed networking and security groups, with roles, policies, and SigV4 auth applied across every service
|
||||
|
||||
### Viperblock (Block Storage – EBS Alternative)
|
||||
|
||||
[Viperblock](https://github.com/mulgadc/viperblock) is a high-performance, WAL-backed block storage service that replicates volumes across multiple nodes. It's built for reliability and speed, with support for snapshots, recovery, and direct connection to QEMU instances using NBD or virtio-blk.
|
||||
|
||||
- Fast, durable virtual disks
|
||||
- Replication for resilience
|
||||
- Exposed over NBD or embedded in VMs
|
||||
- High performance WAL logs on local NVMe drives, reducing IO traffic to S3
|
||||
- In-memory read/write block cache
|
||||
|
||||
### Predastore (Object Storage – S3-Compatible)
|
||||
|
||||
[Predastore](https://github.com/mulgadc/predastore) is a fully S3-compatible object storage system. It supports the AWS S3 API, including Signature V4 authentication, multipart uploads, and Terraform provisioning. Predastore chunks data and distributes it across nodes using Reed-Solomon erasure coding, so it survives node loss and suits large-scale or low-bandwidth sites.
|
||||
|
||||
- S3-compatible API and auth
|
||||
- Multipart uploads, streaming reads/writes
|
||||
- Data redundancy with Reed-Solomon encoding
|
||||
|
||||
### Northstar (Authoritative DNS)
|
||||
|
||||
[Northstar](https://github.com/mulgadc/northstar) is a lightweight authoritative DNS server that handles both internal service discovery and public-facing DNS for a Spinifex cluster. Zones are human-readable TOML files, served from the local filesystem or synced from Predastore, keeping DNS configuration in the same object storage as the rest of your infrastructure.
|
||||
|
||||
- Authoritative DNS over UDP, TCP, and DNS-over-TLS
|
||||
- Service discovery for cluster components via SRV records
|
||||
- Zone files stored locally or in S3-compatible storage
|
||||
|
||||
## Architecture at a Glance
|
||||
|
||||
<p align="center">
|
||||
<img src=".github/assets/architecture.svg" alt="Spinifex message-driven architecture: standard AWS clients call the AWS Gateway over HTTPS with SigV4, which publishes to a NATS bus answered by stateless ec2/ebs/s3/vpc/iam daemons over local backends." width="900">
|
||||
</p>
|
||||
|
||||
Every AWS API call is authenticated at the gateway, published to a NATS subject, and answered by whichever daemon claims it. Daemons are stateless: scale horizontally by starting more. No etcd, no Kubernetes, no external control plane. Just systemd units, a NATS cluster, and your hardware. Deep dive: **[`docs/DESIGN.md`](docs/DESIGN.md)**.
|
||||
|
||||
## Self-hosted Installation
|
||||
|
||||
Installation requires an Ubuntu 26.04 or Debian 13 system. See the detailed documentation at [docs.mulgadc.com](https://docs.mulgadc.com) for installing Spinifex.
|
||||
|
||||
### Bare Metal ISO
|
||||
|
||||
The recommended installation is a [bootable x86 installer](https://iso.mulgadc.com/spinifex.iso) for bare-metal hardware.
|
||||
**Bare metal (recommended):** download the [bootable x86 installer](https://iso.mulgadc.com/spinifex.iso) and follow the [USB install guide](https://docs.mulgadc.com/docs/install-usb).
|
||||
|
||||
```bash
|
||||
curl -fLO https://iso.mulgadc.com/spinifex.iso
|
||||
```
|
||||
|
||||
Follow the [USB install guide](https://docs.mulgadc.com/docs/install-usb) to write the ISO to USB and install on your hardware.
|
||||
|
||||
### Single Node Install
|
||||
|
||||
>*Prerequisite:* Spinifex requires a Linux bridge configured on the host for VM networking. See the [single-node install guide](https://docs.mulgadc.com/docs/install#prerequisites) for setup details.
|
||||
**Single node:**
|
||||
|
||||
```bash
|
||||
curl -fsSL https://install.mulgadc.com | bash
|
||||
|
||||
sudo /usr/local/share/spinifex/setup-ovn.sh --management
|
||||
|
||||
sudo spx admin init --node node1 --nodes 1
|
||||
|
||||
sudo systemctl start spinifex.target
|
||||
|
||||
export AWS_PROFILE=spinifex
|
||||
|
||||
aws ec2 describe-instance-types
|
||||
```
|
||||
|
||||
### Development Setup
|
||||
**From source:** see the [Source Install](https://docs.mulgadc.com/docs/install-source) guide.
|
||||
|
||||
For a complete development environment see the [Source Install](https://docs.mulgadc.com/docs/install-source) documentation.
|
||||
|
||||
## Spinifex UI
|
||||
|
||||
Spinifex ships with a built-in web console, an optional alternative to the AWS CLI, SDKs, and Terraform. If you know the AWS Management Console, the Spinifex UI fills the same role: a browser-based view of your instances, volumes, buckets, VPCs, and IAM resources, without leaving your own network.
|
||||
## Web Console
|
||||
|
||||
<p align="center">
|
||||
<img src=".github/assets/spinifex-ui.jpg" alt="Spinifex web console dashboard" width="900">
|
||||
</p>
|
||||
|
||||
Each node serves the console on port `3000` over TLS, and it comes up with `spinifex.target`:
|
||||
Spinifex ships with a built-in web console for your instances, volumes, buckets, networks, and IAM. See [Launching the Web UI](https://docs.mulgadc.com/docs/setting-up-your-cluster#7-launching-the-web-ui).
|
||||
|
||||
- **Same API, different surface.** The UI issues the same AWS SigV4 calls the CLI does, so RBAC, audit trails, and IAM policies apply the same way.
|
||||
- **Single sign-on against your AWS credentials.** Log in with the access keys from `~/.aws/credentials` on the node where Spinifex is installed. There is no separate user database.
|
||||
- **Self-hosted, works offline.** The Spinifex binary embeds the UI and each node serves it directly, with no external CDN or analytics calls.
|
||||
## Architecture
|
||||
|
||||
For the full walkthrough, covering first-time TLS certificate trust, login, and a feature tour, see [**Launching the Web UI**](https://docs.mulgadc.com/docs/setting-up-your-cluster#7-launching-the-web-ui) in the cluster setup guide.
|
||||
<p align="center">
|
||||
<img src=".github/assets/architecture.svg" alt="Spinifex message-driven architecture: standard AWS clients call the AWS Gateway over HTTPS with SigV4, which publishes to a NATS bus answered by stateless ec2/ebs/s3/vpc/iam daemons over local backends." width="900">
|
||||
</p>
|
||||
|
||||
## Development Philosophy
|
||||
Every AWS API call is authenticated at the gateway, published to a NATS subject, and answered by whichever daemon claims it. Daemons are stateless, so you scale horizontally by starting more. Deep dive: **[`docs/DESIGN.md`](docs/DESIGN.md)**.
|
||||
|
||||
### Built by Engineers, For Engineers
|
||||
## How We Build Spinifex
|
||||
|
||||
Spinifex is developed by experienced infrastructure engineers with deep AWS expertise, including former AWS team members who understand the intricacies of building production-grade cloud services. Our team brings decades of combined experience from AWS, enterprise infrastructure, and edge computing environments.
|
||||
Spinifex is built by infrastructure engineers with deep AWS experience, including former AWS team members, drawing on decades of combined work across AWS, enterprise infrastructure, and edge computing. Engineers architect and implement it, with **Claude Code** accelerating parts of the work.
|
||||
|
||||
### AI-Assisted Development
|
||||
|
||||
Spinifex is architected and implemented by experienced engineers, with **Claude Code** accelerating certain development tasks. The split keeps the quality and reliability standards that production infrastructure demands:
|
||||
|
||||
**How We Use Claude Code:**
|
||||
|
||||
- **Code Generation**: Boilerplate AWS API structures and handlers
|
||||
- **Documentation**: Comprehensive development guides and API documentation
|
||||
- **Testing**: Test case generation and validation scenarios
|
||||
- **Refactoring**: Large-scale code restructuring and optimization
|
||||
|
||||
**What Remains Human-Driven:**
|
||||
|
||||
- **Architecture Decisions**: Core system design and scalability choices
|
||||
- **Security Implementation**: Authentication, encryption, and threat modeling
|
||||
- **Performance Optimization**: Real-world performance tuning and benchmarking
|
||||
- **Production Operations**: Deployment strategies and operational procedures
|
||||
| Claude Code assists with | Engineers own |
|
||||
| --- | --- |
|
||||
| Boilerplate AWS API structures and handlers | Architecture and scalability decisions |
|
||||
| Development guides and API documentation | Authentication, encryption, and threat modelling |
|
||||
| Test case generation and validation scenarios | Real-world performance tuning and benchmarking |
|
||||
| Large-scale restructuring and refactoring | Deployment strategy and production operations |
|
||||
|
||||
## Trademarks
|
||||
|
||||
@@ -230,4 +143,4 @@ Spinifex is architected and implemented by experienced engineers, with **Claude
|
||||
|
||||
## License
|
||||
|
||||
Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE), with a commercial license available. You're free to use, modify, and deploy it anywhere you need reliable infrastructure without depending on centralized cloud platforms.
|
||||
Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE), with a commercial license available.
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
package instancestate
|
||||
|
||||
// allowance is one recorded production consumer of raw instance state.
|
||||
type allowance struct {
|
||||
File, Symbol string
|
||||
Inv string // inventory item in docs/PACKAGE_BOUNDARY_MIGRATION.md
|
||||
}
|
||||
|
||||
// allowlist mirrors the ADR-0007 slice 0 inventory in
|
||||
// docs/PACKAGE_BOUNDARY_MIGRATION.md. It only shrinks: delete an entry as soon
|
||||
// as its use is gone, and never add one to make a new consumer pass.
|
||||
var allowlist = []allowance{
|
||||
{File: "spinifex/runtime/daemon/daemon.go", Symbol: "daemon.LocalStatePath", Inv: "INV-01"},
|
||||
{File: "spinifex/runtime/daemon/daemon.go", Symbol: "daemon.ReadLocalState", Inv: "INV-01"},
|
||||
{File: "spinifex/runtime/daemon/daemon.go", Symbol: "daemon.MarshalLocalState", Inv: "INV-01"},
|
||||
{File: "spinifex/runtime/daemon/daemon.go", Symbol: "daemon.WriteLocalStateBytes", Inv: "INV-01"},
|
||||
{File: "spinifex/runtime/daemon/daemon.go", Symbol: "JetStreamManager.InitKVBucket", Inv: "INV-02"},
|
||||
{File: "spinifex/runtime/daemon/daemon.go", Symbol: "JetStreamManager.InitTerminatedInstanceBucket", Inv: "INV-02"},
|
||||
{File: "spinifex/runtime/daemon/daemon.go", Symbol: "JetStreamManager.WriteNodeMarkerBestEffort", Inv: "INV-03"},
|
||||
{File: "spinifex/runtime/daemon/daemon.go", Symbol: "JetStreamManager.WriteRunningSet", Inv: "INV-03"},
|
||||
{File: "spinifex/runtime/daemon/instance_record_repair.go", Symbol: "JetStreamManager.WriteRunningSet", Inv: "INV-03"},
|
||||
{File: "spinifex/runtime/daemon/daemon.go", Symbol: "JetStreamManager.ListTerminatedInstances", Inv: "INV-04"},
|
||||
|
||||
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.LoadState", Inv: "INV-05"},
|
||||
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.WriteStoppedInstance", Inv: "INV-05"},
|
||||
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.LoadStoppedInstance", Inv: "INV-05"},
|
||||
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.DeleteStoppedInstance", Inv: "INV-05"},
|
||||
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.UpdateStoppedInstance", Inv: "INV-05"},
|
||||
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.ClaimStoppedInstance", Inv: "INV-05"},
|
||||
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.ListStoppedInstances", Inv: "INV-05"},
|
||||
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.WriteTerminatedInstance", Inv: "INV-05"},
|
||||
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.UpdateTerminatedInstance", Inv: "INV-05"},
|
||||
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.ListTerminatedInstances", Inv: "INV-05"},
|
||||
{File: "spinifex/runtime/daemon/vm_adapters.go", Symbol: "JetStreamManager.DeleteTerminatedInstance", Inv: "INV-05"},
|
||||
|
||||
{File: "spinifex/runtime/compute/vm/migrate.go", Symbol: "JetStreamManager.WriteStoppedInstance", Inv: "INV-06"},
|
||||
{File: "spinifex/runtime/compute/vm/migrate.go", Symbol: "JetStreamManager.WriteTerminatedInstance", Inv: "INV-06"},
|
||||
{File: "spinifex/runtime/compute/vm/shutdown.go", Symbol: "JetStreamManager.WriteTerminatedInstance", Inv: "INV-06"},
|
||||
{File: "spinifex/runtime/compute/vm/orphan_qemu_reaper.go", Symbol: "JetStreamManager.ListTerminatedInstances", Inv: "INV-06"},
|
||||
{File: "spinifex/runtime/compute/vm/teardown_reaper.go", Symbol: "JetStreamManager.ListTerminatedInstances", Inv: "INV-06"},
|
||||
{File: "spinifex/runtime/compute/vm/teardown_reaper.go", Symbol: "JetStreamManager.UpdateTerminatedInstance", Inv: "INV-06"},
|
||||
{File: "spinifex/runtime/compute/vm/teardown_reaper.go", Symbol: "JetStreamManager.DeleteTerminatedInstance", Inv: "INV-06"},
|
||||
|
||||
{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.LoadStoppedInstance", Inv: "INV-07"},
|
||||
{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.UpdateStoppedInstance", Inv: "INV-07"},
|
||||
{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.ListStoppedInstances", Inv: "INV-07"},
|
||||
{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.ListTerminatedInstances", Inv: "INV-07"},
|
||||
{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.WriteStoppedInstance", Inv: "INV-07"},
|
||||
{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.ClaimStoppedInstance", Inv: "INV-07"},
|
||||
{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.WriteTerminatedInstance", Inv: "INV-07"},
|
||||
{File: "spinifex/domains/ec2/instance/service_impl.go", Symbol: "JetStreamManager.DeleteStoppedInstance", Inv: "INV-07"},
|
||||
|
||||
{File: "spinifex/runtime/daemon/daemon_handlers_image.go", Symbol: "JetStreamManager.LoadStoppedInstance", Inv: "INV-08"},
|
||||
{File: "spinifex/runtime/daemon/daemon_handlers_image.go", Symbol: "JetStreamManager.UpdateStoppedInstance", Inv: "INV-08"},
|
||||
|
||||
{File: "spinifex/runtime/daemon/instance_recovery.go", Symbol: "vm.InstanceRecord", Inv: "INV-10"},
|
||||
{File: "spinifex/runtime/daemon/instance_recovery.go", Symbol: "JetStreamManager.ListInstanceRecords", Inv: "INV-10"},
|
||||
{File: "spinifex/runtime/daemon/instance_recovery.go", Symbol: "JetStreamManager.ClaimRecoverableInstance", Inv: "INV-10"},
|
||||
{File: "spinifex/runtime/daemon/instance_recovery.go", Symbol: "JetStreamManager.ReleaseRecoveredInstance", Inv: "INV-10"},
|
||||
{File: "spinifex/runtime/daemon/instance_recovery.go", Symbol: "JetStreamManager.AbandonRecovery", Inv: "INV-10"},
|
||||
{File: "spinifex/runtime/daemon/instance_recovery.go", Symbol: "JetStreamManager.LoadInstanceRecord", Inv: "INV-10"},
|
||||
|
||||
{File: "spinifex/runtime/daemon/dns_reconcile.go", Symbol: "daemon.InstanceRecordPrefix", Inv: "INV-11"},
|
||||
{File: "spinifex/runtime/daemon/dns_reconcile.go", Symbol: "daemon.InstanceStateBucket", Inv: "INV-11"},
|
||||
{File: "spinifex/runtime/daemon/dns_reconcile.go", Symbol: "JetStreamManager.ListInstanceRecords", Inv: "INV-11"},
|
||||
|
||||
{File: "spinifex/runtime/daemon/eni_orphan_reaper.go", Symbol: "vm.InstanceRecord", Inv: "INV-12"},
|
||||
{File: "spinifex/runtime/daemon/eni_orphan_reaper.go", Symbol: "JetStreamManager.ListInstanceRecords", Inv: "INV-12"},
|
||||
{File: "spinifex/runtime/daemon/eni_orphan_reaper.go", Symbol: "JetStreamManager.ListTerminatedInstanceRecords", Inv: "INV-12"},
|
||||
|
||||
{File: "spinifex/domains/ec2/instance/state/imds_readers.go", Symbol: "daemon.LocalState", Inv: "INV-13"},
|
||||
{File: "spinifex/domains/ec2/instance/state/imds_readers.go", Symbol: "daemon.LocalStatePath", Inv: "INV-13"},
|
||||
{File: "spinifex/domains/ec2/instance/state/imds_readers.go", Symbol: "daemon.ReadLocalState", Inv: "INV-13"},
|
||||
{File: "spinifex/domains/ec2/instance/state/imds_readers.go", Symbol: "daemon.InstanceStateBucket", Inv: "INV-13"},
|
||||
{File: "spinifex/domains/ec2/instance/state/imds_readers.go", Symbol: "JetStreamManager.InitKVBucket", Inv: "INV-13"},
|
||||
|
||||
{File: "spinifex/domains/ec2/guestmetadata/instance_lookup.go", Symbol: "vm.InstanceRecord", Inv: "INV-14"},
|
||||
{File: "spinifex/domains/ec2/guestmetadata/instance_lookup.go", Symbol: "vm.VMFromRecord", Inv: "INV-14"},
|
||||
{File: "spinifex/domains/ec2/guestmetadata/instance_lookup.go", Symbol: "JetStreamManager.LoadInstanceRecord", Inv: "INV-14"},
|
||||
|
||||
{File: "spinifex/domains/network/reconcile/intent.go", Symbol: `literal "spinifex-instance-state"`, Inv: "INV-15"},
|
||||
{File: "spinifex/domains/network/reconcile/intent.go", Symbol: "vm.InstanceRecord", Inv: "INV-15"},
|
||||
|
||||
{File: "spinifex/runtime/roles/awsgw/awsgw.go", Symbol: "daemon.InstanceStateBucket", Inv: "INV-16"},
|
||||
{File: "spinifex/runtime/roles/awsgw/awsgw.go", Symbol: "daemon.InstanceRecordPrefix", Inv: "INV-16"},
|
||||
{File: "spinifex/runtime/roles/awsgw/awsgw.go", Symbol: "vm.InstanceRecord", Inv: "INV-16"},
|
||||
|
||||
{File: "spinifex/runtime/compute/cache/cache.go", Symbol: "vm.InstanceRecord", Inv: "INV-17"},
|
||||
{File: "spinifex/runtime/compute/cache/cache.go", Symbol: "vm.VMFromRecord", Inv: "INV-17"},
|
||||
|
||||
{File: "spinifex/domains/admission/quota/records.go", Symbol: "vm.InstanceRecord", Inv: "INV-18"},
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
package instancestate
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type report struct {
|
||||
unrecorded []use
|
||||
stale []allowance
|
||||
malformed []string
|
||||
}
|
||||
|
||||
// check compares the scanned uses with allow.
|
||||
func check(s scan, allow []allowance) report {
|
||||
var rep report
|
||||
allowed := map[use]bool{}
|
||||
for _, a := range allow {
|
||||
u := use{File: a.File, Symbol: a.Symbol}
|
||||
switch {
|
||||
case allowed[u]:
|
||||
rep.malformed = append(rep.malformed, "duplicate entry "+u.String())
|
||||
case a.Inv == "":
|
||||
rep.malformed = append(rep.malformed, "entry "+u.String()+" cites no inventory item")
|
||||
}
|
||||
allowed[u] = true
|
||||
}
|
||||
for _, u := range s.sorted() {
|
||||
if !allowed[u] {
|
||||
rep.unrecorded = append(rep.unrecorded, u)
|
||||
}
|
||||
}
|
||||
for _, a := range allow {
|
||||
if _, ok := s[use{File: a.File, Symbol: a.Symbol}]; !ok {
|
||||
rep.stale = append(rep.stale, a)
|
||||
}
|
||||
}
|
||||
return rep
|
||||
}
|
||||
|
||||
// TestADR0007_S6_NoNewRawInstanceStateConsumers is the blocking ratchet: no
|
||||
// production file may start naming the daemon's instance-state bucket, keys,
|
||||
// local file, raw record or JetStreamManager instance methods.
|
||||
func TestADR0007_S6_NoNewRawInstanceStateConsumers(t *testing.T) {
|
||||
root, err := moduleRoot()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s, files, err := scanModule(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Guards against a walk that silently finds nothing and passes vacuously.
|
||||
if files < 500 || len(s) < 20 {
|
||||
t.Fatalf("scanned %d files and found %d uses under %s; the walk has lost the tree", files, len(s), root)
|
||||
}
|
||||
if len(allowlist) == 0 {
|
||||
t.Fatal("the allowlist is empty, so the stale check would pass vacuously")
|
||||
}
|
||||
rep := check(s, allowlist)
|
||||
|
||||
t.Run("NoUnrecordedConsumer", func(t *testing.T) {
|
||||
for _, u := range rep.unrecorded {
|
||||
t.Errorf("ADR-0007 S6 (consumers receive narrow state capabilities rather than raw stores): %s at %s; "+
|
||||
"consume an instance-domain capability instead of the daemon's raw instance state", u, s[u])
|
||||
}
|
||||
})
|
||||
t.Run("Allowlist_NoStaleEntries", func(t *testing.T) {
|
||||
for _, a := range rep.stale {
|
||||
t.Errorf("allowlist entry %s uses %s (%s) no longer matches; delete it and update the inventory",
|
||||
a.File, a.Symbol, a.Inv)
|
||||
}
|
||||
})
|
||||
t.Run("Allowlist_WellFormed", func(t *testing.T) {
|
||||
for _, m := range rep.malformed {
|
||||
t.Errorf("allowlist_test.go: %s", m)
|
||||
}
|
||||
t.Logf("%d allow-listed uses", len(allowlist))
|
||||
})
|
||||
t.Run("OwnerFilesExist", func(t *testing.T) {
|
||||
for _, f := range ownerFiles {
|
||||
if _, err := os.Stat(filepath.Join(root, f)); err != nil {
|
||||
t.Errorf("owner file %s is gone; remove it from ownerFiles and re-scan its consumers: %v", f, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
const faultSource = `package %s
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
spxd "github.com/mulgadc/spinifex/spinifex/runtime/daemon"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
|
||||
)
|
||||
|
||||
type holder struct {
|
||||
LoadInstanceRecord func() // a field name, not a use
|
||||
m *spxd.JetStreamManager
|
||||
}
|
||||
|
||||
func (h holder) WriteRunningSet() {} // a declaration, not a use
|
||||
|
||||
func use(h holder, raw []byte, jsManager *spxd.JetStreamManager, other interface{ LoadState() }) {
|
||||
_ = spxd.InstanceStateBucket
|
||||
_ = spxd.LocalStatePath("")
|
||||
var r vm.InstanceRecord
|
||||
_ = json.Unmarshal(raw, &r)
|
||||
_ = vm.VMFromRecord(&r)
|
||||
_ = (&vm.VM{}).Record()
|
||||
_, _ = h.m.LoadInstanceRecord("i-1")
|
||||
_, _, _ = jsManager.LoadState("node")
|
||||
other.LoadState()
|
||||
_ = "spinifex-instance-state"
|
||||
cluster, _ := spxd.NewJetStreamManager(nil)
|
||||
_ = cluster.InitClusterStateBucket()
|
||||
_ = cluster.WriteRunningSet("node", nil)
|
||||
}
|
||||
`
|
||||
|
||||
func parseFault(t *testing.T, rel, pkg string) scan {
|
||||
t.Helper()
|
||||
fset := token.NewFileSet()
|
||||
f, err := parser.ParseFile(fset, rel, fmt.Sprintf(faultSource, pkg), parser.SkipObjectResolution)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := scan{}
|
||||
scanFile(fset, rel, f, out)
|
||||
return out
|
||||
}
|
||||
|
||||
// TestADR0007_S6_RatchetDetectsFaults injects a new consumer, a stale entry
|
||||
// and a malformed entry, and checks the matcher's declaration exclusions.
|
||||
func TestADR0007_S6_RatchetDetectsFaults(t *testing.T) {
|
||||
const rel = "spinifex/domains/widget/w.go"
|
||||
s := parseFault(t, rel, "widget")
|
||||
|
||||
want := []string{
|
||||
"daemon.InstanceStateBucket", "daemon.LocalStatePath",
|
||||
"vm.InstanceRecord", "vm.VMFromRecord", "VM.Record()",
|
||||
"JetStreamManager.LoadInstanceRecord", "JetStreamManager.LoadState",
|
||||
"JetStreamManager.WriteRunningSet",
|
||||
`literal "spinifex-instance-state"`,
|
||||
}
|
||||
for _, sym := range want {
|
||||
if _, ok := s[use{File: rel, Symbol: sym}]; !ok {
|
||||
t.Errorf("injected use of %s was not detected; got %v", sym, s.sorted())
|
||||
}
|
||||
}
|
||||
if len(s) != len(want) {
|
||||
t.Errorf("want exactly %d uses (the manager type, its constructor, cluster-state methods, field names, method declarations and an unrelated LoadState excluded), got %v",
|
||||
len(want), s.sorted())
|
||||
}
|
||||
|
||||
stale := allowance{File: "spinifex/domains/gone/g.go", Symbol: "vm.InstanceRecord", Inv: "INV-00"}
|
||||
recorded := allowance{File: rel, Symbol: "vm.InstanceRecord", Inv: "INV-00"}
|
||||
rep := check(s, []allowance{recorded, stale, {File: rel, Symbol: "vm.VMFromRecord"}, recorded})
|
||||
if len(rep.unrecorded) != len(want)-2 {
|
||||
t.Errorf("want %d unrecorded uses, got %v", len(want)-2, rep.unrecorded)
|
||||
}
|
||||
if len(rep.stale) != 1 || rep.stale[0] != stale {
|
||||
t.Errorf("want stale entry %+v, got %+v", stale, rep.stale)
|
||||
}
|
||||
if len(rep.malformed) != 2 {
|
||||
t.Errorf("want a missing-inventory and a duplicate entry reported, got %v", rep.malformed)
|
||||
}
|
||||
}
|
||||
|
||||
// Inside the owning packages the symbols are bare identifiers, and the scan
|
||||
// must see those too.
|
||||
func TestADR0007_S6_RatchetSeesBareIdentifiersInOwnerPackages(t *testing.T) {
|
||||
src := map[string]string{
|
||||
"spinifex/runtime/daemon/new_consumer.go": "package daemon\n\nfunc f() string { return InstanceStateBucket + LocalStatePath(\"\") }\n",
|
||||
"spinifex/runtime/compute/vm/new_reader.go": "package vm\n\nfunc g(r *InstanceRecord) *VM { return VMFromRecord(r) }\n",
|
||||
}
|
||||
want := map[string][]string{
|
||||
"spinifex/runtime/daemon/new_consumer.go": {"daemon.InstanceStateBucket", "daemon.LocalStatePath"},
|
||||
"spinifex/runtime/compute/vm/new_reader.go": {"vm.InstanceRecord", "vm.VMFromRecord"},
|
||||
}
|
||||
for rel, code := range src {
|
||||
fset := token.NewFileSet()
|
||||
f, err := parser.ParseFile(fset, rel, code, parser.SkipObjectResolution)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := scan{}
|
||||
scanFile(fset, rel, f, out)
|
||||
for _, sym := range want[rel] {
|
||||
if _, ok := out[use{File: rel, Symbol: sym}]; !ok {
|
||||
t.Errorf("%s: bare use of %s not detected; got %v", rel, sym, out.sorted())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,302 @@
|
||||
// Package instancestate freezes the production consumers of the daemon's raw
|
||||
// instance-state persistence, so ADR-0007 S6 and S7 start from a known set
|
||||
// that can only shrink.
|
||||
package instancestate
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"go/ast"
|
||||
"go/build/constraint"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
daemonPath = "github.com/mulgadc/spinifex/spinifex/runtime/daemon"
|
||||
vmPath = "github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
|
||||
daemonDir = "spinifex/runtime/daemon"
|
||||
vmDir = "spinifex/runtime/compute/vm"
|
||||
)
|
||||
|
||||
// daemonSymbols are the daemon's instance-state bucket, key and local-file
|
||||
// identifiers. The JetStreamManager type and constructor are not listed: they
|
||||
// also serve cluster state, so only the instance-state methods below count.
|
||||
var daemonSymbols = set(
|
||||
"InstanceStateBucket", "InstanceStateBucketVersion",
|
||||
"TerminatedInstanceBucket", "TerminatedInstanceBucketVersion",
|
||||
"InstanceRecordPrefix", "InstanceStatePrefix", "NodePresencePrefix",
|
||||
"StoppedInstancePrefix", "TerminatedInstancePrefix",
|
||||
"LocalState", "LocalStateSchemaVersion", "LocalStatePath", "ReadLocalState",
|
||||
"MarshalLocalState", "WriteLocalStateBytes", "LocalStateFileName",
|
||||
"LocalStateFileMode", "DefaultLocalStateDir",
|
||||
)
|
||||
|
||||
// vmSymbols are the raw persisted record and its conversion.
|
||||
var vmSymbols = set("InstanceRecord", "InstanceSpec", "InstanceStatus", "VMFromRecord")
|
||||
|
||||
// jsmMethods are JetStreamManager's instance-state methods. Matched by name, so
|
||||
// a call through an interface that mirrors them counts as a consumer too.
|
||||
var jsmMethods = set(
|
||||
"WriteInstanceRecord", "LoadInstanceRecord", "UpdateInstanceRecord",
|
||||
"DeleteInstanceRecord", "ListInstanceRecords",
|
||||
"WriteTerminatedInstanceRecord", "LoadTerminatedInstanceRecord",
|
||||
"UpdateTerminatedInstanceRecord", "DeleteTerminatedInstanceRecord",
|
||||
"ListTerminatedInstanceRecords",
|
||||
"WriteNodeMarker", "WriteNodeMarkerBestEffort", "WriteRunningSet",
|
||||
"WriteStoppedInstance", "LoadStoppedInstance", "DeleteStoppedInstance",
|
||||
"ClaimStoppedInstance", "ClaimRecoverableInstance", "ReleaseRecoveredInstance",
|
||||
"AbandonRecovery", "UpdateStoppedInstance", "ListStoppedInstances",
|
||||
"WriteTerminatedInstance", "UpdateTerminatedInstance", "ListTerminatedInstances",
|
||||
"DeleteTerminatedInstance", "LoadTerminatedInstance",
|
||||
"InitKVBucket", "InitTerminatedInstanceBucket",
|
||||
)
|
||||
|
||||
// ambiguousJSMMethods share a name with unrelated methods, so they count only
|
||||
// when called on a field or variable named like a JetStreamManager handle.
|
||||
var ambiguousJSMMethods = set("LoadState", "DeleteState")
|
||||
|
||||
// literals duplicate a bucket or file name instead of naming the owner's symbol.
|
||||
var literals = set(`"spinifex-instance-state"`, `"spinifex-terminated-instances"`, `"instance-state.json"`)
|
||||
|
||||
// ownerFiles define the symbols above. They are the implementation being
|
||||
// migrated, not consumers of it, and are listed so a deleted one is noticed.
|
||||
var ownerFiles = []string{
|
||||
"spinifex/runtime/daemon/instance_membership.go",
|
||||
"spinifex/runtime/daemon/instance_records.go",
|
||||
"spinifex/runtime/daemon/instance_records_migrate.go",
|
||||
"spinifex/runtime/daemon/instance_running_set.go",
|
||||
"spinifex/runtime/daemon/jetstream.go",
|
||||
"spinifex/runtime/daemon/local_state.go",
|
||||
"spinifex/runtime/compute/vm/record.go",
|
||||
}
|
||||
|
||||
func set(names ...string) map[string]bool {
|
||||
m := make(map[string]bool, len(names))
|
||||
for _, n := range names {
|
||||
m[n] = true
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// use is one consumer of one symbol, keyed by file so a new file is a new use.
|
||||
type use struct {
|
||||
File, Symbol string
|
||||
}
|
||||
|
||||
func (u use) String() string { return u.File + " uses " + u.Symbol }
|
||||
|
||||
// scan maps each use to the first file:line it appears at.
|
||||
type scan map[use]string
|
||||
|
||||
func moduleRoot() (string, error) {
|
||||
_, file, _, ok := runtime.Caller(0)
|
||||
if !ok {
|
||||
return "", errors.New("cannot locate test source")
|
||||
}
|
||||
return filepath.Abs(filepath.Join(filepath.Dir(file), "..", ".."))
|
||||
}
|
||||
|
||||
var excludedTopDirs = map[string]bool{"tests": true, "docs": true, "architecture": true}
|
||||
|
||||
// scanModule parses every non-test production Go file under root.
|
||||
func scanModule(root string) (scan, int, error) {
|
||||
out := scan{}
|
||||
files := 0
|
||||
fset := token.NewFileSet()
|
||||
owners := set(ownerFiles...)
|
||||
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel = filepath.ToSlash(rel)
|
||||
if d.IsDir() {
|
||||
if rel == "." {
|
||||
return nil
|
||||
}
|
||||
name := d.Name()
|
||||
if strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_") ||
|
||||
name == "testdata" || name == "vendor" || name == "node_modules" || excludedTopDirs[rel] {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(path, "go.mod")); err == nil {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
|
||||
return nil
|
||||
}
|
||||
src, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f, err := parser.ParseFile(fset, path, src, parser.ParseComments|parser.SkipObjectResolution)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ignoredByBuildTag(f.Comments, f.Package) {
|
||||
return nil
|
||||
}
|
||||
files++
|
||||
if owners[rel] {
|
||||
return nil
|
||||
}
|
||||
scanFile(fset, rel, f, out)
|
||||
return nil
|
||||
})
|
||||
return out, files, err
|
||||
}
|
||||
|
||||
// scanFile records every use in f, which lives at module-relative path rel.
|
||||
func scanFile(fset *token.FileSet, rel string, f *ast.File, out scan) {
|
||||
dir := filepath.ToSlash(filepath.Dir(rel))
|
||||
daemonName, vmName := importName(f, daemonPath, "daemon"), importName(f, vmPath, "vm")
|
||||
inDaemon, inVM := dir == daemonDir, dir == vmDir
|
||||
seesVM := inDaemon || inVM || daemonName != "" || vmName != ""
|
||||
|
||||
record := func(symbol string, pos token.Pos) {
|
||||
u := use{File: rel, Symbol: symbol}
|
||||
if _, ok := out[u]; !ok {
|
||||
out[u] = fmt.Sprintf("%s:%d", rel, fset.Position(pos).Line)
|
||||
}
|
||||
}
|
||||
|
||||
var visit func(n ast.Node) bool
|
||||
visit = func(n ast.Node) bool {
|
||||
switch n := n.(type) {
|
||||
case *ast.ImportSpec:
|
||||
return false
|
||||
case *ast.BasicLit:
|
||||
if n.Kind == token.STRING && literals[n.Value] {
|
||||
record("literal "+n.Value, n.Pos())
|
||||
}
|
||||
case *ast.CallExpr:
|
||||
if sel, ok := n.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "Record" && len(n.Args) == 0 && seesVM {
|
||||
record("VM.Record()", sel.Sel.Pos())
|
||||
}
|
||||
case *ast.SelectorExpr:
|
||||
if x, ok := n.X.(*ast.Ident); ok {
|
||||
switch {
|
||||
case daemonName != "" && x.Name == daemonName && daemonSymbols[n.Sel.Name]:
|
||||
record("daemon."+n.Sel.Name, n.Sel.Pos())
|
||||
case vmName != "" && x.Name == vmName && vmSymbols[n.Sel.Name]:
|
||||
record("vm."+n.Sel.Name, n.Sel.Pos())
|
||||
}
|
||||
}
|
||||
if jsmMethods[n.Sel.Name] || (ambiguousJSMMethods[n.Sel.Name] && jsmReceivers[receiverName(n.X)]) {
|
||||
record("JetStreamManager."+n.Sel.Name, n.Sel.Pos())
|
||||
}
|
||||
// The selected name is a field or method, never a bare package symbol.
|
||||
ast.Inspect(n.X, visit)
|
||||
return false
|
||||
case *ast.FuncDecl:
|
||||
// A method declared with one of these names is an implementation,
|
||||
// not a use; its body is still scanned.
|
||||
if n.Recv != nil {
|
||||
ast.Inspect(n.Recv, visit)
|
||||
}
|
||||
if n.Body != nil {
|
||||
ast.Inspect(n.Body, visit)
|
||||
}
|
||||
if n.Type != nil {
|
||||
ast.Inspect(n.Type, visit)
|
||||
}
|
||||
return false
|
||||
case *ast.Field:
|
||||
// Interface method and struct field names are declarations.
|
||||
if n.Type != nil {
|
||||
ast.Inspect(n.Type, visit)
|
||||
}
|
||||
if n.Tag != nil {
|
||||
ast.Inspect(n.Tag, visit)
|
||||
}
|
||||
return false
|
||||
case *ast.Ident:
|
||||
if inDaemon && daemonSymbols[n.Name] {
|
||||
record("daemon."+n.Name, n.Pos())
|
||||
}
|
||||
if inVM && vmSymbols[n.Name] {
|
||||
record("vm."+n.Name, n.Pos())
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
ast.Inspect(f, visit)
|
||||
}
|
||||
|
||||
var jsmReceivers = set("jsManager", "jsm", "js")
|
||||
|
||||
// receiverName is the last name in x: "js" for both js and a.js.
|
||||
func receiverName(x ast.Expr) string {
|
||||
switch x := x.(type) {
|
||||
case *ast.Ident:
|
||||
return x.Name
|
||||
case *ast.SelectorExpr:
|
||||
return x.Sel.Name
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// importName returns the name f refers to path by, or "" if f does not import it.
|
||||
func importName(f *ast.File, path, def string) string {
|
||||
for _, imp := range f.Imports {
|
||||
p, err := strconv.Unquote(imp.Path.Value)
|
||||
if err != nil || p != path {
|
||||
continue
|
||||
}
|
||||
if imp.Name != nil {
|
||||
if imp.Name.Name == "_" || imp.Name.Name == "." {
|
||||
return ""
|
||||
}
|
||||
return imp.Name.Name
|
||||
}
|
||||
return def
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// ignoredByBuildTag reports a `//go:build ignore` file, which is a standalone
|
||||
// program rather than part of its directory's package.
|
||||
func ignoredByBuildTag(groups []*ast.CommentGroup, pkgPos token.Pos) bool {
|
||||
for _, cg := range groups {
|
||||
if cg.Pos() >= pkgPos {
|
||||
break
|
||||
}
|
||||
for _, c := range cg.List {
|
||||
if !constraint.IsGoBuild(c.Text) {
|
||||
continue
|
||||
}
|
||||
expr, err := constraint.Parse(c.Text)
|
||||
if err == nil && expr.String() == "ignore" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s scan) sorted() []use {
|
||||
out := make([]use, 0, len(s))
|
||||
for u := range s {
|
||||
out = append(out, u)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].File != out[j].File {
|
||||
return out[i].File < out[j].File
|
||||
}
|
||||
return out[i].Symbol < out[j].Symbol
|
||||
})
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
package layering
|
||||
|
||||
// allowance is one recorded forbidden import, allowed until its debt clears.
|
||||
type allowance struct {
|
||||
From, To string
|
||||
Debt string // subject of the migration record's debt entry or close-out row
|
||||
Reason string
|
||||
}
|
||||
|
||||
// allowlist mirrors the dependency-rule debt and close-out table in
|
||||
// docs/PACKAGE_BOUNDARY_MIGRATION.md. It only shrinks: add an edge only with
|
||||
// recorded debt, and delete an entry as soon as its import is gone.
|
||||
var allowlist = []allowance{
|
||||
{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/eip", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
|
||||
{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/igw", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
|
||||
{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/image", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
|
||||
{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/instance", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
|
||||
{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/key", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
|
||||
{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/launchtemplate", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
|
||||
{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/placementgroup", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
|
||||
{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/routetable", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
|
||||
{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/snapshot", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
|
||||
{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/volume", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
|
||||
{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/vpc", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
|
||||
|
||||
{From: "agents/ecs/agent", To: "domains/ecs", Debt: "agents/ecs/agent", Reason: "assignment/state wire types from the ECS implementation; needs a guest/controller contract"},
|
||||
|
||||
{From: "agents/eks/tokenwebhook", To: "domains/eks/cluster", Debt: "agents/eks/tokenwebhook", Reason: "WebhookTokenReviewResult from the EKS implementation; needs an EKS contract or projection"},
|
||||
|
||||
{From: "agents/rds/agent", To: "domains/rds/instance", Debt: "agents/rds/agent (dependency, narrowed)", Reason: "command protocol, wire and record types; needs a guest/controller wire contract"},
|
||||
|
||||
{From: "cmd/aws-model-coverage", To: "domains/acm/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
|
||||
{From: "cmd/aws-model-coverage", To: "domains/ecr/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
|
||||
{From: "cmd/aws-model-coverage", To: "domains/route53/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
|
||||
{From: "cmd/aws-model-coverage", To: "domains/elbv2/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
|
||||
{From: "cmd/aws-model-coverage", To: "domains/rds/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
|
||||
{From: "cmd/aws-model-coverage", To: "domains/ec2/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
|
||||
{From: "cmd/aws-model-coverage", To: "domains/ecs/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
|
||||
{From: "cmd/aws-model-coverage", To: "domains/eks/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
|
||||
{From: "cmd/aws-model-coverage", To: "domains/iam/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
|
||||
{From: "cmd/aws-model-coverage", To: "domains/sts/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
|
||||
|
||||
{From: "runtime/daemon", To: "domains/ec2/awsapi/instance", Debt: "Instance adapter consumers outside EC2", Reason: "runtime/daemon/eks_cp_control.go and rds_deps.go call the instance adapter; needs an EC2 capability"},
|
||||
|
||||
{From: "domains/acm", To: "domains/iam", Debt: "ACM: the IAM AES-GCM helper import (ACM relocation residual)", Reason: "IAM AES-GCM helpers; needs a shared crypto boundary"},
|
||||
|
||||
{From: "domains/admission/quota", To: "domains/ec2/awsapi/eip", Debt: "domains/admission/quota (dependency, moved from handlers/quota unchanged)", Reason: "describe adapter used as a counter; needs a reader capability"},
|
||||
{From: "domains/admission/quota", To: "domains/ec2/awsapi/instance", Debt: "domains/admission/quota (dependency, moved from handlers/quota unchanged)", Reason: "vcpu.go counts instances through the instance adapter; needs an EC2 reader capability"},
|
||||
{From: "domains/admission/quota", To: "domains/ec2/awsapi/volume", Debt: "domains/admission/quota (dependency, moved from handlers/quota unchanged)", Reason: "describe adapter used as a counter; needs a reader capability"},
|
||||
{From: "domains/admission/quota", To: "domains/ec2/awsapi/vpc", Debt: "domains/admission/quota (dependency, moved from handlers/quota unchanged)", Reason: "describe adapter used as a counter; needs a reader capability"},
|
||||
{From: "domains/admission/quota", To: "domains/ec2/instancetypes", Debt: "domains/admission/quota → domains/ec2/instancetypes", Reason: "DefaultVCPUs sizing; needs a consumer-owned capability or EC2 projection"},
|
||||
{From: "domains/admission/quota", To: "domains/elbv2/awsapi", Debt: "domains/admission/quota (dependency, moved from handlers/quota unchanged)", Reason: "live ELBv2 counter; needs a reader capability"},
|
||||
{From: "domains/admission/quota", To: "domains/rds/instance", Debt: "domains/admission/quota (dependency, moved from handlers/quota unchanged)", Reason: "live RDS counter; needs a reader capability"},
|
||||
{From: "domains/admission/quota", To: "runtime/compute/vm", Debt: "domains/admission/quota (dependency, moved from handlers/quota unchanged)", Reason: "instance-record vCPU sweep; needs a reader capability"},
|
||||
|
||||
{From: "domains/dns", To: "domains/network/reconcile", Debt: "domains/dns", Reason: "takes the reconcile lock from network's implementation"},
|
||||
{From: "domains/dns", To: "runtime/compute/vm", Debt: "domains/dns", Reason: "domain-to-runtime import of VM state"},
|
||||
|
||||
{From: "domains/ec2/awsapi/instance", To: "domains/iam", Debt: "domains/ec2/awsapi/instance (dependency)", Reason: "IAM edge needs an IAM capability"},
|
||||
{From: "domains/ec2/awsapi/instance", To: "runtime/compute/cache", Debt: "domains/ec2/awsapi/instance (dependency)", Reason: "node liveness and VM state; needs an EC2-owned projection"},
|
||||
{From: "domains/ec2/awsapi/instance", To: "runtime/compute/vm", Debt: "domains/ec2/awsapi/instance (dependency)", Reason: "node liveness and VM state; needs an EC2-owned projection"},
|
||||
|
||||
{From: "domains/ec2/eip", To: "domains/network/topology", Debt: "EC2 → network implementation", Reason: "EC2 reads network implementation; clears with the EC2-owned projection"},
|
||||
|
||||
{From: "domains/ec2/guestmetadata", To: "domains/dns", Debt: "Guest metadata: the cross-domain and runtime imports above", Reason: "guest-metadata relocation residual; needs capabilities, projections or contracts"},
|
||||
{From: "domains/ec2/guestmetadata", To: "domains/iam", Debt: "Guest metadata: the cross-domain and runtime imports above", Reason: "guest-metadata relocation residual; needs capabilities, projections or contracts"},
|
||||
{From: "domains/ec2/guestmetadata", To: "domains/sts", Debt: "Guest metadata: the cross-domain and runtime imports above", Reason: "guest-metadata relocation residual; needs capabilities, projections or contracts"},
|
||||
{From: "domains/ec2/guestmetadata", To: "runtime/compute/vm", Debt: "Guest metadata: the cross-domain and runtime imports above", Reason: "guest-metadata relocation residual; needs capabilities, projections or contracts"},
|
||||
|
||||
{From: "domains/ec2/instance", To: "domains/dns", Debt: "domains/ec2/instance", Reason: "cross-domain implementation import of DNS"},
|
||||
{From: "domains/ec2/instance", To: "domains/network/topology", Debt: "EC2 → network implementation", Reason: "EC2 reads network implementation; clears with the EC2-owned projection"},
|
||||
{From: "domains/ec2/instance", To: "runtime/compute/gpu", Debt: "domains/ec2/instance", Reason: "VM/GPU runtime state read directly; needs a domain-owned projection"},
|
||||
{From: "domains/ec2/instance", To: "runtime/compute/vm", Debt: "domains/ec2/instance", Reason: "VM/GPU runtime state read directly; needs a domain-owned projection"},
|
||||
{From: "domains/ec2/instance/state", To: "runtime/daemon", Debt: "domains/ec2/instance/state (dependency, IMDS readers)", Reason: "wraps the daemon's local-file reader and JetStreamManager bucket open unchanged for vpcd's IMDS lookup"},
|
||||
{From: "domains/ec2/instance/state", To: "runtime/compute/vm", Debt: "domains/ec2/instance/state (dependency)", Reason: "the vpcd IMDS local reader returns vm.VM"},
|
||||
|
||||
{From: "domains/ec2/volume", To: "runtime/compute/vm", Debt: "Domain → runtime", Reason: "VM runtime state read directly; needs a domain-owned projection"},
|
||||
|
||||
{From: "domains/ec2/vpc", To: "domains/network/external", Debt: "EC2 → network implementation", Reason: "EC2 reads network implementation; clears with the EC2-owned projection"},
|
||||
{From: "domains/ec2/vpc", To: "domains/network/external/dhcp", Debt: "EC2 → network implementation", Reason: "EC2 reads network implementation; clears with the EC2-owned projection"},
|
||||
{From: "domains/ec2/vpc", To: "domains/network/identifiers", Debt: "EC2 → network implementation", Reason: "EC2 reads network implementation; clears with the EC2-owned projection"},
|
||||
{From: "domains/ec2/vpc", To: "domains/network/topology", Debt: "EC2 → network implementation", Reason: "EC2 reads network implementation; clears with the EC2-owned projection"},
|
||||
|
||||
{From: "domains/ecr/auth", To: "domains/iam", Debt: "Cross-domain", Reason: "IAM implementation import; needs an IAM capability"},
|
||||
|
||||
{From: "domains/network/host", To: "runtime/compute/vm", Debt: "domains/network", Reason: "domain-to-runtime import"},
|
||||
{From: "domains/network/host", To: "runtime/host/command", Debt: "domains/network", Reason: "domain-to-runtime import"},
|
||||
|
||||
{From: "domains/network/reconcile", To: "domains/ec2/eip", Debt: "domains/network", Reason: "reconcile reads EC2 private records; clears with the EC2-owned projection"},
|
||||
{From: "domains/network/reconcile", To: "domains/ec2/igw", Debt: "domains/network", Reason: "reconcile reads EC2 private records; clears with the EC2-owned projection"},
|
||||
{From: "domains/network/reconcile", To: "domains/ec2/natgw", Debt: "domains/network", Reason: "reconcile reads EC2 private records; clears with the EC2-owned projection"},
|
||||
{From: "domains/network/reconcile", To: "domains/ec2/routetable", Debt: "domains/network", Reason: "reconcile reads EC2 private records; clears with the EC2-owned projection"},
|
||||
{From: "domains/network/reconcile", To: "domains/ec2/vpc", Debt: "domains/network", Reason: "reconcile reads EC2 private records; clears with the EC2-owned projection"},
|
||||
{From: "domains/network/reconcile", To: "runtime/compute/vm", Debt: "domains/network", Reason: "domain-to-runtime import"},
|
||||
|
||||
{From: "domains/ochre", To: "domains/ec2/systeminstance", Debt: "Cross-domain", Reason: "consumes another domain's implementation rather than a capability"},
|
||||
{From: "domains/ochre", To: "domains/iam", Debt: "IAM implementation consumers in STS, Ochre and ELBv2", Reason: "secret encrypt/decrypt helpers; needs a shared crypto boundary"},
|
||||
{From: "domains/ochre", To: "domains/network/systemvpc", Debt: "Cross-domain", Reason: "consumes another domain's implementation rather than a capability"},
|
||||
{From: "domains/ochre", To: "runtime/compute/gpu", Debt: "Domain → runtime", Reason: "GPU runtime state read directly; needs a domain-owned projection"},
|
||||
|
||||
{From: "domains/ochre/vector", To: "domains/iam", Debt: "Cross-domain", Reason: "consumes another domain's implementation rather than a capability"},
|
||||
|
||||
{From: "domains/rds/awsapi", To: "domains/ec2/awsapi/instance", Debt: "Instance adapter consumers outside EC2", Reason: "enginecatalog.go calls the instance adapter; needs an EC2 capability"},
|
||||
|
||||
{From: "domains/sts/awsapi", To: "domains/iam", Debt: "IAM implementation consumers in STS, Ochre and ELBv2", Reason: "GetCallerIdentity takes IAMService; needs an IAM capability"},
|
||||
|
||||
{From: "domains/ecs", To: "domains/ec2/ebs/metadata", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
|
||||
{From: "domains/ecs", To: "domains/ec2/eip", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
|
||||
{From: "domains/ecs", To: "domains/ec2/instancetypes", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
|
||||
{From: "domains/ecs", To: "domains/elbv2", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
|
||||
{From: "domains/ecs", To: "domains/iam", Debt: "close-out: handlers/ecs (ADR-0004 blocked-dependency list; inventory-ecs boundary imports)", Reason: "legacy ECS calls another domain's implementation as an internal API"},
|
||||
|
||||
{From: "domains/eks/cluster", To: "domains/dns", Debt: "close-out: handlers/eks (ADR-0004 blocked-dependency list; inventory-eks boundary imports)", Reason: "legacy EKS calls another domain's or runtime implementation directly"},
|
||||
{From: "domains/eks/cluster", To: "domains/ec2/ebs/metadata", Debt: "close-out: handlers/eks (ADR-0004 blocked-dependency list; inventory-eks boundary imports)", Reason: "legacy EKS calls another domain's or runtime implementation directly"},
|
||||
{From: "domains/eks/cluster", To: "domains/ec2/instancetypes", Debt: "close-out: handlers/eks (ADR-0004 blocked-dependency list; inventory-eks boundary imports)", Reason: "legacy EKS calls another domain's or runtime implementation directly"},
|
||||
{From: "domains/eks/cluster", To: "domains/ec2/placementgroup", Debt: "close-out: handlers/eks (ADR-0004 blocked-dependency list; inventory-eks boundary imports)", Reason: "legacy EKS calls another domain's or runtime implementation directly"},
|
||||
{From: "domains/eks/cluster", To: "domains/ec2/systeminstance", Debt: "close-out: handlers/eks (ADR-0004 blocked-dependency list; inventory-eks boundary imports)", Reason: "legacy EKS calls another domain's or runtime implementation directly"},
|
||||
{From: "domains/eks/cluster", To: "domains/network/systemvpc", Debt: "close-out: handlers/eks (ADR-0004 blocked-dependency list; inventory-eks boundary imports)", Reason: "legacy EKS calls another domain's or runtime implementation directly"},
|
||||
{From: "domains/eks/cluster", To: "domains/iam", Debt: "close-out: handlers/eks (ADR-0004 blocked-dependency list; inventory-eks boundary imports)", Reason: "legacy EKS calls another domain's or runtime implementation directly"},
|
||||
{From: "domains/eks/cluster", To: "runtime/compute/vm", Debt: "close-out: handlers/eks (ADR-0004 blocked-dependency list; inventory-eks boundary imports)", Reason: "legacy EKS calls another domain's or runtime implementation directly"},
|
||||
|
||||
{From: "domains/elbv2", To: "domains/acm", Debt: "domains/elbv2 (cross-domain)", Reason: "ELBv2 cross-domain import; clears when ELBv2 is extracted behind consumer-owned capabilities"},
|
||||
{From: "domains/elbv2", To: "domains/dns", Debt: "domains/elbv2 (cross-domain)", Reason: "ELBv2 cross-domain import; clears when ELBv2 is extracted behind consumer-owned capabilities"},
|
||||
{From: "domains/elbv2", To: "domains/ec2/systeminstance", Debt: "domains/elbv2 (cross-domain)", Reason: "ELBv2 cross-domain import; clears when ELBv2 is extracted behind consumer-owned capabilities"},
|
||||
{From: "domains/elbv2", To: "domains/ec2/vpc", Debt: "domains/elbv2 (cross-domain)", Reason: "ELBv2 cross-domain import; clears when ELBv2 is extracted behind consumer-owned capabilities"},
|
||||
{From: "domains/elbv2", To: "domains/network/topology", Debt: "domains/elbv2 (cross-domain)", Reason: "ELBv2 cross-domain import; clears when ELBv2 is extracted behind consumer-owned capabilities"},
|
||||
{From: "domains/elbv2", To: "domains/iam", Debt: "IAM implementation consumers in STS, Ochre and ELBv2", Reason: "system-instance role ensurer; needs an IAM capability"},
|
||||
{From: "domains/elbv2", To: "agents/elbv2", Debt: "domains/elbv2/{agent_types,haproxy,health_checker,nginx}.go import lbagent", Reason: "health-report and cert/PID-path vocabulary; needs a guest/controller contract"},
|
||||
|
||||
{From: "domains/rds/instance", To: "domains/dns", Debt: "close-out: handlers/rds (ADR-0004 blocked-dependency list; inventory-rds boundary imports)", Reason: "legacy RDS calls another domain's or runtime implementation directly"},
|
||||
{From: "domains/rds/instance", To: "domains/ec2/instance", Debt: "close-out: handlers/rds (ADR-0004 blocked-dependency list; inventory-rds boundary imports)", Reason: "legacy RDS calls another domain's or runtime implementation directly"},
|
||||
{From: "domains/rds/instance", To: "domains/ec2/instancetypes", Debt: "close-out: handlers/rds (ADR-0004 blocked-dependency list; inventory-rds boundary imports)", Reason: "legacy RDS calls another domain's or runtime implementation directly"},
|
||||
{From: "domains/rds/instance", To: "domains/ec2/systeminstance", Debt: "close-out: handlers/rds (ADR-0004 blocked-dependency list; inventory-rds boundary imports)", Reason: "legacy RDS calls another domain's or runtime implementation directly"},
|
||||
{From: "domains/rds/instance", To: "domains/network/systemvpc", Debt: "close-out: handlers/rds (ADR-0004 blocked-dependency list; inventory-rds boundary imports)", Reason: "legacy RDS calls another domain's or runtime implementation directly"},
|
||||
{From: "domains/rds/instance", To: "domains/iam", Debt: "close-out: handlers/rds (ADR-0004 blocked-dependency list; inventory-rds boundary imports)", Reason: "legacy RDS calls another domain's or runtime implementation directly"},
|
||||
{From: "domains/rds/instance", To: "runtime/compute/vm", Debt: "close-out: handlers/rds (ADR-0004 blocked-dependency list; inventory-rds boundary imports)", Reason: "legacy RDS calls another domain's or runtime implementation directly"},
|
||||
|
||||
{From: "domains/sts", To: "domains/iam", Debt: "IAM implementation consumers in STS, Ochre and ELBv2", Reason: "STS reads IAM service, trust-policy and OIDC-provider state; needs an IAM capability"},
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
// Package layering enforces ADR-0001's package-boundary dependency rules over
|
||||
// the module's production import graph, ratcheted by a recorded-debt allowlist.
|
||||
package layering
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"go/ast"
|
||||
"go/build/constraint"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const modulePath = "github.com/mulgadc/spinifex"
|
||||
|
||||
// edge is one package-to-package import, both sides named as in the allowlist:
|
||||
// application packages relative to spinifex/, module-root ones (cmd/,
|
||||
// contracts/, internal/) relative to the module root.
|
||||
type edge struct {
|
||||
From, To string
|
||||
}
|
||||
|
||||
func (e edge) String() string { return e.From + " -> " + e.To }
|
||||
|
||||
// graph is the production import graph restricted to this module.
|
||||
type graph struct {
|
||||
// pkgs maps a package name to its module-relative directory.
|
||||
pkgs map[string]string
|
||||
// imports records the first file:line at which each edge appears.
|
||||
imports map[edge]string
|
||||
}
|
||||
|
||||
func newGraph() *graph {
|
||||
return &graph{pkgs: map[string]string{}, imports: map[edge]string{}}
|
||||
}
|
||||
|
||||
// shortName drops the application-root prefix so names read like the ADR and
|
||||
// the migration record; classify rejects app-root dirs that would collide.
|
||||
func shortName(full string) string {
|
||||
if rest, ok := strings.CutPrefix(full, "spinifex/"); ok {
|
||||
return rest
|
||||
}
|
||||
return full
|
||||
}
|
||||
|
||||
func moduleRoot() (string, error) {
|
||||
_, file, _, ok := runtime.Caller(0)
|
||||
if !ok {
|
||||
return "", errors.New("cannot locate test source")
|
||||
}
|
||||
// architecture/layering/graph_test.go -> module root.
|
||||
return filepath.Abs(filepath.Join(filepath.Dir(file), "..", ".."))
|
||||
}
|
||||
|
||||
// excludedTopDirs hold no production code: tests/ is test-only, docs/ only
|
||||
// embeds documentation and architecture/ is repository governance tooling.
|
||||
var excludedTopDirs = map[string]bool{"tests": true, "docs": true, "architecture": true}
|
||||
|
||||
// loadGraph parses the imports of every non-test Go file under root, so it
|
||||
// sees every build-tag variant and needs no Go toolchain or workspace.
|
||||
func loadGraph(root string) (*graph, error) {
|
||||
g := newGraph()
|
||||
fset := token.NewFileSet()
|
||||
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel = filepath.ToSlash(rel)
|
||||
if d.IsDir() {
|
||||
if rel == "." {
|
||||
return nil
|
||||
}
|
||||
name := d.Name()
|
||||
if strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_") ||
|
||||
name == "testdata" || name == "vendor" || name == "node_modules" || excludedTopDirs[rel] {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(path, "go.mod")); err == nil {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
|
||||
return nil
|
||||
}
|
||||
f, err := parser.ParseFile(fset, path, nil, parser.ImportsOnly|parser.ParseComments)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ignoredByBuildTag(f.Comments, f.Package) {
|
||||
return nil
|
||||
}
|
||||
dir := filepath.ToSlash(filepath.Dir(rel))
|
||||
if dir == "." {
|
||||
dir = ""
|
||||
}
|
||||
from := shortName(dir)
|
||||
g.pkgs[from] = dir
|
||||
for _, imp := range f.Imports {
|
||||
p, err := strconv.Unquote(imp.Path.Value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
target, ok := strings.CutPrefix(p, modulePath+"/")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
e := edge{From: from, To: shortName(target)}
|
||||
if _, seen := g.imports[e]; !seen {
|
||||
pos := fset.Position(imp.Pos())
|
||||
g.imports[e] = fmt.Sprintf("%s:%d", rel, pos.Line)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return g, err
|
||||
}
|
||||
|
||||
// ignoredByBuildTag reports a `//go:build ignore` file, which is a standalone
|
||||
// program (a generator) rather than part of its directory's package.
|
||||
func ignoredByBuildTag(groups []*ast.CommentGroup, pkgPos token.Pos) bool {
|
||||
for _, cg := range groups {
|
||||
if cg.Pos() >= pkgPos {
|
||||
break
|
||||
}
|
||||
for _, c := range cg.List {
|
||||
if !constraint.IsGoBuild(c.Text) {
|
||||
continue
|
||||
}
|
||||
expr, err := constraint.Parse(c.Text)
|
||||
if err == nil && expr.String() == "ignore" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// sortedEdges returns the graph's edges in a stable order.
|
||||
func (g *graph) sortedEdges() []edge {
|
||||
out := make([]edge, 0, len(g.imports))
|
||||
for e := range g.imports {
|
||||
out = append(out, e)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].From != out[j].From {
|
||||
return out[i].From < out[j].From
|
||||
}
|
||||
return out[i].To < out[j].To
|
||||
})
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,240 @@
|
||||
package layering
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type violation struct {
|
||||
rule string
|
||||
edge edge
|
||||
pos string
|
||||
}
|
||||
|
||||
type report struct {
|
||||
unmapped []string
|
||||
violations []violation // every rule hit, allowlisted or not
|
||||
unrecorded []violation // hits whose edge has no allowlist entry
|
||||
stale []allowance // entries whose edge no rule rejects any more
|
||||
malformed []string
|
||||
}
|
||||
|
||||
// check classifies g, applies every rule and compares the result with allow.
|
||||
func check(g *graph, allow []allowance) report {
|
||||
var rep report
|
||||
classes := map[string]class{}
|
||||
for name, dir := range g.pkgs {
|
||||
c, ok := classify(dir)
|
||||
if !ok || c.layer == layerTests {
|
||||
rep.unmapped = append(rep.unmapped, fmt.Sprintf("%s (directory %s)", name, dir))
|
||||
continue
|
||||
}
|
||||
classes[name] = c
|
||||
}
|
||||
|
||||
allowed := map[edge]bool{}
|
||||
for _, a := range allow {
|
||||
e := edge{From: a.From, To: a.To}
|
||||
switch {
|
||||
case allowed[e]:
|
||||
rep.malformed = append(rep.malformed, "duplicate entry "+e.String())
|
||||
case a.Debt == "" || a.Reason == "":
|
||||
rep.malformed = append(rep.malformed, "entry "+e.String()+" needs both Debt and Reason")
|
||||
}
|
||||
allowed[e] = true
|
||||
}
|
||||
|
||||
hit := map[edge]bool{}
|
||||
for _, e := range g.sortedEdges() {
|
||||
from, ok := classes[e.From]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
to, ok := classify(fullPath(e.To))
|
||||
if !ok {
|
||||
rep.unmapped = append(rep.unmapped, fmt.Sprintf("%s (imported by %s at %s)", e.To, e.From, g.imports[e]))
|
||||
continue
|
||||
}
|
||||
for _, r := range rules {
|
||||
if !r.subject(from, e.From) || !r.forbids(from, to, e.From) {
|
||||
continue
|
||||
}
|
||||
v := violation{rule: r.id, edge: e, pos: g.imports[e]}
|
||||
rep.violations = append(rep.violations, v)
|
||||
hit[e] = true
|
||||
if !allowed[e] {
|
||||
rep.unrecorded = append(rep.unrecorded, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, a := range allow {
|
||||
if !hit[edge{From: a.From, To: a.To}] {
|
||||
rep.stale = append(rep.stale, a)
|
||||
}
|
||||
}
|
||||
sort.Strings(rep.unmapped)
|
||||
return rep
|
||||
}
|
||||
|
||||
func ruleByID(id string) rule {
|
||||
for _, r := range rules {
|
||||
if r.id == id {
|
||||
return r
|
||||
}
|
||||
}
|
||||
panic("unknown rule " + id)
|
||||
}
|
||||
|
||||
// TestADR0001_Layering is the blocking ratchet over the real import graph.
|
||||
func TestADR0001_Layering(t *testing.T) {
|
||||
root, err := moduleRoot()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g, err := loadGraph(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Guards against a walk that silently finds nothing and passes vacuously.
|
||||
if len(g.pkgs) < 100 || len(g.imports) < 500 {
|
||||
t.Fatalf("scanned %d packages and %d module imports under %s; the walk has lost the tree and every rule would pass vacuously",
|
||||
len(g.pkgs), len(g.imports), root)
|
||||
}
|
||||
rep := check(g, allowlist)
|
||||
|
||||
t.Run("Classification_EveryPackageMapped", func(t *testing.T) {
|
||||
for _, u := range rep.unmapped {
|
||||
t.Errorf("ADR-0001 current-to-target map: package %s has no layer; place it under an ADR-0001 home or classify it deliberately in layers_test.go", u)
|
||||
}
|
||||
})
|
||||
|
||||
for _, r := range rules {
|
||||
t.Run(r.id, func(t *testing.T) {
|
||||
subjects := 0
|
||||
for name, dir := range g.pkgs {
|
||||
if c, ok := classify(dir); ok && r.subject(c, name) {
|
||||
subjects++
|
||||
}
|
||||
}
|
||||
if subjects == 0 {
|
||||
t.Fatalf("no scanned package is governed by %s; the rule would pass vacuously", r.id)
|
||||
}
|
||||
var snippets []string
|
||||
for _, v := range rep.unrecorded {
|
||||
if v.rule != r.id {
|
||||
continue
|
||||
}
|
||||
t.Errorf("ADR-0001 %s: %s imports %s (%s); %s",
|
||||
r.clause, v.edge.From, v.edge.To, v.pos, r.hint)
|
||||
snippets = append(snippets, fmt.Sprintf("{From: %q, To: %q, Debt: \"<migration-record debt subject>\", Reason: \"<why>\"},", v.edge.From, v.edge.To))
|
||||
}
|
||||
if len(snippets) > 0 {
|
||||
t.Logf("fix the import; only debt already recorded in docs/PACKAGE_BOUNDARY_MIGRATION.md may be allow-listed:\n%s",
|
||||
strings.Join(snippets, "\n"))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("Allowlist_NoStaleEntries", func(t *testing.T) {
|
||||
for _, a := range rep.stale {
|
||||
t.Errorf("allowlist entry %s -> %s (%s) no longer matches a forbidden import; the debt is paid, so delete the entry from allowlist_test.go",
|
||||
a.From, a.To, a.Debt)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Allowlist_WellFormed", func(t *testing.T) {
|
||||
for _, m := range rep.malformed {
|
||||
t.Errorf("allowlist_test.go: %s", m)
|
||||
}
|
||||
t.Logf("%d allow-listed edges", len(allowlist))
|
||||
})
|
||||
}
|
||||
|
||||
// TestADR0001_LayeringCheckerDetectsFaults injects a forbidden edge, a stale
|
||||
// allowlist entry and an unmapped package into a synthetic graph.
|
||||
func TestADR0001_LayeringCheckerDetectsFaults(t *testing.T) {
|
||||
g := newGraph()
|
||||
for _, dir := range []string{
|
||||
"spinifex/foundation/widget", "spinifex/domains/alpha/thing",
|
||||
"spinifex/domains/beta/store", "spinifex/misc",
|
||||
} {
|
||||
g.pkgs[shortName(dir)] = dir
|
||||
}
|
||||
forbidden := edge{From: "foundation/widget", To: "domains/alpha/thing"}
|
||||
g.imports[forbidden] = "spinifex/foundation/widget/w.go:7"
|
||||
allowedEdge := edge{From: "domains/alpha/thing", To: "domains/beta/store"}
|
||||
g.imports[allowedEdge] = "spinifex/domains/alpha/thing/t.go:9"
|
||||
|
||||
staleEntry := allowance{From: "domains/beta/store", To: "domains/alpha/thing", Debt: "synthetic", Reason: "stale"}
|
||||
rep := check(g, []allowance{
|
||||
{From: allowedEdge.From, To: allowedEdge.To, Debt: "synthetic", Reason: "recorded"},
|
||||
staleEntry,
|
||||
})
|
||||
|
||||
if len(rep.unrecorded) != 1 || rep.unrecorded[0].edge != forbidden ||
|
||||
rep.unrecorded[0].rule != ruleByID("R1_FoundationImportsOnlyFoundation").id {
|
||||
t.Errorf("want only %s reported under R1, got %+v", forbidden, rep.unrecorded)
|
||||
}
|
||||
if len(rep.stale) != 1 || rep.stale[0] != staleEntry {
|
||||
t.Errorf("want stale entry %+v, got %+v", staleEntry, rep.stale)
|
||||
}
|
||||
if len(rep.unmapped) != 1 || !strings.HasPrefix(rep.unmapped[0], "misc ") {
|
||||
t.Errorf("want unmapped package misc, got %v", rep.unmapped)
|
||||
}
|
||||
var allowedHit bool
|
||||
for _, v := range rep.violations {
|
||||
allowedHit = allowedHit || v.edge == allowedEdge
|
||||
}
|
||||
if !allowedHit {
|
||||
t.Errorf("allow-listed cross-domain edge %s was not detected by R3, so the allowlist entry would read as stale", allowedEdge)
|
||||
}
|
||||
}
|
||||
|
||||
// TestADR0001_InterimDaemonDirectionCoversRuntimeDaemon: the daemon left its
|
||||
// legacy root, so gateway importing runtime/daemon must still be rejected, and
|
||||
// the former handlers packages are held to it by R5 at their domain homes.
|
||||
func TestADR0001_InterimDaemonDirectionCoversRuntimeDaemon(t *testing.T) {
|
||||
daemon, _ := classify("spinifex/runtime/daemon")
|
||||
checks := []struct{ rule, from string }{
|
||||
{"Interim_DaemonGatewayHandlersDirection", "spinifex/gateway"},
|
||||
{"R5_DomainImportsNoRuntimeOrOperator", "spinifex/domains/ecs"},
|
||||
}
|
||||
for _, c := range checks {
|
||||
r := ruleByID(c.rule)
|
||||
f, _ := classify(c.from)
|
||||
if !r.subject(f, c.from) || !r.forbids(f, daemon, c.from) {
|
||||
t.Errorf("%s -> runtime/daemon is not rejected by %s", c.from, c.rule)
|
||||
}
|
||||
}
|
||||
other, _ := classify("spinifex/runtime/host/process")
|
||||
if gw, _ := classify("spinifex/gateway"); ruleByID("Interim_DaemonGatewayHandlersDirection").forbids(gw, other, "spinifex/gateway") {
|
||||
t.Error("gateway -> runtime/host/process is rejected, but it is not the daemon")
|
||||
}
|
||||
}
|
||||
|
||||
// TestADR0002_S7_RequestRecordRejectsOutsiders: the request record is readable
|
||||
// by the gateway and ingress/aws only.
|
||||
func TestADR0002_S7_RequestRecordRejectsOutsiders(t *testing.T) {
|
||||
record, ok := classify("spinifex/ingress/aws/reqctx")
|
||||
if !ok || !record.requestRecord {
|
||||
t.Fatalf("ingress/aws/reqctx is not classified as the request record: %+v", record)
|
||||
}
|
||||
r := ruleByID("S7_RequestRecordPrivateToIngress")
|
||||
for name, wantRejected := range map[string]bool{
|
||||
"domains/ecs/awsapi": true,
|
||||
"runtime/roles/awsgw": true,
|
||||
"operator/admin": true,
|
||||
"gateway": false,
|
||||
"ingress/aws/dispatch": false,
|
||||
} {
|
||||
from, ok := classify(fullPath(name))
|
||||
if !ok {
|
||||
t.Fatalf("%s is unclassified", name)
|
||||
}
|
||||
if got := r.forbids(from, record, name); got != wantRejected {
|
||||
t.Errorf("%s -> ingress/aws/reqctx rejected=%v, want %v", name, got, wantRejected)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package layering
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type layer string
|
||||
|
||||
const (
|
||||
layerFoundation layer = "foundation"
|
||||
layerContracts layer = "contracts"
|
||||
layerDomain layer = "domains"
|
||||
layerIngress layer = "ingress"
|
||||
layerRuntime layer = "runtime"
|
||||
layerOperator layer = "operator"
|
||||
layerAgents layer = "agents"
|
||||
layerProviders layer = "providers"
|
||||
layerBootstrap layer = "bootstrap"
|
||||
layerCmd layer = "cmd"
|
||||
layerInternal layer = "internal"
|
||||
layerTestkit layer = "testkit"
|
||||
layerLegacy layer = "legacy"
|
||||
// layerTests is valid only as an import target; tests/ is never scanned.
|
||||
layerTests layer = "tests"
|
||||
)
|
||||
|
||||
// domainKind distinguishes the parts of a domain the rules treat differently.
|
||||
type domainKind int
|
||||
|
||||
const (
|
||||
notDomain domainKind = iota
|
||||
domainImpl // domains/<d>/... resource implementation
|
||||
domainAWSAPI // domains/<d>/awsapi/... AWS action adapters
|
||||
domainProj // domains/<d>/projection/... authorised view
|
||||
legacyImpl // handlers/<d>/... legacy domain implementation
|
||||
legacyAdapter // gateway/<d>/... legacy AWS action adapters
|
||||
)
|
||||
|
||||
type class struct {
|
||||
layer layer
|
||||
legacy string // legacy root name when layer is layerLegacy
|
||||
domain string // owning domain for domain-shaped and agent packages
|
||||
kind domainKind
|
||||
daemon bool // runtime/daemon, the interim daemon -> gateway -> handlers root
|
||||
// requestRecord marks ingress/aws/reqctx, the request record ADR-0002:S7 keeps private to ingress.
|
||||
requestRecord bool
|
||||
}
|
||||
|
||||
func (c class) domainShaped() bool { return c.kind != notDomain }
|
||||
|
||||
// isLegacy reports whether c sits under one of the named legacy roots.
|
||||
func (c class) isLegacy(roots ...string) bool {
|
||||
if c.layer != layerLegacy {
|
||||
return false
|
||||
}
|
||||
if len(roots) == 0 {
|
||||
return true
|
||||
}
|
||||
return slices.Contains(roots, c.legacy)
|
||||
}
|
||||
|
||||
// legacyRoots are pre-ADR-0001 application roots still standing.
|
||||
var legacyRoots = map[string]bool{
|
||||
"gateway": true,
|
||||
}
|
||||
|
||||
// gatewayDomains names the domain each gateway/<service> adapter belongs to;
|
||||
// Bedrock is the AWS surface of the Ochre domain.
|
||||
var gatewayDomains = map[string]string{
|
||||
"bedrock": "ochre", "ecs": "ecs", "eks": "eks", "elbv2": "elbv2", "iam": "iam",
|
||||
"rds": "rds", "sts": "sts",
|
||||
}
|
||||
|
||||
// classify maps a module-relative package directory to its layer. ok is false
|
||||
// for a package that has no deliberate classification.
|
||||
func classify(full string) (class, bool) {
|
||||
parts := strings.Split(full, "/")
|
||||
switch parts[0] {
|
||||
case "cmd":
|
||||
return class{layer: layerCmd}, len(parts) >= 2
|
||||
case "contracts":
|
||||
return class{layer: layerContracts}, len(parts) >= 2
|
||||
case "internal":
|
||||
if len(parts) < 2 {
|
||||
return class{}, false
|
||||
}
|
||||
if parts[1] == "testkit" || parts[1] == "tooling" {
|
||||
return class{layer: layerTestkit}, true
|
||||
}
|
||||
return class{layer: layerInternal}, true
|
||||
case "tests":
|
||||
return class{layer: layerTests}, true
|
||||
case "spinifex":
|
||||
return classifyApp(parts[1:])
|
||||
}
|
||||
return class{}, false
|
||||
}
|
||||
|
||||
func classifyApp(app []string) (class, bool) {
|
||||
if len(app) == 0 {
|
||||
return class{}, false
|
||||
}
|
||||
switch app[0] {
|
||||
case "foundation":
|
||||
return class{layer: layerFoundation}, true
|
||||
case "ingress":
|
||||
return class{layer: layerIngress, requestRecord: len(app) >= 3 && app[1] == "aws" && app[2] == "reqctx"}, true
|
||||
case "runtime":
|
||||
return class{layer: layerRuntime, daemon: len(app) >= 2 && app[1] == "daemon"}, true
|
||||
case "operator":
|
||||
return class{layer: layerOperator}, true
|
||||
case "providers":
|
||||
return class{layer: layerProviders}, true
|
||||
case "bootstrap":
|
||||
return class{layer: layerBootstrap}, true
|
||||
case "agents":
|
||||
c := class{layer: layerAgents}
|
||||
if len(app) >= 2 {
|
||||
c.domain = app[1]
|
||||
}
|
||||
return c, true
|
||||
case "domains":
|
||||
if len(app) < 2 {
|
||||
return class{}, false
|
||||
}
|
||||
c := class{layer: layerDomain, domain: app[1], kind: domainImpl}
|
||||
if len(app) >= 3 {
|
||||
switch app[2] {
|
||||
case "awsapi":
|
||||
c.kind = domainAWSAPI
|
||||
case "projection":
|
||||
c.kind = domainProj
|
||||
}
|
||||
}
|
||||
return c, true
|
||||
}
|
||||
if !legacyRoots[app[0]] {
|
||||
return class{}, false
|
||||
}
|
||||
c := class{layer: layerLegacy, legacy: app[0]}
|
||||
switch {
|
||||
case app[0] == "handlers" && len(app) >= 2:
|
||||
c.domain, c.kind = app[1], legacyImpl
|
||||
case app[0] == "gateway" && len(app) >= 2 && gatewayDomains[app[1]] != "":
|
||||
c.domain, c.kind = gatewayDomains[app[1]], legacyAdapter
|
||||
}
|
||||
return c, true
|
||||
}
|
||||
|
||||
// fullPath reverses shortName for classification.
|
||||
func fullPath(name string) string {
|
||||
top, _, _ := strings.Cut(name, "/")
|
||||
switch top {
|
||||
case "cmd", "contracts", "internal", "tests", "docs":
|
||||
return name
|
||||
}
|
||||
return "spinifex/" + name
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package layering
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// frozenLegacyPackages is every production package still under a legacy root.
|
||||
// It only shrinks: a new entry is a new legacy package, a missing one is a
|
||||
// completed move whose line must be deleted.
|
||||
var frozenLegacyPackages = []string{
|
||||
"gateway",
|
||||
}
|
||||
|
||||
// retiredLegacyRoots were legacy roots whose every package has moved to its
|
||||
// ADR-0001 target; recreating one would reopen a closed migration row.
|
||||
var retiredLegacyRoots = []string{"accountteardown", "admin", "daemon", "handlers", "lbagent", "services", "utils", "vpcd"}
|
||||
|
||||
// TestADR0001_LegacyRootsFrozen: the current-to-target map gives every legacy
|
||||
// root a destination, so none may gain a package and no retired root may return.
|
||||
func TestADR0001_LegacyRootsFrozen(t *testing.T) {
|
||||
root, err := moduleRoot()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g, err := loadGraph(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
present := map[string]bool{}
|
||||
for name, dir := range g.pkgs {
|
||||
if c, ok := classify(dir); ok && c.layer == layerLegacy {
|
||||
present[name] = true
|
||||
}
|
||||
}
|
||||
if len(present) == 0 {
|
||||
t.Fatal("found no legacy packages; the freeze would pass vacuously")
|
||||
}
|
||||
frozen := map[string]bool{}
|
||||
for _, p := range frozenLegacyPackages {
|
||||
frozen[p] = true
|
||||
if !present[p] {
|
||||
t.Errorf("legacy package %s no longer exists; delete it from frozenLegacyPackages", p)
|
||||
}
|
||||
}
|
||||
for p := range present {
|
||||
if !frozen[p] {
|
||||
t.Errorf("ADR-0001 current-to-target map: %s is a new package under a legacy root; create it at its ADR-0001 target home instead", p)
|
||||
}
|
||||
}
|
||||
|
||||
for _, r := range retiredLegacyRoots {
|
||||
entries, err := os.ReadDir(filepath.Join(root, "spinifex", r))
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(entries) > 0 {
|
||||
t.Errorf("ADR-0001 current-to-target map: spinifex/%s is a retired legacy root; give the code its target home instead", r)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
package layering
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// rule is one ADR-0001 dependency rule over a classified import edge.
|
||||
type rule struct {
|
||||
id string // subtest name, led by the ADR-0001 clause it encodes
|
||||
clause string // the ADR text the rule enforces, quoted in failures
|
||||
hint string // one-line fix direction
|
||||
// subject reports whether an importer is governed by the rule; the
|
||||
// non-empty guard requires at least one governed package per rule.
|
||||
subject func(from class, fromName string) bool
|
||||
// forbids reports whether the rule rejects the edge.
|
||||
forbids func(from, to class, fromName string) bool
|
||||
}
|
||||
|
||||
// awsgwRole is the composition root ADR-0002 lets wire domain registrations.
|
||||
const awsgwRole = "runtime/roles/awsgw"
|
||||
|
||||
// qualificationBinaries are the developer/qualification mains ADR-0001 lets
|
||||
// keep reusable source in internal/testkit or internal/tooling.
|
||||
var qualificationBinaries = map[string]bool{
|
||||
"cmd/aws-model-coverage": true, "cmd/dhcptest": true, "cmd/spx-loadgen": true,
|
||||
}
|
||||
|
||||
func isLayer(c class, layers ...layer) bool {
|
||||
return slices.Contains(layers, c.layer)
|
||||
}
|
||||
|
||||
// importsDomainImplementation is any domain-owned package other than a named
|
||||
// projection, in its target or legacy home.
|
||||
func importsDomainImplementation(to class) bool {
|
||||
return to.domainShaped() && to.kind != domainProj
|
||||
}
|
||||
|
||||
var rules = []rule{
|
||||
{
|
||||
id: "R1_FoundationImportsOnlyFoundation",
|
||||
clause: "rule 1: foundation imports neither domains, ingress, runtime nor operator (enforced conservatively: foundation imports no module package above foundation)",
|
||||
hint: "move the dependency's needed primitive into foundation or invert it through a parameter",
|
||||
subject: func(f class, _ string) bool { return f.layer == layerFoundation },
|
||||
forbids: func(_, t class, _ string) bool { return t.layer != layerFoundation },
|
||||
},
|
||||
{
|
||||
id: "R2_IngressImportsNoImplementation",
|
||||
clause: "rule 2: ingress/aws imports no domain implementation; domains supply their own action registration through the ingress registration contract",
|
||||
hint: "take the capability through the registration contract wired by runtime/roles/awsgw",
|
||||
subject: func(f class, _ string) bool { return f.layer == layerIngress },
|
||||
forbids: func(_, t class, _ string) bool {
|
||||
return !isLayer(t, layerFoundation, layerContracts, layerIngress)
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "R3_DomainImportsNoOtherDomainImplementation",
|
||||
clause: "rule 3: a domain does not read another domain's private records, migrations or repository; it uses a consumer-owned capability, a named projection, or a versioned contracts/<domain>/v1 contract",
|
||||
hint: "declare a narrow capability in the consumer, or import the owner's projection or contracts/<domain>/v1",
|
||||
subject: func(f class, _ string) bool { return f.domainShaped() },
|
||||
forbids: func(f, t class, _ string) bool {
|
||||
return f.domainShaped() && importsDomainImplementation(t) && t.domain != f.domain
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "R4_ProviderImportsNoDomainImplementation",
|
||||
clause: "rule 4: a provider adapter implements a provider contract and cannot acquire AWS-visible resource authority by reading a domain's private state",
|
||||
hint: "pass the needed values into the provider contract instead of importing the domain",
|
||||
subject: func(f class, _ string) bool { return f.layer == layerProviders },
|
||||
forbids: func(_, t class, _ string) bool { return t.domainShaped() || t.isLegacy() },
|
||||
},
|
||||
{
|
||||
id: "R5_DomainImportsNoRuntimeOrOperator",
|
||||
clause: "rule 5: operator and runtime are composition consumers; they wire domain modules, so a domain does not import them",
|
||||
hint: "expose the runtime or operator state through a domain-owned projection or a capability wired at the composition root",
|
||||
subject: func(f class, _ string) bool { return f.domainShaped() },
|
||||
forbids: func(f, t class, _ string) bool {
|
||||
return f.domainShaped() && isLayer(t, layerRuntime, layerOperator)
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "R5_AWSAPIAdaptersImportedOnlyByOwnerOrComposition",
|
||||
clause: "rule 5: operator and runtime call public use cases; a domain's awsapi action adapters are owned by that domain and wired only by the composition root",
|
||||
hint: "call an owner-provided capability instead of the AWS action adapter",
|
||||
subject: func(_ class, _ string) bool { return true },
|
||||
forbids: func(f, t class, fromName string) bool {
|
||||
if t.kind != domainAWSAPI || fromName == awsgwRole {
|
||||
return false
|
||||
}
|
||||
return !f.domainShaped() || f.domain != t.domain
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "R5_OperatorImportsNoLegacyImplementation",
|
||||
clause: "rule 5 and the cmd/spinifex map row: operator use cases reach domains through public capabilities and do not read domain-private state",
|
||||
hint: "reach the domain through its public capability; do not import a legacy implementation root",
|
||||
subject: func(f class, _ string) bool { return f.layer == layerOperator },
|
||||
forbids: func(f, t class, _ string) bool { return f.layer == layerOperator && t.isLegacy() },
|
||||
},
|
||||
{
|
||||
id: "R6_ProductionImportsNoTestkit",
|
||||
clause: "rule 6: test support is under internal/testkit; production packages do not import it",
|
||||
hint: "move the shared code to its production owner or keep the import in a _test.go file",
|
||||
subject: func(f class, n string) bool { return f.layer != layerTestkit && !qualificationBinaries[n] },
|
||||
forbids: func(f, t class, n string) bool {
|
||||
return f.layer != layerTestkit && !qualificationBinaries[n] && isLayer(t, layerTestkit, layerTests)
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "Contracts_ImportOnlyFoundationAndContracts",
|
||||
clause: "decision item 1 and the area table: contracts own cross-process vocabulary, not application interfaces, domain persistence or transport implementation",
|
||||
hint: "keep wire types self-contained; depend only on foundation or another contract",
|
||||
subject: func(f class, _ string) bool { return f.layer == layerContracts },
|
||||
forbids: func(_, t class, _ string) bool { return !isLayer(t, layerFoundation, layerContracts) },
|
||||
},
|
||||
{
|
||||
id: "Domains_ImportNoAgentsIngressOrLegacy",
|
||||
clause: "area table and domain module shape: a domain owns no other domain's implementation; only its awsapi adapters may use the generic ingress/aws machinery",
|
||||
hint: "depend on a contract or capability; leave ingress use to the domain's awsapi package",
|
||||
subject: func(f class, _ string) bool { return f.domainShaped() },
|
||||
forbids: func(f, t class, _ string) bool {
|
||||
if !f.domainShaped() {
|
||||
return false
|
||||
}
|
||||
switch {
|
||||
case t.layer == layerAgents:
|
||||
return true
|
||||
case t.layer == layerIngress:
|
||||
return f.kind != domainAWSAPI && f.kind != legacyAdapter
|
||||
case t.isLegacy("gateway") && !t.domainShaped():
|
||||
return true
|
||||
case t.isLegacy("handlers", "gateway"):
|
||||
return f.layer == layerDomain
|
||||
}
|
||||
return false
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "Agents_ImportNoDomainOrLegacyImplementation",
|
||||
clause: "area table: agents/<domain> own in-guest and service-role behaviour, not tenant API authority or another domain's private state",
|
||||
hint: "move the shared wire vocabulary behind a guest/controller contract under contracts/",
|
||||
subject: func(f class, _ string) bool { return f.layer == layerAgents },
|
||||
forbids: func(f, t class, _ string) bool {
|
||||
return f.layer == layerAgents && (t.domainShaped() || t.isLegacy())
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "Q22_STSImportsNoEKSImplementation",
|
||||
clause: "Q-22 prohibits the STS-to-EKS dependency in its present form; cut it through an identity-owned contract or an EKS projection",
|
||||
hint: "consume an EKS projection or an identity-owned contract instead of the EKS implementation",
|
||||
subject: func(f class, _ string) bool { return f.domainShaped() && f.domain == "sts" },
|
||||
forbids: func(f, t class, _ string) bool {
|
||||
return f.domainShaped() && f.domain == "sts" && t.domain == "eks" &&
|
||||
(importsDomainImplementation(t) || t.layer == layerAgents)
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "S7_RequestRecordPrivateToIngress",
|
||||
clause: "ADR-0002:S7: the request record is private to ingress, so no domain, runtime or operator package reads it; only the gateway and packages under ingress/aws may import ingress/aws/reqctx",
|
||||
hint: "receive what you need through dispatch.Invocation instead of the request record",
|
||||
subject: func(_ class, _ string) bool { return true },
|
||||
forbids: func(_, t class, fromName string) bool {
|
||||
if !t.requestRecord {
|
||||
return false
|
||||
}
|
||||
return fromName != "gateway" && !strings.HasPrefix(fromName, "ingress/aws/")
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "Interim_DaemonGatewayHandlersDirection",
|
||||
clause: "during migration the existing daemon -> gateway -> handlers rule remains the guardrail (Q-20, Q-29)",
|
||||
hint: "keep the dependency pointing daemon -> gateway -> handlers; invert it through a callback or capability",
|
||||
subject: func(f class, _ string) bool { return f.isLegacy("handlers", "gateway") },
|
||||
forbids: func(f, t class, _ string) bool {
|
||||
switch {
|
||||
case f.isLegacy("handlers"):
|
||||
return t.isLegacy("gateway") || t.daemon
|
||||
case f.isLegacy("gateway"):
|
||||
return t.daemon
|
||||
}
|
||||
return false
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
ARG VERSION=dev
|
||||
|
||||
FROM golang:1.27-bookworm AS builder
|
||||
FROM golang:1.27.3-bookworm@sha256:972b4110ba0f47539a975c31e52c64765e1ff19f2d74469ae4a595c6676e80a5 AS builder
|
||||
|
||||
ARG VERSION
|
||||
|
||||
@@ -35,16 +35,23 @@ COPY bluebottle/ ./bluebottle/
|
||||
# Build spinifex binary (spx)
|
||||
RUN COMMIT=$(cd spinifex && git rev-parse --short HEAD 2>/dev/null || echo "unknown") && \
|
||||
cd spinifex && GOFIPS140=v1.0.0 go build -ldflags "-s -w \
|
||||
-X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Version=${VERSION} \
|
||||
-X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Commit=${COMMIT}" \
|
||||
-X github.com/mulgadc/spinifex/spinifex/operator/cli.Version=${VERSION} \
|
||||
-X github.com/mulgadc/spinifex/spinifex/operator/cli.Commit=${COMMIT}" \
|
||||
-o /out/spx cmd/spinifex/main.go
|
||||
|
||||
# Build nbdkit viperblock plugin (.so)
|
||||
RUN cd viperblock && GOFIPS140=v1.0.0 go build -o /out/nbdkit-viperblock-plugin.so -buildmode=c-shared nbd/viperblock.go
|
||||
|
||||
# One tarball ships to every OS. The ceiling is what both binaries need today,
|
||||
# so a rise is a decision rather than an accident; every supported OS has 2.39+.
|
||||
RUN spinifex/scripts/check-glibc-ceiling.sh 2.34 /out/spx /out/nbdkit-viperblock-plugin.so
|
||||
|
||||
# Copy setup scripts and systemd units
|
||||
RUN cp spinifex/scripts/setup.sh /out/setup.sh 2>/dev/null || true
|
||||
RUN cp spinifex/scripts/setup-ovn.sh /out/setup-ovn.sh
|
||||
RUN cd spinifex/scripts && cp install-node.sh smoke-test.sh node-reset.sh uninstall-spx.sh /out/
|
||||
RUN mkdir -p /out/udev && cp spinifex/build/udev/* /out/udev/
|
||||
RUN mkdir -p /out/lib && cp spinifex/scripts/lib/distro.sh /out/lib/distro.sh
|
||||
RUN mkdir -p /out/systemd && cp spinifex/build/systemd/* /out/systemd/
|
||||
RUN mkdir -p /out/scripts && cp spinifex/build/scripts/* /out/scripts/
|
||||
RUN cp spinifex/build/logrotate/spinifex /out/logrotate-spinifex
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
#!/bin/sh
|
||||
mount -t proc proc /proc
|
||||
mount -t sysfs sys /sys
|
||||
mount -t devtmpfs dev /dev
|
||||
|
||||
# virtio_mmio is built-in (CONFIG_VIRTIO_MMIO=y) so virtio-net devices are
|
||||
# already enumerated by the time PID 1 runs — no modprobe needed.
|
||||
# qemu_fw_cfg and virtio_net are =m in stock Alpine linux-virt, so load them.
|
||||
#
|
||||
# modprobe needs a usable modules.dep; when the image is built on a host whose
|
||||
# depmod can't produce one, busybox modprobe silently no-ops and the module
|
||||
# never loads — fw_cfg then never appears and init panics. insmod by explicit
|
||||
# path needs no modules.dep, so fall back to it. Handles .ko and .ko.gz.
|
||||
load_mod() {
|
||||
modprobe "$1" 2>/dev/null && return 0
|
||||
for ko in $(find /lib/modules -name "$1.ko" -o -name "$1.ko.gz" 2>/dev/null); do
|
||||
insmod "$ko" 2>/dev/null && return 0
|
||||
case "$ko" in
|
||||
*.gz) gunzip -c "$ko" >/tmp/mod.ko 2>/dev/null \
|
||||
&& insmod /tmp/mod.ko 2>/dev/null && return 0 ;;
|
||||
esac
|
||||
done
|
||||
return 1
|
||||
}
|
||||
load_mod qemu_fw_cfg || load_mod fw_cfg_sysfs || true
|
||||
# virtio_net depends on net_failover, which depends on failover. The load_mod
|
||||
# insmod fallback loads a single .ko with no dependency resolution, and
|
||||
# modules.dep is absent when the image is built on a kmod-less host (depmod
|
||||
# skipped), so modprobe can't resolve the chain either. Load it explicitly
|
||||
# bottom-up so the NIC comes up regardless of how the image was built —
|
||||
# without this the agent has no network and the LB hangs in provisioning.
|
||||
load_mod failover || true
|
||||
load_mod net_failover || true
|
||||
load_mod virtio_net || true
|
||||
|
||||
NETCFG=/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw
|
||||
i=0
|
||||
while [ "$i" -lt 25 ]; do
|
||||
[ -f "$NETCFG" ] && break
|
||||
sleep 0.04
|
||||
i=$((i + 1))
|
||||
done
|
||||
|
||||
if [ ! -f "$NETCFG" ]; then
|
||||
# Dump host/guest fw_cfg state to the serial console before init exits so a
|
||||
# "netcfg not available" panic is diagnosable from the captured console log
|
||||
# (module-not-loaded vs qemu-not-exposing-by_name vs wrong machine type).
|
||||
echo "[init] FATAL: fw_cfg netcfg not available" >&2
|
||||
echo "[init] dmi: vendor=$(cat /sys/class/dmi/id/sys_vendor 2>/dev/null) product=$(cat /sys/class/dmi/id/product_name 2>/dev/null) bios=$(cat /sys/class/dmi/id/bios_version 2>/dev/null)" >&2
|
||||
echo "[init] qemu_fw_cfg dir present: $([ -d /sys/firmware/qemu_fw_cfg ] && echo yes || echo NO)" >&2
|
||||
echo "[init] modules: $(lsmod 2>/dev/null | awk 'NR>1{print $1}' | tr '\n' ',')" >&2
|
||||
echo "[init] by_name listing:" >&2
|
||||
ls -la /sys/firmware/qemu_fw_cfg/by_name/ 2>&1 | sed 's/^/[init] /' >&2
|
||||
ls -la /sys/firmware/qemu_fw_cfg/by_name/opt/ 2>&1 | sed 's/^/[init] /' >&2
|
||||
echo "[init] dmesg fw_cfg:" >&2
|
||||
dmesg 2>/dev/null | grep -i fw_cfg | sed 's/^/[init] /' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
. "$NETCFG"
|
||||
|
||||
for n in 0 1 2 3 4 5; do
|
||||
eval mac=\$NIC${n}_MAC
|
||||
[ -z "$mac" ] && continue
|
||||
|
||||
iface=""
|
||||
for attempt in 1 2 3 4 5; do
|
||||
iface=$(ip -o link | awk -v m="$mac" 'tolower($0) ~ tolower(m) {print $2}' | tr -d :)
|
||||
[ -n "$iface" ] && break
|
||||
sleep 0.02
|
||||
done
|
||||
|
||||
if [ -z "$iface" ]; then
|
||||
echo "[init] WARNING: no interface found for MAC $mac (NIC$n), skipping" >&2
|
||||
continue
|
||||
fi
|
||||
|
||||
eval cidr=\$NIC${n}_CIDR
|
||||
eval gw=\$NIC${n}_GW
|
||||
eval default=\$NIC${n}_DEFAULT
|
||||
eval rdst=\$NIC${n}_ROUTE_DST
|
||||
eval rvia=\$NIC${n}_ROUTE_VIA
|
||||
|
||||
ip link set "$iface" up
|
||||
[ -n "$cidr" ] && ip addr add "$cidr" dev "$iface"
|
||||
[ "$default" = "1" ] && [ -n "$gw" ] && ip route add default via "$gw" dev "$iface"
|
||||
[ -n "$rdst" ] && [ -n "$rvia" ] && ip route add "$rdst" via "$rvia" dev "$iface"
|
||||
done
|
||||
|
||||
# The store image takes no lb-agent environment. A CA bundle is optional here:
|
||||
# nothing in this image dials out yet.
|
||||
CA_CERT=/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/ca-cert/raw
|
||||
mkdir -p /etc/ssl/certs
|
||||
[ -f "$CA_CERT" ] && cp "$CA_CERT" /etc/ssl/certs/ca-certificates.crt
|
||||
|
||||
exec /sbin/init
|
||||
@@ -0,0 +1,10 @@
|
||||
# The power button loads here, not in /init: busybox init has its SIGUSR2
|
||||
# (poweroff) handler installed before sysinit runs, so a press can never reach
|
||||
# the pre-exec shell PID 1. A press before this line is lost; the host re-sends.
|
||||
::sysinit:/sbin/modprobe tiny_power_button power_signal=12
|
||||
::sysinit:/usr/local/lib/sqs-store/sqs-store-prepare
|
||||
::respawn:/usr/local/lib/sqs-store/sqs-store-run
|
||||
::ctrlaltdel:/sbin/reboot
|
||||
# One action: the store stops and its disk unmounts first, and the generic
|
||||
# umount -a -r runs only if the store is gone.
|
||||
::shutdown:/usr/local/lib/sqs-store/sqs-store-stop
|
||||
@@ -0,0 +1,224 @@
|
||||
# sqs-store-lib.sh — shared by the SQS store guest's prepare, run and stop
|
||||
# steps. Sourced, never executed. Every path and command is overridable so the
|
||||
# host test can drive the same code against a fake fw_cfg, sysfs and tools.
|
||||
#
|
||||
# Guest contract, read from fw_cfg under opt/spinifex/sqs-store/:
|
||||
# data-serial virtio-blk serial of the already-formatted ext4 data disk
|
||||
# listen-addr IPv4 address on a configured guest NIC; the only listener
|
||||
# client-nkey NATS user NKey public key allowed to connect; never a seed
|
||||
# Anything absent, malformed or ambiguous refuses the store. Nothing formats.
|
||||
|
||||
SQS_FWCFG=${SQS_FWCFG:-/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/sqs-store}
|
||||
SQS_SYSBLOCK=${SQS_SYSBLOCK:-/sys/block}
|
||||
SQS_DEVDIR=${SQS_DEVDIR:-/dev}
|
||||
# The host test cannot create block nodes, so it relaxes this to -e.
|
||||
SQS_NODE_TEST=${SQS_NODE_TEST:--b}
|
||||
SQS_RUN=${SQS_RUN:-/run/sqs-store}
|
||||
SQS_DATA=${SQS_DATA:-/data}
|
||||
SQS_CONSOLE=${SQS_CONSOLE:-/dev/console}
|
||||
SQS_MOUNTS=${SQS_MOUNTS:-/proc/mounts}
|
||||
SQS_PORT=${SQS_PORT:-4222}
|
||||
SQS_STOP_BUDGET_TENTHS=${SQS_STOP_BUDGET_TENTHS:-120}
|
||||
NATS_BIN=${NATS_BIN:-/usr/local/bin/nats-server}
|
||||
E2FSCK=${E2FSCK:-e2fsck}
|
||||
MOUNT=${MOUNT:-mount}
|
||||
UMOUNT=${UMOUNT:-umount}
|
||||
IP=${IP:-ip}
|
||||
|
||||
sqs_log() { echo "[sqs-store] $*" >> "$SQS_CONSOLE"; }
|
||||
sqs_fatal() { sqs_log "FATAL: $*"; return 1; }
|
||||
|
||||
# sqs_fwcfg <name>: print one fw_cfg value, refusing a missing, empty, oversized
|
||||
# or multi-token file. The value is not echoed on failure.
|
||||
sqs_fwcfg() {
|
||||
f="$SQS_FWCFG/$1/raw"
|
||||
[ -f "$f" ] || { sqs_fatal "fw_cfg $1 not supplied"; return 1; }
|
||||
[ "$(wc -c < "$f")" -le 128 ] || { sqs_fatal "fw_cfg $1 is oversized"; return 1; }
|
||||
v=$(cat "$f")
|
||||
case "$v" in
|
||||
"" | *[[:space:]]*) sqs_fatal "fw_cfg $1 must be exactly one value"; return 1 ;;
|
||||
esac
|
||||
printf '%s' "$v"
|
||||
}
|
||||
|
||||
# A volume serial is the volume ID with dashes stripped: vol + 17 hex.
|
||||
sqs_valid_serial() {
|
||||
[ "${#1}" -eq 20 ] || return 1
|
||||
case "$1" in vol*) ;; *) return 1 ;; esac
|
||||
case "${1#vol}" in *[!0-9a-f]*) return 1 ;; esac
|
||||
}
|
||||
|
||||
# A NATS user public NKey: "U" then 55 base32 characters. The checksum is
|
||||
# left to the pinned server's own config test.
|
||||
sqs_valid_user_nkey() {
|
||||
[ "${#1}" -eq 56 ] || return 1
|
||||
case "$1" in U*) ;; *) return 1 ;; esac
|
||||
case "$1" in *[!A-Z2-7]*) return 1 ;; esac
|
||||
}
|
||||
|
||||
sqs_valid_ipv4() {
|
||||
case "$1" in *[!0-9.]* | .* | *. | *..*) return 1 ;; esac
|
||||
old_ifs=$IFS; IFS=.
|
||||
# shellcheck disable=SC2086 # split on dots on purpose
|
||||
set -- $1
|
||||
IFS=$old_ifs
|
||||
[ "$#" -eq 4 ] || return 1
|
||||
for o in "$@"; do
|
||||
[ "${#o}" -le 3 ] && [ "$o" -le 255 ] || return 1
|
||||
case "$o" in 0?*) return 1 ;; esac
|
||||
done
|
||||
}
|
||||
|
||||
# sqs_find_disk <serial>: print the /dev node of the one block device whose
|
||||
# sysfs serial matches exactly. Zero or several matches refuse.
|
||||
sqs_find_disk() {
|
||||
found="" n=0
|
||||
for b in "$SQS_SYSBLOCK"/*; do
|
||||
[ -r "$b/serial" ] || continue
|
||||
[ "$(cat "$b/serial")" = "$1" ] || continue
|
||||
found=${b##*/}
|
||||
n=$((n + 1))
|
||||
done
|
||||
[ "$n" -eq 1 ] || { sqs_fatal "expected one disk with serial $1, found $n"; return 1; }
|
||||
test "$SQS_NODE_TEST" "$SQS_DEVDIR/$found" || { sqs_fatal "disk $found has no block device node"; return 1; }
|
||||
printf '%s' "$SQS_DEVDIR/$found"
|
||||
}
|
||||
|
||||
# sqs_has_addr <ipv4>: the address is assigned to a non-loopback interface.
|
||||
sqs_has_addr() {
|
||||
"$IP" -o -4 addr show 2>/dev/null \
|
||||
| awk -v a="$1" '$2 != "lo" { split($4, p, "/"); if (p[1] == a) f = 1 } END { exit !f }'
|
||||
}
|
||||
|
||||
# sqs_alive <pid>: running or stopped, not exited (a zombie counts as exited).
|
||||
sqs_alive() { [ -r "/proc/$1/status" ] && ! grep -q '^State:[[:space:]]*Z' "/proc/$1/status" 2>/dev/null; }
|
||||
|
||||
# sqs_starttime <pid>: kernel start time (stat field 22). It survives exec, and
|
||||
# pairing it with the pid reduces PID-reuse risk; without a pidfd the check and
|
||||
# the signal stay separate steps, so it is not an absolute guarantee.
|
||||
sqs_starttime() { sed 's/.*) //' "/proc/$1/stat" 2>/dev/null | cut -d' ' -f20; }
|
||||
|
||||
# sqs_same <pid> <starttime>: the recorded process is still alive.
|
||||
sqs_same() { [ -n "$2" ] && sqs_alive "$1" && [ "$(sqs_starttime "$1")" = "$2" ]; }
|
||||
|
||||
# sqs_write_config <addr> <nkey> <file>: the dedicated server's config. Its
|
||||
# max_payload admits a 1 MiB SQS message with its headers (ADR-0011:S4); the
|
||||
# 1MB NATS default does not.
|
||||
sqs_write_config() {
|
||||
cat > "$3" <<EOF
|
||||
listen: "$1:$SQS_PORT"
|
||||
server_name: "sqs-store"
|
||||
max_payload: 2MiB
|
||||
jetstream {
|
||||
store_dir: "$SQS_DATA"
|
||||
sync_interval: always
|
||||
}
|
||||
authorization {
|
||||
users = [ { nkey: "$2" } ]
|
||||
}
|
||||
EOF
|
||||
}
|
||||
|
||||
# sqs_prepare: validate the contract, check and mount the data disk, write the
|
||||
# server config and mark the store ready. Returns non-zero, unready, on any refusal.
|
||||
sqs_prepare() {
|
||||
rm -f "$SQS_RUN/ready"
|
||||
mkdir -p "$SQS_RUN"
|
||||
serial=$(sqs_fwcfg data-serial) || return 1
|
||||
sqs_valid_serial "$serial" || { sqs_fatal "data-serial is not a volume serial"; return 1; }
|
||||
addr=$(sqs_fwcfg listen-addr) || return 1
|
||||
sqs_valid_ipv4 "$addr" || { sqs_fatal "listen-addr is not an IPv4 address"; return 1; }
|
||||
nkey=$(sqs_fwcfg client-nkey) || return 1
|
||||
sqs_valid_user_nkey "$nkey" || { sqs_fatal "client-nkey is not a user public NKey"; return 1; }
|
||||
sqs_has_addr "$addr" || { sqs_fatal "listen-addr $addr is not on any guest NIC"; return 1; }
|
||||
|
||||
# The server checks the config, including the NKey checksum, before the
|
||||
# disk is touched.
|
||||
umask 077
|
||||
sqs_write_config "$addr" "$nkey" "$SQS_RUN/nats.conf" || return 1
|
||||
if ! "$NATS_BIN" -t -c "$SQS_RUN/nats.conf" > "$SQS_RUN/config-test.out" 2>&1; then
|
||||
sqs_fatal "server config test failed: $(sed 's/U[A-Z2-7]\{55\}/<nkey>/g' "$SQS_RUN/config-test.out" | tr '\n' ' ')"
|
||||
return 1
|
||||
fi
|
||||
|
||||
dev=$(sqs_find_disk "$serial") || return 1
|
||||
rc=0
|
||||
"$E2FSCK" -p "$dev" > "$SQS_RUN/fsck.out" 2>&1 || rc=$?
|
||||
sqs_log "e2fsck -p $dev rc=$rc"
|
||||
# 0 clean, 1 corrected: proceed. Anything higher, including a disk that is
|
||||
# not ext4 at all, refuses; the guest never formats.
|
||||
[ "$rc" -le 1 ] || { sqs_fatal "e2fsck refused $dev: $(tr '\n' ' ' < "$SQS_RUN/fsck.out")"; return 1; }
|
||||
mkdir -p "$SQS_DATA"
|
||||
"$MOUNT" -t ext4 -o nodev,nosuid,noexec "$dev" "$SQS_DATA" || { sqs_fatal "mount $dev failed"; return 1; }
|
||||
: > "$SQS_RUN/ready"
|
||||
sqs_log "prepared: $dev mounted at $SQS_DATA, config written for $addr:$SQS_PORT; server not started yet"
|
||||
}
|
||||
|
||||
# sqs_run: the respawned supervisor body. Its identity is recorded before the
|
||||
# stopping check: a stop that misses it set stopping first, so this parks; a
|
||||
# stop that reads it signals this process whether or not it has exec'd yet.
|
||||
sqs_run() {
|
||||
mkdir -p "$SQS_RUN"
|
||||
echo "$$ $(sqs_starttime $$)" > "$SQS_RUN/pid"
|
||||
if [ -e "$SQS_RUN/stopping" ]; then
|
||||
sqs_log "run: shutdown in progress, not starting"
|
||||
return 2
|
||||
fi
|
||||
if [ ! -e "$SQS_RUN/ready" ]; then
|
||||
sqs_log "run: store not prepared, not starting"
|
||||
return 2
|
||||
fi
|
||||
sqs_log "run: starting pid=$$"
|
||||
# ADR-0011:S11 bounds the store's Go heap; the guest must be larger.
|
||||
GOMEMLIMIT=512MiB exec "$NATS_BIN" -c "$SQS_RUN/nats.conf"
|
||||
}
|
||||
|
||||
# sqs_stop: first shutdown action. Block any respawn, SIGTERM the recorded run
|
||||
# process (a server, or a start not yet exec'd), SIGKILL only after the budget,
|
||||
# then sync and unmount. Each signal rechecks the identity; a survivor keeps the disk mounted.
|
||||
sqs_stop() {
|
||||
mkdir -p "$SQS_RUN"
|
||||
: > "$SQS_RUN/stopping"
|
||||
sqs_log "stop: begin"
|
||||
rec=$(cat "$SQS_RUN/pid" 2>/dev/null || true)
|
||||
pid=${rec%% *} st=${rec#* }
|
||||
[ "$st" != "$rec" ] || st=""
|
||||
if [ -n "$pid" ] && sqs_same "$pid" "$st"; then
|
||||
kill -TERM "$pid" 2>/dev/null || true
|
||||
sqs_log "stop: SIGTERM sent to run pid=$pid"
|
||||
i=0
|
||||
while sqs_same "$pid" "$st" && [ "$i" -lt "$SQS_STOP_BUDGET_TENTHS" ]; do sleep 0.1; i=$((i + 1)); done
|
||||
if sqs_same "$pid" "$st"; then
|
||||
sqs_log "stop: run pid=$pid still running after $((SQS_STOP_BUDGET_TENTHS / 10))s, SIGKILL"
|
||||
kill -KILL "$pid" 2>/dev/null || true
|
||||
j=0
|
||||
while sqs_same "$pid" "$st" && [ "$j" -lt 30 ]; do sleep 0.1; j=$((j + 1)); done
|
||||
fi
|
||||
if sqs_same "$pid" "$st"; then
|
||||
sqs_log "stop: run pid=$pid STILL ALIVE, leaving $SQS_DATA mounted"
|
||||
return 1
|
||||
fi
|
||||
sqs_log "stop: store exited after $((i / 10)).$((i % 10))s"
|
||||
else
|
||||
sqs_log "stop: no running store"
|
||||
fi
|
||||
sync
|
||||
if grep -q " $SQS_DATA " "$SQS_MOUNTS" 2>/dev/null; then
|
||||
rc=0
|
||||
"$UMOUNT" "$SQS_DATA" || rc=$?
|
||||
sqs_log "stop: umount $SQS_DATA rc=$rc"
|
||||
fi
|
||||
}
|
||||
|
||||
# sqs_shutdown: the only shutdown action. The generic read-only remount runs
|
||||
# only after the store stopped; a surviving writer keeps every mount as it is.
|
||||
sqs_shutdown() {
|
||||
if ! sqs_stop; then
|
||||
sync
|
||||
sqs_log "shutdown: store survived, skipping umount -a -r"
|
||||
return 1
|
||||
fi
|
||||
rc=0
|
||||
"$UMOUNT" -a -r || rc=$?
|
||||
echo "[init] shutdown: umount -a -r rc=$rc" >> "$SQS_CONSOLE"
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
# sysinit: load the disk drivers, then validate the guest contract, check and
|
||||
# mount the data disk and write the server config. A refusal leaves the store
|
||||
# unprepared, so the run step never starts it.
|
||||
. /usr/local/lib/sqs-store/sqs-store-lib.sh
|
||||
modprobe virtio_blk 2>/dev/null || true
|
||||
modprobe ext4 2>/dev/null || true
|
||||
sqs_prepare || sqs_log "store refused; it will not start"
|
||||
exit 0
|
||||
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
# Respawned by init. A store that must not run parks here instead of exiting,
|
||||
# so init does not respawn it in a tight loop.
|
||||
. /usr/local/lib/sqs-store/sqs-store-lib.sh
|
||||
sqs_run
|
||||
exec sleep 2147483647
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
# The only shutdown action: stop the store and unmount its disk, then run the
|
||||
# generic read-only remount only if the store is gone.
|
||||
. /usr/local/lib/sqs-store/sqs-store-lib.sh
|
||||
sqs_shutdown
|
||||
@@ -0,0 +1,8 @@
|
||||
# The power button loads here, not in /init: busybox init has its SIGUSR2
|
||||
# (poweroff) handler installed before sysinit runs, so a press can never reach
|
||||
# the pre-exec shell PID 1. A press before this line is lost; the host re-sends.
|
||||
::sysinit:/sbin/modprobe tiny_power_button power_signal=12
|
||||
::sysinit:/usr/local/lib/sqs-worker/sqs-worker-prepare
|
||||
::respawn:/usr/local/lib/sqs-worker/sqs-worker-ready
|
||||
::ctrlaltdel:/sbin/reboot
|
||||
::shutdown:/usr/local/lib/sqs-worker/sqs-worker-stop
|
||||
@@ -0,0 +1,115 @@
|
||||
# sqs-worker-lib.sh — shared by the SQS worker guest's readiness and stop
|
||||
# steps. Sourced, never executed. Every path and command is overridable so the
|
||||
# host test can drive the same code against test-owned processes and stub tools.
|
||||
#
|
||||
# store-ready exists only while the running watcher's last check passed. A
|
||||
# check proves the store accepted the worker key, not the store server's
|
||||
# identity, and a store failure is noticed up to one interval plus call time late.
|
||||
|
||||
SQSW_RUN=${SQSW_RUN:-/run/sqs-worker}
|
||||
SQSW_DATA=${SQSW_DATA:-/data}
|
||||
SQSW_CONSOLE=${SQSW_CONSOLE:-/dev/console}
|
||||
SQSW_MOUNTS=${SQSW_MOUNTS:-/proc/mounts}
|
||||
SQSW_BIN=${SQSW_BIN:-/usr/local/bin/sqs-worker}
|
||||
SQSW_STOP_BUDGET_TENTHS=${SQSW_STOP_BUDGET_TENTHS:-100}
|
||||
SQSW_RESPAWN_GAP=${SQSW_RESPAWN_GAP:-5}
|
||||
SQSW_PARK_INTERVAL=${SQSW_PARK_INTERVAL:-30}
|
||||
UMOUNT=${UMOUNT:-umount}
|
||||
|
||||
sqsw_log() { echo "[sqs-worker] $*" >> "$SQSW_CONSOLE"; }
|
||||
|
||||
# sqsw_alive <pid>: running or stopped, not exited (a zombie counts as exited).
|
||||
sqsw_alive() { [ -r "/proc/$1/status" ] && ! grep -q '^State:[[:space:]]*Z' "/proc/$1/status" 2>/dev/null; }
|
||||
|
||||
# sqsw_starttime <pid>: kernel start time (stat field 22). It survives exec, and
|
||||
# pairing it with the pid reduces PID-reuse risk; it is not an absolute guarantee.
|
||||
sqsw_starttime() { sed 's/.*) //' "/proc/$1/stat" 2>/dev/null | cut -d' ' -f20; }
|
||||
|
||||
# sqsw_same <pid> <starttime>: the recorded process is still alive.
|
||||
sqsw_same() { [ -n "$2" ] && sqsw_alive "$1" && [ "$(sqsw_starttime "$1")" = "$2" ]; }
|
||||
|
||||
# sqsw_run: the respawned readiness body. A predecessor that was killed or
|
||||
# crashed could not remove its marker, so every start removes it first. Its
|
||||
# identity is recorded before the stopping check, as the store's run step does.
|
||||
sqsw_run() {
|
||||
mkdir -p "$SQSW_RUN"
|
||||
echo "$$ $(sqsw_starttime $$)" > "$SQSW_RUN/ready.pid"
|
||||
if [ -e "$SQSW_RUN/store-ready" ]; then
|
||||
rm -f "$SQSW_RUN/store-ready"
|
||||
sqsw_log "ready: removed a store-ready marker left by an earlier watcher"
|
||||
fi
|
||||
# Returning here would make init respawn and log at once, forever. Park
|
||||
# instead, logging once: shutdown stops this recorded process, and a key
|
||||
# that appears later lets the watcher start.
|
||||
if [ -e "$SQSW_RUN/stopping" ] || [ ! -s "$SQSW_RUN/public-key" ]; then
|
||||
if [ -e "$SQSW_RUN/stopping" ]; then
|
||||
sqsw_log "ready: shutdown in progress, not starting"
|
||||
else
|
||||
sqsw_log "ready: no worker key; readiness not started, waiting"
|
||||
fi
|
||||
while [ -e "$SQSW_RUN/stopping" ] || [ ! -s "$SQSW_RUN/public-key" ]; do
|
||||
sleep "$SQSW_PARK_INTERVAL"
|
||||
done
|
||||
fi
|
||||
# Init respawns at once; a watcher that keeps exiting restarts at most
|
||||
# once per gap instead of in a tight loop.
|
||||
now=$(date +%s)
|
||||
last=$(cat "$SQSW_RUN/ready.last" 2>/dev/null || echo 0)
|
||||
wait=$((last + SQSW_RESPAWN_GAP - now))
|
||||
if [ "$wait" -gt 0 ] && [ "$wait" -le "$SQSW_RESPAWN_GAP" ]; then sleep "$wait"; fi
|
||||
date +%s > "$SQSW_RUN/ready.last"
|
||||
sqsw_log "ready: starting watcher pid=$$"
|
||||
exec "$SQSW_BIN" ready >> "$SQSW_CONSOLE" 2>&1
|
||||
}
|
||||
|
||||
# sqsw_stop: block any respawn, drop the marker, SIGTERM the recorded watcher,
|
||||
# SIGKILL only after the budget, and unmount the worker disk only once the
|
||||
# watcher is confirmed gone. A survivor keeps the disk mounted.
|
||||
sqsw_stop() {
|
||||
mkdir -p "$SQSW_RUN"
|
||||
: > "$SQSW_RUN/stopping"
|
||||
rm -f "$SQSW_RUN/store-ready"
|
||||
rec=$(cat "$SQSW_RUN/ready.pid" 2>/dev/null || true)
|
||||
pid=${rec%% *} st=${rec#* }
|
||||
[ "$st" != "$rec" ] || st=""
|
||||
if [ -n "$pid" ] && sqsw_same "$pid" "$st"; then
|
||||
kill -TERM "$pid" 2>/dev/null || true
|
||||
sqsw_log "stop: SIGTERM sent to readiness watcher pid=$pid"
|
||||
i=0
|
||||
while sqsw_same "$pid" "$st" && [ "$i" -lt "$SQSW_STOP_BUDGET_TENTHS" ]; do sleep 0.1; i=$((i + 1)); done
|
||||
if sqsw_same "$pid" "$st"; then
|
||||
sqsw_log "stop: readiness watcher pid=$pid still running after $((SQSW_STOP_BUDGET_TENTHS / 10))s, SIGKILL"
|
||||
kill -KILL "$pid" 2>/dev/null || true
|
||||
j=0
|
||||
while sqsw_same "$pid" "$st" && [ "$j" -lt 30 ]; do sleep 0.1; j=$((j + 1)); done
|
||||
fi
|
||||
if sqsw_same "$pid" "$st"; then
|
||||
rm -f "$SQSW_RUN/store-ready"
|
||||
sqsw_log "stop: readiness watcher pid=$pid STILL ALIVE, leaving $SQSW_DATA mounted"
|
||||
return 1
|
||||
fi
|
||||
sqsw_log "stop: readiness watcher pid=$pid exited"
|
||||
else
|
||||
sqsw_log "stop: no running readiness watcher"
|
||||
fi
|
||||
rm -f "$SQSW_RUN/store-ready" "$SQSW_RUN/ready.pid"
|
||||
sync
|
||||
if grep -q " $SQSW_DATA " "$SQSW_MOUNTS" 2>/dev/null; then
|
||||
rc=0
|
||||
"$UMOUNT" "$SQSW_DATA" || rc=$?
|
||||
sqsw_log "stop: umount $SQSW_DATA rc=$rc"
|
||||
fi
|
||||
}
|
||||
|
||||
# sqsw_shutdown: the only shutdown action. The generic read-only remount runs
|
||||
# only after the watcher stopped; a survivor keeps every mount as it is.
|
||||
sqsw_shutdown() {
|
||||
if ! sqsw_stop; then
|
||||
sync
|
||||
sqsw_log "shutdown: readiness watcher survived, skipping umount -a -r"
|
||||
return 1
|
||||
fi
|
||||
rc=0
|
||||
"$UMOUNT" -a -r || rc=$?
|
||||
echo "[init] shutdown: umount -a -r rc=$rc" >> "$SQSW_CONSOLE"
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
# sysinit: load the disk drivers, then select, check and mount the worker disk
|
||||
# and load or create the worker's store key on it. Any refusal is logged; the
|
||||
# key is never read or created on a disk that is not proven mounted.
|
||||
modprobe virtio_blk 2>/dev/null || true
|
||||
modprobe ext4 2>/dev/null || true
|
||||
/usr/local/bin/sqs-worker boot >> /dev/console 2>&1 || echo "[sqs-worker] boot refused" >> /dev/console
|
||||
exit 0
|
||||
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
# Respawned by init. A watcher that must not run parks here instead of exiting,
|
||||
# so init does not respawn it in a tight loop.
|
||||
. /usr/local/lib/sqs-worker/sqs-worker-lib.sh
|
||||
sqsw_run
|
||||
exec sleep 2147483647
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
# The only shutdown action: stop the readiness watcher and unmount the worker
|
||||
# disk, then run the generic read-only remount only if the watcher is gone.
|
||||
. /usr/local/lib/sqs-worker/sqs-worker-lib.sh
|
||||
sqsw_shutdown
|
||||
@@ -1,10 +1,10 @@
|
||||
# spinifex-unit-version: 3
|
||||
# spinifex-unit-version: 5
|
||||
[Unit]
|
||||
Description=Spinifex Daemon (VM Orchestration)
|
||||
PartOf=spinifex.target
|
||||
After=spinifex-nats.service spinifex-viperblock.service openvswitch-switch.service ovn-controller.service openvswitch-ipsec.service
|
||||
# Wants (not Requires): stay bootable when openvswitch-ipsec is absent
|
||||
Wants=openvswitch-ipsec.service
|
||||
After=spinifex-nats.service spinifex-viperblock.service ovsdb-server.service ovs-vswitchd.service ovn-controller.service openvswitch-ipsec.service spinifex-hostnet.service
|
||||
# Wants (not Requires): boot without openvswitch-ipsec, survive OVS restarts via hostnet
|
||||
Wants=openvswitch-ipsec.service spinifex-hostnet.service
|
||||
|
||||
[Service]
|
||||
EnvironmentFile=-/etc/spinifex/telemetry.env
|
||||
@@ -14,6 +14,7 @@ Type=simple
|
||||
Slice=spinifex-system.slice
|
||||
User=spinifex-daemon
|
||||
Group=spinifex
|
||||
ExecStartPre=/usr/local/bin/spx admin hostnet check
|
||||
ExecStart=/usr/local/bin/spx service spinifex start
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
# spinifex-unit-version: 2
|
||||
[Unit]
|
||||
Description=Spinifex host network wiring (veth pairs, mgmt address, encap route)
|
||||
# PartOf/WantedBy ovs-vswitchd: re-applied whenever OVS starts; daemon and vpcd only Want it
|
||||
Requires=ovs-vswitchd.service
|
||||
After=ovs-vswitchd.service NetworkManager-wait-online.service
|
||||
PartOf=ovs-vswitchd.service
|
||||
# Stopping OVS stops ovn-controller through its Requires= chain and starting OVS
|
||||
# does not bring it back, so the start that re-applies hostnet restarts it too
|
||||
Wants=ovn-controller.service
|
||||
Before=spinifex.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/usr/local/bin/spx admin hostnet apply
|
||||
# Covers the 120s wait behind setup-ovn.sh plus the 60s wait for links
|
||||
TimeoutStartSec=240
|
||||
TimeoutStopSec=10
|
||||
ProtectSystem=full
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
|
||||
[Install]
|
||||
WantedBy=spinifex.target ovs-vswitchd.service
|
||||
@@ -1,9 +1,9 @@
|
||||
# spinifex-unit-version: 1
|
||||
# spinifex-unit-version: 3
|
||||
[Unit]
|
||||
Description=Spinifex VPC Daemon (OVN Networking)
|
||||
PartOf=spinifex.target
|
||||
After=spinifex-nats.service spinifex-daemon.service systemd-networkd.service openvswitch-switch.service ovn-controller.service
|
||||
Wants=spinifex-nats.service ovn-controller.service
|
||||
After=spinifex-nats.service spinifex-daemon.service systemd-networkd.service ovsdb-server.service ovs-vswitchd.service ovn-controller.service spinifex-hostnet.service
|
||||
Wants=spinifex-nats.service ovn-controller.service spinifex-hostnet.service
|
||||
|
||||
[Service]
|
||||
EnvironmentFile=-/etc/spinifex/telemetry.env
|
||||
@@ -11,6 +11,7 @@ Type=simple
|
||||
Slice=spinifex-system.slice
|
||||
User=spinifex-vpcd
|
||||
Group=spinifex
|
||||
ExecStartPre=/usr/local/bin/spx admin hostnet check
|
||||
ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
|
||||
ExecStart=/usr/local/bin/spx service vpcd start
|
||||
Restart=on-failure
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
# spinifex-unit-version: 1
|
||||
# spinifex-unit-version: 4
|
||||
[Unit]
|
||||
Description=Spinifex Infrastructure Platform
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
Wants=spinifex-nats.service spinifex-predastore.service spinifex-northstar.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer openvswitch-switch.service ovn-controller.service
|
||||
Wants=spinifex-nats.service spinifex-predastore.service spinifex-northstar.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer ovsdb-server.service ovs-vswitchd.service ovn-controller.service
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
@@ -4,10 +4,22 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
|
||||
"github.com/mulgadc/predastore/s3api"
|
||||
"github.com/mulgadc/spinifex/internal/awsmodel"
|
||||
acmawsapi "github.com/mulgadc/spinifex/spinifex/domains/acm/awsapi"
|
||||
ec2awsapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi"
|
||||
awsapi "github.com/mulgadc/spinifex/spinifex/domains/ecr/awsapi"
|
||||
ecsawsapi "github.com/mulgadc/spinifex/spinifex/domains/ecs/awsapi"
|
||||
eksawsapi "github.com/mulgadc/spinifex/spinifex/domains/eks/awsapi"
|
||||
elbv2awsapi "github.com/mulgadc/spinifex/spinifex/domains/elbv2/awsapi"
|
||||
iamawsapi "github.com/mulgadc/spinifex/spinifex/domains/iam/awsapi"
|
||||
rdsawsapi "github.com/mulgadc/spinifex/spinifex/domains/rds/awsapi"
|
||||
route53awsapi "github.com/mulgadc/spinifex/spinifex/domains/route53/awsapi"
|
||||
stsawsapi "github.com/mulgadc/spinifex/spinifex/domains/sts/awsapi"
|
||||
"github.com/mulgadc/spinifex/spinifex/gateway"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
|
||||
_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
|
||||
)
|
||||
@@ -43,10 +55,30 @@ func fail(err error) {
|
||||
}
|
||||
|
||||
func compareAll() ([]awsmodel.OperationCoverage, error) {
|
||||
dispatch := gateway.AWSOperationInventory()
|
||||
// Registered services contribute their declared inventory, which needs no
|
||||
// live dependencies; nothing here serves a request.
|
||||
inventories := gateway.AWSOperationInventory(map[string]dispatch.Inventory{
|
||||
awsapi.ServiceName: awsapi.OperationInventory(),
|
||||
acmawsapi.ServiceName: acmawsapi.OperationInventory(),
|
||||
route53awsapi.ServiceName: route53awsapi.OperationInventory(),
|
||||
elbv2awsapi.InventoryName: elbv2awsapi.OperationInventory(),
|
||||
rdsawsapi.ServiceName: rdsawsapi.OperationInventory(),
|
||||
ec2awsapi.ServiceName: ec2awsapi.OperationInventory(),
|
||||
ecsawsapi.ServiceName: ecsawsapi.OperationInventory(),
|
||||
eksawsapi.ServiceName: eksawsapi.OperationInventory(),
|
||||
iamawsapi.ServiceName: iamawsapi.OperationInventory(),
|
||||
stsawsapi.ServiceName: stsawsapi.OperationInventory(),
|
||||
})
|
||||
// The loop below walks the embedded models, so a served service with no
|
||||
// model would otherwise drop out of the report without a word.
|
||||
for name := range inventories {
|
||||
if !slices.Contains(awsmodel.Services(), awsmodel.Service(name)) {
|
||||
return nil, fmt.Errorf("no embedded AWS model for served service %s", name)
|
||||
}
|
||||
}
|
||||
coverages := make([]awsmodel.OperationCoverage, 0, len(awsmodel.Services()))
|
||||
for _, service := range awsmodel.Services() {
|
||||
inventory, ok := dispatch[string(service)]
|
||||
inventory, ok := inventories[string(service)]
|
||||
var modelInventory awsmodel.DispatchInventory
|
||||
switch {
|
||||
case ok:
|
||||
|
||||
@@ -23,7 +23,7 @@ import (
|
||||
|
||||
"github.com/insomniacslk/dhcp/dhcpv4"
|
||||
"github.com/insomniacslk/dhcp/dhcpv4/nclient4"
|
||||
"github.com/mulgadc/spinifex/spinifex/network/external/dhcp"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/network/external/dhcp"
|
||||
|
||||
_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
|
||||
)
|
||||
|
||||
@@ -1,175 +1,15 @@
|
||||
// ecr-credential-provider is a kubelet exec credential provider that lets a
|
||||
// Spinifex EKS worker node pull images from the internal Spinifex ECR registry.
|
||||
//
|
||||
// kubelet invokes it per-pull, feeding a CredentialProviderRequest on STDIN and
|
||||
// reading a CredentialProviderResponse on STDOUT. The provider fetches the
|
||||
// node IAM-role credentials from IMDS, SigV4-signs GetAuthorizationToken against
|
||||
// the AWS gateway's ECR endpoint, and returns the minted AWS:<jwt> as Basic auth
|
||||
// for the requested registry host.
|
||||
//
|
||||
// Static node config (gateway URL, CA, region) is read from the cloud-init env
|
||||
// file /etc/spinifex-eks/agent.env (KEY=value); real env vars override it.
|
||||
// ecr-credential-provider is the kubelet exec credential provider for Spinifex
|
||||
// EKS worker nodes; the implementation is in agents/eks/credentialprovider.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/ecrauth"
|
||||
"github.com/mulgadc/spinifex/internal/guestenv"
|
||||
"github.com/mulgadc/spinifex/internal/imdscreds"
|
||||
|
||||
_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/eks/credentialprovider"
|
||||
)
|
||||
|
||||
const (
|
||||
credProviderAPIVersion = "credentialprovider.kubelet.k8s.io/v1"
|
||||
defaultCacheDuration = "10h"
|
||||
// expiryMargin trims the token's real TTL so kubelet re-invokes the provider
|
||||
// before the JWT actually expires mid-run.
|
||||
expiryMargin = 30 * time.Minute
|
||||
|
||||
defaultEnvFile = "/etc/spinifex-eks/agent.env"
|
||||
defaultGatewayCA = "/etc/spinifex-eks/gateway-ca.pem"
|
||||
defaultIMDSBase = "http://169.254.169.254/latest"
|
||||
)
|
||||
|
||||
// CredentialProviderRequest is the kubelet exec-provider request on STDIN.
|
||||
type CredentialProviderRequest struct {
|
||||
APIVersion string `json:"apiVersion"`
|
||||
Kind string `json:"kind"`
|
||||
Image string `json:"image"`
|
||||
}
|
||||
|
||||
// AuthConfig is a single registry credential entry.
|
||||
type AuthConfig struct {
|
||||
Username string `json:"username"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
|
||||
// CredentialProviderResponse is the kubelet exec-provider response on STDOUT.
|
||||
type CredentialProviderResponse struct {
|
||||
APIVersion string `json:"apiVersion"`
|
||||
Kind string `json:"kind"`
|
||||
CacheKeyType string `json:"cacheKeyType"`
|
||||
CacheDuration string `json:"cacheDuration"`
|
||||
Auth map[string]AuthConfig `json:"auth"`
|
||||
}
|
||||
|
||||
// config holds the static node settings the provider needs.
|
||||
type config struct {
|
||||
GatewayURL string
|
||||
GatewayCA string
|
||||
Region string
|
||||
IMDSBase string
|
||||
}
|
||||
|
||||
func main() {
|
||||
cfg := loadConfig(defaultEnvFile)
|
||||
|
||||
httpClient, err := ecrauth.GatewayHTTPClient(cfg.GatewayCA)
|
||||
if err != nil {
|
||||
slog.Error("ecr-credential-provider: build gateway client", "err", err)
|
||||
emitEmpty(os.Stdout)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err := run(os.Stdin, os.Stdout, cfg, httpClient); err != nil {
|
||||
slog.Error("ecr-credential-provider: run failed", "err", err)
|
||||
// Still emit a valid empty-auth response so kubelet has a parseable reply.
|
||||
emitEmpty(os.Stdout)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// run is the testable core: it reads a CredentialProviderRequest from in,
|
||||
// resolves credentials via IMDS + the gateway, and writes a
|
||||
// CredentialProviderResponse to out. The httpClient must trust the gateway CA.
|
||||
func run(in io.Reader, out io.Writer, cfg config, httpClient *http.Client) error {
|
||||
var req CredentialProviderRequest
|
||||
if err := json.NewDecoder(in).Decode(&req); err != nil {
|
||||
return fmt.Errorf("decode request: %w", err)
|
||||
}
|
||||
if req.Image == "" {
|
||||
return fmt.Errorf("request image is empty")
|
||||
}
|
||||
imageHost := ecrauth.HostFromImage(req.Image)
|
||||
|
||||
creds, err := imdscreds.Fetch(httpClient, cfg.IMDSBase)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fetch IMDS credentials: %w", err)
|
||||
}
|
||||
|
||||
tok, err := ecrauth.GetAuthorizationToken(cfg.Region, cfg.GatewayURL, httpClient,
|
||||
creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken)
|
||||
if err != nil {
|
||||
return fmt.Errorf("get authorization token: %w", err)
|
||||
}
|
||||
|
||||
auth := AuthConfig{Username: tok.Username, Password: tok.Password}
|
||||
authMap := map[string]AuthConfig{imageHost: auth}
|
||||
if tok.ProxyHost != "" && tok.ProxyHost != imageHost {
|
||||
authMap[tok.ProxyHost] = auth
|
||||
}
|
||||
|
||||
resp := CredentialProviderResponse{
|
||||
APIVersion: credProviderAPIVersion,
|
||||
Kind: "CredentialProviderResponse",
|
||||
CacheKeyType: "Registry",
|
||||
CacheDuration: cacheDurationFrom(tok.ExpiresAt),
|
||||
Auth: authMap,
|
||||
}
|
||||
return json.NewEncoder(out).Encode(&resp)
|
||||
}
|
||||
|
||||
// cacheDurationFrom derives a Go-duration string from the token expiry minus a
|
||||
// safety margin, falling back to the default when expiry is unknown or too near.
|
||||
func cacheDurationFrom(expiresAt time.Time) string {
|
||||
if expiresAt.IsZero() {
|
||||
return defaultCacheDuration
|
||||
}
|
||||
d := time.Until(expiresAt) - expiryMargin
|
||||
if d <= 0 {
|
||||
return defaultCacheDuration
|
||||
}
|
||||
return d.Round(time.Second).String()
|
||||
}
|
||||
|
||||
// loadConfig reads the cloud-init env file then lets real env vars override. The
|
||||
// IMDS base is fixed (not in the env file) but overridable for tests.
|
||||
func loadConfig(envFile string) config {
|
||||
get := guestenv.Load(envFile).Get
|
||||
|
||||
cfg := config{
|
||||
GatewayURL: get("EKS_GATEWAY_URL"),
|
||||
GatewayCA: get("EKS_GATEWAY_CA"),
|
||||
Region: get("EKS_REGION"),
|
||||
IMDSBase: get("EKS_IMDS_BASE"),
|
||||
}
|
||||
if cfg.GatewayCA == "" {
|
||||
cfg.GatewayCA = defaultGatewayCA
|
||||
}
|
||||
if cfg.IMDSBase == "" {
|
||||
cfg.IMDSBase = defaultIMDSBase
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// emitEmpty writes a valid CredentialProviderResponse with no auth entries.
|
||||
func emitEmpty(out io.Writer) {
|
||||
resp := CredentialProviderResponse{
|
||||
APIVersion: credProviderAPIVersion,
|
||||
Kind: "CredentialProviderResponse",
|
||||
CacheKeyType: "Registry",
|
||||
CacheDuration: defaultCacheDuration,
|
||||
Auth: map[string]AuthConfig{},
|
||||
}
|
||||
if err := json.NewEncoder(out).Encode(&resp); err != nil {
|
||||
slog.Error("ecr-credential-provider: encode empty response", "err", err)
|
||||
}
|
||||
os.Exit(credentialprovider.Main(os.Stdin, os.Stdout))
|
||||
}
|
||||
|
||||
@@ -1,126 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// fakeBackend serves both the IMDS role/credentials endpoints and the gateway
|
||||
// ECR GetAuthorizationToken on one TLS server, routed by request path.
|
||||
func fakeBackend(t *testing.T) *httptest.Server {
|
||||
t.Helper()
|
||||
mux := http.NewServeMux()
|
||||
|
||||
// IMDSv2 token. Real IMDS echoes the requested TTL back as a response
|
||||
// header; the SDK client parses it from there, not the body.
|
||||
mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPut {
|
||||
w.WriteHeader(http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
|
||||
_, _ = w.Write([]byte("imds-v2-token"))
|
||||
})
|
||||
// IMDS role name.
|
||||
mux.HandleFunc("/latest/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/latest/meta-data/iam/security-credentials/" {
|
||||
if r.Header.Get("X-Aws-Ec2-Metadata-Token") != "imds-v2-token" {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte("node-role"))
|
||||
return
|
||||
}
|
||||
// Role credentials.
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{
|
||||
"Code": "Success",
|
||||
"Type": "AWS-HMAC",
|
||||
"AccessKeyId": "AKIATEST",
|
||||
"SecretAccessKey": "secrettest",
|
||||
"Token": "sessiontoken",
|
||||
"AccountId": "123456789012",
|
||||
})
|
||||
})
|
||||
// Gateway ECR GetAuthorizationToken (X-Amz-Target dispatch).
|
||||
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
if !strings.Contains(r.Header.Get("X-Amz-Target"), "GetAuthorizationToken") {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
authToken := base64.StdEncoding.EncodeToString([]byte("AWS:testjwt"))
|
||||
w.Header().Set("Content-Type", "application/x-amz-json-1.1")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"authorizationData": []map[string]any{{
|
||||
"authorizationToken": authToken,
|
||||
"proxyEndpoint": "https://123456789012.dkr.ecr.us-east-1.spinifex.internal",
|
||||
}},
|
||||
})
|
||||
})
|
||||
|
||||
srv := httptest.NewTLSServer(mux)
|
||||
t.Cleanup(srv.Close)
|
||||
return srv
|
||||
}
|
||||
|
||||
func clientTrusting(srv *httptest.Server) *http.Client {
|
||||
pool := x509.NewCertPool()
|
||||
pool.AddCert(srv.Certificate())
|
||||
return &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}}
|
||||
}
|
||||
|
||||
func TestRun_ReturnsBasicAuth(t *testing.T) {
|
||||
srv := fakeBackend(t)
|
||||
client := clientTrusting(srv)
|
||||
|
||||
cfg := config{
|
||||
GatewayURL: srv.URL,
|
||||
Region: "us-east-1",
|
||||
IMDSBase: srv.URL + "/latest",
|
||||
}
|
||||
|
||||
const image = "123456789012.dkr.ecr.us-east-1.spinifex.internal/spinifex-demo:latest"
|
||||
reqBody, err := json.Marshal(CredentialProviderRequest{
|
||||
APIVersion: credProviderAPIVersion,
|
||||
Kind: "CredentialProviderRequest",
|
||||
Image: image,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal request: %v", err)
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := run(bytes.NewReader(reqBody), &out, cfg, client); err != nil {
|
||||
t.Fatalf("run: %v", err)
|
||||
}
|
||||
|
||||
var resp CredentialProviderResponse
|
||||
if err := json.Unmarshal(out.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("unmarshal response: %v\n%s", err, out.String())
|
||||
}
|
||||
|
||||
if resp.Kind != "CredentialProviderResponse" {
|
||||
t.Errorf("Kind = %q, want CredentialProviderResponse", resp.Kind)
|
||||
}
|
||||
if resp.CacheKeyType != "Registry" {
|
||||
t.Errorf("CacheKeyType = %q, want Registry", resp.CacheKeyType)
|
||||
}
|
||||
|
||||
host := "123456789012.dkr.ecr.us-east-1.spinifex.internal"
|
||||
auth, ok := resp.Auth[host]
|
||||
if !ok {
|
||||
t.Fatalf("auth map missing host %q; got %v", host, resp.Auth)
|
||||
}
|
||||
if auth.Username != "AWS" {
|
||||
t.Errorf("username = %q, want AWS", auth.Username)
|
||||
}
|
||||
if auth.Password != "testjwt" {
|
||||
t.Errorf("password = %q, want testjwt", auth.Password)
|
||||
}
|
||||
}
|
||||
@@ -1,214 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
goruntime "runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
|
||||
ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
|
||||
"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
|
||||
)
|
||||
|
||||
// version is the agent build version, reported in the register message.
|
||||
// Overridable via -ldflags "-X main.version=...".
|
||||
var version = "dev"
|
||||
|
||||
// Agent wires the ecs-agent's runtime seams: a gateway control-plane client, a
|
||||
// container runtime, and an ECR resolver. It registers the host as a container
|
||||
// instance at boot, heartbeats (re-registers) while alive, polls the gateway for
|
||||
// task assignments, and runs them through containerd, reporting state back over
|
||||
// the gateway. It never connects to NATS — the bus stays host-internal.
|
||||
type Agent struct {
|
||||
cfg config
|
||||
id identity
|
||||
cp controlPlane
|
||||
resolver ctrruntime.Resolver
|
||||
|
||||
// rt holds the container runtime, which is resolved by a loop rather than
|
||||
// once at boot. Empty until the containerd socket answers, and emptied again
|
||||
// if it stops answering.
|
||||
rt *runtimeGate
|
||||
|
||||
reg *registrar
|
||||
hb *heartbeater
|
||||
|
||||
// netns builds awsvpc task network namespaces (nil-safe; bridge/host skip it).
|
||||
netns *taskNetns
|
||||
|
||||
// cred serves task IAM role credentials at 169.254.170.2 (nil in unit tests).
|
||||
cred *credEndpoint
|
||||
|
||||
// gpu is the local pinning ledger over the host's discovered GPU UUIDs.
|
||||
// Always non-nil (newAgent seeds an empty one); New() reseeds it from
|
||||
// nvidia-smi discovery.
|
||||
gpu *gpuLedger
|
||||
|
||||
// dialRuntime connects to the container runtime. nil in unit tests, which
|
||||
// drive the gate directly.
|
||||
dialRuntime func() (ctrruntime.Runtime, error)
|
||||
|
||||
closers []func() error
|
||||
}
|
||||
|
||||
// newAgent assembles an Agent from already-built seams. Tests use this directly
|
||||
// with fakes; New builds the production seams and delegates here. rt may be nil
|
||||
// when containerd is unavailable; the assign path then reports the task STOPPED
|
||||
// rather than crashing, and the watch loop keeps dialling for it.
|
||||
func newAgent(cfg config, id identity, cp controlPlane, rt ctrruntime.Runtime, resolver ctrruntime.Resolver) *Agent {
|
||||
gate := newRuntimeGate(rt)
|
||||
a := &Agent{
|
||||
cfg: cfg,
|
||||
id: id,
|
||||
cp: cp,
|
||||
resolver: resolver,
|
||||
rt: gate,
|
||||
netns: newTaskNetns(execNetRunner{}),
|
||||
gpu: newGPULedger(id.Capacity.GPUIDs),
|
||||
}
|
||||
// Registration reports the host's capacity only while a runtime is there to
|
||||
// use it, so an agent that cannot run a container advertises none and takes
|
||||
// no placement.
|
||||
a.reg = newRegistrar(cp, id, gate.available)
|
||||
a.hb = newHeartbeater(cp, id, cfg.Heartbeat, gate.available)
|
||||
return a
|
||||
}
|
||||
|
||||
// New builds an Agent from config: it resolves the host identity from IMDS,
|
||||
// builds the gateway control-plane client, the ECR resolver and (best-effort)
|
||||
// the containerd runtime. A containerd connect failure is logged, not fatal —
|
||||
// registration and heartbeat still run so the instance is visible while the
|
||||
// runtime recovers. The ECR gateway client is built lazily on first image pull
|
||||
// (not here), so a missing or malformed gateway CA does not stop registration.
|
||||
func New(cfg config) (*Agent, error) {
|
||||
imdsClient := &http.Client{Timeout: 5 * time.Second}
|
||||
|
||||
meta, err := fetchInstanceMetadata(imdsClient, cfg.IMDSBase)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("instance metadata: %w", err)
|
||||
}
|
||||
host, _ := os.Hostname()
|
||||
id := identity{
|
||||
AccountID: meta.AccountID,
|
||||
ClusterName: cfg.ClusterName,
|
||||
InstanceID: meta.InstanceID,
|
||||
AZ: meta.AZ,
|
||||
InstanceType: meta.InstanceType,
|
||||
Hostname: host,
|
||||
Capacity: detectCapacity(discoverNvidiaGPUs(execCommandRunner)),
|
||||
AgentVersion: version,
|
||||
}
|
||||
|
||||
creds := credentials.NewIMDSProvider(imdsClient, cfg.IMDSBase)
|
||||
resolver := newLazyECRResolver(creds, cfg.Region, cfg.GatewayURL, cfg.GatewayCA)
|
||||
|
||||
// A dial that fails here is not fatal and not final: the node starts the
|
||||
// container daemon and the agent back to back, so the socket is often not
|
||||
// accepting yet. Run's watch loop keeps dialling.
|
||||
rt, perr := ctrruntime.New(cfg.ContainerdSocket)
|
||||
if perr != nil {
|
||||
slog.Warn("ecs-agent: containerd unavailable at boot, will keep retrying", "err", perr)
|
||||
}
|
||||
|
||||
cp, err := newGatewayControlPlane(cfg, creds)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build gateway control-plane: %w", err)
|
||||
}
|
||||
|
||||
a := newAgent(cfg, id, cp, rt, resolver)
|
||||
a.cred = newCredEndpoint(creds, cfg.Region, cfg.GatewayURL, cfg.GatewayCA,
|
||||
cfg.CredEndpointIP, cfg.CredEndpointPort, execNetRunner{})
|
||||
a.dialRuntime = func() (ctrruntime.Runtime, error) { return ctrruntime.New(cfg.ContainerdSocket) }
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// Run registers the instance, starts the heartbeat and assignment-poll loops,
|
||||
// and blocks until ctx is cancelled, then tears down.
|
||||
func (a *Agent) Run(ctx context.Context) error {
|
||||
if err := a.reg.Register(); err != nil {
|
||||
slog.Error("ecs-agent: initial registration failed", "err", err)
|
||||
} else {
|
||||
slog.Info("ecs-agent: registered", "cluster", a.id.ClusterName, "instance", a.id.InstanceID)
|
||||
}
|
||||
|
||||
if a.cred != nil {
|
||||
if err := a.cred.Start(); err != nil {
|
||||
slog.Error("ecs-agent: credential endpoint start failed", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
go a.hb.Run(ctx)
|
||||
|
||||
// Keeps the runtime current for the life of the process: dials while there
|
||||
// is none, and drops one that stops answering.
|
||||
if a.dialRuntime != nil {
|
||||
go a.rt.watch(ctx, a.dialRuntime, a.cfg.RuntimeRetry)
|
||||
}
|
||||
|
||||
// Re-adopt containers still running from before this restart, then poll with
|
||||
// their tasks pre-seeded so re-delivered assignments are acked, not re-run.
|
||||
go a.pollAssignments(ctx, a.reconcile(ctx))
|
||||
|
||||
<-ctx.Done()
|
||||
return a.Stop()
|
||||
}
|
||||
|
||||
// Stop closes the runtime. Safe to call once.
|
||||
func (a *Agent) Stop() error {
|
||||
var firstErr error
|
||||
if a.cred != nil {
|
||||
_ = a.cred.Stop()
|
||||
}
|
||||
for _, c := range a.closers {
|
||||
if err := c(); err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
}
|
||||
a.rt.clear()
|
||||
return firstErr
|
||||
}
|
||||
|
||||
// detectCapacity reports the host's total CPU units (1 vCPU = 1024), memory,
|
||||
// and GPU count/UUIDs from the caller's nvidia-smi discovery (nil on a
|
||||
// non-GPU host).
|
||||
func detectCapacity(gpuUUIDs []string) bus.InstanceCapacity {
|
||||
return bus.InstanceCapacity{
|
||||
CPU: goruntime.NumCPU() * 1024,
|
||||
MemoryMiB: memTotalMiB(),
|
||||
GPU: len(gpuUUIDs),
|
||||
GPUIDs: gpuUUIDs,
|
||||
}
|
||||
}
|
||||
|
||||
// memTotalMiB reads MemTotal from /proc/meminfo, returning 0 if unavailable.
|
||||
func memTotalMiB() int {
|
||||
f, err := os.Open("/proc/meminfo")
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
defer f.Close()
|
||||
scanner := bufio.NewScanner(f)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if !strings.HasPrefix(line, "MemTotal:") {
|
||||
continue
|
||||
}
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 2 {
|
||||
return 0
|
||||
}
|
||||
kb, err := strconv.Atoi(fields[1])
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return kb / 1024
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -1,66 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
|
||||
)
|
||||
|
||||
func TestAgent_RunRegistersThenStopsOnContext(t *testing.T) {
|
||||
cp := &fakeCP{}
|
||||
cfg := config{Heartbeat: 5 * time.Millisecond, PollInterval: 5 * time.Millisecond}
|
||||
puller := &ctrruntime.FakePuller{}
|
||||
a := newAgent(cfg, testIdentity(), cp, puller, nil)
|
||||
a.closers = append(a.closers, puller.Close)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- a.Run(ctx) }()
|
||||
|
||||
// Give it time to register + re-register (heartbeat) a few times.
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
cancel()
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatalf("Run returned error: %v", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Run did not return after cancel")
|
||||
}
|
||||
|
||||
// One boot register + at least one heartbeat re-register.
|
||||
if cp.registerCount() < 2 {
|
||||
t.Errorf("registers = %d, want >= 2 (boot + heartbeat)", cp.registerCount())
|
||||
}
|
||||
if !puller.Closed {
|
||||
t.Error("Stop did not close the puller")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectCapacity_Positive(t *testing.T) {
|
||||
c := detectCapacity(nil)
|
||||
if c.CPU <= 0 {
|
||||
t.Errorf("CPU = %d, want > 0", c.CPU)
|
||||
}
|
||||
// MemoryMiB may be 0 on platforms without /proc/meminfo; just assert non-negative.
|
||||
if c.MemoryMiB < 0 {
|
||||
t.Errorf("MemoryMiB = %d, want >= 0", c.MemoryMiB)
|
||||
}
|
||||
if c.GPU != 0 || c.GPUIDs != nil {
|
||||
t.Errorf("no GPU UUIDs given: want GPU=0/GPUIDs=nil, got %d/%v", c.GPU, c.GPUIDs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectCapacity_WithGPUUUIDs(t *testing.T) {
|
||||
c := detectCapacity([]string{"GPU-aaa", "GPU-bbb"})
|
||||
if c.GPU != 2 {
|
||||
t.Errorf("GPU = %d, want 2", c.GPU)
|
||||
}
|
||||
if len(c.GPUIDs) != 2 {
|
||||
t.Errorf("GPUIDs = %v, want 2 entries", c.GPUIDs)
|
||||
}
|
||||
}
|
||||
@@ -1,96 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/guestenv"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultEnvFile = "/etc/spinifex-ecs/agent.env"
|
||||
defaultGatewayCA = "/etc/spinifex-ecs/gateway-ca.pem"
|
||||
defaultIMDSBase = "http://169.254.169.254/latest"
|
||||
defaultContainerdSocket = "/run/containerd/containerd.sock"
|
||||
defaultHeartbeat = 30 * time.Second
|
||||
defaultPollInterval = 5 * time.Second
|
||||
)
|
||||
|
||||
// config holds the static settings the agent reads at boot. Cluster identity
|
||||
// (account ID, instance ID) is discovered at runtime from IMDS, not configured;
|
||||
// only the cluster *name* the instance was launched into is static. AccessKey /
|
||||
// SecretKey are legacy seeded creds, optional now that the agent SigV4-signs
|
||||
// gateway calls with instance-role credentials from IMDS; the agent never holds
|
||||
// a NATS token.
|
||||
type config struct {
|
||||
GatewayURL string
|
||||
GatewayCA string
|
||||
Region string
|
||||
ClusterName string
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
IMDSBase string
|
||||
ContainerdSocket string
|
||||
Heartbeat time.Duration
|
||||
PollInterval time.Duration
|
||||
RuntimeRetry time.Duration
|
||||
CredEndpointIP string
|
||||
CredEndpointPort int
|
||||
}
|
||||
|
||||
// loadConfig reads the cloud-init env file then lets real env vars override.
|
||||
func loadConfig(envFile string) config {
|
||||
get := guestenv.Load(envFile).Get
|
||||
|
||||
cfg := config{
|
||||
GatewayURL: get("ECS_GATEWAY_URL"),
|
||||
GatewayCA: get("ECS_GATEWAY_CA"),
|
||||
Region: get("ECS_REGION"),
|
||||
ClusterName: get("ECS_CLUSTER"),
|
||||
AccessKey: get("ECS_ACCESS_KEY"),
|
||||
SecretKey: get("ECS_SECRET_KEY"),
|
||||
IMDSBase: get("ECS_IMDS_BASE"),
|
||||
ContainerdSocket: get("ECS_CONTAINERD_SOCKET"),
|
||||
}
|
||||
if cfg.GatewayCA == "" {
|
||||
cfg.GatewayCA = defaultGatewayCA
|
||||
}
|
||||
if cfg.IMDSBase == "" {
|
||||
cfg.IMDSBase = defaultIMDSBase
|
||||
}
|
||||
if cfg.ContainerdSocket == "" {
|
||||
cfg.ContainerdSocket = defaultContainerdSocket
|
||||
}
|
||||
if cfg.ClusterName == "" {
|
||||
cfg.ClusterName = "default"
|
||||
}
|
||||
cfg.Heartbeat = defaultHeartbeat
|
||||
if v := get("ECS_HEARTBEAT_INTERVAL"); v != "" {
|
||||
if d, err := time.ParseDuration(v); err == nil && d > 0 {
|
||||
cfg.Heartbeat = d
|
||||
}
|
||||
}
|
||||
cfg.PollInterval = defaultPollInterval
|
||||
if v := get("ECS_POLL_INTERVAL"); v != "" {
|
||||
if d, err := time.ParseDuration(v); err == nil && d > 0 {
|
||||
cfg.PollInterval = d
|
||||
}
|
||||
}
|
||||
cfg.RuntimeRetry = defaultRuntimeRetry
|
||||
if v := get("ECS_RUNTIME_RETRY"); v != "" {
|
||||
if d, err := time.ParseDuration(v); err == nil && d > 0 {
|
||||
cfg.RuntimeRetry = d
|
||||
}
|
||||
}
|
||||
cfg.CredEndpointIP = get("ECS_CRED_ENDPOINT_IP")
|
||||
if cfg.CredEndpointIP == "" {
|
||||
cfg.CredEndpointIP = defaultCredEndpointIP
|
||||
}
|
||||
cfg.CredEndpointPort = defaultCredEndpointPort
|
||||
if v := get("ECS_CRED_ENDPOINT_PORT"); v != "" {
|
||||
if p, err := strconv.Atoi(v); err == nil && p > 0 {
|
||||
cfg.CredEndpointPort = p
|
||||
}
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestLoadConfig_Defaults(t *testing.T) {
|
||||
// Ensure env vars don't leak in from the host.
|
||||
for _, k := range []string{"ECS_GATEWAY_URL", "ECS_GATEWAY_CA", "ECS_REGION",
|
||||
"ECS_CLUSTER", "ECS_ACCESS_KEY", "ECS_SECRET_KEY", "ECS_IMDS_BASE",
|
||||
"ECS_CONTAINERD_SOCKET", "ECS_HEARTBEAT_INTERVAL", "ECS_POLL_INTERVAL"} {
|
||||
t.Setenv(k, "")
|
||||
}
|
||||
cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
|
||||
if cfg.GatewayCA != defaultGatewayCA {
|
||||
t.Errorf("GatewayCA = %q, want default", cfg.GatewayCA)
|
||||
}
|
||||
if cfg.IMDSBase != defaultIMDSBase {
|
||||
t.Errorf("IMDSBase = %q, want default", cfg.IMDSBase)
|
||||
}
|
||||
if cfg.ContainerdSocket != defaultContainerdSocket {
|
||||
t.Errorf("ContainerdSocket = %q, want default", cfg.ContainerdSocket)
|
||||
}
|
||||
if cfg.ClusterName != "default" {
|
||||
t.Errorf("ClusterName = %q, want default", cfg.ClusterName)
|
||||
}
|
||||
if cfg.Heartbeat != defaultHeartbeat {
|
||||
t.Errorf("Heartbeat = %v, want %v", cfg.Heartbeat, defaultHeartbeat)
|
||||
}
|
||||
if cfg.PollInterval != defaultPollInterval {
|
||||
t.Errorf("PollInterval = %v, want %v", cfg.PollInterval, defaultPollInterval)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfig_FileThenEnvOverride(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
envFile := filepath.Join(dir, "agent.env")
|
||||
body := "# comment\nECS_GATEWAY_URL=https://gw.file\nECS_REGION=us-west-2\nECS_CLUSTER=prod\n"
|
||||
if err := os.WriteFile(envFile, []byte(body), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("ECS_GATEWAY_URL", "https://gw.env")
|
||||
t.Setenv("ECS_ACCESS_KEY", "AKIAENV")
|
||||
|
||||
cfg := loadConfig(envFile)
|
||||
if cfg.GatewayURL != "https://gw.env" {
|
||||
t.Errorf("GatewayURL = %q, want env override", cfg.GatewayURL)
|
||||
}
|
||||
if cfg.Region != "us-west-2" {
|
||||
t.Errorf("Region = %q, want from file", cfg.Region)
|
||||
}
|
||||
if cfg.ClusterName != "prod" {
|
||||
t.Errorf("ClusterName = %q, want from file", cfg.ClusterName)
|
||||
}
|
||||
if cfg.AccessKey != "AKIAENV" {
|
||||
t.Errorf("AccessKey = %q, want env", cfg.AccessKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfig_HeartbeatOverride(t *testing.T) {
|
||||
t.Setenv("ECS_HEARTBEAT_INTERVAL", "5s")
|
||||
if cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env")); cfg.Heartbeat != 5*time.Second {
|
||||
t.Errorf("Heartbeat = %v, want 5s", cfg.Heartbeat)
|
||||
}
|
||||
// Garbage value falls back to the default, not zero.
|
||||
t.Setenv("ECS_HEARTBEAT_INTERVAL", "not-a-duration")
|
||||
if cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env")); cfg.Heartbeat != defaultHeartbeat {
|
||||
t.Errorf("Heartbeat = %v, want default on bad value", cfg.Heartbeat)
|
||||
}
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
)
|
||||
|
||||
// heartbeater keeps the container instance alive by periodically re-registering
|
||||
// it through the gateway. Real ECS folds liveness into ACS; v1 folds it into an
|
||||
// idempotent RegisterContainerInstance, which refreshes the instance's LastSeen
|
||||
// so the scheduler can tell live instances from dead ones (no bus.Heartbeat).
|
||||
type heartbeater struct {
|
||||
cp controlPlane
|
||||
id identity
|
||||
interval time.Duration
|
||||
// runtimeReady reports whether a container runtime is resolved, so a beat
|
||||
// advertises capacity only while there is something to run it with. nil
|
||||
// means always ready.
|
||||
runtimeReady func() bool
|
||||
}
|
||||
|
||||
func newHeartbeater(cp controlPlane, id identity, interval time.Duration, runtimeReady func() bool) *heartbeater {
|
||||
if interval <= 0 {
|
||||
interval = defaultHeartbeat
|
||||
}
|
||||
return &heartbeater{cp: cp, id: id, interval: interval, runtimeReady: runtimeReady}
|
||||
}
|
||||
|
||||
// beat re-registers the instance once, refreshing its LastSeen.
|
||||
func (h *heartbeater) beat() error {
|
||||
ready := h.runtimeReady == nil || h.runtimeReady()
|
||||
return h.cp.Register(advertisedIdentity(h.id, ready))
|
||||
}
|
||||
|
||||
// Run beats every interval until ctx is cancelled. A failed beat is logged and
|
||||
// retried on the next tick rather than killing the loop.
|
||||
func (h *heartbeater) Run(ctx context.Context) {
|
||||
ticker := time.NewTicker(h.interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := h.beat(); err != nil {
|
||||
slog.Warn("ecs-agent: heartbeat (re-register) failed", "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,13 +1,6 @@
|
||||
// ecs-agent runs inside a Spinifex ECS container instance (a guest VM booted from
|
||||
// the ECS-AMI, with containerd baked in). It registers the host with the ECS
|
||||
// scheduler through the AWS gateway over TLS+SigV4 (never NATS), heartbeats by
|
||||
// re-registering, polls the gateway for task assignments, runs them through
|
||||
// containerd, and reports state back over the gateway.
|
||||
//
|
||||
// Static config (gateway URL, CA, region, cluster, seeded IAM creds, containerd
|
||||
// socket) is read from the cloud-init env file /etc/spinifex-ecs/agent.env
|
||||
// (KEY=value); real env vars override it. Host identity (account, instance, AZ)
|
||||
// comes from IMDS at boot.
|
||||
// Command ecs-agent runs inside a Spinifex ECS container instance; the
|
||||
// implementation is in agents/ecs/agent. This binary owns process lifecycle:
|
||||
// signal handling, startup and run failure logging, and the exit code.
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -18,12 +11,19 @@ import (
|
||||
"syscall"
|
||||
|
||||
_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
|
||||
|
||||
ecsagent "github.com/mulgadc/spinifex/spinifex/agents/ecs/agent"
|
||||
)
|
||||
|
||||
func main() {
|
||||
cfg := loadConfig(defaultEnvFile)
|
||||
// version is the agent build version, reported in the register message.
|
||||
// Overridable via -ldflags "-X main.version=...".
|
||||
var version = "dev"
|
||||
|
||||
agent, err := New(cfg)
|
||||
func main() {
|
||||
cfg := ecsagent.LoadConfig(ecsagent.DefaultEnvFile)
|
||||
cfg.AgentVersion = version
|
||||
|
||||
agent, err := ecsagent.New(cfg)
|
||||
if err != nil {
|
||||
slog.Error("ecs-agent: startup failed", "err", err)
|
||||
os.Exit(1)
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
|
||||
"github.com/mulgadc/spinifex/internal/imdscreds"
|
||||
)
|
||||
|
||||
// instanceMetadata is the IMDS-derived identity of the host the agent runs on.
|
||||
type instanceMetadata struct {
|
||||
AccountID string
|
||||
InstanceID string
|
||||
AZ string
|
||||
InstanceType string
|
||||
}
|
||||
|
||||
// fetchInstanceMetadata reads the instance-id, AZ and account-id from IMDSv2
|
||||
// via the SDK IMDS client. base is the IMDS root, e.g.
|
||||
// "http://169.254.169.254/latest".
|
||||
func fetchInstanceMetadata(client *http.Client, base string) (instanceMetadata, error) {
|
||||
imdsClient := imdscreds.NewClient(client, base)
|
||||
ctx := context.Background()
|
||||
|
||||
instanceID, err := getMetadata(ctx, imdsClient, "instance-id")
|
||||
if err != nil {
|
||||
return instanceMetadata{}, fmt.Errorf("instance-id: %w", err)
|
||||
}
|
||||
az, err := getMetadata(ctx, imdsClient, "placement/availability-zone")
|
||||
if err != nil {
|
||||
return instanceMetadata{}, fmt.Errorf("availability-zone: %w", err)
|
||||
}
|
||||
|
||||
doc, err := imdsClient.GetInstanceIdentityDocument(ctx, &imds.GetInstanceIdentityDocumentInput{})
|
||||
if err != nil {
|
||||
return instanceMetadata{}, fmt.Errorf("identity document: %w", err)
|
||||
}
|
||||
|
||||
// The instance type comes off the identity document already in hand rather
|
||||
// than a further meta-data read; the scheduler sizes the host's awsvpc ENI
|
||||
// capacity from it.
|
||||
return instanceMetadata{
|
||||
AccountID: doc.AccountID,
|
||||
InstanceID: strings.TrimSpace(instanceID),
|
||||
AZ: strings.TrimSpace(az),
|
||||
InstanceType: strings.TrimSpace(doc.InstanceType),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// getMetadata reads a single meta-data path and returns its body as a string.
|
||||
func getMetadata(ctx context.Context, imdsClient *imds.Client, path string) (string, error) {
|
||||
out, err := imdsClient.GetMetadata(ctx, &imds.GetMetadataInput{Path: path})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer out.Content.Close()
|
||||
body, err := io.ReadAll(out.Content)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(body), nil
|
||||
}
|
||||
@@ -1,82 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func metadataStub(t *testing.T, enforceV2 bool) *httptest.Server {
|
||||
t.Helper()
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPut {
|
||||
w.WriteHeader(http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
// Real IMDS echoes the requested TTL back as a response header; the SDK
|
||||
// client parses it from there, not the body.
|
||||
w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
|
||||
_, _ = w.Write([]byte("v2-token"))
|
||||
})
|
||||
guard := func(w http.ResponseWriter, r *http.Request) bool {
|
||||
if enforceV2 && r.Header.Get("X-Aws-Ec2-Metadata-Token") != "v2-token" {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
mux.HandleFunc("/latest/meta-data/instance-id", func(w http.ResponseWriter, r *http.Request) {
|
||||
if guard(w, r) {
|
||||
_, _ = w.Write([]byte("i-0abc123"))
|
||||
}
|
||||
})
|
||||
mux.HandleFunc("/latest/meta-data/placement/availability-zone", func(w http.ResponseWriter, r *http.Request) {
|
||||
if guard(w, r) {
|
||||
_, _ = w.Write([]byte("us-east-1a"))
|
||||
}
|
||||
})
|
||||
mux.HandleFunc("/latest/dynamic/instance-identity/document", func(w http.ResponseWriter, r *http.Request) {
|
||||
if guard(w, r) {
|
||||
_, _ = w.Write([]byte(`{"accountId":"123456789012","region":"us-east-1"}`))
|
||||
}
|
||||
})
|
||||
srv := httptest.NewServer(mux)
|
||||
t.Cleanup(srv.Close)
|
||||
return srv
|
||||
}
|
||||
|
||||
func TestFetchInstanceMetadata_OK(t *testing.T) {
|
||||
srv := metadataStub(t, true)
|
||||
got, err := fetchInstanceMetadata(srv.Client(), srv.URL+"/latest")
|
||||
if err != nil {
|
||||
t.Fatalf("fetch: %v", err)
|
||||
}
|
||||
if got.InstanceID != "i-0abc123" || got.AZ != "us-east-1a" || got.AccountID != "123456789012" {
|
||||
t.Errorf("metadata mismatch: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchInstanceMetadata_TokenRequired(t *testing.T) {
|
||||
// Server returns 405 for non-PUT token; a GET-only client would 401 downstream.
|
||||
srv := metadataStub(t, true)
|
||||
if _, err := fetchInstanceMetadata(srv.Client(), srv.URL+"/latest"); err != nil {
|
||||
t.Fatalf("v2 path should succeed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchInstanceMetadata_BadDocument(t *testing.T) {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
|
||||
_, _ = w.Write([]byte("t"))
|
||||
})
|
||||
mux.HandleFunc("/latest/meta-data/instance-id", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("i-1")) })
|
||||
mux.HandleFunc("/latest/meta-data/placement/availability-zone", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("z")) })
|
||||
mux.HandleFunc("/latest/dynamic/instance-identity/document", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("not json")) })
|
||||
srv := httptest.NewServer(mux)
|
||||
t.Cleanup(srv.Close)
|
||||
if _, err := fetchInstanceMetadata(srv.Client(), srv.URL+"/latest"); err == nil {
|
||||
t.Fatal("expected parse error on bad identity document")
|
||||
}
|
||||
}
|
||||
@@ -1,115 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
|
||||
ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
|
||||
"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
|
||||
)
|
||||
|
||||
// reconcile re-syncs the gateway with what this instance is actually running
|
||||
// after an agent restart. It lists the runtime's containers and, for each task
|
||||
// in this cluster, either re-adopts it (still running: re-register IAM
|
||||
// credentials, re-attach the exit-wait, refresh RUNNING) or reports it STOPPED
|
||||
// and clears its leftovers. The returned set seeds pollAssignments so the
|
||||
// gateway's re-delivered assignments for these tasks are acked but not re-run.
|
||||
//
|
||||
// Reporting the dead ones is the load-bearing half. The exit-wait is cancelled
|
||||
// when the agent shuts down, so a container that dies while the agent is down
|
||||
// has nobody to report its exit; the gateway holds the task RUNNING forever,
|
||||
// its service never replaces it, and the target never recovers.
|
||||
//
|
||||
// It is best-effort: a nil runner or a List error returns an empty set and the
|
||||
// agent falls back to its pre-reconcile behaviour (poll + run).
|
||||
func (a *Agent) reconcile(ctx context.Context) map[string]bool {
|
||||
adopted := map[string]bool{}
|
||||
runner := a.rt.get()
|
||||
if runner == nil {
|
||||
return adopted
|
||||
}
|
||||
containers, err := runner.List(ctx)
|
||||
if err != nil {
|
||||
slog.Warn("ecs-agent: reconcile list failed", "err", err)
|
||||
return adopted
|
||||
}
|
||||
|
||||
// Group this cluster's containers back into their tasks, running or not. The
|
||||
// Assign is reconstructed from labels — enough to re-register creds and tear
|
||||
// down netns.
|
||||
type task struct {
|
||||
as *bus.Assign
|
||||
running []ctrruntime.Container
|
||||
dead []ctrruntime.Container
|
||||
}
|
||||
tasks := map[string]*task{}
|
||||
for _, c := range containers {
|
||||
if c.Labels[labelClusterName] != a.id.ClusterName {
|
||||
continue
|
||||
}
|
||||
taskID := c.Labels[labelTaskID]
|
||||
if taskID == "" {
|
||||
continue
|
||||
}
|
||||
t := tasks[taskID]
|
||||
if t == nil {
|
||||
t = &task{as: &bus.Assign{
|
||||
TaskID: taskID,
|
||||
ClusterName: a.id.ClusterName,
|
||||
CredID: c.Labels[labelCredID],
|
||||
TaskRoleARN: c.Labels[labelTaskRoleARN],
|
||||
ENIMacAddress: c.Labels[labelENIMac],
|
||||
}}
|
||||
tasks[taskID] = t
|
||||
}
|
||||
if c.Running {
|
||||
t.running = append(t.running, c)
|
||||
} else {
|
||||
t.dead = append(t.dead, c)
|
||||
}
|
||||
}
|
||||
|
||||
for taskID, t := range tasks {
|
||||
// A task with nothing running is over, whatever the gateway still believes.
|
||||
// Mixed running/dead is left to the running path: the exit-wait re-attached
|
||||
// below reports the individual container when it is reaped.
|
||||
if len(t.running) == 0 {
|
||||
a.reportStoppedTask(ctx, t.as, t.dead)
|
||||
adopted[taskID] = true
|
||||
continue
|
||||
}
|
||||
|
||||
a.registerTaskCreds(t.as)
|
||||
statuses := make([]bus.ContainerStatus, 0, len(t.running))
|
||||
for _, c := range t.running {
|
||||
name := c.Labels[labelContainerName]
|
||||
go a.waitContainer(ctx, t.as, name, c.ID)
|
||||
statuses = append(statuses, bus.ContainerStatus{Name: name, Status: bus.TaskStatusRunning, ContainerID: c.ID})
|
||||
}
|
||||
a.reportTaskState(t.as, bus.TaskStatusRunning, "", statuses)
|
||||
adopted[taskID] = true
|
||||
slog.Info("ecs-agent: re-adopted running task", "task", taskID, "containers", len(t.running))
|
||||
}
|
||||
return adopted
|
||||
}
|
||||
|
||||
// reportStoppedTask clears a task's leftover containers and tells the gateway it
|
||||
// is STOPPED, so the scheduler stops counting a task this instance is not
|
||||
// running and its service can place a replacement.
|
||||
func (a *Agent) reportStoppedTask(ctx context.Context, as *bus.Assign, dead []ctrruntime.Container) {
|
||||
runner := a.rt.get()
|
||||
statuses := make([]bus.ContainerStatus, 0, len(dead))
|
||||
for _, c := range dead {
|
||||
if runner == nil {
|
||||
break
|
||||
}
|
||||
if err := runner.Remove(ctx, c.ID); err != nil {
|
||||
slog.Warn("ecs-agent: reconcile remove failed", "task", as.TaskID, "container", c.ID, "err", err)
|
||||
}
|
||||
statuses = append(statuses, bus.ContainerStatus{
|
||||
Name: c.Labels[labelContainerName], Status: bus.TaskStatusStopped, ContainerID: c.ID,
|
||||
})
|
||||
}
|
||||
a.reportTaskState(as, bus.TaskStatusStopped, "Container exited while the agent was not running", statuses)
|
||||
slog.Info("ecs-agent: reported stopped task", "task", as.TaskID, "containers", len(dead))
|
||||
}
|
||||
@@ -1,250 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
|
||||
"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
|
||||
)
|
||||
|
||||
const testTaskRole = "arn:aws:iam::111122223333:role/task"
|
||||
|
||||
// adoptedContainer builds a runtime.Container as the reconciler would discover it,
|
||||
// stamping only the mulga.ecs.* labels that are non-empty.
|
||||
func adoptedContainer(taskID, name, cluster, credID, role, mac string, running bool) ctrruntime.Container {
|
||||
labels := map[string]string{
|
||||
labelTaskID: taskID,
|
||||
labelContainerName: name,
|
||||
labelClusterName: cluster,
|
||||
}
|
||||
if credID != "" {
|
||||
labels[labelCredID] = credID
|
||||
}
|
||||
if role != "" {
|
||||
labels[labelTaskRoleARN] = role
|
||||
}
|
||||
if mac != "" {
|
||||
labels[labelENIMac] = mac
|
||||
}
|
||||
return ctrruntime.Container{ID: containerID(taskID, name), Labels: labels, Running: running}
|
||||
}
|
||||
|
||||
func waitedFor(t *testing.T, rt *ctrruntime.FakePuller, id string) {
|
||||
t.Helper()
|
||||
deadline := time.After(time.Second)
|
||||
for !slices.Contains(rt.Waits(), id) {
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatalf("exit-wait never re-attached for %s; waited=%v", id, rt.Waits())
|
||||
case <-time.After(2 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A running, correctly-labeled container is re-adopted: its credentials are
|
||||
// re-registered, its RUNNING state refreshed, and its exit-wait re-attached.
|
||||
func TestReconcile_AdoptsRunningLabeledContainer(t *testing.T) {
|
||||
cp := &fakeCP{}
|
||||
rt := &ctrruntime.FakePuller{
|
||||
WaitErr: errors.New("blocked"),
|
||||
Containers: []ctrruntime.Container{
|
||||
adoptedContainer("t-001", "web", "default", "t-001", testTaskRole, "52:54:00:de:ad:01", true),
|
||||
},
|
||||
}
|
||||
a := newAgent(config{}, testIdentity(), cp, rt, nil)
|
||||
a.cred = newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
|
||||
|
||||
adopted := a.reconcile(context.Background())
|
||||
|
||||
if !adopted["t-001"] {
|
||||
t.Fatalf("t-001 not adopted; got %v", adopted)
|
||||
}
|
||||
if got := a.cred.roles["t-001"]; got != testTaskRole {
|
||||
t.Errorf("cred not re-registered: roles[t-001] = %q, want %q", got, testTaskRole)
|
||||
}
|
||||
if n := countStatus(cp.taskStates(), "t-001", bus.TaskStatusRunning); n != 1 {
|
||||
t.Errorf("RUNNING reports for t-001 = %d, want 1", n)
|
||||
}
|
||||
waitedFor(t, rt, containerID("t-001", "web"))
|
||||
}
|
||||
|
||||
// A task whose container died while the agent was down must be reported STOPPED,
|
||||
// not silently skipped. The exit-wait is cancelled on agent shutdown, so nothing
|
||||
// else ever reports that exit: skipping it leaves the gateway holding the task
|
||||
// RUNNING forever, its service never places a replacement, and the load
|
||||
// balancer target stays unhealthy with no path back.
|
||||
func TestReconcile_ReportsTaskWhoseContainerDied(t *testing.T) {
|
||||
cp := &fakeCP{}
|
||||
rt := &ctrruntime.FakePuller{
|
||||
Containers: []ctrruntime.Container{
|
||||
adoptedContainer("t-dead", "web", "default", "t-dead", testTaskRole, "52:54:00:de:ad:02", false),
|
||||
},
|
||||
}
|
||||
a := newAgent(config{}, testIdentity(), cp, rt, nil)
|
||||
a.cred = newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
|
||||
|
||||
adopted := a.reconcile(context.Background())
|
||||
|
||||
if n := countStatus(cp.taskStates(), "t-dead", bus.TaskStatusStopped); n != 1 {
|
||||
t.Errorf("STOPPED reports for t-dead = %d, want 1", n)
|
||||
}
|
||||
if n := countStatus(cp.taskStates(), "t-dead", bus.TaskStatusRunning); n != 0 {
|
||||
t.Errorf("t-dead must not be reported RUNNING, got %d such reports", n)
|
||||
}
|
||||
// Seeded so a re-delivered assignment for the dead task is acked, not re-run.
|
||||
if !adopted["t-dead"] {
|
||||
t.Errorf("t-dead not seeded into the adopted set; got %v", adopted)
|
||||
}
|
||||
if want := containerID("t-dead", "web"); !slices.Contains(rt.Removed, want) {
|
||||
t.Errorf("leftover container %s not removed; removed=%v", want, rt.Removed)
|
||||
}
|
||||
}
|
||||
|
||||
// A task with one live and one dead container is still running: the live half is
|
||||
// re-adopted and the re-attached exit-wait reports the dead one when it is
|
||||
// reaped, so reconcile must not declare the whole task STOPPED.
|
||||
func TestReconcile_PartiallyDeadTaskStaysRunning(t *testing.T) {
|
||||
cp := &fakeCP{}
|
||||
rt := &ctrruntime.FakePuller{
|
||||
WaitErr: errors.New("blocked"),
|
||||
Containers: []ctrruntime.Container{
|
||||
adoptedContainer("t-mixed", "web", "default", "t-mixed", testTaskRole, "52:54:00:de:ad:03", true),
|
||||
adoptedContainer("t-mixed", "sidecar", "default", "t-mixed", testTaskRole, "52:54:00:de:ad:03", false),
|
||||
},
|
||||
}
|
||||
a := newAgent(config{}, testIdentity(), cp, rt, nil)
|
||||
a.cred = newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
|
||||
|
||||
a.reconcile(context.Background())
|
||||
|
||||
if n := countStatus(cp.taskStates(), "t-mixed", bus.TaskStatusRunning); n != 1 {
|
||||
t.Errorf("RUNNING reports for t-mixed = %d, want 1", n)
|
||||
}
|
||||
if n := countStatus(cp.taskStates(), "t-mixed", bus.TaskStatusStopped); n != 0 {
|
||||
t.Errorf("t-mixed must not be reported STOPPED while a container still runs, got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
// pollAssignments seeded with an adopted task acks its re-delivered assignment
|
||||
// but does not re-run it, while a genuinely new task still dispatches.
|
||||
func TestPollAssignments_SeededTaskNotRerun(t *testing.T) {
|
||||
mk := func(id string) bus.Assign {
|
||||
return bus.Assign{
|
||||
AccountID: "123456789012", ClusterName: "default", InstanceID: "i-abc123", TaskID: id,
|
||||
Containers: []bus.AssignContainer{{Name: "web", Image: "registry/web:1", Command: []string{"/bin/true"}}},
|
||||
}
|
||||
}
|
||||
adopted, fresh := mk("t-001"), mk("t-002")
|
||||
cp := &fakeCP{pollReplies: [][]bus.Assign{
|
||||
{adopted, fresh},
|
||||
{adopted, fresh},
|
||||
nil,
|
||||
}}
|
||||
rt := &ctrruntime.FakePuller{WaitErr: errors.New("blocked")}
|
||||
a := newAgent(config{PollInterval: 5 * time.Millisecond}, testIdentity(), cp, rt, nil)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
go a.pollAssignments(ctx, map[string]bool{"t-001": true})
|
||||
|
||||
deadline := time.After(time.Second)
|
||||
for countStatus(cp.taskStates(), "t-002", bus.TaskStatusRunning) < 1 {
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatal("fresh task t-002 was never dispatched")
|
||||
case <-time.After(2 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
cancel()
|
||||
|
||||
if n := countStatus(cp.taskStates(), "t-001", bus.TaskStatusRunning); n != 0 {
|
||||
t.Errorf("adopted task t-001 was re-run %d times, want 0", n)
|
||||
}
|
||||
if n := countStatus(cp.taskStates(), "t-002", bus.TaskStatusRunning); n != 1 {
|
||||
t.Errorf("fresh task t-002 dispatched %d times, want 1", n)
|
||||
}
|
||||
var t1Acked bool
|
||||
for _, ack := range cp.acks() {
|
||||
for _, id := range ack {
|
||||
if id == "t-001" {
|
||||
t1Acked = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !t1Acked {
|
||||
t.Errorf("adopted task t-001 was never acked; acks=%v", cp.acks())
|
||||
}
|
||||
}
|
||||
|
||||
// Containers belonging to another cluster are ignored entirely: this agent
|
||||
// speaks for its own cluster only, and must neither adopt nor report them.
|
||||
//
|
||||
// A stopped container of this cluster IS handled — see
|
||||
// TestReconcile_ReportsTaskWhoseContainerDied. It lands in the adopted set so
|
||||
// its re-delivered assignment is acked rather than re-run, after reconcile has
|
||||
// reported the task STOPPED.
|
||||
func TestReconcile_IgnoresOtherClusterContainers(t *testing.T) {
|
||||
cp := &fakeCP{}
|
||||
rt := &ctrruntime.FakePuller{
|
||||
WaitErr: errors.New("blocked"),
|
||||
Containers: []ctrruntime.Container{
|
||||
adoptedContainer("t-other", "web", "other", "", "", "", true),
|
||||
adoptedContainer("t-other-dead", "web", "other", "", "", "", false),
|
||||
adoptedContainer("t-ok", "web", "default", "", "", "", true),
|
||||
},
|
||||
}
|
||||
a := newAgent(config{}, testIdentity(), cp, rt, nil)
|
||||
|
||||
adopted := a.reconcile(context.Background())
|
||||
|
||||
if len(adopted) != 1 || !adopted["t-ok"] {
|
||||
t.Fatalf("adopted = %v, want only t-ok", adopted)
|
||||
}
|
||||
for _, id := range []string{"t-other", "t-other-dead"} {
|
||||
if n := countStatus(cp.taskStates(), id, bus.TaskStatusStopped); n != 0 {
|
||||
t.Errorf("reported %d STOPPED states for another cluster's task %s, want 0", n, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A nil runner or a List error degrades to an empty set without panicking.
|
||||
func TestReconcile_DegradesGracefully(t *testing.T) {
|
||||
noRunner := newAgent(config{}, testIdentity(), &fakeCP{}, nil, nil)
|
||||
if got := noRunner.reconcile(context.Background()); len(got) != 0 {
|
||||
t.Errorf("nil runner: adopted = %v, want empty", got)
|
||||
}
|
||||
|
||||
rt := &ctrruntime.FakePuller{ListErr: errors.New("boom")}
|
||||
listErr := newAgent(config{}, testIdentity(), &fakeCP{}, rt, nil)
|
||||
if got := listErr.reconcile(context.Background()); len(got) != 0 {
|
||||
t.Errorf("list error: adopted = %v, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
// taskLabels emits the optional cred/role/MAC labels only when set.
|
||||
func TestTaskLabels_OmitsEmptyOptionalLabels(t *testing.T) {
|
||||
full := taskLabels(&bus.Assign{
|
||||
TaskID: "t1", ClusterName: "c", CredID: "x", TaskRoleARN: testTaskRole, ENIMacAddress: "mac",
|
||||
}, "web")
|
||||
for k, want := range map[string]string{
|
||||
labelCredID: "x", labelTaskRoleARN: testTaskRole, labelENIMac: "mac",
|
||||
} {
|
||||
if full[k] != want {
|
||||
t.Errorf("full labels[%q] = %q, want %q", k, full[k], want)
|
||||
}
|
||||
}
|
||||
|
||||
bare := taskLabels(&bus.Assign{TaskID: "t1", ClusterName: "c"}, "web")
|
||||
if len(bare) != 3 {
|
||||
t.Errorf("bare labels = %v, want only taskID/containerName/clusterName", bare)
|
||||
}
|
||||
for _, k := range []string{labelCredID, labelTaskRoleARN, labelENIMac} {
|
||||
if _, ok := bare[k]; ok {
|
||||
t.Errorf("bare labels unexpectedly carry %q", k)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ecs"
|
||||
handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
|
||||
"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
|
||||
)
|
||||
|
||||
// identity is the container instance's stable identity, assembled at boot from
|
||||
// IMDS (account, instance, AZ) and config (cluster). It is the join key the
|
||||
// scheduler uses to track this instance.
|
||||
type identity struct {
|
||||
AccountID string
|
||||
ClusterName string
|
||||
InstanceID string
|
||||
AZ string
|
||||
InstanceType string
|
||||
Hostname string
|
||||
Capacity bus.InstanceCapacity
|
||||
AgentVersion string
|
||||
}
|
||||
|
||||
// instanceAttributes builds the built-in container-instance attributes from the
|
||||
// host's identity. Only attributes whose value was actually discovered are
|
||||
// reported, so an empty one is absent rather than present and blank.
|
||||
func instanceAttributes(id identity) []*ecs.Attribute {
|
||||
var attrs []*ecs.Attribute
|
||||
add := func(name, value string) {
|
||||
if value == "" {
|
||||
return
|
||||
}
|
||||
attrs = append(attrs, &ecs.Attribute{Name: aws.String(name), Value: aws.String(value)})
|
||||
}
|
||||
add(handlers_ecs.AttrInstanceType, id.InstanceType)
|
||||
add(handlers_ecs.AttrAvailabilityZone, id.AZ)
|
||||
return attrs
|
||||
}
|
||||
|
||||
// advertisedIdentity is the identity to register with, given whether a
|
||||
// container runtime is available. A host with no runtime can run nothing, so it
|
||||
// advertises no capacity: the scheduler places nothing on it, and the control
|
||||
// plane reads the zero as an involuntary drain. The identity is otherwise
|
||||
// unchanged, so the instance stays visible rather than disappearing.
|
||||
func advertisedIdentity(id identity, runtimeReady bool) identity {
|
||||
if runtimeReady {
|
||||
return id
|
||||
}
|
||||
id.Capacity = bus.InstanceCapacity{}
|
||||
return id
|
||||
}
|
||||
|
||||
// registrar registers the host as a container instance through the gateway.
|
||||
type registrar struct {
|
||||
cp controlPlane
|
||||
id identity
|
||||
// runtimeReady reports whether a container runtime is resolved. nil means
|
||||
// always ready, which is what a test with no gate wants.
|
||||
runtimeReady func() bool
|
||||
}
|
||||
|
||||
func newRegistrar(cp controlPlane, id identity, runtimeReady func() bool) *registrar {
|
||||
return ®istrar{cp: cp, id: id, runtimeReady: runtimeReady}
|
||||
}
|
||||
|
||||
// ready reports the runtime state, defaulting to ready when no gate is wired.
|
||||
func (r *registrar) ready() bool { return r.runtimeReady == nil || r.runtimeReady() }
|
||||
|
||||
// Register calls the gateway's RegisterContainerInstance. The scheduler records
|
||||
// (or refreshes) the container instance on receipt.
|
||||
func (r *registrar) Register() error {
|
||||
if err := r.cp.Register(advertisedIdentity(r.id, r.ready())); err != nil {
|
||||
return fmt.Errorf("register instance %s: %w", r.id.InstanceID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,257 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"slices"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
|
||||
"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
|
||||
)
|
||||
|
||||
// fakeCP is a test double for the gateway control-plane. It records registers,
|
||||
// task-state reports and GPU reports, and replays a scripted queue of poll
|
||||
// responses.
|
||||
type fakeCP struct {
|
||||
mu sync.Mutex
|
||||
|
||||
registers int
|
||||
lastRegistered identity
|
||||
states []bus.TaskState
|
||||
gpuReports []gpuReport
|
||||
pollAcks [][]string
|
||||
pollStopAcks [][]string
|
||||
pollReplies [][]bus.Assign
|
||||
stopReplies [][]bus.StopDirective
|
||||
stopPending []bus.StopDirective
|
||||
pollCalls int
|
||||
|
||||
registerErr bool
|
||||
submitErr bool
|
||||
}
|
||||
|
||||
// gpuReport is one recorded ReportTaskGPU call.
|
||||
type gpuReport struct {
|
||||
cluster string
|
||||
task string
|
||||
containers []handlers_ecs.ContainerGPUReport
|
||||
}
|
||||
|
||||
var _ controlPlane = (*fakeCP)(nil)
|
||||
|
||||
func (f *fakeCP) Register(id identity) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.registerErr {
|
||||
return errors.New("register boom")
|
||||
}
|
||||
f.registers++
|
||||
f.lastRegistered = id
|
||||
return nil
|
||||
}
|
||||
|
||||
// registered returns the identity of the most recent Register call.
|
||||
func (f *fakeCP) registered() identity {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.lastRegistered
|
||||
}
|
||||
|
||||
func (f *fakeCP) SubmitTaskState(st bus.TaskState) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.submitErr {
|
||||
return errors.New("submit boom")
|
||||
}
|
||||
f.states = append(f.states, st)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeCP) PollAssignments(_, _ string, ackAssigns, ackStops []string) ([]bus.Assign, []bus.StopDirective, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.pollAcks = append(f.pollAcks, append([]string(nil), ackAssigns...))
|
||||
f.pollStopAcks = append(f.pollStopAcks, append([]string(nil), ackStops...))
|
||||
var out []bus.Assign
|
||||
if f.pollCalls < len(f.pollReplies) {
|
||||
out = f.pollReplies[f.pollCalls]
|
||||
}
|
||||
var stops []bus.StopDirective
|
||||
if f.pollCalls < len(f.stopReplies) {
|
||||
stops = f.stopReplies[f.pollCalls]
|
||||
}
|
||||
// stopPending is the durable inbox the gateway keeps: an acked directive is
|
||||
// deleted, and everything still in it is redelivered on every poll.
|
||||
f.stopPending = slices.DeleteFunc(f.stopPending, func(sd bus.StopDirective) bool {
|
||||
return slices.Contains(ackStops, sd.TaskID)
|
||||
})
|
||||
stops = slices.Concat(stops, f.stopPending)
|
||||
f.pollCalls++
|
||||
return out, stops, nil
|
||||
}
|
||||
|
||||
func (f *fakeCP) ReportTaskGPU(cluster, task string, containers []handlers_ecs.ContainerGPUReport) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.gpuReports = append(f.gpuReports, gpuReport{cluster: cluster, task: task, containers: containers})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeCP) gpuReportsFor(taskID string) []gpuReport {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var out []gpuReport
|
||||
for _, r := range f.gpuReports {
|
||||
if r.task == taskID {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (f *fakeCP) registerCount() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.registers
|
||||
}
|
||||
|
||||
func (f *fakeCP) taskStates() []bus.TaskState {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([]bus.TaskState(nil), f.states...)
|
||||
}
|
||||
|
||||
func (f *fakeCP) acks() [][]string {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([][]string(nil), f.pollAcks...)
|
||||
}
|
||||
|
||||
func (f *fakeCP) stopAcks() [][]string {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([][]string(nil), f.pollStopAcks...)
|
||||
}
|
||||
|
||||
func testIdentity() identity {
|
||||
return identity{
|
||||
AccountID: "123456789012",
|
||||
ClusterName: "default",
|
||||
InstanceID: "i-abc123",
|
||||
AZ: "us-east-1a",
|
||||
Hostname: "ecs-host",
|
||||
Capacity: bus.InstanceCapacity{CPU: 2048, MemoryMiB: 4096},
|
||||
AgentVersion: "test",
|
||||
}
|
||||
}
|
||||
|
||||
// testIdentityWithGPUs is testIdentity seeded with discovered GPU UUIDs, so
|
||||
// newAgent's ledger has real devices to pin from.
|
||||
func testIdentityWithGPUs(uuids ...string) identity {
|
||||
id := testIdentity()
|
||||
id.Capacity.GPU = len(uuids)
|
||||
id.Capacity.GPUIDs = uuids
|
||||
return id
|
||||
}
|
||||
|
||||
func TestRegistrar_Register(t *testing.T) {
|
||||
cp := &fakeCP{}
|
||||
if err := newRegistrar(cp, testIdentity(), nil).Register(); err != nil {
|
||||
t.Fatalf("Register: %v", err)
|
||||
}
|
||||
if cp.registerCount() != 1 {
|
||||
t.Errorf("registers = %d, want 1", cp.registerCount())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistrar_RegisterError(t *testing.T) {
|
||||
cp := &fakeCP{registerErr: true}
|
||||
if err := newRegistrar(cp, testIdentity(), nil).Register(); err == nil {
|
||||
t.Fatal("expected register error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeartbeater_Beat(t *testing.T) {
|
||||
cp := &fakeCP{}
|
||||
if err := newHeartbeater(cp, testIdentity(), time.Second, nil).beat(); err != nil {
|
||||
t.Fatalf("beat: %v", err)
|
||||
}
|
||||
if cp.registerCount() != 1 {
|
||||
t.Errorf("beat should re-register once, got %d", cp.registerCount())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeartbeater_DefaultInterval(t *testing.T) {
|
||||
h := newHeartbeater(&fakeCP{}, testIdentity(), 0, nil)
|
||||
if h.interval != defaultHeartbeat {
|
||||
t.Errorf("interval = %v, want default", h.interval)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeartbeater_RunStopsOnContext(t *testing.T) {
|
||||
cp := &fakeCP{}
|
||||
h := newHeartbeater(cp, testIdentity(), 5*time.Millisecond, nil)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() { h.Run(ctx); close(done) }()
|
||||
|
||||
time.Sleep(25 * time.Millisecond)
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Run did not stop on context cancel")
|
||||
}
|
||||
if cp.registerCount() == 0 {
|
||||
t.Error("expected at least one re-register before cancel")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRegistrar_NoRuntimeAdvertisesNoCapacity is the placement half of the boot
|
||||
// bug: an agent with no container runtime kept registering its full CPU and
|
||||
// memory, so the scheduler placed tasks it could only fail to start. Capacity is
|
||||
// advertised only while a runtime is there to use it.
|
||||
func TestRegistrar_NoRuntimeAdvertisesNoCapacity(t *testing.T) {
|
||||
cp := &fakeCP{}
|
||||
ready := false
|
||||
r := newRegistrar(cp, testIdentity(), func() bool { return ready })
|
||||
|
||||
if err := r.Register(); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
if got := cp.registered().Capacity; got.CPU != 0 || got.MemoryMiB != 0 {
|
||||
t.Fatalf("advertised capacity with no runtime: cpu=%d memory=%d", got.CPU, got.MemoryMiB)
|
||||
}
|
||||
|
||||
ready = true
|
||||
if err := r.Register(); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
want := testIdentity().Capacity
|
||||
if got := cp.registered().Capacity; got.CPU != want.CPU || got.MemoryMiB != want.MemoryMiB {
|
||||
t.Fatalf("capacity not restored once the runtime resolved: cpu=%d memory=%d", got.CPU, got.MemoryMiB)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHeartbeater_NoRuntimeAdvertisesNoCapacity is the same rule on the beat,
|
||||
// which is how a runtime lost after boot reaches the control plane.
|
||||
func TestHeartbeater_NoRuntimeAdvertisesNoCapacity(t *testing.T) {
|
||||
cp := &fakeCP{}
|
||||
h := newHeartbeater(cp, testIdentity(), time.Second, func() bool { return false })
|
||||
|
||||
if err := h.beat(); err != nil {
|
||||
t.Fatalf("beat: %v", err)
|
||||
}
|
||||
if got := cp.registered().Capacity; got.CPU != 0 || got.MemoryMiB != 0 {
|
||||
t.Fatalf("beat advertised capacity with no runtime: cpu=%d memory=%d", got.CPU, got.MemoryMiB)
|
||||
}
|
||||
// The instance still identifies itself, so it stays visible as drained rather
|
||||
// than vanishing from the cluster.
|
||||
if cp.registered().InstanceID != testIdentity().InstanceID {
|
||||
t.Error("beat dropped the instance identity")
|
||||
}
|
||||
}
|
||||
@@ -1,196 +1,43 @@
|
||||
// eks-gateway-fetch runs inside the EKS K3s control-plane VM and fetches a
|
||||
// control-plane resource from the host through the AWS gateway, instead of
|
||||
// dialing core NATS directly. It is the read-side companion to
|
||||
// eks-gateway-publish.
|
||||
//
|
||||
// It SigV4-signs (service "eks") an HTTPS GET to the gateway, retrying with
|
||||
// backoff until the gateway returns 2xx or the attempt budget is exhausted (so
|
||||
// a degraded link surfaces as a non-zero exit), then emits the staged add-ons
|
||||
// as tab-separated lines the on-VM addon-sync shell agent consumes without a
|
||||
// JSON parser:
|
||||
//
|
||||
// <addonName>\t<addonVersion>\t<serviceAccountRoleArn>\t<base64(configurationValues)>
|
||||
//
|
||||
// The configuration values are base64-encoded so embedded tabs/newlines cannot
|
||||
// break the line framing.
|
||||
// control-plane resource from the host through the AWS gateway, writing it to
|
||||
// stdout as TSV; the implementation is in agents/eks/gatewayfetch.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// eks-gateway-fetch -resource addons # GET /clusters/{cluster}/internal-addons/{accountId}
|
||||
// eks-gateway-fetch -resource recovery -instance-id i-… # GET /clusters/{cluster}/internal-recovery/{accountId}/{instanceId}
|
||||
//
|
||||
// The recovery resource returns a single epoch\taction\tsnapshot line the on-VM
|
||||
// k3s-recovery agent applies before k3s starts (etcd cluster-reset / wipe-rejoin).
|
||||
//
|
||||
// Flags default to environment variables seeded by cloud-init:
|
||||
// EKS_GATEWAY_URL, EKS_GATEWAY_CA, EKS_ACCESS_KEY, EKS_SECRET_KEY, EKS_REGION,
|
||||
// EKS_ACCOUNT_ID, EKS_CLUSTER_NAME, EKS_INSTANCE_ID.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/eksgw"
|
||||
|
||||
_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
|
||||
)
|
||||
|
||||
// stagedAddon mirrors the gateway's StagedAddonManifest wire shape; duplicated
|
||||
// here to keep this tiny VM binary free of the handler package's dependency
|
||||
// graph.
|
||||
type stagedAddon struct {
|
||||
AddonName string `json:"addonName"`
|
||||
AddonVersion string `json:"addonVersion"`
|
||||
ServiceAccountRoleArn string `json:"serviceAccountRoleArn"`
|
||||
ConfigurationValues string `json:"configurationValues"`
|
||||
}
|
||||
|
||||
type internalAddonsResponse struct {
|
||||
Addons []stagedAddon `json:"addons"`
|
||||
}
|
||||
|
||||
const (
|
||||
maxAttempts = 30
|
||||
retryDelay = 5 * time.Second
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/eks/gatewayfetch"
|
||||
)
|
||||
|
||||
func main() {
|
||||
var (
|
||||
gatewayURL string
|
||||
gatewayCA string
|
||||
accessKey string
|
||||
secretKey string
|
||||
region string
|
||||
accountID string
|
||||
clusterName string
|
||||
resource string
|
||||
instanceID string
|
||||
)
|
||||
|
||||
flag.StringVar(&gatewayURL, "gateway", os.Getenv("EKS_GATEWAY_URL"), "Gateway URL (e.g. https://10.15.8.1:9999)")
|
||||
flag.StringVar(&gatewayCA, "gateway-ca", os.Getenv("EKS_GATEWAY_CA"), "Path to gateway TLS CA PEM (optional; falls back to system trust)")
|
||||
flag.StringVar(&accessKey, "access-key", os.Getenv("EKS_ACCESS_KEY"), "AWS access key ID")
|
||||
flag.StringVar(&secretKey, "secret-key", os.Getenv("EKS_SECRET_KEY"), "AWS secret access key")
|
||||
flag.StringVar(®ion, "region", os.Getenv("EKS_REGION"), "AWS region for SigV4 signing")
|
||||
flag.StringVar(&accountID, "account-id", os.Getenv("EKS_ACCOUNT_ID"), "Cluster account ID")
|
||||
flag.StringVar(&clusterName, "cluster", os.Getenv("EKS_CLUSTER_NAME"), "Cluster name")
|
||||
flag.StringVar(&instanceID, "instance-id", os.Getenv("EKS_INSTANCE_ID"), "This member's EC2 instance ID (recovery resource)")
|
||||
flag.StringVar(&resource, "resource", "addons", "Resource to fetch: addons | recovery")
|
||||
cfg := gatewayfetch.Config{Out: os.Stdout}
|
||||
flag.StringVar(&cfg.GatewayURL, "gateway", os.Getenv("EKS_GATEWAY_URL"), "Gateway URL (e.g. https://10.15.8.1:9999)")
|
||||
flag.StringVar(&cfg.GatewayCA, "gateway-ca", os.Getenv("EKS_GATEWAY_CA"), "Path to gateway TLS CA PEM (optional; falls back to system trust)")
|
||||
flag.StringVar(&cfg.AccessKey, "access-key", os.Getenv("EKS_ACCESS_KEY"), "AWS access key ID")
|
||||
flag.StringVar(&cfg.SecretKey, "secret-key", os.Getenv("EKS_SECRET_KEY"), "AWS secret access key")
|
||||
flag.StringVar(&cfg.Region, "region", os.Getenv("EKS_REGION"), "AWS region for SigV4 signing")
|
||||
flag.StringVar(&cfg.AccountID, "account-id", os.Getenv("EKS_ACCOUNT_ID"), "Cluster account ID")
|
||||
flag.StringVar(&cfg.ClusterName, "cluster", os.Getenv("EKS_CLUSTER_NAME"), "Cluster name")
|
||||
flag.StringVar(&cfg.InstanceID, "instance-id", os.Getenv("EKS_INSTANCE_ID"), "This member's EC2 instance ID (recovery resource)")
|
||||
flag.StringVar(&cfg.Resource, "resource", "addons", "Resource to fetch: addons | recovery")
|
||||
flag.Parse()
|
||||
|
||||
switch {
|
||||
case accountID == "":
|
||||
fatal("--account-id is required (or set EKS_ACCOUNT_ID)")
|
||||
case clusterName == "":
|
||||
fatal("--cluster is required (or set EKS_CLUSTER_NAME)")
|
||||
case resource != "addons" && resource != "recovery":
|
||||
fatal("--resource must be addons or recovery")
|
||||
case resource == "recovery" && instanceID == "":
|
||||
fatal("--instance-id is required for --resource recovery (or set EKS_INSTANCE_ID)")
|
||||
if err := gatewayfetch.Run(context.Background(), cfg); err != nil {
|
||||
slog.Error("eks-gateway-fetch: " + err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
client, err := eksgw.New(gatewayURL, gatewayCA, accessKey, secretKey, region)
|
||||
if err != nil {
|
||||
fatal(fmt.Sprintf("build gateway client: %v", err))
|
||||
}
|
||||
|
||||
var (
|
||||
path string
|
||||
emit func(io.Writer, []byte) error
|
||||
)
|
||||
switch resource {
|
||||
case "recovery":
|
||||
path = "/clusters/" + clusterName + "/internal-recovery/" + accountID + "/" + instanceID
|
||||
emit = emitRecoveryTSV
|
||||
default:
|
||||
path = "/clusters/" + clusterName + "/internal-addons/" + accountID
|
||||
emit = emitAddonsTSV
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= maxAttempts; attempt++ {
|
||||
body, err := client.Get(path)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
slog.Warn("eks-gateway-fetch: attempt failed", "resource", resource, "attempt", attempt, "err", err)
|
||||
if attempt < maxAttempts {
|
||||
time.Sleep(retryDelay)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if werr := emit(os.Stdout, body); werr != nil {
|
||||
fatal(fmt.Sprintf("render response: %v", werr))
|
||||
}
|
||||
return
|
||||
}
|
||||
fatal(fmt.Sprintf("fetch failed after %d attempts: %v", maxAttempts, lastErr))
|
||||
}
|
||||
|
||||
// emitAddonsTSV decodes the internal-addons response and writes one TSV line per
|
||||
// staged add-on to w. An empty add-on set writes nothing (the agent then GCs
|
||||
// every locally-rendered manifest), which is the correct steady state for a
|
||||
// cluster with no managed add-ons.
|
||||
func emitAddonsTSV(out io.Writer, body []byte) error {
|
||||
var resp internalAddonsResponse
|
||||
if err := json.Unmarshal(body, &resp); err != nil {
|
||||
return fmt.Errorf("unmarshal internal-addons response: %w", err)
|
||||
}
|
||||
w := bufio.NewWriter(out)
|
||||
for _, a := range resp.Addons {
|
||||
cfg := base64.StdEncoding.EncodeToString([]byte(a.ConfigurationValues))
|
||||
if _, err := fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", a.AddonName, a.AddonVersion, a.ServiceAccountRoleArn, cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return w.Flush()
|
||||
}
|
||||
|
||||
// recoveryDirective mirrors the gateway's RecoveryDirective wire shape; duplicated
|
||||
// here to keep this tiny VM binary free of the handler package's dependency graph.
|
||||
type recoveryDirective struct {
|
||||
Epoch int64 `json:"epoch"`
|
||||
Action string `json:"action"`
|
||||
Snapshot string `json:"snapshot"`
|
||||
SnapshotRequired bool `json:"snapshotRequired"`
|
||||
}
|
||||
|
||||
type internalRecoveryResponse struct {
|
||||
Directive recoveryDirective `json:"directive"`
|
||||
}
|
||||
|
||||
// emitRecoveryTSV decodes the recovery directive and writes a single
|
||||
// epoch\taction\tsnapshot\tsnapshotRequired line the on-VM k3s-recovery agent
|
||||
// consumes without a JSON parser. An unset action defaults to "none" (steady
|
||||
// state); snapshotRequired is 1 when the snapshot MUST restore (fresh DR seed
|
||||
// aborts boot on fetch failure) else 0.
|
||||
func emitRecoveryTSV(out io.Writer, body []byte) error {
|
||||
var resp internalRecoveryResponse
|
||||
if err := json.Unmarshal(body, &resp); err != nil {
|
||||
return fmt.Errorf("unmarshal internal-recovery response: %w", err)
|
||||
}
|
||||
d := resp.Directive
|
||||
if d.Action == "" {
|
||||
d.Action = "none"
|
||||
}
|
||||
required := "0"
|
||||
if d.SnapshotRequired {
|
||||
required = "1"
|
||||
}
|
||||
if _, err := fmt.Fprintf(out, "%d\t%s\t%s\t%s\n", d.Epoch, d.Action, d.Snapshot, required); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func fatal(msg string) {
|
||||
slog.Error("eks-gateway-fetch: " + msg)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -1,44 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestEmitAddonsTSV(t *testing.T) {
|
||||
body := []byte(`{"addons":[
|
||||
{"addonName":"spinifex-noop","addonVersion":"0.1.0"},
|
||||
{"addonName":"aws-load-balancer-controller","addonVersion":"2.8.1",
|
||||
"serviceAccountRoleArn":"arn:aws:iam::111122223333:role/alb",
|
||||
"configurationValues":"{\"replicaCount\":2}"}
|
||||
]}`)
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := emitAddonsTSV(&buf, body); err != nil {
|
||||
t.Fatalf("emitAddonsTSV: %v", err)
|
||||
}
|
||||
cfg := base64.StdEncoding.EncodeToString([]byte(`{"replicaCount":2}`))
|
||||
want := "spinifex-noop\t0.1.0\t\t\n" +
|
||||
"aws-load-balancer-controller\t2.8.1\tarn:aws:iam::111122223333:role/alb\t" + cfg + "\n"
|
||||
if buf.String() != want {
|
||||
t.Fatalf("TSV mismatch:\n got: %q\nwant: %q", buf.String(), want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmitAddonsTSV_Empty(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
if err := emitAddonsTSV(&buf, []byte(`{"addons":[]}`)); err != nil {
|
||||
t.Fatalf("emitAddonsTSV: %v", err)
|
||||
}
|
||||
if buf.Len() != 0 {
|
||||
t.Fatalf("empty addon set must emit nothing, got %q", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmitAddonsTSV_BadJSON(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
if err := emitAddonsTSV(&buf, []byte(`{not-json`)); err == nil {
|
||||
t.Fatal("expected error on malformed JSON")
|
||||
}
|
||||
}
|
||||
@@ -1,13 +1,6 @@
|
||||
// eks-gateway-publish runs inside the EKS K3s control-plane VM and relays a
|
||||
// single publication to the host through the AWS gateway, instead of dialing
|
||||
// core NATS directly.
|
||||
//
|
||||
// It reads a JSON payload on stdin, wraps it as
|
||||
// {accountId, channel, kind, payload}, SigV4-signs (service "eks") an HTTPS
|
||||
// POST to {gateway}/clusters/{cluster}/internal-publish, and retries with
|
||||
// backoff until the gateway returns 2xx or the attempt budget is exhausted —
|
||||
// so a degraded link surfaces as a non-zero exit rather than a silently
|
||||
// dropped message (the failure mode of fire-and-forget `nats pub`).
|
||||
// single JSON publication from stdin to the host through the AWS gateway; the
|
||||
// implementation is in agents/eks/gatewaypublish.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
@@ -20,115 +13,31 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/eksgw"
|
||||
|
||||
_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
|
||||
)
|
||||
|
||||
const (
|
||||
// Retry budget: the control plane reaches readiness before this runs, so a
|
||||
// failing POST means a degraded link, not a cold start. Bounded so a stuck
|
||||
// boot still terminates the OpenRC service.
|
||||
maxAttempts = 30
|
||||
retryDelay = 5 * time.Second
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/eks/gatewaypublish"
|
||||
)
|
||||
|
||||
type publishBody struct {
|
||||
AccountID string `json:"accountId"`
|
||||
Channel string `json:"channel"`
|
||||
Kind string `json:"kind"`
|
||||
Payload json.RawMessage `json:"payload"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
var (
|
||||
gatewayURL string
|
||||
gatewayCA string
|
||||
accessKey string
|
||||
secretKey string
|
||||
region string
|
||||
accountID string
|
||||
clusterName string
|
||||
channel string
|
||||
kind string
|
||||
)
|
||||
|
||||
flag.StringVar(&gatewayURL, "gateway", os.Getenv("EKS_GATEWAY_URL"), "Gateway URL (e.g. https://10.15.8.1:9999)")
|
||||
flag.StringVar(&gatewayCA, "gateway-ca", os.Getenv("EKS_GATEWAY_CA"), "Path to gateway TLS CA PEM (optional; falls back to system trust)")
|
||||
flag.StringVar(&accessKey, "access-key", os.Getenv("EKS_ACCESS_KEY"), "AWS access key ID")
|
||||
flag.StringVar(&secretKey, "secret-key", os.Getenv("EKS_SECRET_KEY"), "AWS secret access key")
|
||||
flag.StringVar(®ion, "region", os.Getenv("EKS_REGION"), "AWS region for SigV4 signing")
|
||||
flag.StringVar(&accountID, "account-id", os.Getenv("EKS_ACCOUNT_ID"), "Cluster account ID")
|
||||
flag.StringVar(&clusterName, "cluster", os.Getenv("EKS_CLUSTER_NAME"), "Cluster name")
|
||||
flag.StringVar(&channel, "channel", "", "Publish channel: bootstrap|state|addon")
|
||||
flag.StringVar(&kind, "kind", "", "Bootstrap subject kind (bootstrap channel only)")
|
||||
cfg := gatewaypublish.Config{Payload: os.Stdin}
|
||||
flag.StringVar(&cfg.GatewayURL, "gateway", os.Getenv("EKS_GATEWAY_URL"), "Gateway URL (e.g. https://10.15.8.1:9999)")
|
||||
flag.StringVar(&cfg.GatewayCA, "gateway-ca", os.Getenv("EKS_GATEWAY_CA"), "Path to gateway TLS CA PEM (optional; falls back to system trust)")
|
||||
flag.StringVar(&cfg.AccessKey, "access-key", os.Getenv("EKS_ACCESS_KEY"), "AWS access key ID")
|
||||
flag.StringVar(&cfg.SecretKey, "secret-key", os.Getenv("EKS_SECRET_KEY"), "AWS secret access key")
|
||||
flag.StringVar(&cfg.Region, "region", os.Getenv("EKS_REGION"), "AWS region for SigV4 signing")
|
||||
flag.StringVar(&cfg.AccountID, "account-id", os.Getenv("EKS_ACCOUNT_ID"), "Cluster account ID")
|
||||
flag.StringVar(&cfg.ClusterName, "cluster", os.Getenv("EKS_CLUSTER_NAME"), "Cluster name")
|
||||
flag.StringVar(&cfg.Channel, "channel", "", "Publish channel: bootstrap|state|addon")
|
||||
flag.StringVar(&cfg.Kind, "kind", "", "Bootstrap subject kind (bootstrap channel only)")
|
||||
flag.Parse()
|
||||
|
||||
switch {
|
||||
case accountID == "":
|
||||
fatal("--account-id is required (or set EKS_ACCOUNT_ID)")
|
||||
case clusterName == "":
|
||||
fatal("--cluster is required (or set EKS_CLUSTER_NAME)")
|
||||
case channel != "bootstrap" && channel != "state" && channel != "addon":
|
||||
fatal("--channel must be bootstrap, state or addon")
|
||||
case channel == "bootstrap" && kind == "":
|
||||
fatal("--kind is required for the bootstrap channel")
|
||||
if err := gatewaypublish.Run(context.Background(), cfg); err != nil {
|
||||
slog.Error("eks-gateway-publish: " + err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
payload, err := io.ReadAll(os.Stdin)
|
||||
if err != nil {
|
||||
fatal(fmt.Sprintf("read stdin payload: %v", err))
|
||||
}
|
||||
if len(bytes.TrimSpace(payload)) == 0 {
|
||||
fatal("empty stdin payload")
|
||||
}
|
||||
if !json.Valid(payload) {
|
||||
fatal("stdin payload is not valid JSON")
|
||||
}
|
||||
|
||||
body, err := json.Marshal(publishBody{
|
||||
AccountID: accountID,
|
||||
Channel: channel,
|
||||
Kind: kind,
|
||||
Payload: json.RawMessage(payload),
|
||||
})
|
||||
if err != nil {
|
||||
fatal(fmt.Sprintf("marshal request body: %v", err))
|
||||
}
|
||||
|
||||
client, err := eksgw.New(gatewayURL, gatewayCA, accessKey, secretKey, region)
|
||||
if err != nil {
|
||||
fatal(fmt.Sprintf("build gateway client: %v", err))
|
||||
}
|
||||
path := "/clusters/" + clusterName + "/internal-publish"
|
||||
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= maxAttempts; attempt++ {
|
||||
if _, err := client.Post(path, body); err != nil {
|
||||
lastErr = err
|
||||
slog.Warn("eks-gateway-publish: attempt failed",
|
||||
"channel", channel, "kind", kind, "attempt", attempt, "err", err)
|
||||
if attempt < maxAttempts {
|
||||
time.Sleep(retryDelay)
|
||||
}
|
||||
continue
|
||||
}
|
||||
slog.Info("eks-gateway-publish: published", "channel", channel, "kind", kind)
|
||||
return
|
||||
}
|
||||
fatal(fmt.Sprintf("publish failed after %d attempts: %v", maxAttempts, lastErr))
|
||||
}
|
||||
|
||||
func fatal(msg string) {
|
||||
slog.Error("eks-gateway-publish: " + msg)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -1,16 +1,6 @@
|
||||
// Command eks-konnectivity-cert mints, then reuses, an ECDSA-P256 serving
|
||||
// certificate for the on-node konnectivity-server, signed by the K3s cluster CA.
|
||||
//
|
||||
// The konnectivity-agent (DaemonSet on workers) verifies the server it dials
|
||||
// using the in-pod ServiceAccount CA (/var/run/secrets/.../ca.crt), which in
|
||||
// K3s is the same server-ca that signs the apiserver. Signing the konnectivity
|
||||
// serving cert with that CA is therefore what lets the agent validate the
|
||||
// reverse-tunnel endpoint with no separate CA distribution.
|
||||
//
|
||||
// The eks-node AMI carries no openssl, so this is the Go equivalent: load the
|
||||
// K3s server CA cert+key, issue a leaf (ServerAuth) cert carrying the cluster's
|
||||
// konnectivity SANs (the NLB private-endpoint IP, public endpoint IP, NLB DNS),
|
||||
// and cache it under -dir so the OpenRC service is byte-stable across restarts.
|
||||
// Command eks-konnectivity-cert mints, then reuses, a K3s-CA-signed serving
|
||||
// certificate for the on-node konnectivity-server; the implementation is in
|
||||
// agents/eks/konnectivitycert.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
@@ -23,225 +13,27 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/pem"
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math/big"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/eks/konnectivitycert"
|
||||
)
|
||||
|
||||
func main() {
|
||||
dir := flag.String("dir", "", "directory to cache tls.crt/tls.key in")
|
||||
caCert := flag.String("ca-cert", "", "PEM path of the signing (K3s server) CA cert")
|
||||
caKey := flag.String("ca-key", "", "PEM path of the signing CA private key")
|
||||
cn := flag.String("cn", "konnectivity-server", "certificate common name")
|
||||
sansCSV := flag.String("sans", "", "comma-separated SANs (IP or DNS)")
|
||||
cfg := konnectivitycert.Config{Out: os.Stdout}
|
||||
flag.StringVar(&cfg.Dir, "dir", "", "directory to cache tls.crt/tls.key in")
|
||||
flag.StringVar(&cfg.CACert, "ca-cert", "", "PEM path of the signing (K3s server) CA cert")
|
||||
flag.StringVar(&cfg.CAKey, "ca-key", "", "PEM path of the signing CA private key")
|
||||
flag.StringVar(&cfg.CN, "cn", "konnectivity-server", "certificate common name")
|
||||
flag.StringVar(&cfg.SANs, "sans", "", "comma-separated SANs (IP or DNS)")
|
||||
flag.Parse()
|
||||
|
||||
if *dir == "" || *caCert == "" || *caKey == "" || *sansCSV == "" {
|
||||
fatal("eks-konnectivity-cert: -dir, -ca-cert, -ca-key and -sans are required")
|
||||
if err := konnectivitycert.Run(context.Background(), cfg); err != nil {
|
||||
slog.Error("eks-konnectivity-cert: " + err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
dns, ips := splitSANs(*sansCSV)
|
||||
if len(dns) == 0 && len(ips) == 0 {
|
||||
fatal("eks-konnectivity-cert: -sans lists no usable names")
|
||||
}
|
||||
|
||||
certPath, keyPath, err := ensureCert(*dir, *caCert, *caKey, *cn, dns, ips)
|
||||
if err != nil {
|
||||
fatal("eks-konnectivity-cert: " + err.Error())
|
||||
}
|
||||
fmt.Printf("%s\t%s\n", certPath, keyPath)
|
||||
}
|
||||
|
||||
// ensureCert returns the cached cert/key paths under dir, minting and persisting
|
||||
// a fresh CA-signed pair when none is present, readable, and still SAN-current.
|
||||
func ensureCert(dir, caCertPath, caKeyPath, cn string, dns []string, ips []net.IP) (string, string, error) {
|
||||
certPath := filepath.Join(dir, "tls.crt")
|
||||
keyPath := filepath.Join(dir, "tls.key")
|
||||
|
||||
if certCoversSANs(certPath, keyPath, dns, ips) {
|
||||
return certPath, keyPath, nil
|
||||
}
|
||||
|
||||
caCert, caKey, err := loadCA(caCertPath, caKeyPath)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
certPEM, keyPEM, err := issueLeaf(caCert, caKey, cn, dns, ips)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if err = os.MkdirAll(dir, 0o700); err != nil {
|
||||
return "", "", fmt.Errorf("mkdir %s: %w", dir, err)
|
||||
}
|
||||
if err = os.WriteFile(certPath, certPEM, 0o644); err != nil {
|
||||
return "", "", fmt.Errorf("write %s: %w", certPath, err)
|
||||
}
|
||||
if err = os.WriteFile(keyPath, keyPEM, 0o600); err != nil {
|
||||
return "", "", fmt.Errorf("write %s: %w", keyPath, err)
|
||||
}
|
||||
return certPath, keyPath, nil
|
||||
}
|
||||
|
||||
// certCoversSANs reports whether the cached cert exists, parses, has a key, and
|
||||
// already carries every requested SAN — so a cert minted before the endpoint
|
||||
// set changed is re-issued rather than served stale.
|
||||
func certCoversSANs(certPath, keyPath string, dns []string, ips []net.IP) bool {
|
||||
c, err := os.ReadFile(certPath)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if k, kerr := os.ReadFile(keyPath); kerr != nil || len(k) == 0 {
|
||||
return false
|
||||
}
|
||||
der := decodeFirst(c)
|
||||
if der == nil {
|
||||
return false
|
||||
}
|
||||
crt, err := x509.ParseCertificate(der)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
have := make(map[string]struct{}, len(crt.DNSNames)+len(crt.IPAddresses))
|
||||
for _, n := range crt.DNSNames {
|
||||
have[n] = struct{}{}
|
||||
}
|
||||
for _, ip := range crt.IPAddresses {
|
||||
have[ip.String()] = struct{}{}
|
||||
}
|
||||
for _, n := range dns {
|
||||
if _, ok := have[n]; !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
for _, ip := range ips {
|
||||
if _, ok := have[ip.String()]; !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// loadCA parses the PEM CA cert + EC/PKCS8 private key used to sign the leaf.
|
||||
func loadCA(certPath, keyPath string) (*x509.Certificate, *ecdsa.PrivateKey, error) {
|
||||
certPEM, err := os.ReadFile(certPath)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("read ca cert %s: %w", certPath, err)
|
||||
}
|
||||
der := decodeFirst(certPEM)
|
||||
if der == nil {
|
||||
return nil, nil, fmt.Errorf("ca cert %s: no PEM block", certPath)
|
||||
}
|
||||
caCert, err := x509.ParseCertificate(der)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("parse ca cert: %w", err)
|
||||
}
|
||||
|
||||
keyPEM, err := os.ReadFile(keyPath)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("read ca key %s: %w", keyPath, err)
|
||||
}
|
||||
keyDER := decodeFirst(keyPEM)
|
||||
if keyDER == nil {
|
||||
return nil, nil, fmt.Errorf("ca key %s: no PEM block", keyPath)
|
||||
}
|
||||
caKey, err := parseECKey(keyDER)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return caCert, caKey, nil
|
||||
}
|
||||
|
||||
// parseECKey accepts the EC SEC1 or PKCS8 encodings K3s may write for the CA key.
|
||||
func parseECKey(der []byte) (*ecdsa.PrivateKey, error) {
|
||||
if k, err := x509.ParseECPrivateKey(der); err == nil {
|
||||
return k, nil
|
||||
}
|
||||
k8, err := x509.ParsePKCS8PrivateKey(der)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse ca key (sec1 and pkcs8 failed): %w", err)
|
||||
}
|
||||
ec, ok := k8.(*ecdsa.PrivateKey)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("ca key is %T, want ecdsa", k8)
|
||||
}
|
||||
return ec, nil
|
||||
}
|
||||
|
||||
// issueLeaf mints an ECDSA-P256 ServerAuth leaf for cn + SANs, signed by caCert/caKey.
|
||||
func issueLeaf(caCert *x509.Certificate, caKey *ecdsa.PrivateKey, cn string, dns []string, ips []net.IP) (certPEM, keyPEM []byte, err error) {
|
||||
priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("generate key: %w", err)
|
||||
}
|
||||
serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("serial: %w", err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
tmpl := x509.Certificate{
|
||||
SerialNumber: serial,
|
||||
Subject: pkix.Name{CommonName: cn},
|
||||
NotBefore: now.Add(-time.Hour),
|
||||
NotAfter: now.AddDate(10, 0, 0),
|
||||
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
BasicConstraintsValid: true,
|
||||
DNSNames: dns,
|
||||
IPAddresses: ips,
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, &tmpl, caCert, &priv.PublicKey, caKey)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create cert: %w", err)
|
||||
}
|
||||
keyDER, err := x509.MarshalECPrivateKey(priv)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("marshal key: %w", err)
|
||||
}
|
||||
certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
|
||||
keyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
|
||||
return certPEM, keyPEM, nil
|
||||
}
|
||||
|
||||
// splitSANs partitions the CSV into DNS names and IPs (an entry that parses as an
|
||||
// IP goes to IPAddresses, else DNSNames), matching how SAN matching is typed.
|
||||
func splitSANs(s string) (dns []string, ips []net.IP) {
|
||||
for p := range strings.SplitSeq(s, ",") {
|
||||
p = strings.TrimSpace(p)
|
||||
if p == "" {
|
||||
continue
|
||||
}
|
||||
if ip := net.ParseIP(p); ip != nil {
|
||||
ips = append(ips, ip)
|
||||
continue
|
||||
}
|
||||
dns = append(dns, p)
|
||||
}
|
||||
return dns, ips
|
||||
}
|
||||
|
||||
// decodeFirst returns the DER bytes of the first PEM block, or nil.
|
||||
func decodeFirst(pemBytes []byte) []byte {
|
||||
blk, _ := pem.Decode(pemBytes)
|
||||
if blk == nil {
|
||||
return nil
|
||||
}
|
||||
return blk.Bytes
|
||||
}
|
||||
|
||||
func fatal(msg string) {
|
||||
slog.Error(msg)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -1,153 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/pem"
|
||||
"math/big"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// writeTestCA mints a self-signed EC CA (cert + SEC1 key PEM) into dir, returning
|
||||
// the two paths — a stand-in for K3s' /var/lib/rancher/k3s/server/tls/server-ca.*.
|
||||
func writeTestCA(t *testing.T, dir string) (certPath, keyPath string) {
|
||||
t.Helper()
|
||||
priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatalf("gen ca key: %v", err)
|
||||
}
|
||||
tmpl := x509.Certificate{
|
||||
SerialNumber: big.NewInt(1),
|
||||
Subject: pkix.Name{CommonName: "k3s-server-ca"},
|
||||
NotBefore: time.Now().Add(-time.Hour),
|
||||
NotAfter: time.Now().AddDate(10, 0, 0),
|
||||
KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature,
|
||||
BasicConstraintsValid: true,
|
||||
IsCA: true,
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv)
|
||||
if err != nil {
|
||||
t.Fatalf("create ca cert: %v", err)
|
||||
}
|
||||
keyDER, err := x509.MarshalECPrivateKey(priv)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal ca key: %v", err)
|
||||
}
|
||||
certPath = filepath.Join(dir, "server-ca.crt")
|
||||
keyPath = filepath.Join(dir, "server-ca.key")
|
||||
if err := os.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644); err != nil {
|
||||
t.Fatalf("write ca cert: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o600); err != nil {
|
||||
t.Fatalf("write ca key: %v", err)
|
||||
}
|
||||
return certPath, keyPath
|
||||
}
|
||||
|
||||
func leafFrom(t *testing.T, certPath string) *x509.Certificate {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(certPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read leaf: %v", err)
|
||||
}
|
||||
crt, err := x509.ParseCertificate(decodeFirst(b))
|
||||
if err != nil {
|
||||
t.Fatalf("parse leaf: %v", err)
|
||||
}
|
||||
return crt
|
||||
}
|
||||
|
||||
func TestEnsureCertSignsLeafWithSANsThatChainsToCA(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
caCert, caKey := writeTestCA(t, dir)
|
||||
|
||||
dns, ips := splitSANs("10.32.100.4,203.0.113.9,eks-toc.example")
|
||||
certPath, keyPath, err := ensureCert(dir, caCert, caKey, "konnectivity-server", dns, ips)
|
||||
if err != nil {
|
||||
t.Fatalf("ensureCert: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(keyPath); err != nil {
|
||||
t.Fatalf("key not written: %v", err)
|
||||
}
|
||||
|
||||
leaf := leafFrom(t, certPath)
|
||||
if leaf.IsCA {
|
||||
t.Error("konnectivity serving cert must be a leaf, not a CA")
|
||||
}
|
||||
// The agent dials the NLB private-endpoint IP and verifies via the in-pod CA,
|
||||
// so the IP must be a SAN or the tunnel TLS handshake fails.
|
||||
if err := leaf.VerifyHostname("10.32.100.4"); err != nil {
|
||||
t.Errorf("IP SAN 10.32.100.4 missing: %v", err)
|
||||
}
|
||||
|
||||
// The leaf must chain to the K3s server CA — that CA is the in-pod ca.crt the
|
||||
// agent trusts, so no separate CA distribution is needed.
|
||||
caPEM, _ := os.ReadFile(caCert)
|
||||
roots := x509.NewCertPool()
|
||||
if !roots.AppendCertsFromPEM(caPEM) {
|
||||
t.Fatal("append CA failed")
|
||||
}
|
||||
if _, err := leaf.Verify(x509.VerifyOptions{
|
||||
Roots: roots,
|
||||
DNSName: "eks-toc.example",
|
||||
KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
}); err != nil {
|
||||
t.Errorf("leaf does not chain to the K3s server CA: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureCertReusesThenReissuesOnSANChange(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
caCert, caKey := writeTestCA(t, dir)
|
||||
|
||||
dns, ips := splitSANs("10.32.100.4,eks-toc.example")
|
||||
c1, _, err := ensureCert(dir, caCert, caKey, "konnectivity-server", dns, ips)
|
||||
if err != nil {
|
||||
t.Fatalf("first ensureCert: %v", err)
|
||||
}
|
||||
first, _ := os.ReadFile(c1)
|
||||
|
||||
// Same SANs → cached pair reused byte-for-byte (a churning cert would bounce
|
||||
// the konnectivity-server on every restart).
|
||||
if _, _, err := ensureCert(dir, caCert, caKey, "konnectivity-server", dns, ips); err != nil {
|
||||
t.Fatalf("second ensureCert: %v", err)
|
||||
}
|
||||
if second, _ := os.ReadFile(c1); string(first) != string(second) {
|
||||
t.Error("ensureCert re-minted instead of reusing the SAN-current cached cert")
|
||||
}
|
||||
|
||||
// A new endpoint in the SAN set must force a re-issue.
|
||||
dns2, ips2 := splitSANs("10.32.100.4,eks-toc.example,10.32.100.99")
|
||||
if _, _, err := ensureCert(dir, caCert, caKey, "konnectivity-server", dns2, ips2); err != nil {
|
||||
t.Fatalf("third ensureCert: %v", err)
|
||||
}
|
||||
if third, _ := os.ReadFile(c1); string(first) == string(third) {
|
||||
t.Error("ensureCert served a stale cert that does not cover the new SAN")
|
||||
}
|
||||
if err := leafFrom(t, c1).VerifyHostname("10.32.100.99"); err != nil {
|
||||
t.Errorf("re-issued cert missing new SAN: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitSANsPartitionsIPsAndDNS(t *testing.T) {
|
||||
dns, ips := splitSANs(" 10.0.0.1 , host.example , , 2001:db8::1 ")
|
||||
if len(dns) != 1 || dns[0] != "host.example" {
|
||||
t.Errorf("dns = %v, want [host.example]", dns)
|
||||
}
|
||||
wantIPs := []net.IP{net.ParseIP("10.0.0.1"), net.ParseIP("2001:db8::1")}
|
||||
if len(ips) != len(wantIPs) {
|
||||
t.Fatalf("ips = %v, want %v", ips, wantIPs)
|
||||
}
|
||||
for i, ip := range ips {
|
||||
if !ip.Equal(wantIPs[i]) {
|
||||
t.Errorf("ips[%d] = %v, want %v", i, ip, wantIPs[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,121 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/base64"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"net"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/bluebottle/pkg/tlsconfig"
|
||||
)
|
||||
|
||||
// serverTLSConfig is the webhook's loopback serving TLS config.
|
||||
func serverTLSConfig(cert tls.Certificate) *tls.Config {
|
||||
return &tls.Config{
|
||||
Certificates: []tls.Certificate{cert},
|
||||
MinVersion: tls.VersionTLS13,
|
||||
CurvePreferences: tlsconfig.Curves,
|
||||
}
|
||||
}
|
||||
|
||||
// ensureServingCert loads the persisted self-signed serving cert/key, or mints
|
||||
// a fresh ECDSA-P256 pair (SAN 127.0.0.1 / localhost) and persists it. Reusing
|
||||
// across restarts keeps the CA the apiserver loaded into its webhook kubeconfig
|
||||
// valid even if the webhook process is restarted. Returns the parsed cert and
|
||||
// the certificate PEM (for embedding as the kubeconfig CA bundle).
|
||||
func ensureServingCert(certPath, keyPath string) (tls.Certificate, []byte, error) {
|
||||
// Fall through to regenerate on a missing/corrupt/incompatible persisted
|
||||
// pair. The cert PEM is re-read because tls.Certificate keeps only DER.
|
||||
if cert, err := tls.LoadX509KeyPair(certPath, keyPath); err == nil {
|
||||
if certPEM, err := os.ReadFile(certPath); err == nil {
|
||||
return cert, certPEM, nil
|
||||
}
|
||||
}
|
||||
|
||||
certPEM, keyPEM, err := generateSelfSigned()
|
||||
if err != nil {
|
||||
return tls.Certificate{}, nil, err
|
||||
}
|
||||
if err := os.WriteFile(certPath, certPEM, 0o644); err != nil {
|
||||
return tls.Certificate{}, nil, fmt.Errorf("write cert %s: %w", certPath, err)
|
||||
}
|
||||
if err := os.WriteFile(keyPath, keyPEM, 0o600); err != nil {
|
||||
return tls.Certificate{}, nil, fmt.Errorf("write key %s: %w", keyPath, err)
|
||||
}
|
||||
cert, err := tls.X509KeyPair(certPEM, keyPEM)
|
||||
if err != nil {
|
||||
return tls.Certificate{}, nil, fmt.Errorf("parse minted keypair: %w", err)
|
||||
}
|
||||
return cert, certPEM, nil
|
||||
}
|
||||
|
||||
func generateSelfSigned() (certPEM, keyPEM []byte, err error) {
|
||||
priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("generate key: %w", err)
|
||||
}
|
||||
serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("serial: %w", err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
tmpl := x509.Certificate{
|
||||
SerialNumber: serial,
|
||||
Subject: pkix.Name{CommonName: "eks-token-webhook"},
|
||||
NotBefore: now.Add(-time.Hour),
|
||||
NotAfter: now.AddDate(10, 0, 0),
|
||||
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
BasicConstraintsValid: true,
|
||||
IsCA: true,
|
||||
IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback},
|
||||
DNSNames: []string{"localhost"},
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create cert: %w", err)
|
||||
}
|
||||
keyDER, err := x509.MarshalECPrivateKey(priv)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("marshal key: %w", err)
|
||||
}
|
||||
certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
|
||||
keyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
|
||||
return certPEM, keyPEM, nil
|
||||
}
|
||||
|
||||
// writeAPIServerKubeconfig writes the kubeconfig kube-apiserver loads via
|
||||
// --authentication-token-webhook-config-file. It points the apiserver at the
|
||||
// loopback webhook and trusts its self-signed serving cert as the CA bundle.
|
||||
func writeAPIServerKubeconfig(path, addr string, certPEM []byte) error {
|
||||
caB64 := base64.StdEncoding.EncodeToString(certPEM)
|
||||
kubeconfig := fmt.Sprintf(`apiVersion: v1
|
||||
kind: Config
|
||||
clusters:
|
||||
- name: mulga-eks-token-webhook
|
||||
cluster:
|
||||
certificate-authority-data: %s
|
||||
server: https://%s/authenticate
|
||||
users:
|
||||
- name: kube-apiserver
|
||||
contexts:
|
||||
- name: webhook
|
||||
context:
|
||||
cluster: mulga-eks-token-webhook
|
||||
user: kube-apiserver
|
||||
current-context: webhook
|
||||
`, caB64, addr)
|
||||
if err := os.WriteFile(path, []byte(kubeconfig), 0o600); err != nil {
|
||||
return fmt.Errorf("write %s: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,76 +1,46 @@
|
||||
// eks-token-webhook is the K8s TokenReview webhook authenticator baked into
|
||||
// the eks-server AMI. Per kubectl call the kube-apiserver POSTs the bearer
|
||||
// token (produced by `aws eks get-token`); the webhook relays it through the
|
||||
// AWS gateway broker (SigV4 HTTPS POST to /clusters/{name}/token-review), which
|
||||
// host-side runs the STS verify (cross-cluster x-k8s-aws-id pin) + AccessEntry
|
||||
// KV lookup and returns the resolved identity. The webhook never speaks core
|
||||
// NATS — same gateway-broker model as ELBv2's lb-agent and eks-gateway-publish.
|
||||
//
|
||||
// It binds 127.0.0.1 only; the kube-apiserver is the sole client (loopback).
|
||||
// On startup it writes the apiserver webhook kubeconfig (with its self-signed
|
||||
// serving CA) so k3s can be pointed at it via
|
||||
// --authentication-token-webhook-config-file.
|
||||
//
|
||||
// SigV4 creds come from the AWS SDK chain (IMDS instance-role) when static
|
||||
// EKS_ACCESS_KEY/EKS_SECRET_KEY are absent, matching the sibling helpers.
|
||||
// Command eks-token-webhook is the K8s TokenReview webhook authenticator baked
|
||||
// into the eks-server AMI; the implementation is in agents/eks/tokenwebhook.
|
||||
// It reads its configuration from the first-boot env cloud-init seeds onto the
|
||||
// VM and binds loopback only.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/eksgw"
|
||||
handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
|
||||
|
||||
_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/eks/tokenwebhook"
|
||||
)
|
||||
|
||||
// config is the webhook's runtime configuration, sourced from the first-boot
|
||||
// env (/etc/spinifex-eks/first-boot.env) cloud-init seeds onto the VM.
|
||||
type config struct {
|
||||
addr string
|
||||
gatewayURL string
|
||||
gatewayCA string
|
||||
accessKey string
|
||||
secretKey string
|
||||
region string
|
||||
accountID string
|
||||
clusterName string
|
||||
certPath string
|
||||
keyPath string
|
||||
kubeconfig string
|
||||
}
|
||||
|
||||
func loadConfig() (config, error) {
|
||||
c := config{
|
||||
addr: flagAddr,
|
||||
gatewayURL: os.Getenv("EKS_GATEWAY_URL"),
|
||||
gatewayCA: os.Getenv("EKS_GATEWAY_CA"),
|
||||
accessKey: os.Getenv("EKS_ACCESS_KEY"),
|
||||
secretKey: os.Getenv("EKS_SECRET_KEY"),
|
||||
region: os.Getenv("EKS_REGION"),
|
||||
accountID: os.Getenv("EKS_ACCOUNT_ID"),
|
||||
clusterName: os.Getenv("EKS_CLUSTER_NAME"),
|
||||
certPath: envOr("EKS_WEBHOOK_CERT", "/etc/spinifex-eks/token-webhook.crt"),
|
||||
keyPath: envOr("EKS_WEBHOOK_KEY", "/etc/spinifex-eks/token-webhook.key"),
|
||||
kubeconfig: envOr("EKS_WEBHOOK_KUBECONFIG", "/etc/spinifex-eks/token-webhook.kubeconfig"),
|
||||
func loadConfig() (tokenwebhook.Config, error) {
|
||||
c := tokenwebhook.Config{
|
||||
Addr: flagAddr,
|
||||
GatewayURL: os.Getenv("EKS_GATEWAY_URL"),
|
||||
GatewayCA: os.Getenv("EKS_GATEWAY_CA"),
|
||||
AccessKey: os.Getenv("EKS_ACCESS_KEY"),
|
||||
SecretKey: os.Getenv("EKS_SECRET_KEY"),
|
||||
Region: os.Getenv("EKS_REGION"),
|
||||
AccountID: os.Getenv("EKS_ACCOUNT_ID"),
|
||||
ClusterName: os.Getenv("EKS_CLUSTER_NAME"),
|
||||
CertPath: envOr("EKS_WEBHOOK_CERT", "/etc/spinifex-eks/token-webhook.crt"),
|
||||
KeyPath: envOr("EKS_WEBHOOK_KEY", "/etc/spinifex-eks/token-webhook.key"),
|
||||
Kubeconfig: envOr("EKS_WEBHOOK_KUBECONFIG", "/etc/spinifex-eks/token-webhook.kubeconfig"),
|
||||
ReadyAddr: os.Getenv("EKS_WEBHOOK_READY_ADDR"),
|
||||
}
|
||||
// Static SigV4 creds are optional: when absent, eksgw.New signs with the
|
||||
// AWS SDK chain (IMDS instance-role creds), the same path the CP VM's
|
||||
// sibling helpers use. The CP VM launches on an instance profile, so
|
||||
// buildK3sUserData omits EKS_ACCESS_KEY/EKS_SECRET_KEY by design.
|
||||
switch {
|
||||
case c.gatewayURL == "":
|
||||
case c.GatewayURL == "":
|
||||
return c, errors.New("EKS_GATEWAY_URL not set")
|
||||
case c.accountID == "":
|
||||
case c.AccountID == "":
|
||||
return c, errors.New("EKS_ACCOUNT_ID not set")
|
||||
case c.clusterName == "":
|
||||
case c.ClusterName == "":
|
||||
return c, errors.New("EKS_CLUSTER_NAME not set")
|
||||
}
|
||||
return c, nil
|
||||
@@ -97,132 +67,8 @@ func main() {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err := run(cfg); err != nil {
|
||||
if err := tokenwebhook.Run(context.Background(), cfg); err != nil {
|
||||
slog.Error("eks-token-webhook fatal", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(cfg config) error {
|
||||
// Serving cert + apiserver kubeconfig. Persisted and reused across restarts
|
||||
// so a webhook crash does not invalidate the CA the apiserver already loaded.
|
||||
tlsCert, certPEM, err := ensureServingCert(cfg.certPath, cfg.keyPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("serving cert: %w", err)
|
||||
}
|
||||
if err := writeAPIServerKubeconfig(cfg.kubeconfig, cfg.addr, certPEM); err != nil {
|
||||
return fmt.Errorf("write apiserver kubeconfig: %w", err)
|
||||
}
|
||||
|
||||
client, err := eksgw.New(cfg.gatewayURL, cfg.gatewayCA, cfg.accessKey, cfg.secretKey, cfg.region)
|
||||
if err != nil {
|
||||
return fmt.Errorf("build gateway client: %w", err)
|
||||
}
|
||||
|
||||
authr := &authenticator{
|
||||
accountID: cfg.accountID,
|
||||
clusterName: cfg.clusterName,
|
||||
review: gatewayReviewer(client, cfg.clusterName, cfg.accountID),
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("ok"))
|
||||
})
|
||||
mux.HandleFunc("/authenticate", authr.handle)
|
||||
|
||||
server := &http.Server{
|
||||
Addr: cfg.addr,
|
||||
Handler: mux,
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
TLSConfig: serverTLSConfig(tlsCert),
|
||||
}
|
||||
slog.Info("eks-token-webhook listening", "addr", cfg.addr, "cluster", cfg.clusterName)
|
||||
if err := server.ListenAndServeTLS("", ""); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
return fmt.Errorf("serve: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// tokenReviewRequest is the body POSTed to /clusters/{name}/token-review. The
|
||||
// webhook holds system SigV4 creds (system account, not the cluster account),
|
||||
// so accountId names the cluster account explicitly — same as PublishInternal.
|
||||
type tokenReviewRequest struct {
|
||||
AccountID string `json:"accountId"`
|
||||
Token string `json:"token"`
|
||||
}
|
||||
|
||||
// gatewayReviewer returns the per-call relay: SigV4-POST {accountId,token} to
|
||||
// the gateway and decode the resolved identity. A non-2xx / transport error is
|
||||
// surfaced (handle() turns it into a 5xx so the apiserver retries) rather than a
|
||||
// silent deny, distinguishing "webhook broker down" from "token rejected".
|
||||
func gatewayReviewer(client *eksgw.Client, clusterName, accountID string) func(string) (handlers_eks.WebhookTokenReviewResult, error) {
|
||||
path := "/clusters/" + clusterName + "/token-review"
|
||||
return func(token string) (handlers_eks.WebhookTokenReviewResult, error) {
|
||||
body, err := json.Marshal(tokenReviewRequest{AccountID: accountID, Token: token})
|
||||
if err != nil {
|
||||
return handlers_eks.WebhookTokenReviewResult{}, fmt.Errorf("marshal request: %w", err)
|
||||
}
|
||||
respBody, err := client.Post(path, body)
|
||||
if err != nil {
|
||||
return handlers_eks.WebhookTokenReviewResult{}, err
|
||||
}
|
||||
var res handlers_eks.WebhookTokenReviewResult
|
||||
if err := json.Unmarshal(respBody, &res); err != nil {
|
||||
return handlers_eks.WebhookTokenReviewResult{}, fmt.Errorf("decode response: %w", err)
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
}
|
||||
|
||||
// authenticator resolves a TokenReview by relaying to the gateway broker.
|
||||
type authenticator struct {
|
||||
accountID string
|
||||
clusterName string
|
||||
review func(token string) (handlers_eks.WebhookTokenReviewResult, error)
|
||||
}
|
||||
|
||||
func (a *authenticator) handle(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
var review tokenReview
|
||||
if err := json.NewDecoder(r.Body).Decode(&review); err != nil {
|
||||
http.Error(w, "bad request", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
res, err := a.review(review.Spec.Token)
|
||||
if err != nil {
|
||||
// Broker fault (gateway down, transport error): a 5xx tells the apiserver
|
||||
// to retry rather than treating a transient outage as a hard deny.
|
||||
slog.Error("TokenReview relay failed", "cluster", a.clusterName, "err", err)
|
||||
http.Error(w, "token review unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
status := tokenReviewStatus{Authenticated: res.Authenticated}
|
||||
if res.Authenticated {
|
||||
status.User = userInfo{
|
||||
Username: res.Username,
|
||||
UID: res.UID,
|
||||
Groups: res.Groups,
|
||||
}
|
||||
// Confirm the token for the audiences the apiserver asked about; an empty
|
||||
// status.audiences is rejected when --api-audiences is configured.
|
||||
status.Audiences = review.Spec.Audiences
|
||||
}
|
||||
slog.Info("TokenReview decision", "authenticated", status.Authenticated, "username", status.User.Username, "audiences", status.Audiences)
|
||||
|
||||
resp := tokenReview{
|
||||
APIVersion: "authentication.k8s.io/v1",
|
||||
Kind: "TokenReview",
|
||||
Status: status,
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(resp); err != nil {
|
||||
slog.Error("encode TokenReview response", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,127 +1,12 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// postReview drives authenticator.handle with a TokenReview carrying token and
|
||||
// audiences, returning the decoded response status and HTTP code.
|
||||
func postReview(t *testing.T, a *authenticator, token string, audiences []string) (tokenReviewStatus, int) {
|
||||
t.Helper()
|
||||
body, err := json.Marshal(tokenReview{
|
||||
APIVersion: "authentication.k8s.io/v1",
|
||||
Kind: "TokenReview",
|
||||
Spec: tokenReviewSpec{Token: token, Audiences: audiences},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/authenticate", strings.NewReader(string(body)))
|
||||
w := httptest.NewRecorder()
|
||||
a.handle(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
return tokenReviewStatus{}, w.Code
|
||||
}
|
||||
var resp tokenReview
|
||||
require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp))
|
||||
return resp.Status, w.Code
|
||||
}
|
||||
|
||||
func TestHandle_GrantsAndStampsAudiences(t *testing.T) {
|
||||
a := &authenticator{
|
||||
accountID: "111122223333",
|
||||
clusterName: "alpha",
|
||||
review: func(string) (handlers_eks.WebhookTokenReviewResult, error) {
|
||||
return handlers_eks.WebhookTokenReviewResult{
|
||||
Authenticated: true,
|
||||
Username: "arn:aws:iam::111122223333:role/admin",
|
||||
UID: "AROAEXAMPLE:session",
|
||||
Groups: []string{"system:masters"},
|
||||
}, nil
|
||||
},
|
||||
}
|
||||
|
||||
status, code := postReview(t, a, "tok", []string{"https://kubernetes.default.svc"})
|
||||
require.Equal(t, http.StatusOK, code)
|
||||
require.True(t, status.Authenticated)
|
||||
assert.Equal(t, "arn:aws:iam::111122223333:role/admin", status.User.Username)
|
||||
assert.Equal(t, "AROAEXAMPLE:session", status.User.UID)
|
||||
assert.Equal(t, []string{"system:masters"}, status.User.Groups)
|
||||
assert.Equal(t, []string{"https://kubernetes.default.svc"}, status.Audiences)
|
||||
}
|
||||
|
||||
func TestHandle_DeniesWithoutStampingAudiences(t *testing.T) {
|
||||
a := &authenticator{
|
||||
clusterName: "alpha",
|
||||
review: func(string) (handlers_eks.WebhookTokenReviewResult, error) {
|
||||
return handlers_eks.WebhookTokenReviewResult{Authenticated: false}, nil
|
||||
},
|
||||
}
|
||||
|
||||
status, code := postReview(t, a, "tok", []string{"https://kubernetes.default.svc"})
|
||||
require.Equal(t, http.StatusOK, code)
|
||||
assert.False(t, status.Authenticated)
|
||||
assert.Empty(t, status.User.Username)
|
||||
assert.Empty(t, status.Audiences, "unauthenticated review must not stamp audiences")
|
||||
}
|
||||
|
||||
// A broker fault is a 5xx (retryable), not a silent deny — a transient gateway
|
||||
// outage must not look identical to a rejected token.
|
||||
func TestHandle_BrokerFaultReturns5xx(t *testing.T) {
|
||||
a := &authenticator{
|
||||
clusterName: "alpha",
|
||||
review: func(string) (handlers_eks.WebhookTokenReviewResult, error) {
|
||||
return handlers_eks.WebhookTokenReviewResult{}, errors.New("gateway returned 503")
|
||||
},
|
||||
}
|
||||
|
||||
_, code := postReview(t, a, "tok", nil)
|
||||
assert.Equal(t, http.StatusServiceUnavailable, code)
|
||||
}
|
||||
|
||||
func TestEnsureServingCert_PersistsAndReuses(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
certPath := filepath.Join(dir, "wh.crt")
|
||||
keyPath := filepath.Join(dir, "wh.key")
|
||||
|
||||
c1, pem1, err := ensureServingCert(certPath, keyPath)
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, pem1)
|
||||
require.NotNil(t, serverTLSConfig(c1))
|
||||
|
||||
// Second call reuses the persisted pair (same cert bytes).
|
||||
_, pem2, err := ensureServingCert(certPath, keyPath)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, pem1, pem2)
|
||||
}
|
||||
|
||||
func TestWriteAPIServerKubeconfig(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "wh.kubeconfig")
|
||||
certPEM := []byte("-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----\n")
|
||||
|
||||
require.NoError(t, writeAPIServerKubeconfig(path, "127.0.0.1:8443", certPEM))
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
require.NoError(t, err)
|
||||
body := string(data)
|
||||
assert.Contains(t, body, "server: https://127.0.0.1:8443/authenticate")
|
||||
assert.Contains(t, body, base64.StdEncoding.EncodeToString(certPEM))
|
||||
assert.Contains(t, body, "current-context: webhook")
|
||||
}
|
||||
|
||||
// The CP VM launches on an instance profile, so cloud-init omits the static
|
||||
// EKS_ACCESS_KEY/EKS_SECRET_KEY and eksgw.New signs with IMDS creds. loadConfig
|
||||
// must accept absent static keys; gating on them aborts every IMDS-mode boot,
|
||||
@@ -148,8 +33,8 @@ func TestLoadConfig_AcceptsAbsentStaticCreds(t *testing.T) {
|
||||
})
|
||||
cfg, err := loadConfig()
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, cfg.accessKey)
|
||||
assert.Empty(t, cfg.secretKey)
|
||||
assert.Empty(t, cfg.AccessKey)
|
||||
assert.Empty(t, cfg.SecretKey)
|
||||
|
||||
// Genuinely-required vars are still enforced.
|
||||
setEnv(t, map[string]string{
|
||||
|
||||
@@ -1,12 +1,6 @@
|
||||
// Command eks-webhook-cert mints, then reuses, a self-signed ECDSA-P256 serving
|
||||
// certificate for an in-cluster admission webhook and prints its base64 PEM so
|
||||
// the addon renderer can embed it as the webhook Secret + each caBundle.
|
||||
//
|
||||
// The eks-node AMI carries no openssl, so this is the Go equivalent of the
|
||||
// upstream Helm chart's install-time genSelfSignedCert: one cert per cluster,
|
||||
// cached on disk under -dir, stable across addon-sync ticks. The single cert is
|
||||
// its own CA (IsCA + ServerAuth), so the same PEM serves as both the TLS leaf
|
||||
// and the caBundle the apiserver trusts.
|
||||
// Command eks-webhook-cert mints, then reuses, a self-signed serving certificate
|
||||
// for an in-cluster admission webhook and prints its base64 PEM; the
|
||||
// implementation is in agents/eks/webhookcert.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
@@ -17,133 +11,25 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/base64"
|
||||
"encoding/pem"
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math/big"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/agents/eks/webhookcert"
|
||||
)
|
||||
|
||||
func main() {
|
||||
dir := flag.String("dir", "", "directory to cache tls.crt/tls.key in")
|
||||
cn := flag.String("cn", "", "certificate common name")
|
||||
dnsCSV := flag.String("dns", "", "comma-separated DNS SANs")
|
||||
cfg := webhookcert.Config{Out: os.Stdout}
|
||||
flag.StringVar(&cfg.Dir, "dir", "", "directory to cache tls.crt/tls.key in")
|
||||
flag.StringVar(&cfg.CN, "cn", "", "certificate common name")
|
||||
flag.StringVar(&cfg.DNS, "dns", "", "comma-separated DNS SANs")
|
||||
flag.Parse()
|
||||
|
||||
if *dir == "" || *cn == "" || *dnsCSV == "" {
|
||||
fatal("eks-webhook-cert: -dir, -cn and -dns are required")
|
||||
if err := webhookcert.Run(context.Background(), cfg); err != nil {
|
||||
slog.Error("eks-webhook-cert: " + err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
dnsNames := splitCSV(*dnsCSV)
|
||||
if len(dnsNames) == 0 {
|
||||
fatal("eks-webhook-cert: -dns lists no names")
|
||||
}
|
||||
|
||||
certPEM, keyPEM, err := ensureCert(*dir, *cn, dnsNames)
|
||||
if err != nil {
|
||||
fatal("eks-webhook-cert: " + err.Error())
|
||||
}
|
||||
b := base64.StdEncoding.EncodeToString
|
||||
fmt.Printf("%s\t%s\t%s\n", b(certPEM), b(certPEM), b(keyPEM))
|
||||
}
|
||||
|
||||
// ensureCert returns the cached cert/key under dir, minting and persisting a
|
||||
// fresh self-signed pair (for cn + dnsNames) when none is present or readable.
|
||||
func ensureCert(dir, cn string, dnsNames []string) (certPEM, keyPEM []byte, err error) {
|
||||
certPath := filepath.Join(dir, "tls.crt")
|
||||
keyPath := filepath.Join(dir, "tls.key")
|
||||
|
||||
if c, cerr := os.ReadFile(certPath); cerr == nil {
|
||||
if k, kerr := os.ReadFile(keyPath); kerr == nil {
|
||||
if _, perr := x509.ParseCertificate(decodeFirst(c)); perr == nil && len(k) > 0 {
|
||||
return c, k, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
certPEM, keyPEM, err = generateSelfSigned(cn, dnsNames)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if err = os.MkdirAll(dir, 0o700); err != nil {
|
||||
return nil, nil, fmt.Errorf("mkdir %s: %w", dir, err)
|
||||
}
|
||||
if err = os.WriteFile(certPath, certPEM, 0o644); err != nil {
|
||||
return nil, nil, fmt.Errorf("write %s: %w", certPath, err)
|
||||
}
|
||||
if err = os.WriteFile(keyPath, keyPEM, 0o600); err != nil {
|
||||
return nil, nil, fmt.Errorf("write %s: %w", keyPath, err)
|
||||
}
|
||||
return certPEM, keyPEM, nil
|
||||
}
|
||||
|
||||
// generateSelfSigned mints an ECDSA-P256 self-signed cert valid as both a CA and
|
||||
// a server cert, with the given common name and DNS SANs.
|
||||
func generateSelfSigned(cn string, dnsNames []string) (certPEM, keyPEM []byte, err error) {
|
||||
priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("generate key: %w", err)
|
||||
}
|
||||
serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("serial: %w", err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
tmpl := x509.Certificate{
|
||||
SerialNumber: serial,
|
||||
Subject: pkix.Name{CommonName: cn},
|
||||
NotBefore: now.Add(-time.Hour),
|
||||
NotAfter: now.AddDate(10, 0, 0),
|
||||
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
BasicConstraintsValid: true,
|
||||
IsCA: true,
|
||||
DNSNames: dnsNames,
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create cert: %w", err)
|
||||
}
|
||||
keyDER, err := x509.MarshalECPrivateKey(priv)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("marshal key: %w", err)
|
||||
}
|
||||
certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
|
||||
keyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
|
||||
return certPEM, keyPEM, nil
|
||||
}
|
||||
|
||||
// decodeFirst returns the DER bytes of the first PEM block, or nil.
|
||||
func decodeFirst(pemBytes []byte) []byte {
|
||||
blk, _ := pem.Decode(pemBytes)
|
||||
if blk == nil {
|
||||
return nil
|
||||
}
|
||||
return blk.Bytes
|
||||
}
|
||||
|
||||
func splitCSV(s string) []string {
|
||||
var out []string
|
||||
for p := range strings.SplitSeq(s, ",") {
|
||||
if p = strings.TrimSpace(p); p != "" {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func fatal(msg string) {
|
||||
slog.Error(msg)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/x509"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const (
|
||||
testCN = "aws-load-balancer-webhook-service.kube-system.svc"
|
||||
testDNS = "aws-load-balancer-webhook-service.kube-system.svc.cluster.local"
|
||||
)
|
||||
|
||||
func parseLeaf(t *testing.T, certPEM []byte) *x509.Certificate {
|
||||
t.Helper()
|
||||
leaf, err := x509.ParseCertificate(decodeFirst(certPEM))
|
||||
if err != nil {
|
||||
t.Fatalf("parse cert: %v", err)
|
||||
}
|
||||
return leaf
|
||||
}
|
||||
|
||||
func TestEnsureCertMintsServingCert(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
certPEM, keyPEM, err := ensureCert(dir, testCN, []string{testCN, testDNS})
|
||||
if err != nil {
|
||||
t.Fatalf("ensureCert: %v", err)
|
||||
}
|
||||
if len(keyPEM) == 0 {
|
||||
t.Fatal("empty key PEM")
|
||||
}
|
||||
leaf := parseLeaf(t, certPEM)
|
||||
|
||||
// Both webhook service DNS names must be present, or the apiserver's TLS
|
||||
// dial to the webhook fails SAN verification.
|
||||
for _, want := range []string{testCN, testDNS} {
|
||||
if err := leaf.VerifyHostname(want); err != nil {
|
||||
t.Errorf("SAN %q missing: %v", want, err)
|
||||
}
|
||||
}
|
||||
if !leaf.IsCA {
|
||||
t.Error("cert must be a CA so the same PEM serves as the webhook caBundle")
|
||||
}
|
||||
|
||||
// The single self-signed cert must verify against itself as a root — this is
|
||||
// exactly what the apiserver does with caBundle == the serving cert.
|
||||
roots := x509.NewCertPool()
|
||||
roots.AddCert(leaf)
|
||||
if _, err := leaf.Verify(x509.VerifyOptions{
|
||||
Roots: roots,
|
||||
DNSName: testCN,
|
||||
KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
}); err != nil {
|
||||
t.Errorf("self-signed cert does not verify as its own CA: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureCertReusesCachedPair(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cert1, key1, err := ensureCert(dir, testCN, []string{testCN})
|
||||
if err != nil {
|
||||
t.Fatalf("first ensureCert: %v", err)
|
||||
}
|
||||
cert2, key2, err := ensureCert(dir, testCN, []string{testCN})
|
||||
if err != nil {
|
||||
t.Fatalf("second ensureCert: %v", err)
|
||||
}
|
||||
if string(cert1) != string(cert2) || string(key1) != string(key2) {
|
||||
t.Error("ensureCert regenerated instead of reusing the cached pair; a churning cert re-applies the addon every tick")
|
||||
}
|
||||
|
||||
// The private key must not be world-readable on disk.
|
||||
info, err := os.Stat(filepath.Join(dir, "tls.key"))
|
||||
if err != nil {
|
||||
t.Fatalf("stat key: %v", err)
|
||||
}
|
||||
if info.Mode().Perm()&0o077 != 0 {
|
||||
t.Errorf("tls.key mode %o is group/other-accessible", info.Mode().Perm())
|
||||
}
|
||||
}
|
||||
@@ -6,9 +6,9 @@ import (
|
||||
"strings"
|
||||
|
||||
_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
|
||||
"github.com/mulgadc/spinifex/cmd/installer/autoinstall"
|
||||
"github.com/mulgadc/spinifex/cmd/installer/install"
|
||||
"github.com/mulgadc/spinifex/cmd/installer/ui"
|
||||
"github.com/mulgadc/spinifex/spinifex/operator/installer/autoinstall"
|
||||
"github.com/mulgadc/spinifex/spinifex/operator/installer/install"
|
||||
"github.com/mulgadc/spinifex/spinifex/operator/installer/ui"
|
||||
)
|
||||
|
||||
func main() {
|
||||
|
||||
@@ -1,696 +0,0 @@
|
||||
package ui
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/charmbracelet/bubbles/textinput"
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
"github.com/charmbracelet/lipgloss"
|
||||
"github.com/mulgadc/spinifex/cmd/installer/install"
|
||||
"github.com/mulgadc/spinifex/cmd/installer/netprobe"
|
||||
)
|
||||
|
||||
// foldedNIC marks a role with no interface of its own, which collapses onto
|
||||
// the plane above it (vpc <- lan <- wan).
|
||||
const foldedNIC = -1
|
||||
|
||||
// roleField identifies one editable element on the per-role screen. Which of
|
||||
// these are actually shown depends on the plane, the addressing mode, whether
|
||||
// the NIC is wireless, and whether Advanced is on.
|
||||
type roleField int
|
||||
|
||||
const (
|
||||
roleFieldNIC roleField = iota
|
||||
roleFieldMethod
|
||||
roleFieldIP
|
||||
roleFieldMask
|
||||
roleFieldGateway
|
||||
roleFieldDNS
|
||||
roleFieldVLAN
|
||||
roleFieldMTU
|
||||
roleFieldSSID
|
||||
roleFieldWiFiPass
|
||||
)
|
||||
|
||||
// roleForm is the editable state for one plane. The three forms are the whole
|
||||
// of the installer's network configuration.
|
||||
type roleForm struct {
|
||||
plane install.Plane
|
||||
|
||||
// nic indexes into model.nics, or foldedNIC when the role is folded.
|
||||
nic int
|
||||
dhcp bool
|
||||
focus roleField
|
||||
|
||||
ip textinput.Model
|
||||
mask textinput.Model
|
||||
gateway textinput.Model
|
||||
dns textinput.Model
|
||||
vlan textinput.Model
|
||||
mtu textinput.Model
|
||||
ssid textinput.Model
|
||||
wifiPass textinput.Model
|
||||
}
|
||||
|
||||
func newRoleForm(plane install.Plane, nic int) roleForm {
|
||||
text := func(placeholder string, limit int) textinput.Model {
|
||||
in := textinput.New()
|
||||
in.Prompt = ""
|
||||
in.Placeholder = placeholder
|
||||
if limit > 0 {
|
||||
in.CharLimit = limit
|
||||
}
|
||||
return in
|
||||
}
|
||||
|
||||
f := roleForm{
|
||||
plane: plane,
|
||||
nic: nic,
|
||||
dhcp: plane == install.PlaneWAN,
|
||||
ip: text(defaultIPPlaceholder(plane), 0),
|
||||
mask: text("255.255.255.0", 0),
|
||||
gateway: text("192.168.1.1", 0),
|
||||
dns: text("1.1.1.1, 8.8.8.8", 0),
|
||||
vlan: text("untagged", 4),
|
||||
mtu: text("1500", 5),
|
||||
ssid: text("Network SSID", 64),
|
||||
wifiPass: text("WiFi password", 128),
|
||||
}
|
||||
f.wifiPass.EchoMode = textinput.EchoPassword
|
||||
return f
|
||||
}
|
||||
|
||||
// defaultIPPlaceholder hints the addressing convention for each plane so an
|
||||
// operator filling in a rack does not have to remember it.
|
||||
func defaultIPPlaceholder(plane install.Plane) string {
|
||||
switch plane {
|
||||
case install.PlaneLAN:
|
||||
return "10.0.0.3"
|
||||
case install.PlaneVPC:
|
||||
return "10.1.0.3"
|
||||
default:
|
||||
return "192.168.1.10"
|
||||
}
|
||||
}
|
||||
|
||||
func (f *roleForm) folded() bool { return f.nic == foldedNIC }
|
||||
|
||||
// canFold reports whether this role may be left unbound. wan always needs an
|
||||
// uplink, so only lan and vpc can fold.
|
||||
func (f *roleForm) canFold() bool { return f.plane != install.PlaneWAN }
|
||||
|
||||
// visibleFields returns the fields shown for this form, in tab order. Hiding
|
||||
// rather than disabling keeps the default screen narrow: a DHCP role shows two
|
||||
// lines, and VLAN/MTU only appear once Advanced is on.
|
||||
func (f *roleForm) visibleFields(isWiFi, advanced bool) []roleField {
|
||||
if f.folded() {
|
||||
return []roleField{roleFieldNIC}
|
||||
}
|
||||
fields := []roleField{roleFieldNIC, roleFieldMethod}
|
||||
if !f.dhcp {
|
||||
fields = append(fields, roleFieldIP, roleFieldMask)
|
||||
// Default route and resolvers are properties of the node, not of each
|
||||
// plane, and wan is the only one that carries either: it holds the
|
||||
// default route, so it is the only plane that can reach an upstream
|
||||
// resolver. On DHCP the lease supplies both.
|
||||
if f.plane == install.PlaneWAN {
|
||||
fields = append(fields, roleFieldGateway, roleFieldDNS)
|
||||
}
|
||||
}
|
||||
if advanced {
|
||||
fields = append(fields, roleFieldVLAN, roleFieldMTU)
|
||||
}
|
||||
if isWiFi {
|
||||
fields = append(fields, roleFieldSSID, roleFieldWiFiPass)
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
// input returns the textinput backing a field, or nil for the non-text fields.
|
||||
func (f *roleForm) input(field roleField) *textinput.Model {
|
||||
switch field {
|
||||
case roleFieldIP:
|
||||
return &f.ip
|
||||
case roleFieldMask:
|
||||
return &f.mask
|
||||
case roleFieldGateway:
|
||||
return &f.gateway
|
||||
case roleFieldDNS:
|
||||
return &f.dns
|
||||
case roleFieldVLAN:
|
||||
return &f.vlan
|
||||
case roleFieldMTU:
|
||||
return &f.mtu
|
||||
case roleFieldSSID:
|
||||
return &f.ssid
|
||||
case roleFieldWiFiPass:
|
||||
return &f.wifiPass
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// blurAll drops focus from every input so only the focused field has a cursor.
|
||||
func (f *roleForm) blurAll() {
|
||||
for _, field := range []roleField{
|
||||
roleFieldIP, roleFieldMask, roleFieldGateway, roleFieldDNS,
|
||||
roleFieldVLAN, roleFieldMTU, roleFieldSSID, roleFieldWiFiPass,
|
||||
} {
|
||||
f.input(field).Blur()
|
||||
}
|
||||
}
|
||||
|
||||
// focusCurrent gives the cursor to the focused field, if it is a text field.
|
||||
func (f *roleForm) focusCurrent() {
|
||||
f.blurAll()
|
||||
if in := f.input(f.focus); in != nil {
|
||||
in.Focus()
|
||||
}
|
||||
}
|
||||
|
||||
// toRole converts the form into the install package's role model.
|
||||
func (f *roleForm) toRole(nics []netprobe.NIC) install.NetworkRole {
|
||||
if f.folded() || f.nic >= len(nics) {
|
||||
return install.NetworkRole{}
|
||||
}
|
||||
role := install.NetworkRole{
|
||||
Interface: nics[f.nic].Name,
|
||||
DHCPMode: f.dhcp,
|
||||
VLAN: atoiOr(f.vlan.Value(), 0),
|
||||
MTU: atoiOr(f.mtu.Value(), 0),
|
||||
}
|
||||
if !f.dhcp {
|
||||
role.Address = strings.TrimSpace(f.ip.Value())
|
||||
role.Mask = strings.TrimSpace(f.mask.Value())
|
||||
role.DNS = parseDNS(f.dns.Value())
|
||||
if f.plane == install.PlaneWAN {
|
||||
role.Gateway = strings.TrimSpace(f.gateway.Value())
|
||||
}
|
||||
}
|
||||
if nics[f.nic].IsWiFi {
|
||||
role.WiFiSSID = strings.TrimSpace(f.ssid.Value())
|
||||
role.WiFiPass = f.wifiPass.Value()
|
||||
}
|
||||
return role
|
||||
}
|
||||
|
||||
func atoiOr(s string, fallback int) int {
|
||||
n, err := strconv.Atoi(strings.TrimSpace(s))
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// ── Roles overview screen ─────────────────────────────────────────────────────
|
||||
|
||||
// continueRow is the cursor position below the three role rows.
|
||||
const continueRow = 3
|
||||
|
||||
func (m model) handleRolesKey(key string) (tea.Model, tea.Cmd) {
|
||||
switch key {
|
||||
case "up", "k":
|
||||
if m.roleCursor > 0 {
|
||||
m.roleCursor--
|
||||
}
|
||||
case "down", "j":
|
||||
if m.roleCursor < continueRow {
|
||||
m.roleCursor++
|
||||
}
|
||||
case "a":
|
||||
// Advanced reveals VLAN and MTU on every role at once, rather than
|
||||
// per-role, so the operator flips it on for the whole node.
|
||||
m.advanced = !m.advanced
|
||||
m.validationErr = ""
|
||||
case "enter":
|
||||
if m.roleCursor == continueRow {
|
||||
if err := m.buildConfig().Validate(); err != nil {
|
||||
m.validationErr = capitalise(err.Error())
|
||||
return m, nil
|
||||
}
|
||||
m.validationErr = ""
|
||||
m.screen = screenIdentity
|
||||
m.hostnameInput.Focus()
|
||||
return m, nil
|
||||
}
|
||||
m.validationErr = ""
|
||||
m.screen = screenNetworkRole
|
||||
m.roles[m.roleCursor].focus = roleFieldNIC
|
||||
m.roles[m.roleCursor].blurAll()
|
||||
case "esc":
|
||||
m.screen = screenDiskConfirm
|
||||
m.eraseInput.Focus()
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m model) viewNetworkRoles(w int) string {
|
||||
title := styleTitle.Render("Network Planes")
|
||||
subtitle := styleMuted.Render("A role left unbound folds onto the plane above it.")
|
||||
|
||||
adv := styleMuted.Render("Advanced: off")
|
||||
if m.advanced {
|
||||
adv = styleSuccess.Render("Advanced: on")
|
||||
}
|
||||
|
||||
header := fmt.Sprintf(" %-*s%-*s", colRole, "ROLE", colIface, "INTERFACE")
|
||||
if m.advanced {
|
||||
header += fmt.Sprintf("%-*s%-*s", colVLAN, "VLAN", colMTU, "MTU")
|
||||
}
|
||||
header += "ADDRESS"
|
||||
|
||||
lines := []string{title, subtitle, "", adv, "", styleMuted.Render(header)}
|
||||
|
||||
for i := range m.roles {
|
||||
row := m.renderRoleRow(i)
|
||||
if i == m.roleCursor {
|
||||
lines = append(lines, styleSelected.Render("> "+row))
|
||||
} else {
|
||||
lines = append(lines, " "+row)
|
||||
}
|
||||
if detail := m.roleHardware(i, bodyWidth(w)); detail != "" {
|
||||
lines = append(lines, detail)
|
||||
}
|
||||
}
|
||||
|
||||
lines = append(lines, "")
|
||||
cont := " Continue"
|
||||
if m.roleCursor == continueRow {
|
||||
cont = styleSelected.Render("> Continue")
|
||||
}
|
||||
lines = append(lines, cont)
|
||||
|
||||
if m.validationErr != "" {
|
||||
lines = append(lines, "", styleError.Render(m.validationErr))
|
||||
}
|
||||
lines = append(lines, styleHelp.Render("↑/↓ move • Enter edit • a advanced • Esc back"))
|
||||
|
||||
body := lipgloss.JoinVertical(lipgloss.Left, lines...)
|
||||
return lipgloss.Place(w, m.height, lipgloss.Center, lipgloss.Center,
|
||||
styleBox.Width(min(w-4, 78)).Render(body),
|
||||
)
|
||||
}
|
||||
|
||||
// renderRoleRow renders one line of the overview table. A folded role names
|
||||
// the plane it collapsed onto rather than showing a blank, so the operator can
|
||||
// always see where the traffic will actually go.
|
||||
func (m model) renderRoleRow(i int) string {
|
||||
f := &m.roles[i]
|
||||
name := fmt.Sprintf("%-5s", f.plane)
|
||||
|
||||
if f.folded() {
|
||||
_, landed := m.buildConfig().Resolve(f.plane)
|
||||
return fmt.Sprintf("%-*s%s", colRole, name, styleMuted.Render("folds onto "+string(landed)))
|
||||
}
|
||||
|
||||
iface := styleMuted.Render("—")
|
||||
if f.nic < len(m.nics) {
|
||||
iface = nicSummary(m.nics[f.nic])
|
||||
}
|
||||
|
||||
addr := "DHCP"
|
||||
if !f.dhcp {
|
||||
addr = formatCIDR(f.ip.Value(), f.mask.Value())
|
||||
if addr == "" {
|
||||
addr = styleWarning.Render("not set")
|
||||
}
|
||||
}
|
||||
|
||||
row := padCell(name, colRole) + padCell(iface, colIface)
|
||||
if m.advanced {
|
||||
row += padCell(valueOr(f.vlan.Value(), "—"), colVLAN) + padCell(valueOr(f.mtu.Value(), "—"), colMTU)
|
||||
}
|
||||
return row + addr
|
||||
}
|
||||
|
||||
// roleHardware is the vendor/model line beneath a role row, indented under the
|
||||
// interface column. It is what tells an operator that wan really is the 25gbe
|
||||
// Mellanox and not the onboard 1gbe.
|
||||
func (m model) roleHardware(i, width int) string {
|
||||
f := &m.roles[i]
|
||||
if f.folded() || f.nic >= len(m.nics) {
|
||||
return ""
|
||||
}
|
||||
indent := 2 + colRole
|
||||
return strings.Repeat(" ", indent) + styleMuted.Render(nicHardware(m.nics[f.nic], width-indent))
|
||||
}
|
||||
|
||||
// nicSummary renders name, speed and link state, colouring the state so a port
|
||||
// with no cable stands out rather than having to be read for.
|
||||
func nicSummary(n netprobe.NIC) string {
|
||||
state := styleWarning.Render(n.State)
|
||||
if n.Carrier {
|
||||
state = styleSuccess.Render(n.State)
|
||||
}
|
||||
speed := n.Speed
|
||||
if speed == "" {
|
||||
speed = "—"
|
||||
}
|
||||
return fmt.Sprintf("%s %s %s", n.Name, speed, state)
|
||||
}
|
||||
|
||||
// nicHardware is the detail line: what the card is, plus the predictable name
|
||||
// it also answers to, which is the one written on cabling notes. The alt name
|
||||
// is dropped rather than truncated when the two will not fit together.
|
||||
func nicHardware(n netprobe.NIC, width int) string {
|
||||
hw := n.Hardware()
|
||||
const separator = " · "
|
||||
if n.AltName != "" && len([]rune(hw))+len(separator)+len([]rune(n.AltName)) <= width {
|
||||
return hw + separator + n.AltName
|
||||
}
|
||||
return truncate(hw, width)
|
||||
}
|
||||
|
||||
// bodyWidth is the text width inside the framed body. Detail lines are trimmed
|
||||
// to it rather than left to wrap, which would push the table out of alignment.
|
||||
func bodyWidth(w int) int {
|
||||
return min(w-4, 78) - 4
|
||||
}
|
||||
|
||||
// padCell pads a cell to width by display width, which the %-*s verb cannot do
|
||||
// once a cell carries the ANSI escapes of a style.
|
||||
func padCell(s string, width int) string {
|
||||
return lipgloss.PlaceHorizontal(width, lipgloss.Left, s)
|
||||
}
|
||||
|
||||
// Column widths for the overview table, shared by the header and the rows.
|
||||
const (
|
||||
colRole = 6
|
||||
colIface = 32
|
||||
colVLAN = 7
|
||||
colMTU = 7
|
||||
)
|
||||
|
||||
// focusMark gives the focused text field a visible caret without every field
|
||||
// rendering its own prompt.
|
||||
func focusMark(focused bool) string {
|
||||
if focused {
|
||||
return styleLabel.Render("> ")
|
||||
}
|
||||
return " "
|
||||
}
|
||||
|
||||
// formatCIDR renders an address and mask the way an operator writes them,
|
||||
// falling back to the raw mask when it is not dotted-decimal.
|
||||
func formatCIDR(addr, mask string) string {
|
||||
addr = strings.TrimSpace(addr)
|
||||
mask = strings.TrimSpace(mask)
|
||||
if addr == "" {
|
||||
return ""
|
||||
}
|
||||
if mask == "" {
|
||||
return addr
|
||||
}
|
||||
if ip := net.ParseIP(mask); ip != nil && ip.To4() != nil {
|
||||
if ones, bits := net.IPMask(ip.To4()).Size(); bits != 0 {
|
||||
return fmt.Sprintf("%s/%d", addr, ones)
|
||||
}
|
||||
}
|
||||
return addr + "/" + strings.TrimPrefix(mask, "/")
|
||||
}
|
||||
|
||||
func valueOr(v, fallback string) string {
|
||||
if v = strings.TrimSpace(v); v != "" {
|
||||
return v
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
// ── Per-role editor screen ────────────────────────────────────────────────────
|
||||
|
||||
func (m model) handleRoleKey(key string, msg tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
f := &m.roles[m.roleCursor]
|
||||
fields := f.visibleFields(m.nicIsWiFi(f.nic), m.advanced)
|
||||
pos := fieldIndex(fields, f.focus)
|
||||
|
||||
switch key {
|
||||
case "esc":
|
||||
f.blurAll()
|
||||
m.screen = screenNetworkRoles
|
||||
m.validationErr = ""
|
||||
return m, nil
|
||||
|
||||
case "enter":
|
||||
if err := m.validateRole(m.roleCursor); err != "" {
|
||||
m.validationErr = err
|
||||
return m, nil
|
||||
}
|
||||
f.blurAll()
|
||||
m.screen = screenNetworkRoles
|
||||
m.validationErr = ""
|
||||
return m, nil
|
||||
|
||||
case "tab", "down":
|
||||
pos = (pos + 1) % len(fields)
|
||||
f.focus = fields[pos]
|
||||
f.focusCurrent()
|
||||
return m, nil
|
||||
|
||||
case "shift+tab", "up":
|
||||
pos = (pos - 1 + len(fields)) % len(fields)
|
||||
f.focus = fields[pos]
|
||||
f.focusCurrent()
|
||||
return m, nil
|
||||
|
||||
case "left", "right":
|
||||
// Only the two selector fields consume arrows; text fields need them
|
||||
// for cursor movement.
|
||||
switch f.focus {
|
||||
case roleFieldNIC:
|
||||
return m.cycleNIC(m.roleCursor, key == "right"), nil
|
||||
case roleFieldMethod:
|
||||
f.dhcp = key == "left"
|
||||
f.focusCurrent()
|
||||
return m, nil
|
||||
}
|
||||
}
|
||||
|
||||
if in := f.input(f.focus); in != nil {
|
||||
var cmd tea.Cmd
|
||||
*in, cmd = in.Update(msg)
|
||||
return m, cmd
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// cycleNIC steps the interface selection, including the fold option for lan
|
||||
// and vpc. Wrapping keeps a single key usable on a node with one NIC.
|
||||
func (m model) cycleNIC(idx int, forward bool) model {
|
||||
f := &m.roles[idx]
|
||||
low := 0
|
||||
if f.canFold() {
|
||||
low = foldedNIC
|
||||
}
|
||||
high := len(m.nics) - 1
|
||||
|
||||
if forward {
|
||||
if f.nic >= high {
|
||||
f.nic = low
|
||||
} else {
|
||||
f.nic++
|
||||
}
|
||||
return m
|
||||
}
|
||||
if f.nic <= low {
|
||||
f.nic = high
|
||||
} else {
|
||||
f.nic--
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func (m model) nicIsWiFi(idx int) bool {
|
||||
return idx >= 0 && idx < len(m.nics) && m.nics[idx].IsWiFi
|
||||
}
|
||||
|
||||
func (m model) viewNetworkRole(w int) string {
|
||||
f := &m.roles[m.roleCursor]
|
||||
title := styleTitle.Render(fmt.Sprintf("Configure %s plane", strings.ToUpper(string(f.plane))))
|
||||
subtitle := styleMuted.Render(planeDescription(f.plane))
|
||||
|
||||
lines := []string{title, subtitle, ""}
|
||||
for _, field := range f.visibleFields(m.nicIsWiFi(f.nic), m.advanced) {
|
||||
lines = append(lines, m.renderRoleField(f, field, bodyWidth(w)), "")
|
||||
}
|
||||
|
||||
if m.validationErr != "" {
|
||||
lines = append(lines, styleError.Render(m.validationErr), "")
|
||||
}
|
||||
lines = append(lines, styleHelp.Render("Tab/↑↓ move • ←/→ change • Enter done • Esc cancel"))
|
||||
|
||||
body := lipgloss.JoinVertical(lipgloss.Left, lines...)
|
||||
return lipgloss.Place(w, m.height, lipgloss.Center, lipgloss.Center,
|
||||
styleBox.Width(min(w-4, 78)).Render(body),
|
||||
)
|
||||
}
|
||||
|
||||
// planeDescription explains what rides each plane, so the choice is not just a
|
||||
// three-letter name.
|
||||
func planeDescription(p install.Plane) string {
|
||||
switch p {
|
||||
case install.PlaneWAN:
|
||||
return "Public ingress and egress, EIPs and NAT (br-wan)."
|
||||
case install.PlaneLAN:
|
||||
return "Storage replication, NATS mesh and OVN control (br-lan)."
|
||||
default:
|
||||
return "Geneve tunnels carrying EC2 traffic between nodes (br-vpc)."
|
||||
}
|
||||
}
|
||||
|
||||
// roleLabelWidth is the field-label column on the editor screen, shared so a
|
||||
// wrapped value can indent to line up underneath it.
|
||||
const roleLabelWidth = 14
|
||||
|
||||
func (m model) renderRoleField(f *roleForm, field roleField, width int) string {
|
||||
focused := f.focus == field
|
||||
label := styleLabel.Render(fmt.Sprintf("%-*s", roleLabelWidth, roleFieldLabel(field)))
|
||||
|
||||
switch field {
|
||||
case roleFieldNIC:
|
||||
return label + m.renderNICChoice(f, focused, width)
|
||||
|
||||
case roleFieldMethod:
|
||||
return label + renderToggle([]string{"DHCP (automatic)", "Static"}, boolToIdx(!f.dhcp), focused)
|
||||
|
||||
default:
|
||||
if in := f.input(field); in != nil {
|
||||
return label + focusMark(focused) + in.View()
|
||||
}
|
||||
return label
|
||||
}
|
||||
}
|
||||
|
||||
func roleFieldLabel(field roleField) string {
|
||||
switch field {
|
||||
case roleFieldNIC:
|
||||
return "Interface"
|
||||
case roleFieldMethod:
|
||||
return "IP method"
|
||||
case roleFieldIP:
|
||||
return "IP address"
|
||||
case roleFieldMask:
|
||||
return "Subnet mask"
|
||||
case roleFieldGateway:
|
||||
return "Gateway"
|
||||
case roleFieldDNS:
|
||||
return "DNS"
|
||||
case roleFieldVLAN:
|
||||
return "VLAN id"
|
||||
case roleFieldMTU:
|
||||
return "MTU"
|
||||
case roleFieldSSID:
|
||||
return "WiFi SSID"
|
||||
default:
|
||||
return "WiFi password"
|
||||
}
|
||||
}
|
||||
|
||||
// renderNICChoice shows the selected interface on the field line and its
|
||||
// hardware identity on a line of its own beneath. Vendor and model are what
|
||||
// make a port identifiable, and they do not fit alongside the selector.
|
||||
func (m model) renderNICChoice(f *roleForm, focused bool, width int) string {
|
||||
var text, detail string
|
||||
switch {
|
||||
case f.folded():
|
||||
_, landed := m.buildConfig().Resolve(f.plane)
|
||||
text = fmt.Sprintf("(fold onto %s)", landed)
|
||||
case f.nic < len(m.nics):
|
||||
text = nicSummary(m.nics[f.nic])
|
||||
detail = nicHardware(m.nics[f.nic], width-roleLabelWidth-2)
|
||||
default:
|
||||
text = "(none detected)"
|
||||
}
|
||||
|
||||
if focused {
|
||||
text = styleSelected.Render(" ← " + text + " → ")
|
||||
} else {
|
||||
text = styleLabel.Render("[ " + text + " ]")
|
||||
}
|
||||
if detail == "" {
|
||||
return text
|
||||
}
|
||||
return text + "\n" + strings.Repeat(" ", roleLabelWidth+2) + styleMuted.Render(detail)
|
||||
}
|
||||
|
||||
func renderToggle(options []string, selected int, focused bool) string {
|
||||
var parts []string
|
||||
for i, opt := range options {
|
||||
switch {
|
||||
case i == selected && focused:
|
||||
parts = append(parts, styleSelected.Render(" "+opt+" "))
|
||||
case i == selected:
|
||||
parts = append(parts, styleLabel.Render("["+opt+"]"))
|
||||
default:
|
||||
parts = append(parts, styleMuted.Render(opt))
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
func boolToIdx(b bool) int {
|
||||
if b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func fieldIndex(fields []roleField, want roleField) int {
|
||||
for i, f := range fields {
|
||||
if f == want {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// validateRole checks one plane's entries. Cross-role rules (a shared NIC
|
||||
// without distinct VLAN ids, an unbound wan) belong to install.Config.Validate
|
||||
// and run when the operator continues.
|
||||
func (m model) validateRole(idx int) string {
|
||||
f := &m.roles[idx]
|
||||
if f.folded() {
|
||||
return ""
|
||||
}
|
||||
if len(m.nics) == 0 {
|
||||
return "No network interfaces detected"
|
||||
}
|
||||
|
||||
if !f.dhcp {
|
||||
if net.ParseIP(strings.TrimSpace(f.ip.Value())) == nil {
|
||||
return "Enter a valid IP address"
|
||||
}
|
||||
if !validSubnetMask(f.mask.Value()) {
|
||||
return "Enter a dotted-decimal subnet mask, e.g. 255.255.255.0 or 255.255.0.0"
|
||||
}
|
||||
if f.plane == install.PlaneWAN && net.ParseIP(strings.TrimSpace(f.gateway.Value())) == nil {
|
||||
return "Enter a valid gateway address"
|
||||
}
|
||||
}
|
||||
|
||||
if v := strings.TrimSpace(f.vlan.Value()); v != "" {
|
||||
id, err := strconv.Atoi(v)
|
||||
if err != nil || id < 1 || id > 4094 {
|
||||
return "VLAN id must be between 1 and 4094"
|
||||
}
|
||||
}
|
||||
if v := strings.TrimSpace(f.mtu.Value()); v != "" {
|
||||
mtu, err := strconv.Atoi(v)
|
||||
if err != nil || mtu < 68 || mtu > 9216 {
|
||||
return "MTU must be between 68 and 9216"
|
||||
}
|
||||
}
|
||||
if m.nicIsWiFi(f.nic) && strings.TrimSpace(f.ssid.Value()) == "" {
|
||||
return "Enter the WiFi network SSID"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func capitalise(s string) string {
|
||||
if s == "" {
|
||||
return s
|
||||
}
|
||||
return strings.ToUpper(s[:1]) + s[1:]
|
||||
}
|
||||
@@ -1,357 +0,0 @@
|
||||
package ui
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/cmd/installer/install"
|
||||
"github.com/mulgadc/spinifex/cmd/installer/netprobe"
|
||||
)
|
||||
|
||||
func nics(n int) []netprobe.NIC {
|
||||
out := make([]netprobe.NIC, 0, n)
|
||||
names := []string{"eno1", "ens1f0np0", "ens1f1np1", "ens2f0np0"}
|
||||
for i := range n {
|
||||
out = append(out, netprobe.NIC{Name: names[i], Speed: "25Gbps", Vendor: "Mellanox", State: "online"})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// The pre-fill is the whole of the simple path: an operator with a
|
||||
// conventional node should be able to press Continue without editing anything.
|
||||
func TestNewModelPrefillsRolesByNICCount(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
count int
|
||||
wantLAN, wantVPC int
|
||||
}{
|
||||
{"one NIC folds everything onto wan", 1, foldedNIC, foldedNIC},
|
||||
{"two NICs dedicate the second to lan, vpc folds", 2, 1, foldedNIC},
|
||||
{"three NICs give each plane its own", 3, 1, 2},
|
||||
{"four NICs still only bind three", 4, 1, 2},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
m := newModel([]install.Disk{{Path: "/dev/sda"}}, nics(tt.count))
|
||||
|
||||
if got := m.roles[0].nic; got != 0 {
|
||||
t.Errorf("wan nic = %d, want 0", got)
|
||||
}
|
||||
if got := m.roles[1].nic; got != tt.wantLAN {
|
||||
t.Errorf("lan nic = %d, want %d", got, tt.wantLAN)
|
||||
}
|
||||
if got := m.roles[2].nic; got != tt.wantVPC {
|
||||
t.Errorf("vpc nic = %d, want %d", got, tt.wantVPC)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Hiding rather than disabling is what keeps the default screen narrow.
|
||||
func TestVisibleFields(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*roleForm)
|
||||
isWiFi bool
|
||||
advanced bool
|
||||
want []roleField
|
||||
}{
|
||||
{
|
||||
name: "wan on DHCP asks two questions",
|
||||
mutate: func(f *roleForm) { f.dhcp = true },
|
||||
want: []roleField{roleFieldNIC, roleFieldMethod},
|
||||
},
|
||||
{
|
||||
name: "wan static adds a gateway",
|
||||
mutate: func(f *roleForm) { f.dhcp = false },
|
||||
want: []roleField{roleFieldNIC, roleFieldMethod, roleFieldIP, roleFieldMask, roleFieldGateway, roleFieldDNS},
|
||||
},
|
||||
{
|
||||
name: "advanced reveals VLAN and MTU",
|
||||
mutate: func(f *roleForm) { f.dhcp = true },
|
||||
advanced: true,
|
||||
want: []roleField{roleFieldNIC, roleFieldMethod, roleFieldVLAN, roleFieldMTU},
|
||||
},
|
||||
{
|
||||
name: "wireless adds credentials",
|
||||
mutate: func(f *roleForm) { f.dhcp = true },
|
||||
isWiFi: true,
|
||||
want: []roleField{roleFieldNIC, roleFieldMethod, roleFieldSSID, roleFieldWiFiPass},
|
||||
},
|
||||
{
|
||||
name: "a folded role has nothing to configure",
|
||||
mutate: func(f *roleForm) { f.nic = foldedNIC },
|
||||
want: []roleField{roleFieldNIC},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
f := newRoleForm(install.PlaneWAN, 0)
|
||||
tt.mutate(&f)
|
||||
got := f.visibleFields(tt.isWiFi, tt.advanced)
|
||||
|
||||
if len(got) != len(tt.want) {
|
||||
t.Fatalf("visibleFields() = %v, want %v", got, tt.want)
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != tt.want[i] {
|
||||
t.Fatalf("visibleFields() = %v, want %v", got, tt.want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A non-wan role must be able to reach the folded state, and wan must not.
|
||||
func TestCycleNICFoldReachableOnlyOffWAN(t *testing.T) {
|
||||
m := newModel([]install.Disk{{Path: "/dev/sda"}}, nics(2))
|
||||
|
||||
// lan: 1 -> 0 -> folded -> wraps to the last NIC.
|
||||
m.roles[1].nic = 1
|
||||
for _, want := range []int{0, foldedNIC, 1} {
|
||||
m = m.cycleNIC(1, false)
|
||||
if m.roles[1].nic != want {
|
||||
t.Fatalf("lan nic = %d, want %d", m.roles[1].nic, want)
|
||||
}
|
||||
}
|
||||
|
||||
// wan cycles only through real interfaces.
|
||||
m.roles[0].nic = 0
|
||||
for range 4 {
|
||||
m = m.cycleNIC(0, false)
|
||||
if m.roles[0].nic == foldedNIC {
|
||||
t.Fatal("wan reached the folded state, but it always needs an uplink")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestToRoleReadsVLANAndMTU(t *testing.T) {
|
||||
f := newRoleForm(install.PlaneLAN, 0)
|
||||
f.dhcp = false
|
||||
f.ip.SetValue("10.0.0.2")
|
||||
f.mask.SetValue("255.255.0.0")
|
||||
f.vlan.SetValue("10")
|
||||
f.mtu.SetValue("9000")
|
||||
|
||||
role := f.toRole(nics(1))
|
||||
|
||||
if role.VLAN != 10 || role.MTU != 9000 {
|
||||
t.Errorf("VLAN/MTU = %d/%d, want 10/9000", role.VLAN, role.MTU)
|
||||
}
|
||||
if role.Link() != "eno1.10" {
|
||||
t.Errorf("Link() = %q, want eno1.10", role.Link())
|
||||
}
|
||||
// Only wan carries a default route, even if a gateway was typed earlier.
|
||||
f.gateway.SetValue("10.0.0.1")
|
||||
if got := f.toRole(nics(1)).Gateway; got != "" {
|
||||
t.Errorf("lan Gateway = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestToRoleFoldedIsEmpty(t *testing.T) {
|
||||
f := newRoleForm(install.PlaneVPC, foldedNIC)
|
||||
if !f.toRole(nics(2)).Folded() {
|
||||
t.Error("a folded form must produce a folded role")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRole(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
plane install.Plane
|
||||
mutate func(*roleForm)
|
||||
wantErr string
|
||||
}{
|
||||
{"valid static lan", install.PlaneLAN, func(f *roleForm) {
|
||||
f.dhcp = false
|
||||
f.ip.SetValue("10.0.0.2")
|
||||
f.mask.SetValue("255.255.0.0")
|
||||
}, ""},
|
||||
{"bad ip", install.PlaneLAN, func(f *roleForm) {
|
||||
f.dhcp = false
|
||||
f.ip.SetValue("not-an-ip")
|
||||
f.mask.SetValue("255.255.0.0")
|
||||
}, "valid IP"},
|
||||
{"wan needs a gateway", install.PlaneWAN, func(f *roleForm) {
|
||||
f.dhcp = false
|
||||
f.ip.SetValue("216.218.163.99")
|
||||
f.mask.SetValue("255.255.255.224")
|
||||
}, "gateway"},
|
||||
{"VLAN out of range", install.PlaneLAN, func(f *roleForm) {
|
||||
f.dhcp = true
|
||||
f.vlan.SetValue("9000")
|
||||
}, "VLAN id must be"},
|
||||
{"MTU out of range", install.PlaneLAN, func(f *roleForm) {
|
||||
f.dhcp = true
|
||||
f.mtu.SetValue("70000")
|
||||
}, "MTU must be"},
|
||||
{"folded roles skip validation", install.PlaneVPC, func(f *roleForm) {
|
||||
f.nic = foldedNIC
|
||||
f.ip.SetValue("nonsense")
|
||||
}, ""},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
m := newModel([]install.Disk{{Path: "/dev/sda"}}, nics(3))
|
||||
m.roles[0] = newRoleForm(tt.plane, 0)
|
||||
tt.mutate(&m.roles[0])
|
||||
|
||||
got := m.validateRole(0)
|
||||
if tt.wantErr == "" {
|
||||
if got != "" {
|
||||
t.Fatalf("validateRole() = %q, want no error", got)
|
||||
}
|
||||
return
|
||||
}
|
||||
if !strings.Contains(got, tt.wantErr) {
|
||||
t.Fatalf("validateRole() = %q, want it to mention %q", got, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Continue must refuse a layout install.Config.Validate would reject, rather
|
||||
// than failing later during the install itself.
|
||||
func TestContinueBlocksOnSharedNICWithoutVLANs(t *testing.T) {
|
||||
m := newModel([]install.Disk{{Path: "/dev/sda"}}, nics(2))
|
||||
m.screen = screenNetworkRoles
|
||||
m.roleCursor = continueRow
|
||||
m.roles[1].nic = 0 // lan onto the wan NIC, both untagged
|
||||
|
||||
next, _ := m.handleRolesKey("enter")
|
||||
got := next.(model)
|
||||
|
||||
if got.screen != screenNetworkRoles {
|
||||
t.Error("expected to stay on the roles screen when the layout is invalid")
|
||||
}
|
||||
if !strings.Contains(got.validationErr, "both bind eno1") {
|
||||
t.Errorf("validationErr = %q, want it to name the shared interface", got.validationErr)
|
||||
}
|
||||
|
||||
// Distinct VLAN ids make the same pairing legal.
|
||||
got.roles[0].vlan.SetValue("30")
|
||||
got.roles[1].vlan.SetValue("10")
|
||||
next, _ = got.handleRolesKey("enter")
|
||||
if next.(model).screen != screenIdentity {
|
||||
t.Error("distinct VLAN ids should allow two roles to share one NIC")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdvancedToggle(t *testing.T) {
|
||||
m := newModel([]install.Disk{{Path: "/dev/sda"}}, nics(3))
|
||||
m.screen = screenNetworkRoles
|
||||
|
||||
next, _ := m.handleRolesKey("a")
|
||||
if !next.(model).advanced {
|
||||
t.Fatal("'a' should turn Advanced on")
|
||||
}
|
||||
next, _ = next.(model).handleRolesKey("a")
|
||||
if next.(model).advanced {
|
||||
t.Fatal("'a' should turn Advanced back off")
|
||||
}
|
||||
}
|
||||
|
||||
// The default route and the resolvers are node-level, and wan is the only plane
|
||||
// that carries either, so the east-west editors must not ask for them. Their
|
||||
// bridges are link-local and activate manually — a resolver pinned there is
|
||||
// unreachable, and a gateway would fight the real default route.
|
||||
func TestVisibleFieldsOmitsRoutingOffWAN(t *testing.T) {
|
||||
for _, plane := range []install.Plane{install.PlaneLAN, install.PlaneVPC} {
|
||||
t.Run(string(plane), func(t *testing.T) {
|
||||
f := newRoleForm(plane, 1)
|
||||
f.dhcp = false
|
||||
got := f.visibleFields(false, false)
|
||||
|
||||
want := []roleField{roleFieldNIC, roleFieldMethod, roleFieldIP, roleFieldMask}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("visibleFields() = %v, want %v", got, want)
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("visibleFields() = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The mask field must accept only what netgen can consume. It previously
|
||||
// advertised and accepted a prefix length, which passed the TUI and then failed
|
||||
// the install itself with "invalid mask: 24".
|
||||
func TestValidSubnetMaskRejectsPrefixLength(t *testing.T) {
|
||||
for _, bad := range []string{"24", "/24", "8", "", "255.255.0", "255.0.255.0", "::1"} {
|
||||
if validSubnetMask(bad) {
|
||||
t.Errorf("validSubnetMask(%q) = true, want false", bad)
|
||||
}
|
||||
}
|
||||
for _, good := range []string{"255.255.255.0", "255.255.0.0", "255.0.0.0", "255.255.255.252"} {
|
||||
if !validSubnetMask(good) {
|
||||
t.Errorf("validSubnetMask(%q) = false, want true", good)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The hardware detail is the point of the NIC table: an operator has to be able
|
||||
// to tell which physical port each plane landed on. It gets a line of its own,
|
||||
// under the interface column, rather than being squeezed into the row.
|
||||
func TestRoleHardwareRendersOnItsOwnLine(t *testing.T) {
|
||||
m := newModel([]install.Disk{{Path: "/dev/sda"}}, []netprobe.NIC{{
|
||||
Name: "ens1f0np0", Vendor: "Mellanox Technologies", Model: "MT27710",
|
||||
AltName: "enp132s0f0np0", Speed: "25 Gbps", Carrier: true, State: "online",
|
||||
}})
|
||||
|
||||
view := m.viewNetworkRoles(100)
|
||||
if !strings.Contains(view, "ens1f0np0 25 Gbps online") {
|
||||
t.Errorf("expected the identity line in:\n%s", view)
|
||||
}
|
||||
detail := m.roleHardware(0, bodyWidth(100))
|
||||
if !strings.Contains(detail, "Mellanox Technologies MT27710") {
|
||||
t.Errorf("roleHardware = %q, want the vendor and model", detail)
|
||||
}
|
||||
if !strings.Contains(detail, "enp132s0f0np0") {
|
||||
t.Errorf("roleHardware = %q, want the alternative name", detail)
|
||||
}
|
||||
// Every line has to fit the box, or lipgloss wraps it and the columns shear.
|
||||
for line := range strings.SplitSeq(view, "\n") {
|
||||
if len([]rune(line)) > 100 {
|
||||
t.Errorf("line exceeds the terminal width: %q", line)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A narrow box drops the alternative name before it truncates the model, since
|
||||
// half a model name identifies nothing.
|
||||
func TestNICHardwareDropsAltNameBeforeTruncating(t *testing.T) {
|
||||
n := netprobe.NIC{Vendor: "Broadcom Inc. and subsidiaries", Model: "NetXtreme BCM5720", AltName: "enp1s0f1"}
|
||||
|
||||
if got := nicHardware(n, 60); got != "Broadcom Inc. and subsidiaries NetXtreme BCM5720" {
|
||||
t.Errorf("nicHardware(60) = %q, want the name without the alt name", got)
|
||||
}
|
||||
if got := nicHardware(n, 20); got != "Broadcom Inc. and s…" {
|
||||
t.Errorf("nicHardware(20) = %q, want a marked truncation", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A role bound to a port with no hardware database entry still gets something
|
||||
// to identify the card by, never the bare word "unknown".
|
||||
func TestRoleHardwareFallsBackToDriver(t *testing.T) {
|
||||
m := newModel([]install.Disk{{Path: "/dev/sda"}}, []netprobe.NIC{{
|
||||
Name: "eno1", Driver: "bnxt_en", DeviceID: "14e4:165f", State: "no cable",
|
||||
}})
|
||||
|
||||
if got := m.roleHardware(0, bodyWidth(100)); !strings.Contains(got, "bnxt_en [14e4:165f]") {
|
||||
t.Errorf("roleHardware = %q, want the driver and device id", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A folded role has no interface, so it must not render an orphan detail line
|
||||
// under the plane it collapsed onto.
|
||||
func TestRoleHardwareEmptyWhenFolded(t *testing.T) {
|
||||
m := newModel([]install.Disk{{Path: "/dev/sda"}}, nics(1))
|
||||
if got := m.roleHardware(1, bodyWidth(100)); got != "" {
|
||||
t.Errorf("roleHardware for a folded role = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
@@ -1,139 +0,0 @@
|
||||
package ui
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/cmd/installer/install"
|
||||
)
|
||||
|
||||
// diskModel is a storage screen with n unselected disks and ext4 chosen, which
|
||||
// is where the wizard starts.
|
||||
func diskModel(n int) model {
|
||||
m := model{fsCursor: slices.Index(install.AllFSTypes, install.FSExt4)}
|
||||
for i := range n {
|
||||
m.disks = append(m.disks, install.Disk{
|
||||
Path: string(rune('a'+i)) + "-disk",
|
||||
Bytes: 200 << 30,
|
||||
LogicalBlockSize: 512,
|
||||
PhysicalBlockSize: 512,
|
||||
})
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func rolePaths(cfg install.DiskConfig) map[install.DiskRole]string {
|
||||
out := map[install.DiskRole]string{}
|
||||
for _, rm := range cfg.Roles {
|
||||
out[rm.Role] = rm.Disk.Path
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// A single-disk machine should need no storage input at all, so the disk that
|
||||
// newModel preselects has to arrive already holding the os role.
|
||||
func TestPreselectedDiskIsAValidConfiguration(t *testing.T) {
|
||||
m := newModel([]install.Disk{{Path: "/dev/sda", Bytes: 200 << 30, LogicalBlockSize: 512}}, nil)
|
||||
cfg := m.storage()
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("the preselected disk must be installable as-is: %v", err)
|
||||
}
|
||||
if cfg.Primary().Path != "/dev/sda" {
|
||||
t.Errorf("Primary() = %q, want the preselected disk", cfg.Primary().Path)
|
||||
}
|
||||
}
|
||||
|
||||
// Picking disks in order gives the layout an operator almost always wants, so
|
||||
// the role key is only needed for the exceptions.
|
||||
func TestSelectingDisksAssignsRolesInOrder(t *testing.T) {
|
||||
m := diskModel(3)
|
||||
for i := range 3 {
|
||||
m = m.toggleDisk(i)
|
||||
}
|
||||
got := rolePaths(m.storage())
|
||||
for role, want := range map[install.DiskRole]string{
|
||||
install.RoleOS: "a-disk", install.RoleSpinifex: "b-disk", install.RolePredastore: "c-disk",
|
||||
} {
|
||||
if got[role] != want {
|
||||
t.Errorf("%s = %q, want %q", role, got[role], want)
|
||||
}
|
||||
}
|
||||
if err := m.storage().Validate(); err != nil {
|
||||
t.Errorf("the default assignment must be valid: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Deselecting has to drop the role with the disk, or every later role shifts
|
||||
// onto the wrong drive.
|
||||
func TestDeselectingADiskReleasesItsRole(t *testing.T) {
|
||||
m := diskModel(3)
|
||||
for i := range 3 {
|
||||
m = m.toggleDisk(i)
|
||||
}
|
||||
m = m.toggleDisk(1) // drop the spinifex drive
|
||||
|
||||
got := rolePaths(m.storage())
|
||||
if got[install.RoleOS] != "a-disk" || got[install.RolePredastore] != "c-disk" {
|
||||
t.Fatalf("remaining roles moved: %v", got)
|
||||
}
|
||||
if _, ok := got[install.RoleSpinifex]; ok {
|
||||
t.Errorf("the spinifex role should be free again: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The two-drive predastore layout is only reachable by reassigning, so cycling
|
||||
// swaps rather than refusing a role another disk already holds.
|
||||
func TestCycleRoleSwapsWithTheDiskThatHoldsIt(t *testing.T) {
|
||||
m := diskModel(2)
|
||||
m = m.toggleDisk(0).toggleDisk(1)
|
||||
// b-disk: spinifex → predastore, which no disk holds, so no swap.
|
||||
m = m.cycleRole(1)
|
||||
|
||||
got := rolePaths(m.storage())
|
||||
if got[install.RoleOS] != "a-disk" || got[install.RolePredastore] != "b-disk" {
|
||||
t.Fatalf("roles = %v, want os on a-disk and predastore on b-disk", got)
|
||||
}
|
||||
|
||||
// Cycling past the end wraps onto os, which a-disk holds, so they trade.
|
||||
m = m.cycleRole(1)
|
||||
got = rolePaths(m.storage())
|
||||
if got[install.RoleOS] != "b-disk" || got[install.RolePredastore] != "a-disk" {
|
||||
t.Fatalf("roles = %v, want the two disks to have swapped", got)
|
||||
}
|
||||
if err := m.storage().Validate(); err != nil {
|
||||
t.Errorf("a swap must always leave a valid assignment: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ZFS pairs mirrors by selection order and has no roles to assign.
|
||||
func TestStorageLeavesRolesUnsetOnZFS(t *testing.T) {
|
||||
m := diskModel(2)
|
||||
m.fsCursor = slices.Index(install.AllFSTypes, install.FSZFSRAID1)
|
||||
m = m.toggleDisk(0).toggleDisk(1)
|
||||
|
||||
cfg := m.storage()
|
||||
if len(cfg.Roles) != 0 {
|
||||
t.Errorf("Roles = %v, want none on a pool", cfg.Roles)
|
||||
}
|
||||
if !slices.Equal(cfg.Paths(), []string{"a-disk", "b-disk"}) {
|
||||
t.Errorf("Paths() = %v, want the selection order preserved", cfg.Paths())
|
||||
}
|
||||
}
|
||||
|
||||
// A three-drive selection looks like an array and is not one, so the mapping
|
||||
// and the absence of redundancy both have to be on screen before the erase.
|
||||
func TestConfirmScreenStatesTheLayoutAndTheRisk(t *testing.T) {
|
||||
m := diskModel(3)
|
||||
m.height = 40
|
||||
for i := range 3 {
|
||||
m = m.toggleDisk(i)
|
||||
}
|
||||
got := m.viewDiskConfirm(100)
|
||||
|
||||
for _, want := range []string{"/var/lib/spinifex/predastore", "c-disk", "losing one loses"} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("confirm screen is missing %q:\n%s", want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -19,7 +19,7 @@ import (
|
||||
"syscall"
|
||||
|
||||
"github.com/mulgadc/bluebottle/pkg/otelsetup"
|
||||
"github.com/mulgadc/spinifex/spinifex/lbagent"
|
||||
lbagent "github.com/mulgadc/spinifex/spinifex/agents/elbv2"
|
||||
|
||||
_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
|
||||
)
|
||||
|
||||
@@ -1,424 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/gwsign"
|
||||
"github.com/mulgadc/spinifex/internal/rdsgw"
|
||||
handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
|
||||
"github.com/mulgadc/spinifex/spinifex/otelsetup"
|
||||
)
|
||||
|
||||
// Overridable via -ldflags "-X main.version=...".
|
||||
var version = "dev"
|
||||
|
||||
// The gateway resolves the authoritative DB instance from the request's
|
||||
// credentials, so DBInstanceIdentifier is an assertion to be checked, not
|
||||
// trusted.
|
||||
type identity struct {
|
||||
DBInstanceIdentifier string
|
||||
AgentVersion string
|
||||
EngineVersion string
|
||||
}
|
||||
|
||||
// Every method is a SigV4-signed Query request, so the NATS bus stays
|
||||
// host-internal.
|
||||
type controlPlane interface {
|
||||
Register(ctx context.Context, id identity) (*handlers_rds.RegisterDBInstanceOutput, error)
|
||||
SubmitState(ctx context.Context, id identity, health handlers_rds.EngineHealth, message string) (*handlers_rds.SubmitDBStateChangeOutput, error)
|
||||
// Returns Mode=initialize with the master password for as long as a payload
|
||||
// staged for this VM generation is unacknowledged, and Mode=attach after.
|
||||
GetBootstrapConfig(ctx context.Context, id identity) (*handlers_rds.GetDBBootstrapConfigOutput, error)
|
||||
// Reports that PostgreSQL durably applied the staged password, which is what
|
||||
// lets the control plane destroy it.
|
||||
AcknowledgeBootstrap(ctx context.Context, id identity, pending *pendingBootstrap) error
|
||||
PollCommands(ctx context.Context, id identity, replies []handlers_rds.CommandReply, wait time.Duration) ([]handlers_rds.Command, error)
|
||||
}
|
||||
|
||||
// Bounds a register, heartbeat or bootstrap request. The long poll sets its own
|
||||
// deadline from the wait it asked the gateway to hold.
|
||||
const callTimeout = 15 * time.Second
|
||||
|
||||
// Keeps the poll's deadline past the requested wait, so a gateway answering at
|
||||
// the end of its window is not cut off by the client.
|
||||
const pollSlack = 10 * time.Second
|
||||
|
||||
type gatewayControlPlane struct {
|
||||
client *rdsgw.Client
|
||||
}
|
||||
|
||||
var _ controlPlane = (*gatewayControlPlane)(nil)
|
||||
|
||||
// Signs with the instance-role credentials the SDK chain resolves from IMDS,
|
||||
// against the pinned gateway CA.
|
||||
func newGatewayControlPlane(cfg config) (*gatewayControlPlane, error) {
|
||||
signer, err := gwsign.NewIMDS(context.Background(), cfg.Region)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build IMDS signer: %w", err)
|
||||
}
|
||||
// The client timeout is the ceiling for the longest call, the long poll;
|
||||
// shorter calls carry their own deadline.
|
||||
client, err := rdsgw.New(cfg.GatewayURL, cfg.GatewayCA, signer, cfg.Region, cfg.PollWait+pollSlack)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &gatewayControlPlane{client: client}, nil
|
||||
}
|
||||
|
||||
func (g *gatewayControlPlane) Register(ctx context.Context, id identity) (*handlers_rds.RegisterDBInstanceOutput, error) {
|
||||
params := url.Values{}
|
||||
setIfPresent(params, "DBInstanceIdentifier", id.DBInstanceIdentifier)
|
||||
setIfPresent(params, "AgentVersion", id.AgentVersion)
|
||||
setIfPresent(params, "EngineVersion", id.EngineVersion)
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, callTimeout)
|
||||
defer cancel()
|
||||
|
||||
var out handlers_rds.RegisterDBInstanceOutput
|
||||
if err := g.client.Call(ctx, "RegisterDBInstance", params, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
func (g *gatewayControlPlane) SubmitState(ctx context.Context, id identity, health handlers_rds.EngineHealth, message string) (*handlers_rds.SubmitDBStateChangeOutput, error) {
|
||||
params := url.Values{"EngineHealth": {string(health)}}
|
||||
setIfPresent(params, "DBInstanceIdentifier", id.DBInstanceIdentifier)
|
||||
setIfPresent(params, "EngineVersion", id.EngineVersion)
|
||||
setIfPresent(params, "Message", message)
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, callTimeout)
|
||||
defer cancel()
|
||||
|
||||
var out handlers_rds.SubmitDBStateChangeOutput
|
||||
if err := g.client.Call(ctx, "SubmitDBStateChange", params, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
func (g *gatewayControlPlane) GetBootstrapConfig(ctx context.Context, id identity) (*handlers_rds.GetDBBootstrapConfigOutput, error) {
|
||||
params := url.Values{}
|
||||
setIfPresent(params, "DBInstanceIdentifier", id.DBInstanceIdentifier)
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, callTimeout)
|
||||
defer cancel()
|
||||
|
||||
var out handlers_rds.GetDBBootstrapConfigOutput
|
||||
if err := g.client.Call(ctx, "GetDBBootstrapConfig", params, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
func (g *gatewayControlPlane) AcknowledgeBootstrap(ctx context.Context, id identity, pending *pendingBootstrap) error {
|
||||
params := url.Values{
|
||||
"PayloadId": {pending.payloadID},
|
||||
"VMGeneration": {strconv.FormatInt(pending.vmGeneration, 10)},
|
||||
}
|
||||
setIfPresent(params, "DBInstanceIdentifier", id.DBInstanceIdentifier)
|
||||
setIfPresent(params, "DataVolumeId", pending.dataVolumeID)
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, callTimeout)
|
||||
defer cancel()
|
||||
|
||||
var out handlers_rds.AcknowledgeDBBootstrapOutput
|
||||
if err := g.client.Call(ctx, "AcknowledgeDBBootstrap", params, &out); err != nil {
|
||||
return err
|
||||
}
|
||||
if !out.Acknowledged {
|
||||
return fmt.Errorf("the control plane did not acknowledge bootstrap payload %s", pending.payloadID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Declared here so the in-guest binary does not link the server side of the API.
|
||||
type pollDBCommandsOutput struct {
|
||||
Commands []handlers_rds.Command `xml:"Commands>member"`
|
||||
}
|
||||
|
||||
func (g *gatewayControlPlane) PollCommands(ctx context.Context, id identity, replies []handlers_rds.CommandReply, wait time.Duration) ([]handlers_rds.Command, error) {
|
||||
params := url.Values{"WaitTimeSeconds": {strconv.FormatInt(int64(wait.Seconds()), 10)}}
|
||||
setIfPresent(params, "DBInstanceIdentifier", id.DBInstanceIdentifier)
|
||||
// Replies ride the poll rather than a separate action, so replying and
|
||||
// asking for the next command are one round trip.
|
||||
for i, reply := range replies {
|
||||
member := fmt.Sprintf("Replies.member.%d.", i+1)
|
||||
params.Set(member+"CommandId", reply.CommandID)
|
||||
params.Set(member+"Status", reply.Status)
|
||||
setIfPresent(params, member+"Message", reply.Message)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, wait+pollSlack)
|
||||
defer cancel()
|
||||
|
||||
var out pollDBCommandsOutput
|
||||
if err := g.client.Call(ctx, "PollDBCommands", params, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out.Commands, nil
|
||||
}
|
||||
|
||||
// Only sets a parameter that has a value, so the gateway can tell "no
|
||||
// assertion" from "asserted as blank".
|
||||
func setIfPresent(params url.Values, key, value string) {
|
||||
if value != "" {
|
||||
params.Set(key, value)
|
||||
}
|
||||
}
|
||||
|
||||
type Agent struct {
|
||||
cfg config
|
||||
id identity
|
||||
cp controlPlane
|
||||
probe *engineProbe
|
||||
engine engine
|
||||
handoffWriter func(string, *handlers_rds.GetDBBootstrapConfigOutput) error
|
||||
dataMountWaiter func(context.Context, string, string) error
|
||||
|
||||
// Set by bootstrap when the fetch replayed a staged payload; nil leaves the
|
||||
// acknowledgement step a no-op, which is every attach boot.
|
||||
pending *pendingBootstrap
|
||||
|
||||
hb *heartbeater
|
||||
cmd *commander
|
||||
guard *paramGuard
|
||||
}
|
||||
|
||||
// Assembles from already-built parts, so tests can pass fakes; New builds the
|
||||
// production control plane and delegates here.
|
||||
func newAgent(cfg config, cp controlPlane, probe *engineProbe) (*Agent, error) {
|
||||
if cfg.MountsFile == "" {
|
||||
cfg.MountsFile = defaultMountsFile
|
||||
}
|
||||
id := identity{
|
||||
DBInstanceIdentifier: cfg.DBInstanceIdentifier,
|
||||
AgentVersion: version,
|
||||
EngineVersion: cfg.EngineVersion,
|
||||
}
|
||||
a := &Agent{
|
||||
cfg: cfg, id: id, cp: cp, probe: probe,
|
||||
handoffWriter: writeHandoff, dataMountWaiter: waitForDataMount,
|
||||
}
|
||||
eng, err := newEngine(cfg, execCommandRunner, execSessionRunner, probe)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a.engine = eng
|
||||
a.hb = newHeartbeater(cp, probe, a.engine, handlers_rds.HeartbeatInterval)
|
||||
a.cmd = newCommander(cp, newCommandRegistry(a.engine, newGuestStorage(cfg, execCommandRunner)), cfg.PollWait)
|
||||
a.guard = newParamGuard(a.engine, probe, cp)
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// New builds the agent from cfg, failing if no gateway URL is set. It does not
|
||||
// wait for IMDS: the register loop rides out a datapath still coming up.
|
||||
func New(cfg config) (*Agent, error) {
|
||||
if cfg.GatewayURL == "" {
|
||||
return nil, fmt.Errorf("no gateway URL configured (RDS_GATEWAY_URL)")
|
||||
}
|
||||
cp, err := newGatewayControlPlane(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
probe, err := newProbe(cfg, execProbeRunner)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return newAgent(cfg, cp, probe)
|
||||
}
|
||||
|
||||
func (a *Agent) Run(ctx context.Context) error {
|
||||
// Named before the first dial: when register cannot reach the gateway, the
|
||||
// address it is dialing is what separates a mgmt NIC that came up without one
|
||||
// from a control plane that is genuinely down.
|
||||
slog.Info("rds-agent: starting",
|
||||
"gatewayURL", a.cfg.GatewayURL, "dbInstanceIdentifier", a.id.DBInstanceIdentifier,
|
||||
"agentVersion", a.id.AgentVersion)
|
||||
|
||||
// Registration comes first: its response is the agent's identity.
|
||||
if err := a.register(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
a.hb.id, a.cmd.id, a.guard.id = a.id, a.id, a.id
|
||||
|
||||
// Ahead of the handoff, so rds-init never unblocks and no datadir is touched.
|
||||
// An agent running on the wrong image would initialise one engine over
|
||||
// another's data, coming up healthy and empty beside data nothing references.
|
||||
if err := a.refuseEngineMismatch(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Beating before the bootstrap keeps a stuck boot visible as a live VM with
|
||||
// a down engine rather than as silence.
|
||||
go a.hb.Run(ctx)
|
||||
|
||||
if err := a.bootstrap(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// The handoff must land before this wait: rds-datadir needs it to identify
|
||||
// and authorize the volume. Commands and the parameter guard wait because
|
||||
// their durable state belongs on that mount, never on the disposable boot disk.
|
||||
if err := a.dataMountWaiter(ctx, a.cfg.MountsFile, a.cfg.DataMount); err != nil {
|
||||
// A shutdown cancels this wait; main treats the wrapped context.Canceled
|
||||
// as a clean stop, exactly as it does for register and bootstrap above.
|
||||
return fmt.Errorf("wait for data mount %s: %w", a.cfg.DataMount, err)
|
||||
}
|
||||
|
||||
// The receipt it waits on lives on that mount, so this cannot run any earlier.
|
||||
// A cancelled wait is a clean stop here too.
|
||||
if err := a.acknowledgeBootstrap(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Directives are only meaningful against a bootstrapped, mounted engine.
|
||||
go a.cmd.Run(ctx)
|
||||
|
||||
// Started after the handoff, so the window it measures covers the engine's
|
||||
// own start rather than the time rds-init spent waiting for this agent.
|
||||
go a.guard.Run(ctx)
|
||||
|
||||
<-ctx.Done()
|
||||
return nil
|
||||
}
|
||||
|
||||
// The engine cloud-init says this VM was launched as, against the engine the
|
||||
// image bakes. An empty expectation is no assertion rather than a disagreement:
|
||||
// a VM launched before the control plane emitted RDS_ENGINE carries none.
|
||||
func (a *Agent) refuseEngineMismatch(ctx context.Context) error {
|
||||
expected := strings.ToLower(strings.TrimSpace(a.cfg.Engine))
|
||||
if expected == "" || expected == a.cfg.BakedEngine {
|
||||
return nil
|
||||
}
|
||||
|
||||
message := fmt.Sprintf("this VM was launched as engine %q but its image bakes %q; refusing to bootstrap",
|
||||
expected, a.cfg.BakedEngine)
|
||||
slog.ErrorContext(ctx, "rds-agent: "+message)
|
||||
// Reported before returning, so the control plane learns why the instance
|
||||
// never came up rather than only that it never did.
|
||||
if _, err := a.cp.SubmitState(ctx, a.id, handlers_rds.EngineHealthUnhealthy, message); err != nil {
|
||||
slog.ErrorContext(ctx, "rds-agent: reporting the engine mismatch failed", "err", err)
|
||||
}
|
||||
return errors.New(message)
|
||||
}
|
||||
|
||||
const dataMountPollInterval = time.Second
|
||||
|
||||
// waitForDataMount blocks the stateful agent loops until the kernel reports the
|
||||
// configured data mount. Registration, heartbeat, and bootstrap run before it.
|
||||
func waitForDataMount(ctx context.Context, mountsFile, target string) error {
|
||||
ticker := time.NewTicker(dataMountPollInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
mounted, err := mountTableContains(mountsFile, target)
|
||||
if err != nil {
|
||||
slog.WarnContext(ctx, "rds-agent: could not read mount table while waiting for data volume",
|
||||
"mountsFile", mountsFile, "dataMount", target, "err", err)
|
||||
} else if mounted {
|
||||
slog.InfoContext(ctx, "rds-agent: data volume mount is ready", "dataMount", target)
|
||||
return nil
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mountTableContains(path, target string) (bool, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
scanner := bufio.NewScanner(f)
|
||||
for scanner.Scan() {
|
||||
fields := strings.Fields(scanner.Text())
|
||||
if len(fields) >= 2 && unescapeMountField(fields[1]) == target {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Linux mount tables encode whitespace and backslashes as octal escapes.
|
||||
func unescapeMountField(field string) string {
|
||||
replacer := strings.NewReplacer(`\134`, `\`, `\040`, " ", `\011`, "\t", `\012`, "\n")
|
||||
return replacer.Replace(field)
|
||||
}
|
||||
|
||||
// Boot-retry bounds. Tight to start, since the usual cause is a control plane
|
||||
// still creating this instance's record, and capped so an outage does not turn
|
||||
// a booting fleet into a retry storm. Vars so tests can shorten the wait.
|
||||
var (
|
||||
retryMin = 1 * time.Second
|
||||
retryMax = 30 * time.Second
|
||||
)
|
||||
|
||||
// With the production backoff, attempt five is about 15 seconds into the
|
||||
// failure and still precedes the first 30-second heartbeat.
|
||||
const retryErrorAttempt = 5
|
||||
|
||||
// Wrapping an error in this stops retryObserved. For the failures the control
|
||||
// plane decides from its own record, no retry can change the answer, and looping
|
||||
// on one only keeps the caller blocked.
|
||||
var errRetryTerminal = errors.New("terminal failure")
|
||||
|
||||
// Never gives up on its own: an agent that stopped trying would leave the
|
||||
// control plane with no signal at all.
|
||||
func retry(ctx context.Context, what string, fn func(context.Context) error) error {
|
||||
return retryObserved(ctx, what, fn, nil)
|
||||
}
|
||||
|
||||
// onFailure lets boot-critical callers publish the latest error while the retry
|
||||
// remains blocked. It must not block because it runs on the retry path itself.
|
||||
func retryObserved(ctx context.Context, what string, fn func(context.Context) error, onFailure func(error)) error {
|
||||
delay := retryMin
|
||||
for attempt := 1; ; attempt++ {
|
||||
err := fn(ctx)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return fmt.Errorf("%s: %w", what, ctx.Err())
|
||||
}
|
||||
if onFailure != nil {
|
||||
onFailure(err)
|
||||
}
|
||||
if errors.Is(err, errRetryTerminal) {
|
||||
slog.ErrorContext(ctx, "rds-agent: "+what+" failed terminally, not retrying", "err", err)
|
||||
return fmt.Errorf("%s: %w", what, err)
|
||||
}
|
||||
if attempt >= retryErrorAttempt {
|
||||
slog.ErrorContext(ctx, "rds-agent: "+what+" still failing, retrying",
|
||||
"attempt", attempt, "retry_in_ms", otelsetup.Millis(delay), "err", err)
|
||||
} else {
|
||||
slog.WarnContext(ctx, "rds-agent: "+what+" failed, retrying",
|
||||
"attempt", attempt, "retry_in_ms", otelsetup.Millis(delay), "err", err)
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("%s: %w", what, ctx.Err())
|
||||
case <-time.After(delay):
|
||||
}
|
||||
delay = min(delay*2, retryMax)
|
||||
}
|
||||
}
|
||||
@@ -1,873 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/gwsign"
|
||||
"github.com/mulgadc/spinifex/internal/rdsgw"
|
||||
"github.com/mulgadc/spinifex/spinifex/awserrors"
|
||||
gateway_rds "github.com/mulgadc/spinifex/spinifex/gateway/rds"
|
||||
handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
|
||||
"github.com/mulgadc/spinifex/spinifex/utils"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// fakeControlPlane is a scriptable controlPlane. Every call records what the
|
||||
// agent sent, so a test asserts on the request as well as the reaction.
|
||||
type fakeControlPlane struct {
|
||||
mu sync.Mutex
|
||||
|
||||
registerOut *handlers_rds.RegisterDBInstanceOutput
|
||||
registerErrs []error
|
||||
registerReqs []identity
|
||||
|
||||
bootstrapOut *handlers_rds.GetDBBootstrapConfigOutput
|
||||
bootstrapErrs []error
|
||||
bootstrapReqs []identity
|
||||
// Scripted responses ahead of bootstrapOut, so a test can answer the first
|
||||
// fetch differently from the ones the agent retries with.
|
||||
bootstrapQueue []*handlers_rds.GetDBBootstrapConfigOutput
|
||||
|
||||
ackErrs []error
|
||||
ackReqs []pendingBootstrap
|
||||
|
||||
states []submittedState
|
||||
|
||||
pollReplies [][]handlers_rds.CommandReply
|
||||
pollQueue [][]handlers_rds.Command
|
||||
pollErr error
|
||||
polled chan struct{}
|
||||
}
|
||||
|
||||
type submittedState struct {
|
||||
id identity
|
||||
health handlers_rds.EngineHealth
|
||||
message string
|
||||
}
|
||||
|
||||
var _ controlPlane = (*fakeControlPlane)(nil)
|
||||
|
||||
func newFakeControlPlane() *fakeControlPlane {
|
||||
return &fakeControlPlane{
|
||||
registerOut: &handlers_rds.RegisterDBInstanceOutput{
|
||||
DBInstanceIdentifier: "db-resolved", HeartbeatIntervalSeconds: 30,
|
||||
},
|
||||
polled: make(chan struct{}, 64),
|
||||
}
|
||||
}
|
||||
|
||||
// nextErr pops the next scripted error, so a test can fail a call n times and
|
||||
// then let it succeed.
|
||||
func nextErr(errs *[]error) error {
|
||||
if len(*errs) == 0 {
|
||||
return nil
|
||||
}
|
||||
err := (*errs)[0]
|
||||
*errs = (*errs)[1:]
|
||||
return err
|
||||
}
|
||||
|
||||
func (f *fakeControlPlane) Register(_ context.Context, id identity) (*handlers_rds.RegisterDBInstanceOutput, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.registerReqs = append(f.registerReqs, id)
|
||||
if err := nextErr(&f.registerErrs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.registerOut, nil
|
||||
}
|
||||
|
||||
func (f *fakeControlPlane) SubmitState(_ context.Context, id identity, health handlers_rds.EngineHealth, message string) (*handlers_rds.SubmitDBStateChangeOutput, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.states = append(f.states, submittedState{id: id, health: health, message: message})
|
||||
// Zero means "keep the cadence you have", so a test's fast interval is not
|
||||
// reset by every beat.
|
||||
return &handlers_rds.SubmitDBStateChangeOutput{Acknowledged: true}, nil
|
||||
}
|
||||
|
||||
func (f *fakeControlPlane) GetBootstrapConfig(_ context.Context, id identity) (*handlers_rds.GetDBBootstrapConfigOutput, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.bootstrapReqs = append(f.bootstrapReqs, id)
|
||||
if err := nextErr(&f.bootstrapErrs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(f.bootstrapQueue) > 0 {
|
||||
out := f.bootstrapQueue[0]
|
||||
f.bootstrapQueue = f.bootstrapQueue[1:]
|
||||
return out, nil
|
||||
}
|
||||
return f.bootstrapOut, nil
|
||||
}
|
||||
|
||||
func (f *fakeControlPlane) AcknowledgeBootstrap(_ context.Context, _ identity, pending *pendingBootstrap) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.ackReqs = append(f.ackReqs, *pending)
|
||||
return nextErr(&f.ackErrs)
|
||||
}
|
||||
|
||||
func (f *fakeControlPlane) PollCommands(ctx context.Context, _ identity, replies []handlers_rds.CommandReply, _ time.Duration) ([]handlers_rds.Command, error) {
|
||||
f.mu.Lock()
|
||||
f.pollReplies = append(f.pollReplies, slices.Clone(replies))
|
||||
err := f.pollErr
|
||||
var out []handlers_rds.Command
|
||||
if len(f.pollQueue) > 0 {
|
||||
out, f.pollQueue = f.pollQueue[0], f.pollQueue[1:]
|
||||
}
|
||||
f.mu.Unlock()
|
||||
|
||||
select {
|
||||
case f.polled <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(out) > 0 {
|
||||
return out, nil
|
||||
}
|
||||
// An idle window: block until the agent hangs up, as the gateway's long poll
|
||||
// does, rather than spinning the caller's loop.
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
|
||||
func (f *fakeControlPlane) ackRequests() []pendingBootstrap {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return slices.Clone(f.ackReqs)
|
||||
}
|
||||
|
||||
func (f *fakeControlPlane) snapshotStates() []submittedState {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([]submittedState(nil), f.states...)
|
||||
}
|
||||
|
||||
func (f *fakeControlPlane) snapshotReplies() [][]handlers_rds.CommandReply {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
out := make([][]handlers_rds.CommandReply, len(f.pollReplies))
|
||||
for i := range f.pollReplies {
|
||||
out[i] = slices.Clone(f.pollReplies[i])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// testConfig stamps the engine a real rds-postgres image bakes, so the agent
|
||||
// resolves the same guest layout it would in a guest, and points its file output
|
||||
// at a temp dir.
|
||||
func testConfig(t *testing.T) config {
|
||||
t.Helper()
|
||||
cfg := testLoadConfig(t, enginePostgres)
|
||||
cfg.GatewayURL = "https://gw.test"
|
||||
cfg.HandoffDir = filepath.Join(t.TempDir(), "spinifex-rds")
|
||||
cfg.PollWait = time.Second
|
||||
return cfg
|
||||
}
|
||||
|
||||
// newTestAgent builds the agent over a probe the test drives directly.
|
||||
func newTestAgent(t *testing.T, cfg config, cp controlPlane, run probeRunner) *Agent {
|
||||
t.Helper()
|
||||
a, err := newAgent(cfg, cp, newPostgresProbe(cfg, run))
|
||||
if err != nil {
|
||||
t.Fatalf("newAgent: %v", err)
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// staticProbe is a probeRunner that always reports the same exit code.
|
||||
func staticProbe(code int) probeRunner {
|
||||
return func(context.Context, string, ...string) (int, string, error) { return code, "", nil }
|
||||
}
|
||||
|
||||
func bootstrapOutput(password *string) *handlers_rds.GetDBBootstrapConfigOutput {
|
||||
mode := handlers_rds.BootstrapModeAttach
|
||||
if password != nil {
|
||||
mode = handlers_rds.BootstrapModeInitialize
|
||||
}
|
||||
return &handlers_rds.GetDBBootstrapConfigOutput{
|
||||
Mode: mode,
|
||||
DBInstanceIdentifier: "db-resolved",
|
||||
Engine: "postgres",
|
||||
MasterUsername: "master",
|
||||
MasterUserPassword: password,
|
||||
Port: 5432,
|
||||
ListenAddress: testListenAddress,
|
||||
ClientCIDR: testClientCIDR,
|
||||
DataVolumeID: "vol-data-01",
|
||||
DataVolumeSerial: "voldata01",
|
||||
VMGeneration: 1,
|
||||
PayloadID: testPayloadID,
|
||||
BootstrapPending: password != nil,
|
||||
}
|
||||
}
|
||||
|
||||
const testPayloadID = "bp-0123456789abcdef"
|
||||
|
||||
// The customer ENI address the engine binds, and the VPC range its client
|
||||
// authentication admits. writeHandoff refuses a config without either.
|
||||
const (
|
||||
testListenAddress = "10.0.1.20"
|
||||
testClientCIDR = "10.0.0.0/16"
|
||||
)
|
||||
|
||||
// Writes the completion receipt rds-init leaves behind, so the acknowledgement
|
||||
// step finds what a finished initialization would have produced.
|
||||
func writeReceipt(t *testing.T, a *Agent, payloadID, dbInstanceIdentifier string) {
|
||||
t.Helper()
|
||||
path := a.receiptPath()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
t.Fatalf("mkdir receipt dir: %v", err)
|
||||
}
|
||||
body := fmt.Sprintf("%s=%s\n%s=%s\n", receiptPayload, payloadID, receiptDBID, dbInstanceIdentifier)
|
||||
if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
|
||||
t.Fatalf("write receipt: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// runAgent runs the agent until its bootstrap has landed, then cancels. It
|
||||
// returns once Run has exited so no goroutine outlives the test.
|
||||
func runAgent(t *testing.T, a *Agent) error {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- a.Run(ctx) }()
|
||||
|
||||
waitFor(t, func() bool {
|
||||
_, err := os.Stat(filepath.Join(a.cfg.HandoffDir, handoffEnvFile))
|
||||
return err == nil
|
||||
}, "handoff file to be written")
|
||||
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
// The cancellation above is this helper's own, so Run reporting it is the
|
||||
// expected clean stop rather than a failure, exactly as main treats it.
|
||||
if errors.Is(err, context.Canceled) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Run did not exit after cancellation")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// waitFor polls cond until it holds or the test times out.
|
||||
func waitFor(t *testing.T, cond func() bool, what string) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if cond() {
|
||||
return
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("timed out waiting for %s", what)
|
||||
}
|
||||
|
||||
func TestRun_RegistersThenDeliversBootstrap(t *testing.T) {
|
||||
password := "s3cret"
|
||||
cp := newFakeControlPlane()
|
||||
cp.bootstrapOut = bootstrapOutput(&password)
|
||||
|
||||
cfg := testConfig(t)
|
||||
a := newTestAgent(t, cfg, cp, staticProbe(0))
|
||||
if err := runAgent(t, a); err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
|
||||
if len(cp.registerReqs) != 1 {
|
||||
t.Fatalf("registered %d times, want 1", len(cp.registerReqs))
|
||||
}
|
||||
// The agent had no configured identifier, so the gateway's resolution is
|
||||
// what every later call must carry.
|
||||
if got := cp.bootstrapReqs[0].DBInstanceIdentifier; got != "db-resolved" {
|
||||
t.Errorf("bootstrap sent identifier %q, want the registered db-resolved", got)
|
||||
}
|
||||
|
||||
env := readFile(t, filepath.Join(cfg.HandoffDir, handoffEnvFile))
|
||||
if !strings.Contains(env, "RDS_MODE='initialize'") || !strings.Contains(env, "RDS_MASTER_PASSWORD='s3cret'") {
|
||||
t.Errorf("handoff env = %q, want initialize mode carrying the password", env)
|
||||
}
|
||||
// The probe follows the port the control plane assigned, not the default.
|
||||
if got := a.probe.port.Load(); got != 5432 {
|
||||
t.Errorf("probe port = %d, want 5432", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A reboot re-fetches in attach mode. The agent must write that payload as it
|
||||
// stands rather than refusing it or inventing a password.
|
||||
func TestRun_AttachModeNeedsNoPassword(t *testing.T) {
|
||||
cp := newFakeControlPlane()
|
||||
cp.bootstrapOut = bootstrapOutput(nil)
|
||||
|
||||
cfg := testConfig(t)
|
||||
if err := runAgent(t, newTestAgent(t, cfg, cp, staticProbe(0))); err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
|
||||
env := readFile(t, filepath.Join(cfg.HandoffDir, handoffEnvFile))
|
||||
if !strings.Contains(env, "RDS_MODE='attach'") {
|
||||
t.Errorf("handoff env = %q, want attach mode", env)
|
||||
}
|
||||
if strings.Contains(env, "RDS_MASTER_PASSWORD") {
|
||||
t.Errorf("handoff env = %q, want no password key at all in attach mode", env)
|
||||
}
|
||||
}
|
||||
|
||||
// The record may not exist yet when a freshly launched VM first calls, so both
|
||||
// boot-critical calls have to ride out failures instead of exiting.
|
||||
func TestRun_RetriesRegisterAndBootstrap(t *testing.T) {
|
||||
cp := newFakeControlPlane()
|
||||
cp.bootstrapOut = bootstrapOutput(nil)
|
||||
cp.registerErrs = []error{errors.New("DBInstanceNotFound")}
|
||||
cp.bootstrapErrs = []error{errors.New("ServerInternal")}
|
||||
|
||||
cfg := testConfig(t)
|
||||
if err := runAgent(t, newTestAgent(t, cfg, cp, staticProbe(0))); err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
|
||||
if len(cp.registerReqs) != 2 {
|
||||
t.Errorf("register attempts = %d, want 2 (one failure, one success)", len(cp.registerReqs))
|
||||
}
|
||||
if len(cp.bootstrapReqs) != 2 {
|
||||
t.Errorf("bootstrap attempts = %d, want 2 (one failure, one success)", len(cp.bootstrapReqs))
|
||||
}
|
||||
}
|
||||
|
||||
// The fetch mutates nothing, so a handoff that cannot be written is retried
|
||||
// against the payload already in hand rather than costing a second fetch.
|
||||
func TestBootstrap_RetriesHandoffWithoutRefetching(t *testing.T) {
|
||||
cp := newFakeControlPlane()
|
||||
password := "staged-secret"
|
||||
cp.bootstrapOut = bootstrapOutput(&password)
|
||||
cfg := testConfig(t)
|
||||
a := newTestAgent(t, cfg, cp, staticProbe(0))
|
||||
|
||||
writes := 0
|
||||
a.handoffWriter = func(_ string, got *handlers_rds.GetDBBootstrapConfigOutput) error {
|
||||
writes++
|
||||
if got.MasterUserPassword == nil || *got.MasterUserPassword != password {
|
||||
t.Fatalf("handoff attempt %d lost the initialize password", writes)
|
||||
}
|
||||
if writes == 1 {
|
||||
return errors.New("temporary tmpfs write error")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := a.bootstrap(t.Context()); err != nil {
|
||||
t.Fatalf("bootstrap: %v", err)
|
||||
}
|
||||
if len(cp.bootstrapReqs) != 1 {
|
||||
t.Errorf("bootstrap fetches = %d, want 1", len(cp.bootstrapReqs))
|
||||
}
|
||||
if writes != 2 {
|
||||
t.Errorf("handoff writes = %d, want 2", writes)
|
||||
}
|
||||
if a.pending == nil || a.pending.payloadID != testPayloadID {
|
||||
t.Errorf("pending bootstrap = %+v, want the staged payload %s", a.pending, testPayloadID)
|
||||
}
|
||||
}
|
||||
|
||||
// A daemon that predates the encrypted payload answers initialize with nothing
|
||||
// to initialise with. Writing that handoff would let rds-init run initdb with an
|
||||
// empty master password, so it is retried until an upgraded node answers.
|
||||
func TestBootstrap_RetriesInitializeWithNoPassword(t *testing.T) {
|
||||
cp := newFakeControlPlane()
|
||||
empty := ""
|
||||
password := "staged-secret"
|
||||
cp.bootstrapQueue = []*handlers_rds.GetDBBootstrapConfigOutput{{
|
||||
Mode: handlers_rds.BootstrapModeInitialize, DBInstanceIdentifier: "db-resolved",
|
||||
MasterUsername: "master", MasterUserPassword: &empty,
|
||||
}}
|
||||
cp.bootstrapOut = bootstrapOutput(&password)
|
||||
cfg := testConfig(t)
|
||||
a := newTestAgent(t, cfg, cp, staticProbe(0))
|
||||
|
||||
var handed []*handlers_rds.GetDBBootstrapConfigOutput
|
||||
a.handoffWriter = func(_ string, got *handlers_rds.GetDBBootstrapConfigOutput) error {
|
||||
handed = append(handed, got)
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := a.bootstrap(t.Context()); err != nil {
|
||||
t.Fatalf("bootstrap: %v", err)
|
||||
}
|
||||
if len(handed) != 1 {
|
||||
t.Fatalf("handoff writes = %d, want 1 once a usable payload arrived", len(handed))
|
||||
}
|
||||
if handed[0].MasterUserPassword == nil || *handed[0].MasterUserPassword != password {
|
||||
t.Error("the agent wrote a handoff for a fetch that carried no master password")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcknowledgeBootstrap_WaitsForTheReceiptThenRetiresIt(t *testing.T) {
|
||||
cp := newFakeControlPlane()
|
||||
cfg := testConfig(t)
|
||||
cfg.DataMount = t.TempDir()
|
||||
a := newTestAgent(t, cfg, cp, staticProbe(0))
|
||||
a.id = identity{DBInstanceIdentifier: "db-resolved"}
|
||||
a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1, dataVolumeID: "vol-data-01"}
|
||||
|
||||
// rds-init runs after the agent registers, so the receipt is absent when the
|
||||
// acknowledgement step first looks and appears some way into the boot.
|
||||
if err := a.checkBootstrapReceipt(a.pending); err == nil {
|
||||
t.Fatal("a missing receipt must not read as a completed bootstrap")
|
||||
}
|
||||
writeReceipt(t, a, testPayloadID, "db-resolved")
|
||||
|
||||
if err := a.acknowledgeBootstrap(t.Context()); err != nil {
|
||||
t.Fatalf("acknowledgeBootstrap: %v", err)
|
||||
}
|
||||
if len(cp.ackRequests()) != 1 {
|
||||
t.Fatalf("acknowledgements = %d, want 1", len(cp.ackRequests()))
|
||||
}
|
||||
if got := cp.ackRequests()[0]; got != *a.pending {
|
||||
t.Errorf("acknowledged %+v, want %+v", got, *a.pending)
|
||||
}
|
||||
if _, err := os.Stat(a.receiptPath()); !os.IsNotExist(err) {
|
||||
t.Error("the receipt must be removed once the record holds the audit trail")
|
||||
}
|
||||
}
|
||||
|
||||
// The receipt rides along in every snapshot of the data volume, so a restored
|
||||
// instance's volume carries the source instance's receipt. Acknowledging on one
|
||||
// would destroy the ciphertext this instance still needs.
|
||||
func TestAcknowledgeBootstrap_IgnoresAForeignReceipt(t *testing.T) {
|
||||
tests := map[string]struct{ payloadID, dbID string }{
|
||||
"another payload": {payloadID: "bp-somebody-else", dbID: "db-resolved"},
|
||||
"another instance": {payloadID: testPayloadID, dbID: "db-the-snapshot-source"},
|
||||
}
|
||||
|
||||
for name, tc := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
cp := newFakeControlPlane()
|
||||
cfg := testConfig(t)
|
||||
cfg.DataMount = t.TempDir()
|
||||
a := newTestAgent(t, cfg, cp, staticProbe(0))
|
||||
a.id = identity{DBInstanceIdentifier: "db-resolved"}
|
||||
a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1}
|
||||
writeReceipt(t, a, tc.payloadID, tc.dbID)
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond)
|
||||
defer cancel()
|
||||
if err := a.acknowledgeBootstrap(ctx); err == nil {
|
||||
t.Fatal("acknowledgeBootstrap returned nil, want it still waiting on a real receipt")
|
||||
}
|
||||
if n := len(cp.ackRequests()); n != 0 {
|
||||
t.Errorf("acknowledgements = %d, want 0", n)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// An attach boot has nothing staged, so it must not go looking for a receipt the
|
||||
// datadir will never grow.
|
||||
func TestAcknowledgeBootstrap_IsANoOpOnAttach(t *testing.T) {
|
||||
cp := newFakeControlPlane()
|
||||
cfg := testConfig(t)
|
||||
cfg.DataMount = t.TempDir()
|
||||
a := newTestAgent(t, cfg, cp, staticProbe(0))
|
||||
|
||||
if err := a.acknowledgeBootstrap(t.Context()); err != nil {
|
||||
t.Fatalf("acknowledgeBootstrap: %v", err)
|
||||
}
|
||||
if n := len(cp.ackRequests()); n != 0 {
|
||||
t.Errorf("acknowledgements = %d, want 0 when nothing was staged", n)
|
||||
}
|
||||
}
|
||||
|
||||
// The image is the authority on which engine the agent runs. A VM launched as
|
||||
// another engine has to stop ahead of the handoff: rds-init would otherwise
|
||||
// initialise one engine over the other's datadir and come up healthy and empty
|
||||
// beside data nothing references.
|
||||
func TestRun_RefusesAnEngineTheImageDoesNotBake(t *testing.T) {
|
||||
cp := newFakeControlPlane()
|
||||
cp.bootstrapOut = bootstrapOutput(nil)
|
||||
|
||||
cfg := testConfig(t)
|
||||
cfg.Engine = "mariadb"
|
||||
a := newTestAgent(t, cfg, cp, staticProbe(0))
|
||||
|
||||
err := a.Run(t.Context())
|
||||
if err == nil {
|
||||
t.Fatal("Run returned nil for a VM launched as an engine this image does not bake")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "mariadb") {
|
||||
t.Errorf("error = %v, want it to name the engine the VM was launched as", err)
|
||||
}
|
||||
if _, statErr := os.Stat(filepath.Join(cfg.HandoffDir, handoffEnvFile)); !os.IsNotExist(statErr) {
|
||||
t.Error("the agent wrote a bootstrap handoff for an engine it refused to run")
|
||||
}
|
||||
if n := len(cp.bootstrapReqs); n != 0 {
|
||||
t.Errorf("bootstrap fetches = %d, want none before the refusal", n)
|
||||
}
|
||||
// Reported rather than only logged, so the control plane learns why the
|
||||
// instance never came up rather than only that it never did.
|
||||
states := cp.snapshotStates()
|
||||
if len(states) != 1 || states[0].health != handlers_rds.EngineHealthUnhealthy {
|
||||
t.Fatalf("submitted states = %+v, want a single unhealthy report", states)
|
||||
}
|
||||
}
|
||||
|
||||
// The same assertion when it agrees is not a gate: the boot goes on exactly as
|
||||
// it does for a VM launched before the control plane sent one at all.
|
||||
func TestRun_ProceedsWhenTheDeliveredEngineMatchesTheImage(t *testing.T) {
|
||||
cp := newFakeControlPlane()
|
||||
cp.bootstrapOut = bootstrapOutput(nil)
|
||||
|
||||
cfg := testConfig(t)
|
||||
cfg.Engine = enginePostgres
|
||||
if err := runAgent(t, newTestAgent(t, cfg, cp, staticProbe(0))); err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A configured identifier is asserted on the wire so a mis-provisioned VM is
|
||||
// rejected by the gateway instead of adopting whatever it resolves to.
|
||||
func TestRun_SendsConfiguredIdentifier(t *testing.T) {
|
||||
cp := newFakeControlPlane()
|
||||
cp.bootstrapOut = bootstrapOutput(nil)
|
||||
|
||||
cfg := testConfig(t)
|
||||
cfg.DBInstanceIdentifier = "db-configured"
|
||||
cfg.EngineVersion = "18.1"
|
||||
if err := runAgent(t, newTestAgent(t, cfg, cp, staticProbe(0))); err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
|
||||
sent := cp.registerReqs[0]
|
||||
if sent.DBInstanceIdentifier != "db-configured" || sent.EngineVersion != "18.1" {
|
||||
t.Errorf("register sent %+v, want the configured identifier and engine version", sent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRun_WaitsForDataMountBeforePollingCommands(t *testing.T) {
|
||||
cp := newFakeControlPlane()
|
||||
cp.bootstrapOut = bootstrapOutput(nil)
|
||||
cfg := testConfig(t)
|
||||
a := newTestAgent(t, cfg, cp, staticProbe(2))
|
||||
|
||||
waiting := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
a.dataMountWaiter = func(ctx context.Context, mountsFile, target string) error {
|
||||
if mountsFile != defaultMountsFile || target != cfg.DataMount {
|
||||
t.Errorf("mount waiter got %q/%q, want configured defaults", mountsFile, target)
|
||||
}
|
||||
close(waiting)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-release:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- a.Run(ctx) }()
|
||||
|
||||
select {
|
||||
case <-waiting:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("agent did not reach the data-mount wait after writing the handoff")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(cfg.HandoffDir, handoffEnvFile)); err != nil {
|
||||
t.Fatalf("handoff was not present before mount wait: %v", err)
|
||||
}
|
||||
select {
|
||||
case <-cp.polled:
|
||||
t.Fatal("command poll started before the data volume was mounted")
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
|
||||
close(release)
|
||||
select {
|
||||
case <-cp.polled:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("command poll did not start after the data mount became ready")
|
||||
}
|
||||
cancel()
|
||||
if err := <-done; err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMountTableContains(t *testing.T) {
|
||||
mounts := filepath.Join(t.TempDir(), "mounts")
|
||||
require.NoError(t, os.WriteFile(mounts, []byte(
|
||||
"/dev/vda1 / ext4 rw 0 0\n/dev/vdb /var/lib/postgresql ext4 rw 0 0\n"+
|
||||
"/dev/vdc /path\\040with\\040spaces xfs rw 0 0\nmalformed\n"), 0o600))
|
||||
|
||||
tests := []struct {
|
||||
target string
|
||||
want bool
|
||||
}{
|
||||
{target: "/var/lib/postgresql", want: true},
|
||||
{target: "/path with spaces", want: true},
|
||||
{target: "/missing", want: false},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.target, func(t *testing.T) {
|
||||
got, err := mountTableContains(mounts, tc.target)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRun_HeartbeatIncludesBootstrapFailure(t *testing.T) {
|
||||
cp := newFakeControlPlane()
|
||||
cp.bootstrapErrs = []error{errors.New("403 AccessDenied: instance profile is not authorised")}
|
||||
cp.registerOut.HeartbeatIntervalSeconds = 0
|
||||
|
||||
cfg := testConfig(t)
|
||||
a := newTestAgent(t, cfg, cp, staticProbe(2))
|
||||
a.hb.interval = 10 * time.Millisecond
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- a.Run(ctx) }()
|
||||
|
||||
waitFor(t, func() bool {
|
||||
for _, state := range cp.snapshotStates() {
|
||||
if strings.Contains(state.message, "bootstrap fetch failed") &&
|
||||
strings.Contains(state.message, "403 AccessDenied") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}, "the bootstrap failure to reach a heartbeat")
|
||||
cancel()
|
||||
<-done
|
||||
}
|
||||
|
||||
func TestRun_HeartbeatsEngineHealth(t *testing.T) {
|
||||
cp := newFakeControlPlane()
|
||||
cp.bootstrapOut = bootstrapOutput(nil)
|
||||
// Declining to set a cadence leaves the agent's own, which the test shortens
|
||||
// rather than waiting out the 30s production interval.
|
||||
cp.registerOut.HeartbeatIntervalSeconds = 0
|
||||
|
||||
cfg := testConfig(t)
|
||||
a := newTestAgent(t, cfg, cp, staticProbe(0))
|
||||
a.hb.interval = 10 * time.Millisecond
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- a.Run(ctx) }()
|
||||
|
||||
waitFor(t, func() bool { return len(cp.snapshotStates()) > 0 }, "a heartbeat")
|
||||
cancel()
|
||||
<-done
|
||||
|
||||
beat := cp.snapshotStates()[0]
|
||||
if beat.health != handlers_rds.EngineHealthHealthy {
|
||||
t.Errorf("heartbeat health = %q, want healthy", beat.health)
|
||||
}
|
||||
if beat.id.DBInstanceIdentifier != "db-resolved" {
|
||||
t.Errorf("heartbeat identifier = %q, want db-resolved", beat.id.DBInstanceIdentifier)
|
||||
}
|
||||
}
|
||||
|
||||
// The cadence is the control plane's to set: it feeds the staleness window the
|
||||
// recovery reconciler judges an instance against, so the guest adopts it rather
|
||||
// than keeping its own.
|
||||
func TestRegister_AdoptsControlPlaneCadence(t *testing.T) {
|
||||
cp := newFakeControlPlane()
|
||||
cp.bootstrapOut = bootstrapOutput(nil)
|
||||
cp.registerOut.HeartbeatIntervalSeconds = 45
|
||||
|
||||
cfg := testConfig(t)
|
||||
a := newTestAgent(t, cfg, cp, staticProbe(0))
|
||||
if err := runAgent(t, a); err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if a.hb.interval != 45*time.Second {
|
||||
t.Errorf("heartbeat interval = %v, want the 45s the control plane returned", a.hb.interval)
|
||||
}
|
||||
}
|
||||
|
||||
// The agent being up says nothing about the engine. A down engine has to reach
|
||||
// the control plane as such, or auto-recovery has nothing to act on.
|
||||
func TestRun_HeartbeatReportsDownEngine(t *testing.T) {
|
||||
cp := newFakeControlPlane()
|
||||
cp.bootstrapOut = bootstrapOutput(nil)
|
||||
cp.registerOut.HeartbeatIntervalSeconds = 0
|
||||
|
||||
cfg := testConfig(t)
|
||||
probe := newPostgresProbe(cfg, staticProbe(2))
|
||||
// The engine has served before, so a silent one now is a failure rather
|
||||
// than a boot still in progress.
|
||||
probe.seenHealthy = true
|
||||
|
||||
a, err := newAgent(cfg, cp, probe)
|
||||
if err != nil {
|
||||
t.Fatalf("newAgent: %v", err)
|
||||
}
|
||||
a.hb.interval = 10 * time.Millisecond
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- a.Run(ctx) }()
|
||||
|
||||
waitFor(t, func() bool { return len(cp.snapshotStates()) > 0 }, "a heartbeat")
|
||||
cancel()
|
||||
<-done
|
||||
|
||||
beat := cp.snapshotStates()[0]
|
||||
if beat.health != handlers_rds.EngineHealthUnhealthy {
|
||||
t.Errorf("heartbeat health = %q, want unhealthy while the agent is up", beat.health)
|
||||
}
|
||||
if beat.message == "" {
|
||||
t.Error("unhealthy heartbeat carried no message explaining the probe result")
|
||||
}
|
||||
}
|
||||
|
||||
// The mirror the agent decodes poll responses into has to stay aligned with the
|
||||
// gateway type that produces them; nothing else in the build ties the two.
|
||||
func TestPollDBCommandsOutput_MatchesGatewayShape(t *testing.T) {
|
||||
issued := time.Date(2026, 7, 28, 1, 2, 3, 0, time.UTC)
|
||||
// Rendered exactly as the gateway's typed adapter renders it, so a change to
|
||||
// either side's tags fails here.
|
||||
payload := utils.GenerateIAMXMLPayload("PollDBCommands", &gateway_rds.PollDBCommandsOutput{
|
||||
Commands: []handlers_rds.Command{{
|
||||
CommandID: "cmd-1",
|
||||
Type: "reload-parameters",
|
||||
Parameters: []handlers_rds.Parameter{{Name: "work_mem", Value: "8MB"}},
|
||||
IssuedAt: &issued,
|
||||
}},
|
||||
})
|
||||
body, err := utils.MarshalToXML(payload)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal poll result: %v", err)
|
||||
}
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/xml")
|
||||
_, _ = w.Write(body)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
client, err := rdsgw.New(srv.URL, "", gwsign.NewStatic("AKIATEST", "secret"), "us-east-1", time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("rdsgw.New: %v", err)
|
||||
}
|
||||
|
||||
var got pollDBCommandsOutput
|
||||
if err := client.Call(context.Background(), "PollDBCommands", nil, &got); err != nil {
|
||||
t.Fatalf("Call: %v", err)
|
||||
}
|
||||
if len(got.Commands) != 1 {
|
||||
t.Fatalf("decoded %d commands, want 1", len(got.Commands))
|
||||
}
|
||||
cmd := got.Commands[0]
|
||||
if cmd.CommandID != "cmd-1" || cmd.Type != "reload-parameters" {
|
||||
t.Errorf("command = %+v, want cmd-1/reload-parameters", cmd)
|
||||
}
|
||||
if len(cmd.Parameters) != 1 || cmd.Parameters[0].Value != "8MB" {
|
||||
t.Errorf("Parameters = %+v, want the seeded member", cmd.Parameters)
|
||||
}
|
||||
if cmd.IssuedAt == nil || !cmd.IssuedAt.Equal(issued) {
|
||||
t.Errorf("IssuedAt = %v, want %v", cmd.IssuedAt, issued)
|
||||
}
|
||||
}
|
||||
|
||||
func readFile(t *testing.T, path string) string {
|
||||
t.Helper()
|
||||
body, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", path, err)
|
||||
}
|
||||
return string(body)
|
||||
}
|
||||
|
||||
// AccessDenied is decided by the control plane's own record, so no retry can
|
||||
// change it. Looping on one would leave the command poller and the parameter
|
||||
// guard unstarted for the life of a VM whose engine is serving normally.
|
||||
func TestAcknowledgeBootstrap_StopsOnATerminalDenial(t *testing.T) {
|
||||
cp := newFakeControlPlane()
|
||||
cp.ackErrs = []error{&rdsgw.APIError{
|
||||
Action: "AcknowledgeDBBootstrap", StatusCode: 403,
|
||||
Code: awserrors.ErrorAccessDenied,
|
||||
Message: "bootstrap payload bp-x at generation 1 does not match the current state",
|
||||
}}
|
||||
cfg := testConfig(t)
|
||||
cfg.DataMount = t.TempDir()
|
||||
a := newTestAgent(t, cfg, cp, staticProbe(0))
|
||||
a.id = identity{DBInstanceIdentifier: "db-resolved"}
|
||||
a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1}
|
||||
writeReceipt(t, a, testPayloadID, "db-resolved")
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
if err := a.acknowledgeBootstrap(ctx); err != nil {
|
||||
t.Fatalf("acknowledgeBootstrap = %v, want nil so the stateful loops still start", err)
|
||||
}
|
||||
if n := len(cp.ackRequests()); n != 1 {
|
||||
t.Errorf("acknowledgements = %d, want 1 attempt and no retry", n)
|
||||
}
|
||||
if a.hb.bootstrapFailure.Load() == nil {
|
||||
t.Error("a terminal denial must still reach the operator through the heartbeat")
|
||||
}
|
||||
if _, err := os.Stat(a.receiptPath()); err != nil {
|
||||
t.Error("a denied acknowledgement must leave the receipt for a later attempt")
|
||||
}
|
||||
}
|
||||
|
||||
// rds-init runs after this agent, so the receipt is absent for the whole initdb
|
||||
// window. Reporting that would put a failure on every healthy create's beat, and
|
||||
// then into the reason a create that timed out for some other cause records.
|
||||
func TestAcknowledgeBootstrap_DoesNotReportAnAbsentReceiptAsAFailure(t *testing.T) {
|
||||
cp := newFakeControlPlane()
|
||||
cfg := testConfig(t)
|
||||
cfg.DataMount = t.TempDir()
|
||||
a := newTestAgent(t, cfg, cp, staticProbe(0))
|
||||
a.id = identity{DBInstanceIdentifier: "db-resolved"}
|
||||
a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1}
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond)
|
||||
defer cancel()
|
||||
if err := a.acknowledgeBootstrap(ctx); err == nil {
|
||||
t.Fatal("acknowledgeBootstrap returned nil, want it still waiting for rds-init")
|
||||
}
|
||||
if got := a.hb.bootstrapFailure.Load(); got != nil {
|
||||
t.Errorf("bootstrap failure = %q, want none while the receipt has simply not been written", *got)
|
||||
}
|
||||
|
||||
// A receipt that exists but names another instance is a real mismatch and is
|
||||
// reported, which is what separates the two cases.
|
||||
writeReceipt(t, a, testPayloadID, "db-the-snapshot-source")
|
||||
mismatch, cancelMismatch := context.WithTimeout(t.Context(), 50*time.Millisecond)
|
||||
defer cancelMismatch()
|
||||
if err := a.acknowledgeBootstrap(mismatch); err == nil {
|
||||
t.Fatal("a foreign receipt must not be acknowledged")
|
||||
}
|
||||
if a.hb.bootstrapFailure.Load() == nil {
|
||||
t.Error("a receipt naming another instance must be reported")
|
||||
}
|
||||
}
|
||||
@@ -1,237 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
|
||||
)
|
||||
|
||||
// The staged payload this boot has to acknowledge once PostgreSQL has applied
|
||||
// it. Nil on an attach boot, which makes the acknowledgement step a no-op.
|
||||
type pendingBootstrap struct {
|
||||
payloadID string
|
||||
vmGeneration int64
|
||||
dataVolumeID string
|
||||
}
|
||||
|
||||
// Fetches the boot material and writes the handoff rds-init is blocked waiting
|
||||
// for, then points the health probe at the assigned port.
|
||||
func (a *Agent) bootstrap(ctx context.Context) error {
|
||||
var cfg *handlers_rds.GetDBBootstrapConfigOutput
|
||||
if err := retryObserved(ctx, "bootstrap fetch", func(ctx context.Context) error {
|
||||
fetched, err := a.cp.GetBootstrapConfig(ctx, a.id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if fetched == nil {
|
||||
return fmt.Errorf("bootstrap fetch returned no config")
|
||||
}
|
||||
// A control plane predating the encrypted payload answers initialize with
|
||||
// nothing to initialise with. Retrying re-dials the queue group until an
|
||||
// upgraded node answers, so a mixed-version fleet heals within one boot.
|
||||
if fetched.Mode == handlers_rds.BootstrapModeInitialize &&
|
||||
(fetched.MasterUserPassword == nil || *fetched.MasterUserPassword == "") {
|
||||
return fmt.Errorf("bootstrap fetch returned mode=%s with no master password", fetched.Mode)
|
||||
}
|
||||
cfg = fetched
|
||||
return nil
|
||||
}, func(err error) {
|
||||
a.hb.setBootstrapFailure("bootstrap fetch", err)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
a.hb.clearBootstrapFailure()
|
||||
|
||||
// The fetch mutates nothing, so a handoff that cannot be written is retried
|
||||
// against the same staged payload rather than against an attach response.
|
||||
if err := retryObserved(ctx, "bootstrap handoff", func(context.Context) error {
|
||||
return a.handoffWriter(a.cfg.HandoffDir, cfg)
|
||||
}, func(err error) {
|
||||
a.hb.setBootstrapFailure("bootstrap handoff", err)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
a.hb.clearBootstrapFailure()
|
||||
if cfg.Port > 0 {
|
||||
a.probe.setPort(int(cfg.Port))
|
||||
}
|
||||
if cfg.BootstrapPending && cfg.PayloadID != "" {
|
||||
a.pending = &pendingBootstrap{
|
||||
payloadID: cfg.PayloadID,
|
||||
vmGeneration: cfg.VMGeneration,
|
||||
dataVolumeID: cfg.DataVolumeID,
|
||||
}
|
||||
}
|
||||
// Mode is logged but never branched on: rds-init decides whether to initdb
|
||||
// from the state of the datadir.
|
||||
slog.Info("rds-agent: bootstrap config delivered",
|
||||
"mode", cfg.Mode, "port", cfg.Port, "parameters", len(cfg.Parameters),
|
||||
"bootstrapPending", cfg.BootstrapPending)
|
||||
return nil
|
||||
}
|
||||
|
||||
// The handoff files rds-init reads. They live on tmpfs, so nothing survives a
|
||||
// reboot and the next boot re-fetches rather than reusing a stale password or a
|
||||
// re-minted cert.
|
||||
const (
|
||||
handoffEnvFile = "bootstrap.env"
|
||||
handoffParamsFile = "parameters.conf"
|
||||
handoffCertFile = "server.crt"
|
||||
handoffKeyFile = "server.key"
|
||||
|
||||
// Root-only: only rds-init reads these, and one holds the master password.
|
||||
handoffMode = 0o600
|
||||
handoffDirMode = 0o700
|
||||
)
|
||||
|
||||
// bootstrap.env is written last and renamed into place, so its appearance —
|
||||
// which rds-init waits on — means the whole handoff is complete.
|
||||
func writeHandoff(dir string, cfg *handlers_rds.GetDBBootstrapConfigOutput) error {
|
||||
if cfg.MasterUsername == "" {
|
||||
return fmt.Errorf("bootstrap config carries no master username")
|
||||
}
|
||||
// Half a handoff would leave rds-init deciding what to bind and who to admit,
|
||||
// and the only answers it could reach on its own are the wildcards this
|
||||
// contract exists to remove.
|
||||
if cfg.ListenAddress == "" {
|
||||
return fmt.Errorf("bootstrap config carries no listen address for the engine to bind")
|
||||
}
|
||||
if cfg.ClientCIDR == "" {
|
||||
return fmt.Errorf("bootstrap config carries no client CIDR to scope client authentication to")
|
||||
}
|
||||
if err := os.MkdirAll(dir, handoffDirMode); err != nil {
|
||||
return fmt.Errorf("create handoff dir %s: %w", dir, err)
|
||||
}
|
||||
// MkdirAll leaves an existing directory's mode alone, and this one may have
|
||||
// been created more permissively earlier in the boot.
|
||||
if err := os.Chmod(dir, handoffDirMode); err != nil {
|
||||
return fmt.Errorf("secure handoff dir %s: %w", dir, err)
|
||||
}
|
||||
|
||||
params, err := renderParameters(cfg.Engine, cfg.Parameters)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := writeHandoffFile(dir, handoffParamsFile, params); err != nil {
|
||||
return err
|
||||
}
|
||||
// Cert and key are written as a pair; half of one would start the engine
|
||||
// with TLS configured against a key it does not have.
|
||||
if cfg.ServingCertificate != "" && cfg.ServingPrivateKey != "" {
|
||||
if err := writeHandoffFile(dir, handoffCertFile, cfg.ServingCertificate); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := writeHandoffFile(dir, handoffKeyFile, cfg.ServingPrivateKey); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return writeHandoffFile(dir, handoffEnvFile, renderBootstrapEnv(cfg))
|
||||
}
|
||||
|
||||
// Every value is single-quoted because rds-init sources this with `.`: an
|
||||
// unquoted password holding a space, `$` or `;` would be word-split or executed.
|
||||
func renderBootstrapEnv(cfg *handlers_rds.GetDBBootstrapConfigOutput) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("# Written by rds-agent. Regenerated on every boot; edits are lost.\n")
|
||||
writeEnvLine(&b, "RDS_MODE", cfg.Mode)
|
||||
writeEnvLine(&b, "RDS_DB_INSTANCE_IDENTIFIER", cfg.DBInstanceIdentifier)
|
||||
writeEnvLine(&b, "RDS_MASTER_USERNAME", cfg.MasterUsername)
|
||||
writeEnvLine(&b, "RDS_LISTEN_ADDRESS", cfg.ListenAddress)
|
||||
writeEnvLine(&b, "RDS_CLIENT_CIDR", cfg.ClientCIDR)
|
||||
writeEnvLine(&b, "RDS_DATA_VOLUME_ID", cfg.DataVolumeID)
|
||||
writeEnvLine(&b, "RDS_DATA_VOLUME_SERIAL", cfg.DataVolumeSerial)
|
||||
writeEnvLine(&b, "RDS_VM_GENERATION", strconv.FormatInt(cfg.VMGeneration, 10))
|
||||
writeEnvLine(&b, "RDS_FORMAT_AUTHORIZED", strconv.FormatBool(cfg.FormatAuthorized))
|
||||
// Neither needs volume access, which is why the assertion is made here: the
|
||||
// agent runs before rds-datadir mounts the volume, so rds-init is the first
|
||||
// component that can compare it against the completion receipt on disk.
|
||||
writeEnvLine(&b, "RDS_BOOTSTRAP_PENDING", boolFlag(cfg.BootstrapPending))
|
||||
writeEnvLine(&b, "RDS_PAYLOAD_ID", cfg.PayloadID)
|
||||
// Present only in initialize mode: an attach fetch has no password to write
|
||||
// and rds-init must not find a stale one.
|
||||
if cfg.MasterUserPassword != nil {
|
||||
writeEnvLine(&b, "RDS_MASTER_PASSWORD", *cfg.MasterUserPassword)
|
||||
}
|
||||
if cfg.DBName != "" {
|
||||
writeEnvLine(&b, "RDS_DB_NAME", cfg.DBName)
|
||||
}
|
||||
if cfg.Port > 0 {
|
||||
writeEnvLine(&b, "RDS_PORT", strconv.FormatInt(cfg.Port, 10))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// rds-init compares against 1 rather than sourcing a Go boolean, so the two
|
||||
// halves of the guard cannot drift on spelling.
|
||||
func boolFlag(v bool) string {
|
||||
if v {
|
||||
return "1"
|
||||
}
|
||||
return "0"
|
||||
}
|
||||
|
||||
func writeEnvLine(b *strings.Builder, key, value string) {
|
||||
b.WriteString(key)
|
||||
b.WriteString("=")
|
||||
b.WriteString(shellQuote(value))
|
||||
b.WriteString("\n")
|
||||
}
|
||||
|
||||
// Single quotes suppress every shell expansion; an embedded quote is closed,
|
||||
// escaped and reopened.
|
||||
func shellQuote(s string) string {
|
||||
return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
|
||||
}
|
||||
|
||||
// postgresql.conf syntax, which MariaDB's option files also parse. Values are
|
||||
// quoted so a space or unit suffix survives. Names are the engine's startup
|
||||
// spellings, not the customer's: the wrong one leaves the engine unable to boot.
|
||||
func renderParameters(engineName string, params []handlers_rds.Parameter) (string, error) {
|
||||
engine, err := handlers_rds.LookupEngine(engineName)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("render the resolved parameters: %w", err)
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString("# Resolved parameter group, written by rds-agent.\n")
|
||||
for _, p := range params {
|
||||
if p.Name == "" {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(&b, "%s = '%s'\n", engine.OptionFileName(p.Name), strings.ReplaceAll(p.Value, "'", "''"))
|
||||
}
|
||||
return b.String(), nil
|
||||
}
|
||||
|
||||
// Atomic: a temp file in the same directory, renamed over the target. The mode
|
||||
// is set at creation, so content is never briefly readable at the process umask.
|
||||
func writeHandoffFile(dir, name, content string) error {
|
||||
path := filepath.Join(dir, name)
|
||||
tmp := path + ".new"
|
||||
|
||||
f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, handoffMode)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create %s: %w", tmp, err)
|
||||
}
|
||||
// A failed write must not leave the temp file behind holding a password.
|
||||
if _, err := f.WriteString(content); err != nil {
|
||||
f.Close()
|
||||
os.Remove(tmp)
|
||||
return fmt.Errorf("write %s: %w", tmp, err)
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
os.Remove(tmp)
|
||||
return fmt.Errorf("close %s: %w", tmp, err)
|
||||
}
|
||||
if err := os.Rename(tmp, path); err != nil {
|
||||
os.Remove(tmp)
|
||||
return fmt.Errorf("install %s: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,280 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
|
||||
)
|
||||
|
||||
func TestWriteHandoff_WritesEveryFileRootOnly(t *testing.T) {
|
||||
dir := filepath.Join(t.TempDir(), "handoff")
|
||||
password := "s3cret"
|
||||
|
||||
err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
|
||||
Mode: handlers_rds.BootstrapModeInitialize,
|
||||
Engine: "postgres",
|
||||
MasterUsername: "master",
|
||||
ListenAddress: "10.0.1.20",
|
||||
ClientCIDR: "10.0.0.0/16",
|
||||
MasterUserPassword: &password,
|
||||
DBName: "appdb",
|
||||
Port: 5433,
|
||||
DataVolumeID: "vol-data-01",
|
||||
DataVolumeSerial: "voldata01",
|
||||
VMGeneration: 4,
|
||||
FormatAuthorized: true,
|
||||
Parameters: []handlers_rds.Parameter{
|
||||
{Name: "max_connections", Value: "200"},
|
||||
{Name: "shared_buffers", Value: "256MB"},
|
||||
},
|
||||
ServingCertificate: "CERT",
|
||||
ServingPrivateKey: "KEY",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("writeHandoff: %v", err)
|
||||
}
|
||||
|
||||
env := readFile(t, filepath.Join(dir, handoffEnvFile))
|
||||
for _, want := range []string{
|
||||
"RDS_MODE='initialize'",
|
||||
"RDS_MASTER_USERNAME='master'",
|
||||
"RDS_MASTER_PASSWORD='s3cret'",
|
||||
"RDS_DB_NAME='appdb'",
|
||||
"RDS_PORT='5433'",
|
||||
"RDS_DATA_VOLUME_ID='vol-data-01'",
|
||||
"RDS_DATA_VOLUME_SERIAL='voldata01'",
|
||||
"RDS_VM_GENERATION='4'",
|
||||
"RDS_FORMAT_AUTHORIZED='true'",
|
||||
} {
|
||||
if !strings.Contains(env, want) {
|
||||
t.Errorf("handoff env missing %s:\n%s", want, env)
|
||||
}
|
||||
}
|
||||
|
||||
params := readFile(t, filepath.Join(dir, handoffParamsFile))
|
||||
if !strings.Contains(params, "max_connections = '200'") || !strings.Contains(params, "shared_buffers = '256MB'") {
|
||||
t.Errorf("parameters.conf = %q, want both resolved settings", params)
|
||||
}
|
||||
if got := readFile(t, filepath.Join(dir, handoffCertFile)); got != "CERT" {
|
||||
t.Errorf("server.crt = %q, want CERT", got)
|
||||
}
|
||||
if got := readFile(t, filepath.Join(dir, handoffKeyFile)); got != "KEY" {
|
||||
t.Errorf("server.key = %q, want KEY", got)
|
||||
}
|
||||
|
||||
// The handoff holds a master password and a private key; anything readable
|
||||
// beyond root is a disclosure to every process in the guest.
|
||||
if mode := statMode(t, dir); mode != handoffDirMode {
|
||||
t.Errorf("handoff dir mode = %#o, want %#o", mode, handoffDirMode)
|
||||
}
|
||||
for _, name := range []string{handoffEnvFile, handoffParamsFile, handoffCertFile, handoffKeyFile} {
|
||||
if mode := statMode(t, filepath.Join(dir, name)); mode != handoffMode {
|
||||
t.Errorf("%s mode = %#o, want %#o", name, mode, handoffMode)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Attach mode carries no password, and the key must be absent rather than
|
||||
// present and empty — rds-init reads an empty value as "no password" only by
|
||||
// accident, and a later boot would inherit it.
|
||||
func TestWriteHandoff_AttachOmitsPassword(t *testing.T) {
|
||||
dir := filepath.Join(t.TempDir(), "handoff")
|
||||
if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
|
||||
Mode: handlers_rds.BootstrapModeAttach,
|
||||
Engine: "postgres",
|
||||
MasterUsername: "master",
|
||||
ListenAddress: "10.0.1.20",
|
||||
ClientCIDR: "10.0.0.0/16",
|
||||
Port: 5432,
|
||||
}); err != nil {
|
||||
t.Fatalf("writeHandoff: %v", err)
|
||||
}
|
||||
|
||||
env := readFile(t, filepath.Join(dir, handoffEnvFile))
|
||||
if strings.Contains(env, "RDS_MASTER_PASSWORD") {
|
||||
t.Errorf("handoff env = %q, want no RDS_MASTER_PASSWORD key", env)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"RDS_DATA_VOLUME_ID=''",
|
||||
"RDS_DATA_VOLUME_SERIAL=''",
|
||||
"RDS_VM_GENERATION='0'",
|
||||
"RDS_FORMAT_AUTHORIZED='false'",
|
||||
} {
|
||||
if !strings.Contains(env, want) {
|
||||
t.Errorf("handoff env missing fail-closed field %s: %q", want, env)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The env file is sourced by a shell, so a password holding shell metacharacters
|
||||
// must survive as data rather than being split, expanded or executed.
|
||||
func TestWriteHandoff_QuotesShellMetacharacters(t *testing.T) {
|
||||
dir := filepath.Join(t.TempDir(), "handoff")
|
||||
password := `a b$(id);'"\` + "`whoami`"
|
||||
|
||||
if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
|
||||
Mode: handlers_rds.BootstrapModeInitialize,
|
||||
Engine: "postgres",
|
||||
MasterUsername: "master",
|
||||
ListenAddress: "10.0.1.20",
|
||||
ClientCIDR: "10.0.0.0/16",
|
||||
MasterUserPassword: &password,
|
||||
Port: 5432,
|
||||
}); err != nil {
|
||||
t.Fatalf("writeHandoff: %v", err)
|
||||
}
|
||||
|
||||
env := readFile(t, filepath.Join(dir, handoffEnvFile))
|
||||
line := findLine(t, env, "RDS_MASTER_PASSWORD=")
|
||||
if got := shellSource(t, line, "RDS_MASTER_PASSWORD"); got != password {
|
||||
t.Errorf("sourced password = %q, want %q (line was %s)", got, password, line)
|
||||
}
|
||||
}
|
||||
|
||||
// A serving cert with no key would have rds-init configure TLS against a key
|
||||
// that is not there; the pair is delivered together or not at all.
|
||||
func TestWriteHandoff_SkipsHalfATLSPair(t *testing.T) {
|
||||
dir := filepath.Join(t.TempDir(), "handoff")
|
||||
if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
|
||||
Mode: handlers_rds.BootstrapModeAttach,
|
||||
Engine: "postgres",
|
||||
MasterUsername: "master",
|
||||
ListenAddress: "10.0.1.20",
|
||||
ClientCIDR: "10.0.0.0/16",
|
||||
Port: 5432,
|
||||
ServingCertificate: "CERT",
|
||||
}); err != nil {
|
||||
t.Fatalf("writeHandoff: %v", err)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(filepath.Join(dir, handoffCertFile)); !os.IsNotExist(err) {
|
||||
t.Errorf("server.crt exists without a key; stat err = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A config with no master username cannot bootstrap anything, and writing it
|
||||
// would only move the failure into rds-init after the boot has committed to it.
|
||||
func TestWriteHandoff_RejectsConfigWithNoMasterUser(t *testing.T) {
|
||||
dir := filepath.Join(t.TempDir(), "handoff")
|
||||
err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
|
||||
Mode: handlers_rds.BootstrapModeAttach,
|
||||
Engine: "postgres",
|
||||
Port: 5432,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("writeHandoff accepted a config with no master username, want an error")
|
||||
}
|
||||
if _, statErr := os.Stat(filepath.Join(dir, handoffEnvFile)); !os.IsNotExist(statErr) {
|
||||
t.Errorf("handoff env was written despite the rejection; stat err = %v", statErr)
|
||||
}
|
||||
}
|
||||
|
||||
// A directory left over from an earlier boot at a looser mode must be tightened,
|
||||
// not accepted as it is found.
|
||||
func TestWriteHandoff_TightensExistingDirectory(t *testing.T) {
|
||||
dir := filepath.Join(t.TempDir(), "handoff")
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatalf("seed dir: %v", err)
|
||||
}
|
||||
|
||||
if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
|
||||
Mode: handlers_rds.BootstrapModeAttach,
|
||||
Engine: "postgres",
|
||||
MasterUsername: "master",
|
||||
ListenAddress: "10.0.1.20",
|
||||
ClientCIDR: "10.0.0.0/16",
|
||||
Port: 5432,
|
||||
}); err != nil {
|
||||
t.Fatalf("writeHandoff: %v", err)
|
||||
}
|
||||
if mode := statMode(t, dir); mode != handoffDirMode {
|
||||
t.Errorf("handoff dir mode = %#o, want it tightened to %#o", mode, handoffDirMode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShellQuote_EscapesEmbeddedQuotes(t *testing.T) {
|
||||
for _, in := range []string{"", "plain", "it's", `''`, "new\nline", `$HOME`} {
|
||||
if got := shellSource(t, "V="+shellQuote(in), "V"); got != in {
|
||||
t.Errorf("shellQuote(%q) sourced back as %q", in, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// shellSource evaluates one KEY=value assignment in a real shell and returns
|
||||
// what the variable ends up holding. The quoting is only correct if a shell
|
||||
// agrees, so the test asks one instead of re-implementing its rules.
|
||||
func shellSource(t *testing.T, line, name string) string {
|
||||
t.Helper()
|
||||
out, err := exec.Command("/bin/sh", "-c", line+"\nprintf %s \"$"+name+"\"\n").Output()
|
||||
if err != nil {
|
||||
t.Fatalf("sourcing %q: %v", line, err)
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
||||
func statMode(t *testing.T, path string) os.FileMode {
|
||||
t.Helper()
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatalf("stat %s: %v", path, err)
|
||||
}
|
||||
return info.Mode().Perm()
|
||||
}
|
||||
|
||||
func findLine(t *testing.T, body, prefix string) string {
|
||||
t.Helper()
|
||||
for line := range strings.SplitSeq(body, "\n") {
|
||||
if strings.HasPrefix(line, prefix) {
|
||||
return line
|
||||
}
|
||||
}
|
||||
t.Fatalf("no line starting %q in:\n%s", prefix, body)
|
||||
return ""
|
||||
}
|
||||
|
||||
// The include rds-init installs is what the engine starts on, so the spelling
|
||||
// written here is the one that has to boot.
|
||||
func TestRenderParameters_UsesTheEngineStartupSpelling(t *testing.T) {
|
||||
params := []handlers_rds.Parameter{
|
||||
{Name: "time_zone", Value: "SYSTEM"},
|
||||
{Name: "max_connections", Value: "85"},
|
||||
}
|
||||
|
||||
rendered, err := renderParameters("mariadb", params)
|
||||
if err != nil {
|
||||
t.Fatalf("renderParameters: %v", err)
|
||||
}
|
||||
if !strings.Contains(rendered, "default_time_zone = 'SYSTEM'\n") {
|
||||
t.Errorf("time_zone was not written under its startup spelling:\n%s", rendered)
|
||||
}
|
||||
if strings.Contains(rendered, "\ntime_zone = ") {
|
||||
t.Errorf("time_zone reached the option file, which mariadbd refuses:\n%s", rendered)
|
||||
}
|
||||
if !strings.Contains(rendered, "max_connections = '85'\n") {
|
||||
t.Errorf("an unmapped parameter was not written unchanged:\n%s", rendered)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderParameters_PostgresKeepsEveryName(t *testing.T) {
|
||||
rendered, err := renderParameters("postgres", []handlers_rds.Parameter{
|
||||
{Name: "shared_buffers", Value: "262144"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("renderParameters: %v", err)
|
||||
}
|
||||
if !strings.Contains(rendered, "shared_buffers = '262144'\n") {
|
||||
t.Errorf("postgres name was rewritten:\n%s", rendered)
|
||||
}
|
||||
}
|
||||
|
||||
// Failing closed beats writing a file the engine may refuse: the bad include
|
||||
// would already be on the data volume by the time the server rejected it.
|
||||
func TestRenderParameters_RefusesAnUnknownEngine(t *testing.T) {
|
||||
if _, err := renderParameters("", nil); err == nil {
|
||||
t.Fatal("renderParameters accepted a config naming no engine, want an error")
|
||||
}
|
||||
}
|
||||