mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-15 09:01:31 +00:00
Merge remote-tracking branch 'origin/dev' into refactor/spinifex-package-boundaries
# Conflicts: # spinifex/daemon/daemon.go # spinifex/foundation/aws/arn/arn.go # spinifex/foundation/aws/arn/arn_test.go # spinifex/foundation/state/kvlease/bucket.go # spinifex/foundation/state/kvlease/bucket_test.go # spinifex/foundation/state/kvstore/wait.go # spinifex/foundation/state/kvstore/wait_test.go # spinifex/foundation/state/kvutil/bucket_creation_invariants_test.go # spinifex/foundation/state/kvutil/export_test.go # spinifex/foundation/state/kvutil/kvutil.go # spinifex/foundation/state/kvutil/kvutil_test.go # spinifex/foundation/state/kvutil/replicas.go # spinifex/foundation/state/kvutil/replicas_health_test.go # spinifex/foundation/state/kvutil/replicas_test.go # spinifex/foundation/state/kvutil/state_test.go # spinifex/gateway/auth.go # spinifex/gateway/bedrock/logging.go # spinifex/gateway/ec2_authz_test.go # spinifex/gateway/ecrapi/authz.go # spinifex/gateway/eks.go # spinifex/gateway/gateway_test.go # spinifex/handlers/ec2/eigw/service_impl.go # spinifex/handlers/ec2/igw/service_impl.go # spinifex/handlers/ec2/instance/service_impl.go # spinifex/handlers/ec2/launchtemplate/service_impl_test.go # spinifex/handlers/ec2/placementgroup/service_impl.go # spinifex/handlers/ec2/placementgroup/service_impl_test.go # spinifex/handlers/ec2/routetable/service_impl.go # spinifex/handlers/ec2/spotinstance/service_impl.go # spinifex/handlers/ec2/volume/service_impl.go # spinifex/handlers/ec2/vpc/eni.go # spinifex/handlers/ec2/vpc/security_group.go # spinifex/handlers/ec2/vpc/service_impl.go # spinifex/handlers/ecs/tags.go # spinifex/handlers/eks/k3s_ha_control_plane.go # spinifex/handlers/iam/groups.go # spinifex/handlers/iam/retry.go # spinifex/handlers/rds/subnetgroup_test.go # spinifex/handlers/sts/assume_role.go # spinifex/ingress/aws/query/parser_fuzz_test.go # spinifex/runtime/compute/cache/cache_test.go # spinifex/runtime/roles/nats/servername.go # spinifex/services/awsgw/awsgw.go # spinifex/services/viperblockd/volume_dirty.go # spinifex/utils/account.go
This commit is contained in:
joshsiv-mulga
@@ -0,0 +1,79 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="1020" height="836" viewBox="0 0 1020 836" role="img" aria-label="Spinifex on a single Exoscale instance: the instance sits on the zone's public network with one interface, eth0, holding its own address and the default route. Manual Elastic IPs are delivered to eth0 and routed by the host into OVN through the spx-nat transit veth, and an Exoscale block storage volume backs /var/lib/spinifex.">
|
||||
<rect x="0" y="0" width="1020" height="836" rx="20" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="28" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2.5">SPINIFEX ON EXOSCALE — SINGLE NODE</text>
|
||||
<text x="992" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="end">one instance · routed NAT · Elastic IPs</text>
|
||||
<text x="28" y="72" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="23" fill="#F5F3EE" font-weight="800" text-anchor="start">One Exoscale instance, single Spinifex node</text>
|
||||
<rect x="28" y="92" width="964" height="76" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="48" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">INTERNET</text>
|
||||
<text x="972" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">AWS SDKs, SSH and guest egress all arrive here</text>
|
||||
<rect x="46" y="126" width="260.0" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="176.0" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">aws-cli / Terraform / boto3 → :9999</text>
|
||||
<rect x="315.0" y="126" width="171.6" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="400.8" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">ssh → guest Elastic IP</text>
|
||||
<rect x="495.6" y="126" width="205.6" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="598.4" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">guest egress ← same address</text>
|
||||
<line x1="510" y1="170" x2="510" y2="186" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
|
||||
<rect x="28" y="188" width="964" height="582" rx="16" fill="#EE4B12" fill-opacity="0.022" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.45" stroke-dasharray="6 5"/>
|
||||
<text x="50" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2">EXOSCALE ZONE DE-FRA-1 · PUBLIC NETWORK 91.92.116.0/23</text>
|
||||
<text x="970" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">gateway 91.92.116.1 · security group open to all</text>
|
||||
<rect x="48" y="232" width="924" height="520" rx="14" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
|
||||
<text x="68" y="266" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="16" fill="#F5F3EE" font-weight="800" text-anchor="start">standard.huge</text>
|
||||
<text x="212" y="266" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="400" text-anchor="start">spinifex-node-01 · 8 vCPU / 32 GiB · Ubuntu 26.04</text>
|
||||
<text x="952" y="266" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">one host = one Spinifex region</text>
|
||||
<line x1="68" y1="282" x2="952" y2="282" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
|
||||
<rect x="68" y="298" width="434" height="108" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="88" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">eth0 — PUBLIC INTERFACE</text>
|
||||
<text x="482" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">the only NIC</text>
|
||||
<text x="88" y="356" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">91.92.116.250/23 · holds the default route</text>
|
||||
<text x="88" y="378" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">advertise IP · API :9999 · SSH · host masquerade</text>
|
||||
<rect x="518" y="298" width="434" height="108" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="538" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">ELASTIC IPs — MANUAL</text>
|
||||
<text x="932" y="328" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">attached to the instance</text>
|
||||
<text x="538" y="356" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#F3F1EC" font-weight="400" text-anchor="start">89.145.163.1/32 → delivered to eth0</text>
|
||||
<text x="538" y="378" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">the host routes each /32 into OVN, never owns it</text>
|
||||
<line x1="735" y1="406" x2="735" y2="422" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
|
||||
<rect x="68" y="424" width="884" height="40" rx="11" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
|
||||
<text x="88" y="449" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#EE4B12" font-weight="600" text-anchor="start">spx-nat-host ⇄ spx-nat-ovs</text>
|
||||
<text x="296" y="449" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#9AA1A8" font-weight="400" text-anchor="start">transit 100.127.0.0/24 · EIP/32 routed in · masquerade out eth0 for the rest</text>
|
||||
<line x1="510" y1="464" x2="510" y2="480" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
|
||||
<rect x="68" y="482" width="884" height="100" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="88" y="512" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">SPINIFEX CONTROL PLANE</text>
|
||||
<text x="932" y="512" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">every service is one binary: spx service <name></text>
|
||||
<rect x="86" y="530" width="93.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="132.5" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">awsgw :9999</text>
|
||||
<rect x="188.0" y="530" width="58.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="217.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">daemon</text>
|
||||
<rect x="255.0" y="530" width="44.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="277.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">vpcd</text>
|
||||
<rect x="308.0" y="530" width="44.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="330.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">NATS</text>
|
||||
<rect x="361.0" y="530" width="86.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="404.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">predastore</text>
|
||||
<rect x="456.0" y="530" width="86.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="499.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">viperblock</text>
|
||||
<rect x="551.0" y="530" width="79.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="590.5" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">northstar</text>
|
||||
<rect x="639.0" y="530" width="72.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="675.0" y="544.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">ui :3000</text>
|
||||
<text x="86" y="568" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">SigV4 in, NATS in the middle, QEMU/KVM (nested) · OVN/OVS · Predastore · Viperblock underneath</text>
|
||||
<line x1="248" y1="582" x2="248" y2="598" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
|
||||
<line x1="772" y1="582" x2="772" y2="598" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.4" stroke-dasharray="2 4"/>
|
||||
<rect x="68" y="600" width="532" height="130" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="88" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">TENANT VPCs — OVN br-int</text>
|
||||
<text x="580" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">geneve-local</text>
|
||||
<rect x="86" y="644" width="496" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="100" y="666" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">vpc-a 172.31.0.0/16</text>
|
||||
<text x="300" y="666" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-… 172.31.0.4 → 89.145.163.1</text>
|
||||
<rect x="86" y="686" width="496" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="100" y="708" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">vpc-b 10.0.0.0/16</text>
|
||||
<text x="300" y="708" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-… 10.0.1.9 · private, egress only</text>
|
||||
<rect x="616" y="600" width="336" height="130" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="636" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">BLOCK STORAGE VOLUME</text>
|
||||
<text x="932" y="630" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">256 GiB+</text>
|
||||
<text x="636" y="658" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">virtio-blk → /dev/vdb</text>
|
||||
<text x="636" y="680" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">/var/lib/spinifex</text>
|
||||
<text x="636" y="700" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11.5" fill="#5B636C" font-weight="400" text-anchor="start">found by-id, defaults,nofail</text>
|
||||
<text x="636" y="718" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="11.5" fill="#5B636C" font-weight="400" text-anchor="start">viperblock · predastore · JetStream</text>
|
||||
<text x="28" y="796" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">A guest's public address is a manual Exoscale Elastic IP, delivered to eth0 addressed to itself — no private/public pair to translate.</text>
|
||||
<text x="28" y="814" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">The host routes it into OVN, so inbound and outbound are the same address and the guest sees its private IP only.</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 14 KiB |
@@ -0,0 +1,123 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="1020" height="796" viewBox="0 0 1020 796" role="img" aria-label="Spinifex across three Exoscale instances in one zone, spread by an anti-affinity group. Each has a public eth0 carrying its own Elastic IPs, a private-network eth1 for the cluster mesh, its own block storage volume and its share of the guests. Geneve, NATS JetStream, predastore erasure coding and the OVN raft join them east-west over the private network. This topology is not yet tested.">
|
||||
<rect x="0" y="0" width="1020" height="796" rx="20" fill="#14181D" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="28" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2.5">SPINIFEX ON EXOSCALE — THREE NODES</text>
|
||||
<text x="992" y="42" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#9AA1A8" font-weight="400" text-anchor="end">anti-affinity group · one zone · not yet tested</text>
|
||||
<text x="28" y="72" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="23" fill="#F5F3EE" font-weight="800" text-anchor="start">Three Exoscale instances, redundant multi-node install</text>
|
||||
<rect x="28" y="92" width="964" height="76" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="48" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">INTERNET</text>
|
||||
<text x="972" y="122" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">one endpoint, whichever node answers</text>
|
||||
<rect x="46" y="126" width="260.0" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="176.0" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">aws-cli / Terraform / boto3 → :9999</text>
|
||||
<rect x="315.0" y="126" width="171.6" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="400.8" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">ssh → guest Elastic IP</text>
|
||||
<rect x="495.6" y="126" width="205.6" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="598.4" y="140.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">guest egress ← same address</text>
|
||||
<line x1="510" y1="170" x2="510" y2="186" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
|
||||
<rect x="28" y="188" width="964" height="440" rx="16" fill="#EE4B12" fill-opacity="0.022" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.45" stroke-dasharray="6 5"/>
|
||||
<text x="50" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start" letter-spacing="2">EXOSCALE ZONE DE-FRA-1 · PUBLIC NETWORK + PRIVATE NETWORK</text>
|
||||
<text x="970" y="218" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">one security group · one anti-affinity group</text>
|
||||
<rect x="48" y="232" width="300" height="380" rx="14" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
|
||||
<text x="64" y="260" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-01</text>
|
||||
<text x="332" y="260" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">standard.huge</text>
|
||||
<line x1="64" y1="272" x2="332" y2="272" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
|
||||
<rect x="64" y="284" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="78" y="305" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth0 91.92.116.250</text>
|
||||
<text x="78" y="323" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">public · default route · API</text>
|
||||
<rect x="64" y="342" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="78" y="363" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth1 10.10.0.11</text>
|
||||
<text x="78" y="381" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">private network · mesh · advertise · encap</text>
|
||||
<rect x="64" y="400" width="268" height="50" rx="10" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
|
||||
<text x="78" y="421" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start">EIP 89.145.163.1</text>
|
||||
<text x="78" y="439" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">→ eth0 · /32 via spx-nat-host</text>
|
||||
<rect x="64" y="462" width="51.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="89.5" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">awsgw</text>
|
||||
<rect x="124.0" y="462" width="57.6" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="152.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">daemon</text>
|
||||
<rect x="190.6" y="462" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="212.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">vpcd</text>
|
||||
<rect x="64" y="492" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="86.2" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">nats</text>
|
||||
<rect x="117.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="159.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">predastore</text>
|
||||
<rect x="210.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="252.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">viperblock</text>
|
||||
<rect x="64" y="526" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="76" y="548" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-23d2… 172.31.0.4 → 89.145.163.1</text>
|
||||
<rect x="64" y="568" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="76" y="590" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">volume /dev/vdb → /var/lib/spinifex</text>
|
||||
<rect x="360" y="232" width="300" height="380" rx="14" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
|
||||
<text x="376" y="260" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-02</text>
|
||||
<text x="644" y="260" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">standard.huge</text>
|
||||
<line x1="376" y1="272" x2="644" y2="272" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
|
||||
<rect x="376" y="284" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="390" y="305" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth0 91.92.117.14</text>
|
||||
<text x="390" y="323" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">public · default route · API</text>
|
||||
<rect x="376" y="342" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="390" y="363" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth1 10.10.0.12</text>
|
||||
<text x="390" y="381" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">private network · mesh · advertise · encap</text>
|
||||
<rect x="376" y="400" width="268" height="50" rx="10" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
|
||||
<text x="390" y="421" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start">EIP 89.145.160.107</text>
|
||||
<text x="390" y="439" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">→ eth0 · /32 via spx-nat-host</text>
|
||||
<rect x="376" y="462" width="51.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="401.5" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">awsgw</text>
|
||||
<rect x="436.0" y="462" width="57.6" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="464.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">daemon</text>
|
||||
<rect x="502.6" y="462" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="524.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">vpcd</text>
|
||||
<rect x="376" y="492" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="398.2" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">nats</text>
|
||||
<rect x="429.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="471.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">predastore</text>
|
||||
<rect x="522.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="564.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">viperblock</text>
|
||||
<rect x="376" y="526" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="388" y="548" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-6b91… 172.31.0.9 → 89.145.160.107</text>
|
||||
<rect x="376" y="568" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="388" y="590" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">volume /dev/vdb → /var/lib/spinifex</text>
|
||||
<rect x="672" y="232" width="300" height="380" rx="14" fill="#EE4B12" fill-opacity="0.05" stroke="#EE4B12" stroke-width="1.4" stroke-opacity="0.5"/>
|
||||
<text x="688" y="260" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="14" fill="#F5F3EE" font-weight="800" text-anchor="start">spinifex-node-03</text>
|
||||
<text x="956" y="260" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="end">standard.huge</text>
|
||||
<line x1="688" y1="272" x2="956" y2="272" stroke="#EE4B12" stroke-opacity="0.25" stroke-width="1"/>
|
||||
<rect x="688" y="284" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="702" y="305" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth0 91.92.116.88</text>
|
||||
<text x="702" y="323" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">public · default route · API</text>
|
||||
<rect x="688" y="342" width="268" height="50" rx="10" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="702" y="363" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#F3F1EC" font-weight="400" text-anchor="start">eth1 10.10.0.13</text>
|
||||
<text x="702" y="381" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="start">private network · mesh · advertise · encap</text>
|
||||
<rect x="688" y="400" width="268" height="50" rx="10" fill="#1E242B" fill-opacity="1.0" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.45"/>
|
||||
<text x="702" y="421" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11.5" fill="#EE4B12" font-weight="600" text-anchor="start">EIP 89.145.162.131</text>
|
||||
<text x="702" y="439" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">→ eth0 · /32 via spx-nat-host</text>
|
||||
<rect x="688" y="462" width="51.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="713.5" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">awsgw</text>
|
||||
<rect x="748.0" y="462" width="57.6" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="776.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">daemon</text>
|
||||
<rect x="814.6" y="462" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="836.8" y="476.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">vpcd</text>
|
||||
<rect x="688" y="492" width="44.4" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="710.2" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">nats</text>
|
||||
<rect x="741.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="783.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">predastore</text>
|
||||
<rect x="834.4" y="492" width="84.0" height="22" rx="7" fill="#EE4B12" fill-opacity="0.06" stroke="#EE4B12" stroke-width="1" stroke-opacity="0.5"/>
|
||||
<text x="876.4" y="506.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#F3F1EC" font-weight="400" text-anchor="middle">viperblock</text>
|
||||
<rect x="688" y="526" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="700" y="548" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">i-0ae4… 172.31.1.6 → 89.145.162.131</text>
|
||||
<rect x="688" y="568" width="268" height="34" rx="9" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="700" y="590" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#9AA1A8" font-weight="400" text-anchor="start">volume /dev/vdb → /var/lib/spinifex</text>
|
||||
<line x1="510" y1="628" x2="510" y2="644" stroke="#EE4B12" stroke-width="1.3" stroke-opacity="0.55" stroke-dasharray="2 4"/>
|
||||
<rect x="28" y="646" width="964" height="70" rx="13" fill="#181D23" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="48" y="676" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="12" fill="#9AA1A8" font-weight="700" text-anchor="start" letter-spacing="1.5">EAST–WEST, OVER THE EXOSCALE PRIVATE NETWORK</text>
|
||||
<text x="972" y="676" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="10.5" fill="#5B636C" font-weight="400" text-anchor="end">no Elastic IP is involved · MTU 1500</text>
|
||||
<rect x="46" y="684" width="124.0" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="108.0" y="698.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">Geneve UDP 6081</text>
|
||||
<rect x="179.0" y="684" width="137.6" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="247.8" y="698.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">NATS JetStream R3</text>
|
||||
<rect x="325.6" y="684" width="144.4" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="397.8" y="698.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">predastore RS(2,1)</text>
|
||||
<rect x="479.0" y="684" width="151.2" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="554.6" y="698.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">OVN Northbound raft</text>
|
||||
<rect x="639.2" y="684" width="56.0" height="22" rx="7" fill="none" fill-opacity="1.0" stroke="#2A2F36" stroke-width="1" stroke-opacity="1.0"/>
|
||||
<text x="667.2" y="698.8" font-family="ui-monospace, 'SFMono-Regular', Menlo, Consolas, monospace" font-size="11" fill="#9AA1A8" font-weight="400" text-anchor="middle">OVSDB</text>
|
||||
<text x="28" y="744" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Guest-to-guest traffic rides Geneve between the nodes' private-network interfaces — it never touches eth0 or an Elastic IP.</text>
|
||||
<text x="28" y="762" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Each node carries the Elastic IPs of its own guests. Moving one between nodes is an Exoscale API call, not yet automated.</text>
|
||||
<text x="28" y="780" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, system-ui, sans-serif" font-size="12" fill="#5B636C" font-weight="400" text-anchor="start">Three nodes is the smallest cluster that tolerates losing one; the anti-affinity group keeps them on different hypervisors.</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 22 KiB |
@@ -178,5 +178,17 @@
|
||||
"extra": "",
|
||||
"suite": "lbrecovery",
|
||||
"timeout_minutes": 90
|
||||
},
|
||||
{
|
||||
"cell": 33,
|
||||
"install": "binary",
|
||||
"topo": "real-multi",
|
||||
"net": "static",
|
||||
"host": "debian-13",
|
||||
"arch": "x86_64",
|
||||
"runner": "ci-multi",
|
||||
"extra": "",
|
||||
"suite": "kvquorum",
|
||||
"timeout_minutes": 55
|
||||
}
|
||||
]
|
||||
|
||||
@@ -51,7 +51,13 @@ E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota
|
||||
# the load balancer's, so taking that node away does not take the backends with
|
||||
# it. It runs beside instancerecovery at the end because it also takes a node
|
||||
# away and owns its environment while it does.
|
||||
E2E_SUITES_MULTI="multinode lb cert quota lbrecovery instancerecovery storagefault"
|
||||
#
|
||||
# kvquorum is multi-only and needs three, for the same reason as those two: a
|
||||
# surviving majority. It is what proves the control plane's own state is
|
||||
# replicated rather than sitting on one node, so it runs before the pair that
|
||||
# take a node away — a cluster whose buckets are already wrong should report
|
||||
# that, not report a recovery failing.
|
||||
E2E_SUITES_MULTI="multinode lb cert quota kvquorum lbrecovery instancerecovery storagefault"
|
||||
|
||||
# grep -xE alternation form, for narrowing a requested set to the eligible one.
|
||||
E2E_SUITES_SINGLE_RE="$(printf '%s' "${E2E_SUITES_SINGLE}" | tr ' ' '|')"
|
||||
@@ -75,6 +81,10 @@ E2E_SUITES_MULTI_RE="$(printf '%s' "${E2E_SUITES_MULTI}" | tr ' ' '|')"
|
||||
# after a failed seal do not exist on one node, and they are the assertions
|
||||
# worth a nightly.
|
||||
E2E_SUITES_NIGHTLY_SINGLE="single cert iam"
|
||||
#
|
||||
# The multi-node nightly subset carries kvquorum's read-only half through
|
||||
# `multinode`, which sweeps every KV bucket rather than the three the daemon
|
||||
# owns. The node-loss half is too slow for a 35-minute cell and has its own.
|
||||
E2E_SUITES_NIGHTLY_MULTI="multinode cert lb"
|
||||
|
||||
# How long a suite gets, for the same reason the lists above live here: the
|
||||
@@ -112,6 +122,10 @@ e2e_suite_timeout() {
|
||||
# the backends before it forwards anything. Then the node comes back and has
|
||||
# to be shown not to answer for an address it no longer holds.
|
||||
lbrecovery) echo "75m" ;;
|
||||
# kvquorum audits every bucket in seconds and then spends the rest of its
|
||||
# budget on a node stopping and starting: seven units down, a raft election,
|
||||
# and the whole service stack back up with every stream caught up.
|
||||
kvquorum) echo "40m" ;;
|
||||
*) echo "30m" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@ on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
cells:
|
||||
description: "Comma-separated cell numbers to run, no spaces (blank = all). 1-3, 8-10, 17-19 tofu permutation matrix, 20 baremetal, 21-22 ISO, 23 predastore throughput, 24 predastore fault injection, 25-27 service suites (eks, ecs, rds), 28 guest disk performance, 29 storage fault injection, 30 routed NAT on three nodes, 31 instance recovery on three nodes, 32 load balancer recovery on three nodes"
|
||||
description: "Comma-separated cell numbers to run, no spaces (blank = all). 1-3, 8-10, 17-19 tofu permutation matrix, 20 baremetal, 21-22 ISO, 23 predastore throughput, 24 predastore fault injection, 25-27 service suites (eks, ecs, rds), 28 guest disk performance, 29 storage fault injection, 30 routed NAT on three nodes, 31 instance recovery on three nodes, 32 load balancer recovery on three nodes, 33 KV bucket quorum under node loss on three nodes"
|
||||
required: false
|
||||
|
||||
permissions:
|
||||
|
||||
@@ -153,8 +153,9 @@ install-microvm: $(MICROVM_ARTIFACTS) ## Install microVM artifacts to /usr/share
|
||||
# -j2 overlaps the only two heavy gates, lint and test-cover: alone each leaves
|
||||
# most of the machine idle, so pairing them cuts a cold run by about a quarter.
|
||||
# -Otarget keeps each gate's output in one block instead of interleaving them.
|
||||
# manifest-check manifest-lint test-package-check removed, can add back later
|
||||
preflight:
|
||||
@$(MAKE) --no-print-directory -j2 -Otarget QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-package-check test-harness test-build-scripts
|
||||
@$(MAKE) --no-print-directory -j2 -Otarget QUIET=1 lint govulncheck test-cover diff-coverage test-harness test-build-scripts
|
||||
@echo -e "\n ✅ Preflight passed — safe to commit."
|
||||
|
||||
# Shell suites + shellcheck for build/scripts/, the systemd-unit helpers that
|
||||
@@ -317,6 +318,17 @@ bench:
|
||||
@echo -e "\n....Running benchmarks for $(GO_PROJECT_NAME)...."
|
||||
LOG_IGNORE=1 go test -benchmem -run=. -bench=. ./...
|
||||
|
||||
# Fuzz every Fuzz* target for FUZZTIME each; -fuzz takes one target per run.
|
||||
# Plain `go test` already replays the seeds and testdata/fuzz on preflight.
|
||||
FUZZTIME ?= 30s
|
||||
fuzz:
|
||||
@set -e; grep -rlE '^func Fuzz\w+\(' --include='*_test.go' spinifex cmd internal | while read -r file; do \
|
||||
for name in $$(grep -oE '^func Fuzz\w+' "$$file" | cut -d' ' -f2); do \
|
||||
echo -e "\n....Fuzzing $$name ($$(dirname $$file)) for $(FUZZTIME)...."; \
|
||||
LOG_IGNORE=1 go test -run='^$$' -fuzz="^$$name$$" -fuzztime=$(FUZZTIME) ./$$(dirname $$file); \
|
||||
done; \
|
||||
done
|
||||
|
||||
# Fast iteration: build + install binary + restart all services.
|
||||
# Microvm artifacts are reinstalled when they already exist on disk — the rule's
|
||||
# input timestamps drive a rebuild only if anything actually changed. On a fresh
|
||||
@@ -452,7 +464,7 @@ distro-arm64:
|
||||
distro-clean:
|
||||
rm -rf dist/
|
||||
|
||||
.PHONY: test-package-check build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
|
||||
.PHONY: test-package-check build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
|
||||
deploy reinstall clean \
|
||||
install-system install-go install-aws quickinstall \
|
||||
lint fix govulncheck nilaway \
|
||||
|
||||
@@ -870,7 +870,7 @@ func fireInstallCallback() {
|
||||
return
|
||||
}
|
||||
client := &http.Client{Timeout: 10 * time.Second}
|
||||
resp, err := client.Get(url) //nolint:noctx // installer has no context; best-effort fire-and-forget
|
||||
resp, err := client.Get(url)
|
||||
if err != nil {
|
||||
slog.Warn("fireInstallCallback: request failed", "url", url, "err", err)
|
||||
return
|
||||
|
||||
@@ -30,6 +30,7 @@ import (
|
||||
"github.com/mulgadc/bluebottle/pkg/masterkey"
|
||||
"github.com/mulgadc/bluebottle/pkg/safecast"
|
||||
"github.com/mulgadc/spinifex/spinifex/admin"
|
||||
"github.com/mulgadc/spinifex/spinifex/clustersize"
|
||||
"github.com/mulgadc/spinifex/spinifex/config"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/formation"
|
||||
@@ -372,7 +373,7 @@ func init() {
|
||||
adminInitCmd.Flags().String("region", "ap-southeast-2", "Mulga region to create")
|
||||
adminInitCmd.Flags().String("az", "ap-southeast-2a", "Mulga AZ to create")
|
||||
adminInitCmd.Flags().String("node", "node1", "Node name, increment for additional nodes (default, node1)")
|
||||
adminInitCmd.Flags().Int("nodes", 3, "Number of nodes to expect for cluster")
|
||||
adminInitCmd.Flags().Int("nodes", 3, "Number of nodes to expect for cluster (1, or 3 and up)")
|
||||
adminInitCmd.Flags().String("host", "", "Leader node to join (if not specified, tries multicast discovery)")
|
||||
adminInitCmd.Flags().Int("port", 4432, "Port to bind cluster services on")
|
||||
adminInitCmd.Flags().String("bind", "0.0.0.0", "IP address to bind services to (e.g., 10.11.12.1 for multi-node). Default 0.0.0.0 listens on all interfaces.")
|
||||
@@ -1212,6 +1213,26 @@ func runimagesListCmd(cmd *cobra.Command, args []string) {
|
||||
pterm.Println("spx admin images import --name <image-name>")
|
||||
}
|
||||
|
||||
// validateNodeCount accepts the cluster sizes Spinifex supports: one server, or
|
||||
// three and up.
|
||||
//
|
||||
// Two is rejected rather than allowed and warned about. On two servers OVN runs
|
||||
// standalone, the storage metadata quorum has no majority to lose, and the
|
||||
// control plane's own KV state has no replica count that works — one replica
|
||||
// disappears with its node, and two stop accepting writes when either goes. A
|
||||
// cluster that cannot survive the event it was built for is worth refusing at the
|
||||
// point someone asks for it.
|
||||
func validateNodeCount(nodes int) error {
|
||||
if nodes < 1 {
|
||||
return fmt.Errorf("a cluster needs at least one node, got %d", nodes)
|
||||
}
|
||||
if nodes > 1 && nodes < clustersize.MinHANodes {
|
||||
return fmt.Errorf("%d nodes is not a supported cluster size: use 1 for a single server, or %d and up for a cluster that can survive losing one",
|
||||
nodes, clustersize.MinHANodes)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// TODO: Move all logic to a module, use minimal application logic in viper commands.
|
||||
func runAdminInit(cmd *cobra.Command, args []string) {
|
||||
if os.Getuid() != 0 {
|
||||
@@ -1243,6 +1264,15 @@ func runAdminInit(cmd *cobra.Command, args []string) {
|
||||
clusterName, _ := cmd.Flags().GetString("cluster-name")
|
||||
services, _ := cmd.Flags().GetStringSlice("services")
|
||||
|
||||
// Rejected before anything is written, because a two-node cluster has no
|
||||
// correct answer rather than a worse one: no layer of the stack has a
|
||||
// majority to lose, and no replica count for its state both survives a node
|
||||
// going away and still accepts writes.
|
||||
if err := validateNodeCount(nodes); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "--nodes: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Optional operator email — validated up-front so a bad address fails
|
||||
// before we touch any config state. Empty is allowed here; reset / repeat
|
||||
// inits on a box that already has an email in /etc/spinifex/spinifex.toml
|
||||
@@ -2824,7 +2854,7 @@ func initIAMServiceFromConfig() (*handlers_iam.IAMServiceImpl, *config.ClusterCo
|
||||
|
||||
// Background: this runs at CLI top level, where there is no request to
|
||||
// inherit a deadline from and the process exits after the one command.
|
||||
svc, err := handlers_iam.NewIAMServiceImpl(context.Background(), nc, masterKey, len(cfg.Nodes))
|
||||
svc, err := handlers_iam.NewIAMServiceImpl(context.Background(), nc, masterKey)
|
||||
if err != nil {
|
||||
nc.Close()
|
||||
return nil, nil, nil, nil, fmt.Errorf("init IAM service: %w", err)
|
||||
@@ -3469,11 +3499,11 @@ func resolvePublicPoolFlags(source, poolRange, bindBridge, gateway, defaultBindB
|
||||
if gateway == "" {
|
||||
return "", "", "", "", fmt.Errorf("--external-gateway is required with --external-source=static")
|
||||
}
|
||||
parts := strings.SplitN(poolRange, "-", 2)
|
||||
if len(parts) != 2 || net.ParseIP(parts[0]) == nil || net.ParseIP(parts[1]) == nil {
|
||||
var ok bool
|
||||
start, end, ok = strings.Cut(poolRange, "-")
|
||||
if !ok || net.ParseIP(start) == nil || net.ParseIP(end) == nil {
|
||||
return "", "", "", "", fmt.Errorf("--external-pool must be start-end IPs (e.g., 192.168.1.150-192.168.1.250), got: %s", poolRange)
|
||||
}
|
||||
start, end = parts[0], parts[1]
|
||||
default:
|
||||
return "", "", "", "", fmt.Errorf("--external-source must be 'static' or 'dhcp', got: %s", source)
|
||||
}
|
||||
|
||||
@@ -3,7 +3,8 @@ package cmd
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"maps"
|
||||
"slices"
|
||||
"strconv"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/gateway"
|
||||
@@ -166,11 +167,7 @@ func printAccountQuota(resp *gateway.AccountQuotaResponse) {
|
||||
fmt.Printf("%-16s %10s %s\n", "DIMENSION", "LIMIT", "SOURCE")
|
||||
fmt.Printf("%-16s %10s %s\n", "---------", "-----", "------")
|
||||
|
||||
names := make([]string, 0, len(resp.Limits))
|
||||
for name := range resp.Limits {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
names := slices.Sorted(maps.Keys(resp.Limits))
|
||||
for _, name := range names {
|
||||
limit := strconv.Itoa(resp.Limits[name])
|
||||
if resp.Limits[name] == handlers_quota.Unlimited {
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
@@ -420,11 +421,7 @@ func hashStoredMsg(m *jetstream.RawStreamMsg) string {
|
||||
field([]byte(m.Subject))
|
||||
binary.BigEndian.PutUint64(n[:], uint64(m.Time.UnixNano()))
|
||||
h.Write(n[:])
|
||||
keys := make([]string, 0, len(m.Header))
|
||||
for k := range m.Header {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
keys := slices.Sorted(maps.Keys(m.Header))
|
||||
for _, k := range keys {
|
||||
field([]byte(k))
|
||||
for _, v := range m.Header[k] {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
@@ -153,11 +154,7 @@ func compareNodeDigests(nodes []nodeDigest) (kvCompareResult, error) {
|
||||
byName[d.Name] = append(byName[d.Name], v)
|
||||
}
|
||||
}
|
||||
names := make([]string, 0, len(byName))
|
||||
for name := range byName {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
names := slices.Sorted(maps.Keys(byName))
|
||||
|
||||
var res kvCompareResult
|
||||
for _, name := range names {
|
||||
|
||||
@@ -0,0 +1,201 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"text/tabwriter"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var kvReplicasCmd = &cobra.Command{
|
||||
Use: "replicas",
|
||||
Short: "Report how many nodes each KV bucket is replicated across",
|
||||
Long: `Print one line per JetStream KV bucket: how many replicas it has, how many the
|
||||
cluster's size entitles it to, and which nodes currently hold it.
|
||||
|
||||
A bucket with fewer replicas than the cluster has nodes lives on fewer nodes
|
||||
than exist, and every node depends on those. Losing one of them makes the
|
||||
bucket unreadable and unwritable from every node at once, which is a
|
||||
cluster-wide outage caused by a single-node event.
|
||||
|
||||
A bucket also has to be able to serve, which is a different question from how
|
||||
many replicas it was configured for. ONLINE counts the replicas that are caught
|
||||
up and reachable; a bucket whose ONLINE is not a majority of REPLICAS cannot
|
||||
accept a write no matter what its replica count says.
|
||||
|
||||
--repair raises any under-replicated bucket to the cluster's replica count,
|
||||
capped at JetStream's maximum of five. It changes nothing else about a bucket
|
||||
and never lowers a replica count, so it is safe to run on a healthy cluster and
|
||||
safe to run twice.
|
||||
|
||||
Exit status separates the two problems, so this can gate a deployment without
|
||||
parsing the output:
|
||||
|
||||
0 every bucket is at the cluster's replica count and can serve
|
||||
1 at least one bucket is under-replicated — --repair is the answer
|
||||
2 at least one bucket has no quorum, or the report could not be produced —
|
||||
a node needs looking at, and --repair will not help`,
|
||||
Run: runKVReplicas,
|
||||
}
|
||||
|
||||
func init() {
|
||||
kvCmd.AddCommand(kvReplicasCmd)
|
||||
kvReplicasCmd.Flags().Bool("repair", false, "raise every under-replicated bucket to the cluster's node count")
|
||||
kvReplicasCmd.Flags().Bool("json", false, "emit the report as JSON")
|
||||
}
|
||||
|
||||
func runKVReplicas(cmd *cobra.Command, _ []string) {
|
||||
repair, _ := cmd.Flags().GetBool("repair")
|
||||
asJSON, _ := cmd.Flags().GetBool("json")
|
||||
|
||||
// Anything that stops the report being produced exits unreachable rather
|
||||
// than under-replicated: an empty answer is not a cluster with no buckets,
|
||||
// and --repair is not the response to it.
|
||||
_, nc, err := loadConfigAndConnect()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "connect to cluster: %v\n", err)
|
||||
os.Exit(exitUnreachable)
|
||||
}
|
||||
defer nc.Close()
|
||||
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "get JetStream context: %v\n", err)
|
||||
os.Exit(exitUnreachable)
|
||||
}
|
||||
|
||||
// Background: this runs at CLI top level, where there is no request to
|
||||
// inherit a deadline from and the process exits after the one command.
|
||||
ctx := context.Background()
|
||||
reports, err := kvutil.AuditBucketReplicas(ctx, js)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "audit KV bucket replicas: %v\n", err)
|
||||
os.Exit(exitUnreachable)
|
||||
}
|
||||
|
||||
if repair {
|
||||
repairReplicaReports(ctx, js, reports, os.Stderr)
|
||||
}
|
||||
|
||||
if err := writeReplicaReport(os.Stdout, reports, asJSON); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "write report: %v\n", err)
|
||||
os.Exit(exitUnreachable)
|
||||
}
|
||||
|
||||
os.Exit(replicaReportExit(reports))
|
||||
}
|
||||
|
||||
// Exit statuses, documented in the command's own help because a deployment gate
|
||||
// reads them rather than the output.
|
||||
const (
|
||||
exitHealthy = 0
|
||||
// exitUnderReplicated is the repairable problem: the buckets serve, but on
|
||||
// fewer nodes than the cluster has.
|
||||
exitUnderReplicated = 1
|
||||
// exitUnreachable is the problem repair cannot touch — a bucket that cannot
|
||||
// accept a write, or a report that could not be produced at all.
|
||||
exitUnreachable = 2
|
||||
)
|
||||
|
||||
// replicaReportExit is the exit status for a set of reports.
|
||||
//
|
||||
// No quorum outranks under-replicated because they need different responses: one
|
||||
// is fixed by raising a count, the other by finding out why a node is not
|
||||
// answering. A bucket that is both should send the operator to the node.
|
||||
func replicaReportExit(reports []kvutil.BucketReport) int {
|
||||
status := exitHealthy
|
||||
for _, r := range reports {
|
||||
if !r.HasQuorum() {
|
||||
return exitUnreachable
|
||||
}
|
||||
if r.UnderReplicated() {
|
||||
status = exitUnderReplicated
|
||||
}
|
||||
}
|
||||
return status
|
||||
}
|
||||
|
||||
// repairReplicaReports raises every under-replicated bucket and updates reports
|
||||
// in place for the ones that were raised.
|
||||
//
|
||||
// One bucket that cannot be placed must not stop the others being repaired, so a
|
||||
// failure is named on errOut and the sweep continues. The bucket it failed on
|
||||
// stays UNDER-REPLICATED in the report, which is already what makes the exit
|
||||
// status non-zero, so nothing is lost by not returning an error here.
|
||||
func repairReplicaReports(ctx context.Context, js jetstream.JetStream, reports []kvutil.BucketReport, errOut io.Writer) {
|
||||
for i, r := range reports {
|
||||
if !r.UnderReplicated() {
|
||||
continue
|
||||
}
|
||||
if err := kvutil.RaiseBucketReplicas(ctx, js, r.Bucket, r.Want); err != nil {
|
||||
fmt.Fprintf(errOut, "repair %s: %v\n", r.Bucket, err)
|
||||
continue
|
||||
}
|
||||
reports[i].Replicas = r.Want
|
||||
}
|
||||
}
|
||||
|
||||
// underReplicatedCount is how many buckets are below the cluster's replica count.
|
||||
func underReplicatedCount(reports []kvutil.BucketReport) int {
|
||||
under := 0
|
||||
for _, r := range reports {
|
||||
if r.UnderReplicated() {
|
||||
under++
|
||||
}
|
||||
}
|
||||
return under
|
||||
}
|
||||
|
||||
// noQuorumCount is how many buckets cannot currently accept a write.
|
||||
func noQuorumCount(reports []kvutil.BucketReport) int {
|
||||
stuck := 0
|
||||
for _, r := range reports {
|
||||
if !r.HasQuorum() {
|
||||
stuck++
|
||||
}
|
||||
}
|
||||
return stuck
|
||||
}
|
||||
|
||||
// replicaStatus is the one word for a bucket's condition.
|
||||
//
|
||||
// No quorum wins over under-replicated when both are true, matching the exit
|
||||
// status: a bucket that cannot serve is the more urgent of the two and is not
|
||||
// what --repair is for.
|
||||
func replicaStatus(r kvutil.BucketReport) string {
|
||||
switch {
|
||||
case !r.HasQuorum():
|
||||
return "NO-QUORUM"
|
||||
case r.UnderReplicated():
|
||||
return "UNDER-REPLICATED"
|
||||
default:
|
||||
return "ok"
|
||||
}
|
||||
}
|
||||
|
||||
// writeReplicaReport renders reports as a table, or as JSON when asJSON.
|
||||
func writeReplicaReport(out io.Writer, reports []kvutil.BucketReport, asJSON bool) error {
|
||||
if asJSON {
|
||||
return json.NewEncoder(out).Encode(reports)
|
||||
}
|
||||
|
||||
w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
|
||||
fmt.Fprintln(w, "BUCKET\tREPLICAS\tWANT\tONLINE\tSTATUS\tLEADER\tHELD BY")
|
||||
for _, r := range reports {
|
||||
fmt.Fprintf(w, "%s\t%d\t%d\t%d\t%s\t%s\t%s\n",
|
||||
r.Bucket, r.Replicas, r.Want, r.Online, replicaStatus(r), r.Leader, strings.Join(r.Peers, ","))
|
||||
}
|
||||
if err := w.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := fmt.Fprintf(out, "\n%d buckets, %d under-replicated, %d without quorum\n",
|
||||
len(reports), underReplicatedCount(reports), noQuorumCount(reports))
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
//test:in-package — it tests this command's unexported rendering and repair
|
||||
// helpers, which exist only to be exercised without a live cluster.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/spinifex/clustersize"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestWriteReplicaReport_NamesEveryUnderReplicatedBucket(t *testing.T) {
|
||||
reports := []kvutil.BucketReport{
|
||||
{Bucket: "healthy", Replicas: 3, Want: 3, Online: 3, Leader: "a", Peers: []string{"a", "b", "c"}},
|
||||
{Bucket: "short", Replicas: 1, Want: 3, Online: 1, Leader: "a", Peers: []string{"a"}},
|
||||
{Bucket: "stuck", Replicas: 3, Want: 3, Online: 1, Peers: []string{"a", "b", "c"}},
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
require.NoError(t, writeReplicaReport(&out, reports, false))
|
||||
got := out.String()
|
||||
|
||||
assert.Contains(t, got, "BUCKET")
|
||||
assert.Contains(t, got, "HELD BY")
|
||||
assert.Regexp(t, `healthy\s+3\s+3\s+3\s+ok\s+a\s+a,b,c`, got)
|
||||
assert.Regexp(t, `short\s+1\s+3\s+1\s+UNDER-REPLICATED\s+a\s+a`, got)
|
||||
// A bucket that cannot serve reads as NO-QUORUM even though its configured
|
||||
// count is right, because raising a count is not the response to it.
|
||||
assert.Regexp(t, `stuck\s+3\s+3\s+1\s+NO-QUORUM`, got)
|
||||
assert.Contains(t, got, "3 buckets, 1 under-replicated, 1 without quorum")
|
||||
}
|
||||
|
||||
// TestReplicaReportExit pins the three exit classes a deployment gate reads. They
|
||||
// are separate because the responses are: a raise fixes an under-replicated
|
||||
// bucket, and nothing the command can do fixes one that will not answer.
|
||||
func TestReplicaReportExit(t *testing.T) {
|
||||
healthy := kvutil.BucketReport{Bucket: "a", Replicas: 3, Want: 3, Online: 3}
|
||||
short := kvutil.BucketReport{Bucket: "b", Replicas: 1, Want: 3, Online: 1}
|
||||
stuck := kvutil.BucketReport{Bucket: "c", Replicas: 3, Want: 3, Online: 1}
|
||||
|
||||
assert.Equal(t, exitHealthy, replicaReportExit(nil))
|
||||
assert.Equal(t, exitHealthy, replicaReportExit([]kvutil.BucketReport{healthy}))
|
||||
assert.Equal(t, exitUnderReplicated, replicaReportExit([]kvutil.BucketReport{healthy, short}))
|
||||
assert.Equal(t, exitUnreachable, replicaReportExit([]kvutil.BucketReport{healthy, stuck}))
|
||||
// Both problems at once sends the operator to the node rather than to --repair.
|
||||
assert.Equal(t, exitUnreachable, replicaReportExit([]kvutil.BucketReport{short, stuck}))
|
||||
}
|
||||
|
||||
// TestWriteReplicaReport_JSONIsWhatTheE2ESuiteDecodes keeps the machine-readable
|
||||
// form honest: the e2e suite and anything gating a deployment parse these field
|
||||
// names, so a rename here breaks them silently.
|
||||
func TestWriteReplicaReport_JSONIsWhatTheE2ESuiteDecodes(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
require.NoError(t, writeReplicaReport(&out, []kvutil.BucketReport{
|
||||
{Bucket: "short", Replicas: 1, Want: 3, Cluster: "spinifex", Online: 1, Leader: "a", Peers: []string{"a"}},
|
||||
}, true))
|
||||
|
||||
assert.Contains(t, out.String(), `"bucket":"short"`)
|
||||
|
||||
var decoded []kvutil.BucketReport
|
||||
require.NoError(t, json.Unmarshal(out.Bytes(), &decoded))
|
||||
require.Len(t, decoded, 1)
|
||||
assert.Equal(t, "short", decoded[0].Bucket)
|
||||
assert.Equal(t, 1, decoded[0].Replicas)
|
||||
assert.Equal(t, 3, decoded[0].Want)
|
||||
assert.Equal(t, 1, decoded[0].Online)
|
||||
assert.Equal(t, "a", decoded[0].Leader)
|
||||
assert.True(t, decoded[0].UnderReplicated())
|
||||
assert.True(t, decoded[0].HasQuorum())
|
||||
}
|
||||
|
||||
func TestUnderReplicatedCount(t *testing.T) {
|
||||
assert.Equal(t, 0, underReplicatedCount(nil))
|
||||
assert.Equal(t, 2, underReplicatedCount([]kvutil.BucketReport{
|
||||
{Bucket: "a", Replicas: 1, Want: 3},
|
||||
{Bucket: "b", Replicas: 3, Want: 3},
|
||||
{Bucket: "c", Replicas: 2, Want: 3},
|
||||
}))
|
||||
}
|
||||
|
||||
// TestRepairReplicaReports_KeepsGoingPastABucketItCannotPlace is the behaviour
|
||||
// that makes --repair usable mid-growth: the buckets it can raise are raised,
|
||||
// and the one it cannot is named rather than aborting the run.
|
||||
func TestRepairReplicaReports_KeepsGoingPastABucketItCannotPlace(t *testing.T) {
|
||||
_, nc, js := testutil.StartTestJetStream(t)
|
||||
defer nc.Close()
|
||||
|
||||
for _, b := range []string{"repair-alpha", "repair-beta"} {
|
||||
_, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: b, History: 1})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// One embedded server cannot hold two replicas, so every raise fails — and
|
||||
// every bucket must still have been attempted and named.
|
||||
reports := []kvutil.BucketReport{
|
||||
{Bucket: "repair-alpha", Replicas: 1, Want: 2},
|
||||
{Bucket: "repair-beta", Replicas: 1, Want: 2},
|
||||
}
|
||||
var errOut bytes.Buffer
|
||||
repairReplicaReports(t.Context(), js, reports, &errOut)
|
||||
|
||||
assert.Contains(t, errOut.String(), "repair repair-alpha")
|
||||
assert.Contains(t, errOut.String(), "repair repair-beta")
|
||||
assert.Equal(t, 2, underReplicatedCount(reports), "a failed raise must not be reported as repaired")
|
||||
}
|
||||
|
||||
// TestRepairReplicaReports_RaisesWhatItCan covers the success side against a real
|
||||
// server, so the in-place report update is proved rather than assumed.
|
||||
func TestRepairReplicaReports_RaisesWhatItCan(t *testing.T) {
|
||||
_, nc, js := testutil.StartTestJetStream(t)
|
||||
defer nc.Close()
|
||||
clustersize.DeclareForTest(t, 1)
|
||||
|
||||
_, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: "repair-ok", History: 1})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Want 1 against a bucket already at 1: the raise is a no-op that must still
|
||||
// leave the report reading as healthy rather than as repaired-and-wrong.
|
||||
reports := []kvutil.BucketReport{{Bucket: "repair-ok", Replicas: 1, Want: 1}}
|
||||
var errOut bytes.Buffer
|
||||
repairReplicaReports(t.Context(), js, reports, &errOut)
|
||||
|
||||
assert.Empty(t, strings.TrimSpace(errOut.String()))
|
||||
assert.Equal(t, 0, underReplicatedCount(reports))
|
||||
}
|
||||
@@ -279,7 +279,7 @@ func TestKVDigestIgnoresExpiry(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
js, err := jetstream.New(nc)
|
||||
require.NoError(t, err)
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "ttl", TTL: time.Second})
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "ttl", TTL: 200 * time.Millisecond})
|
||||
require.NoError(t, err)
|
||||
_, err = kv.PutString(ctx, "k", "v")
|
||||
require.NoError(t, err)
|
||||
@@ -291,7 +291,7 @@ func TestKVDigestIgnoresExpiry(t *testing.T) {
|
||||
ns.Shutdown()
|
||||
ns.WaitForShutdown()
|
||||
|
||||
time.Sleep(1500 * time.Millisecond)
|
||||
time.Sleep(400 * time.Millisecond)
|
||||
d, err := digestStore(ctx, work, nil, false, metas)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, d, 1)
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
//test:in-package — validateNodeCount is an unexported guard on this command's
|
||||
// flags, and exporting it would widen the package's API for a test's convenience.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestValidateNodeCount pins the supported cluster sizes. Two is rejected rather
|
||||
// than warned about: on two servers no layer of the stack has a majority to lose,
|
||||
// and the control plane's own state has no replica count that both survives a node
|
||||
// going away and still accepts writes.
|
||||
func TestValidateNodeCount(t *testing.T) {
|
||||
for _, nodes := range []int{1, 3, 4, 5, 9} {
|
||||
if err := validateNodeCount(nodes); err != nil {
|
||||
t.Errorf("validateNodeCount(%d) = %v, want nil", nodes, err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, nodes := range []int{0, -1} {
|
||||
if err := validateNodeCount(nodes); err == nil {
|
||||
t.Errorf("validateNodeCount(%d) returned no error", nodes)
|
||||
}
|
||||
}
|
||||
|
||||
err := validateNodeCount(2)
|
||||
if err == nil {
|
||||
t.Fatal("validateNodeCount(2) returned no error")
|
||||
}
|
||||
// The message has to say what to do instead, because the operator typed a
|
||||
// number and needs to know which one to type next.
|
||||
for _, want := range []string{"2", "1", "3"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("error %q does not mention %s", err, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -438,14 +438,10 @@ func parseWeightsS3URI(uri string) (bucket, prefix string, err error) {
|
||||
if trimmed == uri {
|
||||
return "", "", fmt.Errorf("invalid --s3-uri %q: expected s3://bucket/prefix", uri)
|
||||
}
|
||||
parts := strings.SplitN(trimmed, "/", 2)
|
||||
bucket = parts[0]
|
||||
bucket, prefix, _ = strings.Cut(trimmed, "/")
|
||||
if bucket == "" {
|
||||
return "", "", fmt.Errorf("invalid --s3-uri %q: missing bucket", uri)
|
||||
}
|
||||
if len(parts) == 2 {
|
||||
prefix = parts[1]
|
||||
}
|
||||
if prefix != "" && !strings.HasSuffix(prefix, "/") {
|
||||
prefix += "/"
|
||||
}
|
||||
@@ -854,7 +850,7 @@ func runOchreWeightsStage(cmd *cobra.Command, _ []string) {
|
||||
ochreExit(1)
|
||||
return
|
||||
}
|
||||
weightsStore := gateway_bedrock.NewWeightsStore(js, len(appConfig.Nodes))
|
||||
weightsStore := gateway_bedrock.NewWeightsStore(js)
|
||||
store := objectstore.NewS3ObjectStoreFromConfig(node.Predastore.Host, node.Predastore.Region, node.Predastore.AccessKey, node.Predastore.SecretKey)
|
||||
|
||||
provider := ebsprovider.NewNATSProvider(nc, imageImportTimeout)
|
||||
@@ -930,7 +926,7 @@ func runOchreWeightsList(_ *cobra.Command, _ []string) {
|
||||
ochreExit(1)
|
||||
return
|
||||
}
|
||||
weightsStore := gateway_bedrock.NewWeightsStore(js, len(appConfig.Nodes))
|
||||
weightsStore := gateway_bedrock.NewWeightsStore(js)
|
||||
store := objectstore.NewS3ObjectStoreFromConfig(node.Predastore.Host, node.Predastore.Region, node.Predastore.AccessKey, node.Predastore.SecretKey)
|
||||
checkSnapshotLive := newWeightsSnapshotChecker(store, node.Predastore.Bucket)
|
||||
|
||||
@@ -967,7 +963,7 @@ func removeWeights(ctx context.Context, weightsStore *gateway_bedrock.WeightsSto
|
||||
func runOchreWeightsRemove(cmd *cobra.Command, _ []string) {
|
||||
modelID, _ := cmd.Flags().GetString("model-id")
|
||||
|
||||
appConfig, nc, err := loadConfigAndConnectFn()
|
||||
_, nc, err := loadConfigAndConnectFn()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
|
||||
ochreExit(1)
|
||||
@@ -981,7 +977,7 @@ func runOchreWeightsRemove(cmd *cobra.Command, _ []string) {
|
||||
ochreExit(1)
|
||||
return
|
||||
}
|
||||
weightsStore := gateway_bedrock.NewWeightsStore(js, len(appConfig.Nodes))
|
||||
weightsStore := gateway_bedrock.NewWeightsStore(js)
|
||||
|
||||
entry, err := removeWeights(context.Background(), weightsStore, modelID)
|
||||
if err != nil {
|
||||
@@ -1014,7 +1010,7 @@ func resolveHFToken(ctx context.Context, cmd *cobra.Command, cfg *config.Cluster
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
credStore := gateway_bedrock.NewCredentialStore(js, masterKey, len(cfg.Nodes), nil)
|
||||
credStore := gateway_bedrock.NewCredentialStore(js, masterKey, nil)
|
||||
token, ok, err := credStore.Resolve(ctx, utils.GlobalAccountID, vendorHuggingFace)
|
||||
if err != nil || !ok {
|
||||
return ""
|
||||
@@ -1088,7 +1084,7 @@ func ochreCredentialsStore() (*gateway_bedrock.CredentialStore, func(), error) {
|
||||
nc.Close()
|
||||
return nil, nil, fmt.Errorf("jetstream context: %w", err)
|
||||
}
|
||||
return gateway_bedrock.NewCredentialStore(js, masterKey, len(cfg.Nodes), nil), func() { nc.Close() }, nil
|
||||
return gateway_bedrock.NewCredentialStore(js, masterKey, nil), func() { nc.Close() }, nil
|
||||
}
|
||||
|
||||
func runOchreCredentialsSet(cmd *cobra.Command, _ []string) {
|
||||
@@ -1186,7 +1182,7 @@ func init() {
|
||||
// ochreAccessStore connects to the cluster and returns the grant store along
|
||||
// with a cleanup that closes the connection.
|
||||
func ochreAccessStore() (*gateway_bedrock.ModelAccessStore, func(), error) {
|
||||
cfg, nc, err := loadConfigAndConnectFn()
|
||||
_, nc, err := loadConfigAndConnectFn()
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("connect to cluster: %w", err)
|
||||
}
|
||||
@@ -1195,7 +1191,7 @@ func ochreAccessStore() (*gateway_bedrock.ModelAccessStore, func(), error) {
|
||||
nc.Close()
|
||||
return nil, nil, fmt.Errorf("jetstream context: %w", err)
|
||||
}
|
||||
return gateway_bedrock.NewModelAccessStore(js, len(cfg.Nodes)), func() { nc.Close() }, nil
|
||||
return gateway_bedrock.NewModelAccessStore(js), func() { nc.Close() }, nil
|
||||
}
|
||||
|
||||
// ochreTargetModels resolves the --model-id / --all-models pair into the model
|
||||
|
||||
@@ -470,7 +470,7 @@ func TestResolveHFToken_FallsBackToStoredPlatformCredential(t *testing.T) {
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(baseDir, "config"), 0750))
|
||||
require.NoError(t, handlers_iam.SaveMasterKey(filepath.Join(baseDir, "config", "master.key"), masterKey))
|
||||
|
||||
credStore := gateway_bedrock.NewCredentialStore(js, masterKey, 1, nil)
|
||||
credStore := gateway_bedrock.NewCredentialStore(js, masterKey, nil)
|
||||
require.NoError(t, credStore.PutCredential(context.Background(), utils.GlobalAccountID, vendorHuggingFace, "hf_stored_token"))
|
||||
|
||||
cmd := newOchreWeightsPullTestCmd(t, selfHostModelID, testHFRepo, "main", "", "")
|
||||
@@ -621,7 +621,7 @@ func TestRunOchreCredentialsSet_RoundTripsAndDefaultsToPlatformAccount(t *testin
|
||||
defer verifyConn.Close()
|
||||
verifyJS := testutil.NewJetStream(t, verifyConn)
|
||||
|
||||
credStore := gateway_bedrock.NewCredentialStore(verifyJS, masterKey, 1, nil)
|
||||
credStore := gateway_bedrock.NewCredentialStore(verifyJS, masterKey, nil)
|
||||
token, ok, err := credStore.Resolve(context.Background(), utils.GlobalAccountID, vendorHuggingFace)
|
||||
require.NoError(t, err)
|
||||
require.True(t, ok)
|
||||
|
||||
@@ -42,7 +42,7 @@ const (
|
||||
func newWeightsStoreForTest(t *testing.T) *gateway_bedrock.WeightsStore {
|
||||
t.Helper()
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
return gateway_bedrock.NewWeightsStore(testutil.NewJetStream(t, nc), 1)
|
||||
return gateway_bedrock.NewWeightsStore(testutil.NewJetStream(t, nc))
|
||||
}
|
||||
|
||||
// explodingObjectStore fails the test immediately if any of its methods is
|
||||
@@ -775,7 +775,7 @@ func TestRunOchreWeightsList_PrintsStagedModels(t *testing.T) {
|
||||
}
|
||||
stubConnect(t, cfg, nc, nil)
|
||||
|
||||
seedStore := gateway_bedrock.NewWeightsStore(js, 1)
|
||||
seedStore := gateway_bedrock.NewWeightsStore(js)
|
||||
require.NoError(t, seedStore.PutWeights(context.Background(), selfHostModelID, "s3://models/llama-3.2-1b/", "snap-0001"))
|
||||
|
||||
out := captureStdout(t, func() { runOchreWeightsList(nil, nil) })
|
||||
@@ -822,7 +822,7 @@ func TestRunOchreWeightsRemove_RoundTripPrintsSnapshotAndDropsEntry(t *testing.T
|
||||
ns, nc, js := testutil.StartTestJetStream(t)
|
||||
stubConnect(t, fakeClusterConfig(), nc, nil)
|
||||
|
||||
seedStore := gateway_bedrock.NewWeightsStore(js, 1)
|
||||
seedStore := gateway_bedrock.NewWeightsStore(js)
|
||||
require.NoError(t, seedStore.PutWeights(context.Background(), selfHostModelID, "s3://models/llama-3.2-1b/", "snap-0001"))
|
||||
|
||||
cmd := newOchreWeightsRemoveTestCmd(t, selfHostModelID)
|
||||
@@ -835,7 +835,7 @@ func TestRunOchreWeightsRemove_RoundTripPrintsSnapshotAndDropsEntry(t *testing.T
|
||||
verifyNC, err := nats.Connect(ns.ClientURL())
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { verifyNC.Close() })
|
||||
verifyStore := gateway_bedrock.NewWeightsStore(testutil.NewJetStream(t, verifyNC), 1)
|
||||
verifyStore := gateway_bedrock.NewWeightsStore(testutil.NewJetStream(t, verifyNC))
|
||||
|
||||
_, ok, err := verifyStore.GetWeights(context.Background(), selfHostModelID)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -283,11 +283,9 @@ func truncateForTable(s string, maxRunes int) string {
|
||||
}
|
||||
|
||||
// formatQueryResults renders 'ochre vector query' output: a truncated table
|
||||
// by default, or the full response as indented JSON when asJSON is set
|
||||
// (D10's "full text via source ref" -- here, via --json).
|
||||
// by default, or the full response as indented JSON when asJSON is set.
|
||||
func formatQueryResults(results []handlers_ochrevector.QueryResult, asJSON bool) (string, error) {
|
||||
if asJSON {
|
||||
//nolint:musttag // QueryResult is an internal, non-wire Go type (predates this CLI); it round-trips fine on its default field names.
|
||||
data, err := json.MarshalIndent(results, "", " ")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("encode results as JSON: %w", err)
|
||||
@@ -317,7 +315,6 @@ func parseFilterFlag(raw string) (*handlers_ochrevector.Filter, error) {
|
||||
return nil, nil
|
||||
}
|
||||
var f handlers_ochrevector.Filter
|
||||
//nolint:musttag // Filter is the existing filter-AST type (D9); it round-trips fine on its default field names.
|
||||
if err := json.Unmarshal([]byte(raw), &f); err != nil {
|
||||
return nil, fmt.Errorf("decode --filter: %w", err)
|
||||
}
|
||||
|
||||
@@ -188,13 +188,14 @@ func describePrincipal(svc handlers_iam.IAMService, accountID, userName string)
|
||||
}
|
||||
|
||||
// principalPolicyActions returns the actions an inline policy allows, shortened
|
||||
// to the method name. An undecodable document reports nothing rather than
|
||||
// guessing, and the listing shows the user with no grants.
|
||||
// to the method name. A NotAction grant is one entry naming what it excludes.
|
||||
// An undecodable document reports nothing rather than guessing.
|
||||
func principalPolicyActions(document string) []string {
|
||||
var doc struct {
|
||||
Statement []struct {
|
||||
Effect string `json:"Effect"`
|
||||
Action handlers_iam.StringOrArr `json:"Action"`
|
||||
Effect string `json:"Effect"`
|
||||
Action handlers_iam.StringOrArr `json:"Action"`
|
||||
NotAction handlers_iam.StringOrArr `json:"NotAction"`
|
||||
} `json:"Statement"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(document), &doc); err != nil {
|
||||
@@ -209,6 +210,13 @@ func principalPolicyActions(document string) []string {
|
||||
for _, action := range stmt.Action {
|
||||
actions = append(actions, strings.TrimPrefix(action, adminPrincipalService+":"))
|
||||
}
|
||||
if len(stmt.NotAction) > 0 {
|
||||
excluded := make([]string, 0, len(stmt.NotAction))
|
||||
for _, action := range stmt.NotAction {
|
||||
excluded = append(excluded, strings.TrimPrefix(action, adminPrincipalService+":"))
|
||||
}
|
||||
actions = append(actions, "NotAction("+strings.Join(excluded, ", ")+")")
|
||||
}
|
||||
}
|
||||
sort.Strings(actions)
|
||||
return actions
|
||||
|
||||
@@ -102,6 +102,18 @@ func TestPrincipalPolicyActionsRoundTrip(t *testing.T) {
|
||||
cmd.PrincipalPolicyActions(cmd.PrincipalPolicyDocument(grants)))
|
||||
}
|
||||
|
||||
// A NotAction grant covers everything but its list, so rendering only Action
|
||||
// would show the principal as holding nothing.
|
||||
func TestPrincipalPolicyActionsRenderNotAction(t *testing.T) {
|
||||
document := `{"Version":"2012-10-17","Statement":[
|
||||
{"Effect":"Allow","NotAction":["spinifex:DeleteAccount","spinifex:CreateAccount"],"Resource":"*"},
|
||||
{"Effect":"Allow","Action":"spinifex:ListAccounts","Resource":"*"},
|
||||
{"Effect":"Deny","NotAction":"spinifex:ListAccounts","Resource":"*"}]}`
|
||||
|
||||
assert.Equal(t, []string{"ListAccounts", "NotAction(DeleteAccount, CreateAccount)"},
|
||||
cmd.PrincipalPolicyActions(document))
|
||||
}
|
||||
|
||||
// A document nobody can parse is reported as no grants rather than as a guess.
|
||||
func TestPrincipalPolicyActionsIgnoreAnUndecodableDocument(t *testing.T) {
|
||||
assert.Empty(t, cmd.PrincipalPolicyActions("{not json"))
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/migrate"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
|
||||
"github.com/mulgadc/spinifex/spinifex/operator/host/systemd"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/migrate"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
|
||||
"github.com/mulgadc/spinifex/spinifex/operator/host/systemd"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
@@ -167,7 +167,7 @@ func runClusterShutdown(cmd *cobra.Command, args []string) {
|
||||
fmt.Printf("Timeout per phase: %s\n\n", timeout)
|
||||
|
||||
// Write cluster shutdown marker to KV
|
||||
jsm, err := daemon.NewJetStreamManager(nc, 1)
|
||||
jsm, err := daemon.NewJetStreamManager(nc)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: failed to create JetStream manager: %v\n", err)
|
||||
} else {
|
||||
|
||||
@@ -38,7 +38,7 @@ func TestDrainDHCPLeasesSumsReleased(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
fakeDrainResponder(t, nc, 7, "")
|
||||
|
||||
released, responders := drainDHCPLeases(nc, 2*time.Second)
|
||||
released, responders := drainDHCPLeases(nc, 250*time.Millisecond)
|
||||
assert.Equal(t, 7, released)
|
||||
assert.Equal(t, 1, responders)
|
||||
}
|
||||
@@ -49,7 +49,7 @@ func TestDrainDHCPLeasesReportsResponderError(t *testing.T) {
|
||||
|
||||
// An erroring responder still counts as a responder but contributes
|
||||
// zero released leases.
|
||||
released, responders := drainDHCPLeases(nc, 2*time.Second)
|
||||
released, responders := drainDHCPLeases(nc, 250*time.Millisecond)
|
||||
assert.Equal(t, 0, released)
|
||||
assert.Equal(t, 1, responders)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
//test:in-package — the embedded nats.conf template it checks is unexported,
|
||||
// and exporting it would widen the package's API for a test's convenience.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
natssvc "github.com/mulgadc/spinifex/spinifex/services/nats"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestNATSConfServerNameMatchesTheHelper ties the rendered server_name to
|
||||
// nats.ServerName. JetStream reports stream peers by server name, so anything
|
||||
// mapping a replica back to the node holding it reads the prefix from that
|
||||
// helper — and a template that drifted from it would break the mapping silently,
|
||||
// reporting a healthy cluster as one whose buckets name no node at all.
|
||||
func TestNATSConfServerNameMatchesTheHelper(t *testing.T) {
|
||||
var line string
|
||||
for l := range strings.SplitSeq(natsConfTemplate, "\n") {
|
||||
if strings.HasPrefix(strings.TrimSpace(l), "server_name:") {
|
||||
line = strings.TrimSpace(l)
|
||||
break
|
||||
}
|
||||
}
|
||||
require.NotEmpty(t, line, "nats.conf no longer sets server_name")
|
||||
|
||||
got := strings.TrimSpace(strings.TrimPrefix(line, "server_name:"))
|
||||
assert.Equal(t, natssvc.ServerName("{{.Node}}"), got)
|
||||
|
||||
node, ok := natssvc.NodeFromServerName(natssvc.ServerName("node2"))
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, "node2", node)
|
||||
|
||||
_, ok = natssvc.NodeFromServerName("node2")
|
||||
assert.False(t, ok, "a bare node name is not a server name")
|
||||
|
||||
// The bare prefix is a name we could not have generated, and mapping it to the
|
||||
// empty node would put a phantom peer into anything counting nodes.
|
||||
_, ok = natssvc.NodeFromServerName(natssvc.ServerNamePrefix)
|
||||
assert.False(t, ok, "the prefix with no node after it is not a server name")
|
||||
}
|
||||
@@ -225,42 +225,6 @@ var predastoreStartCmd = &cobra.Command{
|
||||
},
|
||||
}
|
||||
|
||||
var predastoreStopCmd = &cobra.Command{
|
||||
Use: "stop",
|
||||
Short: "Stop the predastore service",
|
||||
SilenceErrors: true,
|
||||
SilenceUsage: true,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
fmt.Println("Stopping predastore service...")
|
||||
|
||||
// Only the pid directory matters here: Stop finds the running process
|
||||
// through it, and start wrote the file there.
|
||||
service, err := service.New("predastore", &predastore.Config{
|
||||
BasePath: viper.GetString("predastore-base-path"),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("create predastore service: %w", err)
|
||||
}
|
||||
|
||||
if err = service.Stop(); err != nil {
|
||||
return fmt.Errorf("stop predastore service: %w", err)
|
||||
}
|
||||
|
||||
fmt.Println("Predastore service stopped")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var predastoreStatusCmd = &cobra.Command{
|
||||
Use: "status",
|
||||
Short: "Get status of the predastore service",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
// Add your status logic here
|
||||
fmt.Println("Predastore service status: ...")
|
||||
},
|
||||
}
|
||||
|
||||
// Repeat for viperblock.
|
||||
var viperblockStartCmd = &cobra.Command{
|
||||
Use: "start",
|
||||
@@ -384,7 +348,6 @@ var viperblockStartCmd = &cobra.Command{
|
||||
SecretKey: nodeConfig.Predastore.SecretKey,
|
||||
BaseDir: nodeConfig.Predastore.BaseDir,
|
||||
NodeName: clusterConfig.Node,
|
||||
KVReplicas: len(clusterConfig.Nodes),
|
||||
ShardWAL: shardWAL,
|
||||
GCEnabled: gcEnabled,
|
||||
WALBaseDir: nodeConfig.Viperblock.WALBaseDir,
|
||||
@@ -414,37 +377,6 @@ var viperblockStartCmd = &cobra.Command{
|
||||
},
|
||||
}
|
||||
|
||||
var viperblockStopCmd = &cobra.Command{
|
||||
Use: "stop",
|
||||
Short: "Stop the viperblock service",
|
||||
SilenceErrors: true,
|
||||
SilenceUsage: true,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
fmt.Println("Stopping viperblock service...")
|
||||
|
||||
service, err := service.New("viperblock", &viperblockd.Config{})
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("create viperblock service: %w", err)
|
||||
}
|
||||
|
||||
if err = service.Stop(); err != nil {
|
||||
return fmt.Errorf("stop viperblock service: %w", err)
|
||||
}
|
||||
|
||||
fmt.Println("Viperblock service stopped")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var viperblockStatusCmd = &cobra.Command{
|
||||
Use: "status",
|
||||
Short: "Get status of the viperblock service",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
fmt.Println("Viperblock service status: ...")
|
||||
},
|
||||
}
|
||||
|
||||
var qemunbdCmd = &cobra.Command{
|
||||
Use: "qemunbd",
|
||||
Short: "Manage the qemunbd service",
|
||||
@@ -519,37 +451,6 @@ var qemunbdStartCmd = &cobra.Command{
|
||||
},
|
||||
}
|
||||
|
||||
var qemunbdStopCmd = &cobra.Command{
|
||||
Use: "stop",
|
||||
Short: "Stop the qemunbd service",
|
||||
SilenceErrors: true,
|
||||
SilenceUsage: true,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
fmt.Println("Stopping qemunbd service...")
|
||||
|
||||
service, err := service.New("qemunbd", &qemunbdd.Config{})
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("create qemunbd service: %w", err)
|
||||
}
|
||||
|
||||
if err = service.Stop(); err != nil {
|
||||
return fmt.Errorf("stop qemunbd service: %w", err)
|
||||
}
|
||||
|
||||
fmt.Println("qemunbd service stopped")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var qemunbdStatusCmd = &cobra.Command{
|
||||
Use: "status",
|
||||
Short: "Get status of the qemunbd service",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
fmt.Println("qemunbd service status: ...")
|
||||
},
|
||||
}
|
||||
|
||||
// Repeat for nats.
|
||||
var natsStartCmd = &cobra.Command{
|
||||
Use: "start",
|
||||
@@ -590,37 +491,6 @@ var natsStartCmd = &cobra.Command{
|
||||
},
|
||||
}
|
||||
|
||||
var natsStopCmd = &cobra.Command{
|
||||
Use: "stop",
|
||||
Short: "Stop the nats service",
|
||||
SilenceErrors: true,
|
||||
SilenceUsage: true,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
fmt.Println("Stopping nats service...")
|
||||
|
||||
service, err := service.New("nats", &nats.Config{})
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("create nats service: %w", err)
|
||||
}
|
||||
|
||||
if err = service.Stop(); err != nil {
|
||||
return fmt.Errorf("stop nats service: %w", err)
|
||||
}
|
||||
|
||||
fmt.Println("Nats service stopped")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var natsStatusCmd = &cobra.Command{
|
||||
Use: "status",
|
||||
Short: "Get status of the nats service",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
fmt.Println("Nats service status: ...")
|
||||
},
|
||||
}
|
||||
|
||||
// Repeat for spinifex.
|
||||
var spinifexStartCmd = &cobra.Command{
|
||||
Use: "start",
|
||||
@@ -683,37 +553,6 @@ var spinifexStartCmd = &cobra.Command{
|
||||
},
|
||||
}
|
||||
|
||||
var spinifexStopCmd = &cobra.Command{
|
||||
Use: "stop",
|
||||
Short: "Stop the spinifex service",
|
||||
SilenceErrors: true,
|
||||
SilenceUsage: true,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
fmt.Println("Stopping spinifex service...")
|
||||
|
||||
service, err := service.New("spinifex", &config.ClusterConfig{})
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("create spinifex service: %w", err)
|
||||
}
|
||||
|
||||
if err = service.Stop(); err != nil {
|
||||
return fmt.Errorf("stop spinifex service: %w", err)
|
||||
}
|
||||
|
||||
fmt.Println("Spinifex service stopped")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var spinifexStatusCmd = &cobra.Command{
|
||||
Use: "status",
|
||||
Short: "Get status of the spinifex service",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
fmt.Println("Spinifex service status: ...")
|
||||
},
|
||||
}
|
||||
|
||||
// AWS GW
|
||||
|
||||
var awsgwStartCmd = &cobra.Command{
|
||||
@@ -786,37 +625,6 @@ var awsgwStartCmd = &cobra.Command{
|
||||
},
|
||||
}
|
||||
|
||||
var awsgwStopCmd = &cobra.Command{
|
||||
Use: "stop",
|
||||
Short: "Stop the awsgw service",
|
||||
SilenceErrors: true,
|
||||
SilenceUsage: true,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
fmt.Println("Stopping awsgw service...")
|
||||
|
||||
service, err := service.New("awsgw", &config.ClusterConfig{})
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("create awsgw service: %w", err)
|
||||
}
|
||||
|
||||
if err = service.Stop(); err != nil {
|
||||
return fmt.Errorf("stop awsgw service: %w", err)
|
||||
}
|
||||
|
||||
fmt.Println("AWSGW service stopped")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var awsgwStatusCmd = &cobra.Command{
|
||||
Use: "status",
|
||||
Short: "Get status of the awsgw service",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
fmt.Println("AWSGW service status: ...")
|
||||
},
|
||||
}
|
||||
|
||||
// consoleRegion is the region the console signs with. awsgw pins signature
|
||||
// verification to this same node field, so reading it from anywhere else —
|
||||
// including the cluster-wide [aws] region — would let the two drift and fail
|
||||
@@ -877,50 +685,6 @@ var spinifexUIStartCmd = &cobra.Command{
|
||||
},
|
||||
}
|
||||
|
||||
var spinifexUIStopCmd = &cobra.Command{
|
||||
Use: "stop",
|
||||
Short: "Stop the spinifex-ui service",
|
||||
SilenceErrors: true,
|
||||
SilenceUsage: true,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
fmt.Println("Stopping spinifex-ui service...")
|
||||
|
||||
svc, err := service.New("spinifex-ui", &spinifexui.Config{})
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("create spinifex-ui service: %w", err)
|
||||
}
|
||||
|
||||
if err = svc.Stop(); err != nil {
|
||||
return fmt.Errorf("stop spinifex-ui service: %w", err)
|
||||
}
|
||||
fmt.Println("spinifex-ui service stopped")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var spinifexUIStatusCmd = &cobra.Command{
|
||||
Use: "status",
|
||||
Short: "Get status of the spinifex-ui service",
|
||||
SilenceErrors: true,
|
||||
SilenceUsage: true,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
svc, err := service.New("spinifex-ui", &spinifexui.Config{})
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("create spinifex-ui service: %w", err)
|
||||
}
|
||||
|
||||
status, err := svc.Status()
|
||||
if err != nil {
|
||||
return fmt.Errorf("get spinifex-ui service status: %w", err)
|
||||
}
|
||||
|
||||
fmt.Println("spinifex-ui service status:", status)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
// checkLegacyWanBridgeKey fails vpcd startup if the deprecated `wan_bridge`
|
||||
// TOML key or `SPINIFEX_VPCD_WAN_BRIDGE` env-var is present. There is no silent
|
||||
// alias or auto-rewrite; the operator must remove the key before vpcd will start.
|
||||
@@ -1048,35 +812,6 @@ var vpcdStartCmd = &cobra.Command{
|
||||
},
|
||||
}
|
||||
|
||||
var vpcdStopCmd = &cobra.Command{
|
||||
Use: "stop",
|
||||
Short: "Stop the vpcd service",
|
||||
SilenceErrors: true,
|
||||
SilenceUsage: true,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
fmt.Println("Stopping vpcd service...")
|
||||
|
||||
svc, err := service.New("vpcd", &vpcd.Config{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("create vpcd service: %w", err)
|
||||
}
|
||||
|
||||
if err = svc.Stop(); err != nil {
|
||||
return fmt.Errorf("stop vpcd service: %w", err)
|
||||
}
|
||||
fmt.Println("vpcd service stopped")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var vpcdStatusCmd = &cobra.Command{
|
||||
Use: "status",
|
||||
Short: "Get status of the vpcd service",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
fmt.Println("vpcd service status: ...")
|
||||
},
|
||||
}
|
||||
|
||||
var northstarCmd = &cobra.Command{
|
||||
Use: "northstar",
|
||||
Short: "Manage the northstar (DNS) service",
|
||||
@@ -1199,35 +934,6 @@ func resolveNorthstarConfigPath(nodePath, override string) string {
|
||||
return nodePath
|
||||
}
|
||||
|
||||
var northstarStopCmd = &cobra.Command{
|
||||
Use: "stop",
|
||||
Short: "Stop the northstar service",
|
||||
SilenceErrors: true,
|
||||
SilenceUsage: true,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
fmt.Println("Stopping northstar service...")
|
||||
|
||||
svc, err := service.New("northstar", &northstar.Config{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("create northstar service: %w", err)
|
||||
}
|
||||
|
||||
if err = svc.Stop(); err != nil {
|
||||
return fmt.Errorf("stop northstar service: %w", err)
|
||||
}
|
||||
fmt.Println("northstar service stopped")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var northstarStatusCmd = &cobra.Command{
|
||||
Use: "status",
|
||||
Short: "Get status of the northstar service",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
fmt.Println("northstar service status: ...")
|
||||
},
|
||||
}
|
||||
|
||||
var qmpCollectorCmd = &cobra.Command{
|
||||
Use: "qmp-collector",
|
||||
Short: "Manage the qmp-collector (guest metrics) service",
|
||||
@@ -1271,34 +977,6 @@ var qmpCollectorStartCmd = &cobra.Command{
|
||||
},
|
||||
}
|
||||
|
||||
var qmpCollectorStopCmd = &cobra.Command{
|
||||
Use: "stop",
|
||||
Short: "Stop the qmp-collector service",
|
||||
SilenceErrors: true,
|
||||
SilenceUsage: true,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
fmt.Println("Stopping qmp-collector service...")
|
||||
|
||||
svc, err := service.New("qmp-collector", &qmpcollector.Config{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("create qmp-collector service: %w", err)
|
||||
}
|
||||
if err = svc.Stop(); err != nil {
|
||||
return fmt.Errorf("stop qmp-collector service: %w", err)
|
||||
}
|
||||
fmt.Println("qmp-collector service stopped")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var qmpCollectorStatusCmd = &cobra.Command{
|
||||
Use: "status",
|
||||
Short: "Get status of the qmp-collector service",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
fmt.Println("qmp-collector service status: ...")
|
||||
},
|
||||
}
|
||||
|
||||
// registerPredastoreNamespacedFlags defines predastore's host, base-path and
|
||||
// config-path flags and binds them to viper. It must only run once (from
|
||||
// init()); pflag panics on a redefined flag.
|
||||
@@ -1428,8 +1106,6 @@ func init() {
|
||||
bindPredastoreCollisionEnv()
|
||||
|
||||
predastoreCmd.AddCommand(predastoreStartCmd)
|
||||
predastoreCmd.AddCommand(predastoreStopCmd)
|
||||
predastoreCmd.AddCommand(predastoreStatusCmd)
|
||||
|
||||
serviceCmd.AddCommand(viperblockCmd)
|
||||
|
||||
@@ -1455,8 +1131,6 @@ func init() {
|
||||
viper.BindPFlag("viperblock-debug", viperblockCmd.PersistentFlags().Lookup("debug"))
|
||||
|
||||
viperblockCmd.AddCommand(viperblockStartCmd)
|
||||
viperblockCmd.AddCommand(viperblockStopCmd)
|
||||
viperblockCmd.AddCommand(viperblockStatusCmd)
|
||||
|
||||
serviceCmd.AddCommand(qemunbdCmd)
|
||||
|
||||
@@ -1465,15 +1139,11 @@ func init() {
|
||||
viper.BindPFlag("qemunbd-debug", qemunbdCmd.PersistentFlags().Lookup("debug"))
|
||||
|
||||
qemunbdCmd.AddCommand(qemunbdStartCmd)
|
||||
qemunbdCmd.AddCommand(qemunbdStopCmd)
|
||||
qemunbdCmd.AddCommand(qemunbdStatusCmd)
|
||||
|
||||
// Nats
|
||||
serviceCmd.AddCommand(natsCmd)
|
||||
|
||||
natsCmd.AddCommand(natsStartCmd)
|
||||
natsCmd.AddCommand(natsStopCmd)
|
||||
natsCmd.AddCommand(natsStatusCmd)
|
||||
|
||||
// Add NATS flags
|
||||
natsCmd.PersistentFlags().Int("port", 4222, "NATS server port")
|
||||
@@ -1495,8 +1165,6 @@ func init() {
|
||||
serviceCmd.AddCommand(spinifexCmd)
|
||||
|
||||
spinifexCmd.AddCommand(spinifexStartCmd)
|
||||
spinifexCmd.AddCommand(spinifexStopCmd)
|
||||
spinifexCmd.AddCommand(spinifexStatusCmd)
|
||||
|
||||
spinifexCmd.PersistentFlags().String("wal-dir", "", "Write-ahead log (WAL) directory. Place on high-speed NVMe disk, or tmpfs for development.")
|
||||
viper.BindEnv("wal-dir", "SPINIFEX_WAL_DIR")
|
||||
@@ -1519,8 +1187,6 @@ func init() {
|
||||
bindAwsgwCollisionEnv()
|
||||
|
||||
awsgwCmd.AddCommand(awsgwStartCmd)
|
||||
awsgwCmd.AddCommand(awsgwStopCmd)
|
||||
awsgwCmd.AddCommand(awsgwStatusCmd)
|
||||
|
||||
// spinifex-ui
|
||||
serviceCmd.AddCommand(spinifexUICmd)
|
||||
@@ -1550,15 +1216,11 @@ func init() {
|
||||
viper.BindPFlag("spinifex-ui-ochre-enabled", spinifexUICmd.PersistentFlags().Lookup("ochre-enabled"))
|
||||
|
||||
spinifexUICmd.AddCommand(spinifexUIStartCmd)
|
||||
spinifexUICmd.AddCommand(spinifexUIStopCmd)
|
||||
spinifexUICmd.AddCommand(spinifexUIStatusCmd)
|
||||
|
||||
// vpcd
|
||||
serviceCmd.AddCommand(vpcdCmd)
|
||||
|
||||
vpcdCmd.AddCommand(vpcdStartCmd)
|
||||
vpcdCmd.AddCommand(vpcdStopCmd)
|
||||
vpcdCmd.AddCommand(vpcdStatusCmd)
|
||||
|
||||
// northstar
|
||||
serviceCmd.AddCommand(northstarCmd)
|
||||
@@ -1568,13 +1230,9 @@ func init() {
|
||||
viper.BindPFlag("northstar-config", northstarCmd.PersistentFlags().Lookup("northstar-config"))
|
||||
|
||||
northstarCmd.AddCommand(northstarStartCmd)
|
||||
northstarCmd.AddCommand(northstarStopCmd)
|
||||
northstarCmd.AddCommand(northstarStatusCmd)
|
||||
|
||||
// qmp-collector
|
||||
serviceCmd.AddCommand(qmpCollectorCmd)
|
||||
|
||||
qmpCollectorCmd.AddCommand(qmpCollectorStartCmd)
|
||||
qmpCollectorCmd.AddCommand(qmpCollectorStopCmd)
|
||||
qmpCollectorCmd.AddCommand(qmpCollectorStatusCmd)
|
||||
}
|
||||
|
||||
@@ -62,16 +62,8 @@ func writeMalformedToml(t *testing.T) string {
|
||||
return writeSpinifexToml(t, "[section\nkey = value without quotes\n")
|
||||
}
|
||||
|
||||
// isolateRuntimeDir redirects the pid-file fallback (utils.pidPath, via
|
||||
// XDG_RUNTIME_DIR) to an empty temp dir, so a stop command with no explicit
|
||||
// directory override can't resolve against a real, shared, ambient pid dir.
|
||||
func isolateRuntimeDir(t *testing.T) {
|
||||
t.Helper()
|
||||
t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
|
||||
}
|
||||
|
||||
// TestServiceStartCmdsReturnErrorOnMissingConfig exercises each service
|
||||
// start/stop command's RunE directly and asserts a fatal precondition
|
||||
// start command's RunE directly and asserts a fatal precondition
|
||||
// failure surfaces as a returned error, not a silent zero-exit return.
|
||||
// Cases with a setup func drive further into RunE toward a specific branch.
|
||||
func TestServiceStartCmdsReturnErrorOnMissingConfig(t *testing.T) {
|
||||
@@ -339,99 +331,6 @@ func TestServiceStartCmdsReturnErrorOnMissingConfig(t *testing.T) {
|
||||
},
|
||||
wantErr: "start qmp-collector service",
|
||||
},
|
||||
// The stop commands below resolve their pid directory to an isolated,
|
||||
// always-empty temp dir, so Stop() safely fails to find a pid file
|
||||
// rather than risking a real running process via an ambient path.
|
||||
{
|
||||
name: "predastore stop (no running process)",
|
||||
cmd: predastoreStopCmd,
|
||||
setup: func(t *testing.T) {
|
||||
resetGlobalViper(t)
|
||||
viper.Set("predastore-base-path", t.TempDir())
|
||||
},
|
||||
wantErr: "stop predastore service",
|
||||
},
|
||||
{
|
||||
name: "viperblock stop (no running process)",
|
||||
cmd: viperblockStopCmd,
|
||||
setup: func(t *testing.T) {
|
||||
resetGlobalViper(t)
|
||||
isolateRuntimeDir(t)
|
||||
},
|
||||
wantErr: "stop viperblock service",
|
||||
},
|
||||
{
|
||||
name: "qemunbd stop (no running process)",
|
||||
cmd: qemunbdStopCmd,
|
||||
setup: func(t *testing.T) {
|
||||
resetGlobalViper(t)
|
||||
isolateRuntimeDir(t)
|
||||
},
|
||||
wantErr: "stop qemunbd service",
|
||||
},
|
||||
{
|
||||
name: "nats stop (no running process)",
|
||||
cmd: natsStopCmd,
|
||||
setup: func(t *testing.T) {
|
||||
resetGlobalViper(t)
|
||||
isolateRuntimeDir(t)
|
||||
},
|
||||
wantErr: "stop nats service",
|
||||
},
|
||||
{
|
||||
name: "spinifex stop (no running process)",
|
||||
cmd: spinifexStopCmd,
|
||||
setup: func(t *testing.T) {
|
||||
resetGlobalViper(t)
|
||||
isolateRuntimeDir(t)
|
||||
},
|
||||
wantErr: "stop spinifex service",
|
||||
},
|
||||
{
|
||||
name: "awsgw stop (no running process)",
|
||||
cmd: awsgwStopCmd,
|
||||
setup: func(t *testing.T) {
|
||||
resetGlobalViper(t)
|
||||
isolateRuntimeDir(t)
|
||||
},
|
||||
wantErr: "stop awsgw service",
|
||||
},
|
||||
{
|
||||
name: "spinifex-ui stop (no running process)",
|
||||
cmd: spinifexUIStopCmd,
|
||||
setup: func(t *testing.T) {
|
||||
resetGlobalViper(t)
|
||||
isolateRuntimeDir(t)
|
||||
},
|
||||
wantErr: "stop spinifex-ui service",
|
||||
},
|
||||
{
|
||||
name: "vpcd stop (no running process)",
|
||||
cmd: vpcdStopCmd,
|
||||
setup: func(t *testing.T) {
|
||||
resetGlobalViper(t)
|
||||
isolateRuntimeDir(t)
|
||||
},
|
||||
wantErr: "stop vpcd service",
|
||||
},
|
||||
{
|
||||
name: "northstar stop (no running process)",
|
||||
cmd: northstarStopCmd,
|
||||
setup: func(t *testing.T) {
|
||||
resetGlobalViper(t)
|
||||
isolateRuntimeDir(t)
|
||||
},
|
||||
wantErr: "stop northstar service",
|
||||
},
|
||||
{
|
||||
name: "qmp-collector stop (no running process)",
|
||||
cmd: qmpCollectorStopCmd,
|
||||
setup: func(t *testing.T) {
|
||||
resetGlobalViper(t)
|
||||
isolateRuntimeDir(t)
|
||||
},
|
||||
wantErr: "stop qmp-collector service",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
@@ -451,36 +350,6 @@ func TestServiceStartCmdsReturnErrorOnMissingConfig(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestSpinifexUIStatusCmdReportsStoppedWithNoRunningProcess drives the
|
||||
// status command's success path: with no pid file at an isolated
|
||||
// XDG_RUNTIME_DIR, Status reports "stopped" and RunE must return nil.
|
||||
func TestSpinifexUIStatusCmdReportsStoppedWithNoRunningProcess(t *testing.T) {
|
||||
isolateRuntimeDir(t)
|
||||
|
||||
require.NotNil(t, spinifexUIStatusCmd.RunE)
|
||||
err := spinifexUIStatusCmd.RunE(spinifexUIStatusCmd, nil)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// TestSpinifexUIStatusCmdReturnsErrorOnUnreadablePidDir drives the status
|
||||
// command's other branch: a genuine read failure (permission denied, not
|
||||
// ENOENT) must surface as an error rather than the benign "stopped" case.
|
||||
func TestSpinifexUIStatusCmdReturnsErrorOnUnreadablePidDir(t *testing.T) {
|
||||
if os.Geteuid() == 0 {
|
||||
t.Skip("root bypasses directory permission checks")
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "spinifex-ui.pid"), []byte("123"), 0o600))
|
||||
require.NoError(t, os.Chmod(dir, 0o000))
|
||||
t.Cleanup(func() { _ = os.Chmod(dir, 0o755) })
|
||||
t.Setenv("XDG_RUNTIME_DIR", dir)
|
||||
|
||||
require.NotNil(t, spinifexUIStatusCmd.RunE)
|
||||
err := spinifexUIStatusCmd.RunE(spinifexUIStatusCmd, nil)
|
||||
require.ErrorContains(t, err, "get spinifex-ui service status")
|
||||
}
|
||||
|
||||
// TestServiceStartExitsNonZeroOnFailure proves a fatal service-start error
|
||||
// terminates the process non-zero (systemd's Restart=on-failure depends on
|
||||
// it). os.Exit can't be observed in-process, so this re-execs the test
|
||||
|
||||
@@ -24,6 +24,10 @@ Spinifex implements **127 operations** in the EC2 `2016-11-15` API model.
|
||||
|
||||
Spot Instance Requests are a mock over the on-demand `RunInstances` path. A request synchronously launches real VMs on the operator's own compute and is then reported `active` and `fulfilled`. There is no spot market: no bidding, no price rejection, no interruption and no reclamation, and instances are never taken back.
|
||||
|
||||
### AMI Launch Permissions Are Refused
|
||||
|
||||
An AMI cannot be shared with another account or made public. `ModifyImageAttribute` refuses `LaunchPermission`, `UserIds`, `UserGroups`, `OrganizationArns`, `OrganizationalUnitArns` and `OperationType`, and `ResetImageAttribute` resets only `description`, not `launchPermission`. Each refusal is `InvalidParameterValue`. Terraform's `aws_ami_launch_permission` therefore cannot be created. `description` is the only image attribute that can be modified or reset.
|
||||
|
||||
### Operations
|
||||
|
||||
| Operation |
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
### Spot Instances Are a Mock
|
||||
|
||||
Spot Instance Requests are a mock over the on-demand `RunInstances` path. A request synchronously launches real VMs on the operator's own compute and is then reported `active` and `fulfilled`. There is no spot market: no bidding, no price rejection, no interruption and no reclamation, and instances are never taken back.
|
||||
|
||||
### AMI Launch Permissions Are Refused
|
||||
|
||||
An AMI cannot be shared with another account or made public. `ModifyImageAttribute` refuses `LaunchPermission`, `UserIds`, `UserGroups`, `OrganizationArns`, `OrganizationalUnitArns` and `OperationType`, and `ResetImageAttribute` resets only `description`, not `launchPermission`. Each refusal is `InvalidParameterValue`. Terraform's `aws_ami_launch_permission` therefore cannot be created. `description` is the only image attribute that can be modified or reset.
|
||||
|
||||
@@ -54,7 +54,7 @@ A parameter whose omission would create a false safety, security or availability
|
||||
| `DBSecurityGroups` | EC2-Classic security groups — use `VpcSecurityGroupIds` |
|
||||
| `DBClusterIdentifier`, `DBClusterSnapshotIdentifier` | Clustered engines are not offered |
|
||||
| `EnableCloudwatchLogsExports` | Log export is not implemented |
|
||||
| `EngineVersion` other than the engine's pin, `Engine` on modify | No in-place engine or version change |
|
||||
| `EngineVersion` other than the engine's pin; on modify, any `EngineVersion` and `Engine` | No in-place engine or version change |
|
||||
| `Engine=mysql` (and Aurora engines) | Oracle MySQL is not offered; `mariadb` is a distinct engine, not an alias for it |
|
||||
| `NewDBInstanceIdentifier` | The identifier is the DNS label and the KV key |
|
||||
| `DBPortNumber`, `DBSubnetGroupName` on modify | Both would move the endpoint |
|
||||
@@ -67,6 +67,8 @@ A parameter whose omission would create a false safety, security or availability
|
||||
| `EnableCustomerOwnedIp` | An Outposts feature |
|
||||
| `ForceFailover` (reboot) | No standby to fail over to |
|
||||
| `DBSnapshotIdentifier` (stop) | Snapshot-on-stop is not implemented |
|
||||
| `DBInstanceAutomatedBackupsArn` (DescribeDBInstanceAutomatedBackups) | Cross-region automated backup replication is not offered, so no replicated backup has an ARN |
|
||||
| `dbi-resource-id` filter (DescribeDBSnapshots, DescribeDBInstanceAutomatedBackups) | Snapshots and automated backups do not record their source instance's resource ID |
|
||||
|
||||
### Operations
|
||||
|
||||
|
||||
@@ -31,7 +31,7 @@ A parameter whose omission would create a false safety, security or availability
|
||||
| `DBSecurityGroups` | EC2-Classic security groups — use `VpcSecurityGroupIds` |
|
||||
| `DBClusterIdentifier`, `DBClusterSnapshotIdentifier` | Clustered engines are not offered |
|
||||
| `EnableCloudwatchLogsExports` | Log export is not implemented |
|
||||
| `EngineVersion` other than the engine's pin, `Engine` on modify | No in-place engine or version change |
|
||||
| `EngineVersion` other than the engine's pin; on modify, any `EngineVersion` and `Engine` | No in-place engine or version change |
|
||||
| `Engine=mysql` (and Aurora engines) | Oracle MySQL is not offered; `mariadb` is a distinct engine, not an alias for it |
|
||||
| `NewDBInstanceIdentifier` | The identifier is the DNS label and the KV key |
|
||||
| `DBPortNumber`, `DBSubnetGroupName` on modify | Both would move the endpoint |
|
||||
@@ -44,3 +44,5 @@ A parameter whose omission would create a false safety, security or availability
|
||||
| `EnableCustomerOwnedIp` | An Outposts feature |
|
||||
| `ForceFailover` (reboot) | No standby to fail over to |
|
||||
| `DBSnapshotIdentifier` (stop) | Snapshot-on-stop is not implemented |
|
||||
| `DBInstanceAutomatedBackupsArn` (DescribeDBInstanceAutomatedBackups) | Cross-region automated backup replication is not offered, so no replicated backup has an ARN |
|
||||
| `dbi-resource-id` filter (DescribeDBSnapshots, DescribeDBInstanceAutomatedBackups) | Snapshots and automated backups do not record their source instance's resource ID |
|
||||
|
||||
@@ -0,0 +1,258 @@
|
||||
---
|
||||
title: "Spinifex on Exoscale"
|
||||
seoTitle: "Run Spinifex on Exoscale with Elastic IPs — Spinifex Docs"
|
||||
description: "Install Spinifex on one Exoscale instance and run EC2, load balancers and EKS on real Exoscale Elastic IPs."
|
||||
category: "Install"
|
||||
tags:
|
||||
- install
|
||||
- exoscale
|
||||
- cloud
|
||||
- elastic ip
|
||||
resources:
|
||||
- title: "Single-Node Install"
|
||||
url: "/docs/install"
|
||||
- title: "Spinifex on Oracle Cloud"
|
||||
url: "/docs/oci-integration"
|
||||
- title: "VPC Networking"
|
||||
url: "/docs/vpc-networking"
|
||||
- title: "Spinifex Repository"
|
||||
url: "https://github.com/mulgadc/spinifex"
|
||||
---
|
||||
|
||||
# Running Spinifex on Exoscale
|
||||
|
||||
> [!WARNING]
|
||||
> **Work in progress.** This page is not yet published on docs.mulgadc.com. Do not add it to `scripts/docs-config.json` in `mulga-docs` until the [Limits](#limits) section is settled.
|
||||
|
||||
> One Exoscale instance, the AWS APIs on its address, and guests reachable on real Exoscale Elastic IPs that Spinifex creates and deletes for you.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- [What you get](#what-you-get)
|
||||
- [Before you start](#before-you-start)
|
||||
- [Install](#install)
|
||||
- [Use it from your laptop](#use-it-from-your-laptop)
|
||||
- [Troubleshooting](#troubleshooting)
|
||||
- [Limits](#limits)
|
||||
|
||||
---
|
||||
|
||||
## What you get
|
||||
|
||||
A single Exoscale instance runs the whole Spinifex stack: EC2, EBS, S3, VPC, load balancers and EKS, behind an endpoint the ordinary AWS CLI, SDKs and Terraform provider talk to unmodified.
|
||||
|
||||
Whenever something needs a public address, such as an instance in a public subnet, an Elastic IP, an internet-facing load balancer or a NAT gateway, Spinifex creates an Exoscale Elastic IP, attaches it to your instance, and deletes it again when the resource goes away. You never manage the addresses by hand.
|
||||
|
||||
<p align="center">
|
||||
<img src="../../.github/assets/diagrams/exoscale-single-node.svg" alt="Single node on Exoscale — the instance on the zone's public network with one interface, Elastic IPs routed into OVN through the routed-NAT transit veth, the control plane, tenant VPCs and the block storage volume" width="900">
|
||||
</p>
|
||||
|
||||
---
|
||||
|
||||
## Before you start
|
||||
|
||||
You need:
|
||||
|
||||
- An Exoscale organisation and an **API key** that can create, attach, detach and delete Elastic IPs.
|
||||
- An SSH key pair.
|
||||
- The AWS CLI on your laptop.
|
||||
|
||||
Then create these in one zone, from the Exoscale console or the `exo` CLI. This guide uses `de-fra-1`.
|
||||
|
||||
| Resource | Setting |
|
||||
| --- | --- |
|
||||
| Compute instance | `standard.huge` (8 vCPU / 32 GiB) or larger, template `Linux Ubuntu 26.04 LTS 64-bit`, 100 GiB disk, your SSH key |
|
||||
| Block storage volume | 256 GiB or more, attached to the instance |
|
||||
| Security group | Allow all inbound TCP, UDP and ICMP from `0.0.0.0/0`, attached to the instance |
|
||||
|
||||
Note the instance's **ID** (a UUID, shown on its page in the console) and the volume's **ID**. You do not need to create any Elastic IPs.
|
||||
|
||||
> [!NOTE]
|
||||
> The security group is open on purpose. Guests are filtered by their own Spinifex security groups, and the node by the Spinifex host firewall you turn on in step 5.
|
||||
|
||||
---
|
||||
|
||||
## Install
|
||||
|
||||
Everything below runs on the instance, over SSH as `ubuntu`.
|
||||
|
||||
### 1. Check that the instance can run VMs
|
||||
|
||||
```bash
|
||||
ls -l /dev/kvm && cat /sys/module/kvm_intel/parameters/nested
|
||||
```
|
||||
|
||||
`/dev/kvm` must exist and the second line must print `Y`. If either fails, pick a different instance type before going further.
|
||||
|
||||
### 2. Put the data volume at `/var/lib/spinifex`
|
||||
|
||||
Every guest disk and S3 object lives here, so it belongs on the volume and not on the 100 GiB root disk.
|
||||
|
||||
```bash
|
||||
VOL_ID=<volume ID>
|
||||
DEV=/dev/disk/by-id/virtio-${VOL_ID:0:20}
|
||||
|
||||
sudo mkfs.ext4 -L spinifex-data "$DEV" # only on a new, empty volume
|
||||
sudo mkdir -p /var/lib/spinifex
|
||||
echo "UUID=$(sudo blkid -o value -s UUID "$DEV") /var/lib/spinifex ext4 defaults,nofail 0 2" | sudo tee -a /etc/fstab
|
||||
sudo mount /var/lib/spinifex
|
||||
findmnt /var/lib/spinifex # must print a line
|
||||
```
|
||||
|
||||
### 3. Install the `exo` CLI and give it your key
|
||||
|
||||
Spinifex manages Elastic IPs by running `exo`, so it needs the CLI and a key file.
|
||||
|
||||
```bash
|
||||
curl -fsSLO https://github.com/exoscale/cli/releases/download/v1.101.0/exoscale-cli_1.101.0_linux_amd64.deb
|
||||
sudo dpkg -i exoscale-cli_1.101.0_linux_amd64.deb
|
||||
|
||||
sudo mkdir -p /etc/spinifex/exoscale
|
||||
sudo tee /etc/spinifex/exoscale/exoscale.toml >/dev/null <<'EOF'
|
||||
defaultaccount = "spinifex"
|
||||
|
||||
[[accounts]]
|
||||
name = "spinifex"
|
||||
key = "EXO..."
|
||||
secret = "..."
|
||||
defaultZone = "de-fra-1"
|
||||
EOF
|
||||
sudo chmod 700 /etc/spinifex/exoscale && sudo chmod 600 /etc/spinifex/exoscale/exoscale.toml
|
||||
```
|
||||
|
||||
Check the key works: `sudo exo --config /etc/spinifex/exoscale/exoscale.toml compute elastic-ip list -z de-fra-1`.
|
||||
|
||||
### 4. Install Spinifex
|
||||
|
||||
```bash
|
||||
curl -sfL https://install.mulgadc.com | sudo bash
|
||||
sudo /usr/local/share/spinifex/setup-ovn.sh --management --nat-uplink
|
||||
|
||||
sudo spx admin init --node node1 --nodes 1 --region eu-central-1 --az eu-central-1a \
|
||||
--ipsec=false --external-mode=nat
|
||||
sudo chown -R spinifex-daemon: /etc/spinifex/exoscale
|
||||
```
|
||||
|
||||
Now tell Spinifex where to get public addresses. Add this to the end of `/etc/spinifex/spinifex.toml`, just above the `[bootstrap]` section, using your instance ID:
|
||||
|
||||
```toml
|
||||
[[network.external_pools]]
|
||||
name = "exo"
|
||||
source = "exoscale"
|
||||
exoscale_zone = "de-fra-1"
|
||||
exoscale_instance_id = "<instance ID>"
|
||||
```
|
||||
|
||||
Leave the `nat-transit` pool that `init` wrote where it is.
|
||||
|
||||
> [!NOTE]
|
||||
> `eu-central-1` is the region name your AWS tools will see. It is your choice and does not have to match the Exoscale zone.
|
||||
|
||||
### 5. Turn on the firewall, then start
|
||||
|
||||
```bash
|
||||
sudo env SETUP_STAGES=firewall /usr/local/share/spinifex/setup.sh --firewall on
|
||||
sudo systemctl start spinifex.target
|
||||
systemctl list-units 'spinifex-*' --no-pager
|
||||
```
|
||||
|
||||
Every `spinifex-*` unit should be `active`. The daemon log confirms the Elastic IP integration is working:
|
||||
|
||||
```bash
|
||||
sudo journalctl -u spinifex-daemon | grep -E 'exo CLI|Exoscale allocator ready'
|
||||
```
|
||||
|
||||
### 6. Import an image
|
||||
|
||||
```bash
|
||||
sudo spx admin images import --name ubuntu-26.04-x86_64 --config /etc/spinifex/spinifex.toml
|
||||
```
|
||||
|
||||
For EKS, also import `spinifex-eks-node`. `sudo spx admin images list --config /etc/spinifex/spinifex.toml` shows the full catalogue.
|
||||
|
||||
---
|
||||
|
||||
## Use it from your laptop
|
||||
|
||||
Copy the cluster's CA and the admin credentials from the instance:
|
||||
|
||||
```bash
|
||||
NODE=<instance public IP>
|
||||
mkdir -p ~/.spinifex
|
||||
ssh ubuntu@$NODE 'sudo cat /etc/spinifex/ca.pem' > ~/.spinifex/exo-ca.pem
|
||||
```
|
||||
|
||||
Copy the `[spinifex]` section of `~/.aws/credentials` on the instance into your own `~/.aws/credentials` as `[spinifex-exo]`, then:
|
||||
|
||||
```bash
|
||||
aws configure set --profile spinifex-exo region eu-central-1
|
||||
export AWS_PROFILE=spinifex-exo AWS_CA_BUNDLE=~/.spinifex/exo-ca.pem
|
||||
export AWS_ENDPOINT_URL=https://$NODE:9999
|
||||
|
||||
aws ec2 describe-regions # proves the connection
|
||||
```
|
||||
|
||||
### Launch an instance
|
||||
|
||||
```bash
|
||||
aws ec2 import-key-pair --key-name demo --public-key-material fileb://~/.ssh/id_ed25519.pub
|
||||
SG=$(aws ec2 describe-security-groups --filters Name=group-name,Values=default --query 'SecurityGroups[0].GroupId' --output text)
|
||||
aws ec2 authorize-security-group-ingress --group-id "$SG" --protocol tcp --port 22 --cidr 0.0.0.0/0
|
||||
|
||||
AMI=$(aws ec2 describe-images --filters Name=name,Values='*ubuntu-26.04*' --query 'Images[0].ImageId' --output text)
|
||||
aws ec2 run-instances --image-id "$AMI" --instance-type t3.small --key-name demo \
|
||||
--query 'Instances[0].InstanceId' --output text
|
||||
```
|
||||
|
||||
After about a minute, `describe-instances` shows a public IP. That is a new Exoscale Elastic IP, and it appears in the Exoscale console with a description beginning `spinifex:`.
|
||||
|
||||
```bash
|
||||
ssh ubuntu@<public IP> 'curl -s ifconfig.me' # prints the same address
|
||||
```
|
||||
|
||||
Terminating the instance deletes the Elastic IP at Exoscale.
|
||||
|
||||
### Load balancers and EKS
|
||||
|
||||
The [Terraform workbooks](https://github.com/mulgadc/spinifex/tree/main/docs/terraform-workbooks) run unchanged. Pass the region and endpoint:
|
||||
|
||||
```bash
|
||||
cd nginx-alb
|
||||
terraform init
|
||||
terraform apply -var region=eu-central-1 -var spinifex_endpoint=https://$NODE:9999
|
||||
```
|
||||
|
||||
`nginx-alb` builds two web servers behind an internet-facing load balancer. `eks-quickstart` builds a Kubernetes cluster with one worker. Run `terraform destroy` before starting the next one, because of the quota below.
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **Exoscale allows five Elastic IPs per organisation by default**, and every public address Spinifex hands out uses one. An instance with a public IP uses one, an internet-facing load balancer one, a NAT gateway one, and `eks-quickstart` three. Past the quota, `allocate-address` returns `AddressLimitExceeded` and a launch returns `InsufficientAddressCapacity`. Ask Exoscale support to raise it.
|
||||
>
|
||||
> **Exoscale keeps counting a deleted Elastic IP for a while.** After a `terraform destroy`, `exo limits` still shows the old usage even though `exo compute elastic-ip list` no longer shows the addresses, so an `apply` straight afterwards can hit the quota. Wait until `exo limits` agrees with the list before starting the next workbook.
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**A guest's public IP does not answer from the instance itself.** That is expected: traffic from the host to a guest's Elastic IP is dropped. Test from your laptop.
|
||||
|
||||
**SSH or HTTP is refused straight after launch.** Guests take 30 to 60 seconds to boot. Wait and retry.
|
||||
|
||||
**An EKS cluster goes to `FAILED` within seconds, or a load balancer never becomes active.** Run `exo limits`. If Elastic IP usage is at the maximum, the cluster could not get its public endpoint; destroy it, wait for usage to drop, and apply again.
|
||||
|
||||
**The daemon logs an `exo` error at start.** Run the check at the end of step 3. A wrong key, secret or zone is the usual cause.
|
||||
|
||||
**An Elastic IP you created yourself.** Spinifex only ever touches Elastic IPs whose description starts with `spinifex:<instance ID>:`. Anything else in the organisation is left alone, but it still counts against the quota.
|
||||
|
||||
---
|
||||
|
||||
## Limits
|
||||
|
||||
Proven on one `standard.huge` instance in `de-fra-1` on 2026-09-30: EC2 with a public IP, Elastic IP allocate and release, the `nginx-alb` workbook and the `eks-quickstart` workbook with one worker. Each was reached from the internet and left no Elastic IP behind at Exoscale.
|
||||
|
||||
Not yet tested:
|
||||
|
||||
- **More than one node.** Each node would carry its own guests' Elastic IPs, with cluster traffic over an Exoscale private network.
|
||||
- **Moving an Elastic IP between nodes** with its guest.
|
||||
- **A narrowly scoped API key.** The proof used an unrestricted key.
|
||||
- **Other zones, bare metal and GPU instances.**
|
||||
- **Guest disk performance** on Exoscale block storage.
|
||||
@@ -30,6 +30,7 @@ resources:
|
||||
|
||||
- [Overview](#overview)
|
||||
- [Cluster sizing](#cluster-sizing)
|
||||
- [How the cluster's own state is replicated](#how-the-clusters-own-state-is-replicated)
|
||||
- [Hardware](#hardware)
|
||||
- [Network requirements](#network-requirements)
|
||||
- [Prerequisites](#prerequisites)
|
||||
@@ -65,11 +66,82 @@ A Spinifex cluster distributes services across multiple servers for high availab
|
||||
| **VPC networking** — OVN | Control-plane databases run clustered, surviving the loss of any one node. |
|
||||
| **Object storage** — Predastore (S3) | Objects are erasure coded `RS(2,1)`, surviving the loss of any one node's shards. |
|
||||
| **Block storage** — Viperblock (EBS) | Volumes are stored in Predastore, so they inherit the same durability. |
|
||||
| **Cluster state** — NATS JetStream | Every key-value bucket is replicated onto all three nodes, so the control plane keeps its own state through the loss of any one. |
|
||||
|
||||
On one or two servers none of that holds. OVN runs standalone on the first node, and the storage metadata quorum has no majority to lose. If that node goes down, running instances keep full networking — but nothing can *change*: no new VPCs, no launches, no security group edits. See [OVN control plane on multi-node clusters](/docs/vpc-networking#ovn-control-plane-on-multi-node-clusters).
|
||||
|
||||
Servers beyond the third run the full set of services — storage, gateway and networking agents — and add their capacity to the pool. What they do not do is join the OVN database cluster, which stays at three members, so write latency there stays flat as the cluster grows.
|
||||
|
||||
### How the cluster's own state is replicated
|
||||
|
||||
Guest data has an obvious home: volumes live in Viperblock, objects in Predastore, both erasure coded across the cluster. The cluster's own bookkeeping has a less obvious one, and it matters just as much. Every instance record, every VPC and subnet, the server roster, IAM users and access keys, quotas and DNS records are kept in **NATS JetStream key-value buckets** — Spinifex's control-plane database, replicated across the servers you installed rather than sitting on any one of them. A cluster that has lost its state has lost the ability to describe, launch, terminate or route anything, even while every guest is still running.
|
||||
|
||||
**A bucket is replicated across the largest odd number of nodes the cluster has, up to five.** That number is decided once, by the `--nodes` count you give `spx admin init` in Step 4, and every service on every node reads it back out of `/etc/spinifex/spinifex.toml` before it creates anything.
|
||||
|
||||
| Servers | Replicas per bucket | Quorum | Survives |
|
||||
|---|---|---|---|
|
||||
| 1 | 1 | 1 | nothing — a single node is a single point of failure by definition |
|
||||
| 2 | — | — | rejected by `spx admin init`; see below |
|
||||
| 3 | 3 | 2 | any 1 node lost |
|
||||
| 4 | 3 | 2 | any 1 node lost |
|
||||
| 5 | 5 | 3 | any 2 nodes lost |
|
||||
| 10 | 5 | 3 | any 2 of the 5 nodes holding each bucket |
|
||||
|
||||
**The count is always odd, and that is why four servers replicate three ways rather than four.** Agreement is by majority, and an even number of copies does not buy you one: four copies cost a fourth write every time and still survive exactly one loss, the same as three. On four servers NATS spreads which three nodes hold each bucket, so all four carry their share and none of them is idle.
|
||||
|
||||
Five is JetStream's own ceiling on a stream's replica count, not a choice of ours. Past five servers the buckets stay at five and NATS spreads which five nodes hold each one, so the cluster keeps growing while the cost of a write does not.
|
||||
|
||||
**Two servers is refused rather than allowed with a warning.** There is no replica count that works on two: one disappears with the node holding it, and two stop accepting writes when either member goes, because neither survivor is a majority on its own. OVN and the storage metadata quorum have the same problem on two servers, so `spx admin init --nodes 2` fails and tells you to use one server or three.
|
||||
|
||||
#### Worked example — three servers
|
||||
|
||||
Say you ran Step 4 with `--nodes 3` on `node1`, `node2` and `node3`. Every bucket the cluster creates is held by all three servers, with one of them responsible for coordinating changes to it:
|
||||
|
||||
```
|
||||
KV bucket "spinifex-instance-state" KV bucket "spinifex-vpcd-reconcile"
|
||||
┌──────────┬──────────┬──────────┐ ┌──────────┬──────────┬──────────┐
|
||||
│ node1 │ node2 │ node3 │ │ node1 │ node2 │ node3 │
|
||||
│ leader │ follower │ follower │ │ follower │ leader │ follower │
|
||||
└──────────┴──────────┴──────────┘ └──────────┴──────────┴──────────┘
|
||||
quorum = 2 of 3 quorum = 2 of 3
|
||||
```
|
||||
|
||||
Three things follow from that picture, and they are the whole reason three servers is the floor:
|
||||
|
||||
- **A change is not confirmed until most of the cluster has it.** When you launch an instance, the API does not report success until at least two of the three servers have written the record to disk. Losing a server afterwards cannot lose that instance.
|
||||
- **Losing a server costs a pause, not the data.** If `node1` goes down, `node2` and `node3` are still a majority. They take over the buckets `node1` was responsible for — seconds on a healthy cluster — and the API keeps answering throughout. Nothing becomes unreadable, because nothing lived only on `node1`.
|
||||
- **The cluster keeps repairing itself.** Spinifex runs background workers that fix up VPC state, enforce quotas and publish DNS. Only one server runs each at a time, and they claim that right by writing to a bucket. If those buckets were not replicated, a server failure would stop the repair work at precisely the moment there was repair work to do.
|
||||
|
||||
Contrast that with a bucket on **one** server. It lives wherever it happened to be created, which is not recorded in any config file you can inspect. Stopping, rebooting or losing that one server makes the bucket unreachable from *every* node at once — a single-server event becomes a cluster-wide outage, on a server nobody chose.
|
||||
|
||||
#### What you see when a server fails
|
||||
|
||||
Majority is not an implementation detail. It is what stops two halves of a damaged cluster from both believing they are in charge, and it is the reason the minimum is three servers rather than two.
|
||||
|
||||
**With one server down, the cluster carries on.** `node2` and `node3` are two of three. Instances launch, VPCs change, the API answers normally, and there is nothing to do but replace the failed server. What you have lost is the *next* failure: a second server down leaves no majority and the cluster stops accepting changes until one comes back.
|
||||
|
||||
**A server that is cut off stops accepting changes, on purpose.** If `node1` is not down but isolated — a failed switch, a misconfigured port — it is one server out of three, which is not a majority, so it will not record anything. An operator working on `node1` who tries to launch an instance gets a failure rather than a success. That is the behaviour you want: if `node1` accepted changes while `node2` and `node3` were also accepting them, you would end up with two different versions of the same cluster and no way to merge them. Refusing the isolated side is what makes the surviving side worth trusting.
|
||||
|
||||
Everything else on `node1` keeps running. Guests continue to execute and their volumes keep serving. What stops is anything that has to be recorded: launching, terminating, attaching a volume, changing a security group. Storage writes fail alongside it, so a guest on the isolated server stalls rather than quietly diverging from the rest of the cluster.
|
||||
|
||||
**Be careful reading state from an isolated server.** A server that is cut off can still answer `describe-instances` from its own copy of the data — frozen at the moment it lost contact, with nothing in the answer to say so. **Run operator commands from a server you know is with the majority**, and if one server's answers disagree with its peers', believe the peers.
|
||||
|
||||
**A returning server catches up on its own.** Bring `node1` back and it synchronises from the others automatically: no command to run, no restart, no repair step. (`spx admin kv replicas --repair` exists for a different situation — a bucket configured with too few copies — not a server that is simply behind.)
|
||||
|
||||
#### Growing an existing cluster
|
||||
|
||||
Adding a fourth server does not, on its own, re-replicate the buckets that already exist — the nodes that hold them are perfectly healthy, so nothing prompts a change. Spinifex raises them for you at the next service start: each service checks every bucket it opens against the cluster's current node count and raises any that are short. It never lowers one.
|
||||
|
||||
**A config that names a node before that node is serving is safe.** It is the ordinary middle of growing a cluster, and services start normally through it: a bucket that cannot be raised yet keeps working at the count it has, and the node logs `Could not raise KV bucket to the cluster's replica count`. The raise is retried every time a service opens the bucket, so it completes on its own once the new server is in the cluster.
|
||||
|
||||
To do it immediately rather than waiting for a restart, run the audit with `--repair` from any node:
|
||||
|
||||
```bash
|
||||
sudo spx admin kv replicas --repair
|
||||
```
|
||||
|
||||
It changes nothing about a bucket except the replica count, never lowers one, and is safe to run twice or on a healthy cluster.
|
||||
|
||||
### Hardware
|
||||
|
||||
Per server, for the three-node minimum:
|
||||
@@ -252,6 +324,8 @@ sudo spx admin init --force \
|
||||
|
||||
`--nodes 3` is the number of servers init waits for. Set it to your total node count if you are building a larger cluster.
|
||||
|
||||
It is also what decides how many nodes the cluster's JetStream state is replicated across, so it is worth getting right rather than raising later — see [How the cluster's own state is replicated](#how-the-clusters-own-state-is-replicated). Every joining server takes the count from server 1, so it is chosen once, here. `--nodes 2` is refused: use 1 for a single server, or 3 and up for a cluster that can survive losing one.
|
||||
|
||||
IPsec encrypts the Geneve overlay between servers and is on by default. Joining servers take the setting from server 1, so it is chosen once, on init. On servers that share a trusted private link, `--ipsec=false` leaves the overlay unencrypted in exchange for considerably higher throughput between instances.
|
||||
|
||||
The init output displays the join command including the token:
|
||||
@@ -360,7 +434,40 @@ The top table is per-node CPU, memory and GPU usage. The bottom table is what th
|
||||
|
||||
If capacity looks like a single server rather than the sum of your nodes, the others have not joined.
|
||||
|
||||
**3. The AWS API answers.**
|
||||
**3. The cluster's own state is replicated across every node.**
|
||||
|
||||
```bash
|
||||
sudo spx admin kv replicas
|
||||
```
|
||||
|
||||
```
|
||||
spinifex@node1:~$ sudo spx admin kv replicas
|
||||
BUCKET REPLICAS WANT ONLINE STATUS LEADER HELD BY
|
||||
spinifex-cluster-state 3 3 3 ok spinifex-nats-node1 spinifex-nats-node1,spinifex-nats-node2,spinifex-nats-node3
|
||||
spinifex-dns-reconcile 3 3 3 ok spinifex-nats-node2 spinifex-nats-node2,spinifex-nats-node1,spinifex-nats-node3
|
||||
spinifex-iam 3 3 3 ok spinifex-nats-node3 spinifex-nats-node3,spinifex-nats-node1,spinifex-nats-node2
|
||||
spinifex-instance-state 3 3 3 ok spinifex-nats-node1 spinifex-nats-node1,spinifex-nats-node2,spinifex-nats-node3
|
||||
spinifex-quota-reconcile 3 3 3 ok spinifex-nats-node2 spinifex-nats-node2,spinifex-nats-node3,spinifex-nats-node1
|
||||
spinifex-terminated-instances 3 3 3 ok spinifex-nats-node3 spinifex-nats-node3,spinifex-nats-node2,spinifex-nats-node1
|
||||
spinifex-vpcd-reconcile 3 3 3 ok spinifex-nats-node2 spinifex-nats-node2,spinifex-nats-node1,spinifex-nats-node3
|
||||
|
||||
47 buckets, 0 under-replicated, 0 without quorum
|
||||
```
|
||||
|
||||
Server names are the node name behind a `spinifex-nats-` prefix, which is what NATS itself calls each server.
|
||||
|
||||
What to check:
|
||||
|
||||
- **`0 under-replicated, 0 without quorum`.** These are different problems. Under-replicated means a bucket lives on fewer nodes than depend on it — see [Buckets report as under-replicated](#buckets-report-as-under-replicated), and `--repair` is the fix. Without quorum means a bucket is not able to accept a write right now, whatever its replica count says, and no repair helps until you find out why a node is not answering.
|
||||
- **`WANT` is the largest odd number of nodes you have**, capped at five — 3 on three or four servers, 5 on five or more. If it says 1 on a three-server cluster, the node the command ran on is not configured for three: check `--nodes` in Step 4 and the `[nodes.*]` sections of that host's `/etc/spinifex/spinifex.toml`.
|
||||
- **`ONLINE` equals `REPLICAS`.** It counts the replicas that are caught up and reachable, so a bucket at `3` replicas with `ONLINE 2` is one node away from being unavailable and needs looking at before anything else is done to the cluster.
|
||||
- **`LEADER` is spread across the cluster** rather than parked on one node, and `HELD BY` names as many distinct servers as `REPLICAS`.
|
||||
|
||||
Exit status is what a deployment gate should read rather than the output: `0` healthy, `1` something is under-replicated, `2` something has no quorum or the report could not be produced at all.
|
||||
|
||||
Run it on every node, not just one. It reads cluster-global Raft metadata, so all of them should give the same answer; one that disagrees is not fully part of the cluster.
|
||||
|
||||
**4. The AWS API answers.**
|
||||
|
||||
```bash
|
||||
export AWS_PROFILE=spinifex
|
||||
@@ -403,6 +510,28 @@ The node has its own cluster configuration — normal for anything installed fro
|
||||
|
||||
Safe on a freshly installed node. On one that has been in service it orphans every volume and fragment sealed under the old key, so check before forcing.
|
||||
|
||||
### Buckets Report as Under-Replicated
|
||||
|
||||
```bash
|
||||
sudo spx admin kv replicas
|
||||
```
|
||||
|
||||
A bucket showing `UNDER-REPLICATED` lives on fewer nodes than the cluster has, so losing the node holding it takes it down for the whole cluster. Two things put a bucket in that state, and the fix is the same for both:
|
||||
|
||||
```bash
|
||||
sudo spx admin kv replicas --repair
|
||||
```
|
||||
|
||||
The first cause is **a cluster that grew**: buckets created when there were three servers stay on three replicas until something raises them. Services do this themselves as they restart, and `--repair` does it now.
|
||||
|
||||
The second is **a node that was down when the bucket was created**, so the Raft group formed short. Bring the node back first — `--repair` on an incomplete cluster raises the configured count, but the group cannot place the extra replica until the node is there to hold it. Confirm with `spx get nodes` before repairing.
|
||||
|
||||
If `WANT` itself is wrong — 1 on a three-server cluster — the problem is not the buckets but that host's view of the cluster. Check it has all three `[nodes.*]` sections:
|
||||
|
||||
```bash
|
||||
sudo grep -oP '^\[nodes\.\K[^.\]]+' /etc/spinifex/spinifex.toml | sort -u
|
||||
```
|
||||
|
||||
### OVN Database Cluster Not Forming
|
||||
|
||||
```bash
|
||||
|
||||
@@ -256,7 +256,7 @@ kubectl -n kube-system logs deploy/aws-load-balancer-controller --tail=50
|
||||
|
||||
Confirm the cluster carries `spinifex.io/managed-ingress = "false"` so the LBC owns ingress.
|
||||
|
||||
If the controller logs show `FailedBuildModel ... DescribeAvailabilityZones ... 403 ... AccessDenied`, the node role is missing the LBC permissions. The controller runs with the node instance-profile credentials, so the `${var.cluster_name}-node-lbc` policy (`aws_iam_policy.node_lbc`) must be attached to the node role — re-run `tofu apply` on the parent module if it was provisioned before that policy existed.
|
||||
If the controller logs show `FailedBuildModel ... DescribeAvailabilityZones ... 403 ... UnauthorizedOperation`, the node role is missing the LBC permissions. The controller runs with the node instance-profile credentials, so the `${var.cluster_name}-node-lbc` policy (`aws_iam_policy.node_lbc`) must be attached to the node role — re-run `tofu apply` on the parent module if it was provisioned before that policy existed.
|
||||
|
||||
### Provider Connection Refused
|
||||
|
||||
|
||||
@@ -7,11 +7,13 @@ Suites live in `tests/e2e/<name>/` and build to `tests/e2e/_bin/<name>.test`. Ev
|
||||
| Set | Suites |
|
||||
| --- | --- |
|
||||
| `E2E_SUITES_SINGLE` | `single iam cert eks ecs storagegrowth partialblock rds quota storagefault` |
|
||||
| `E2E_SUITES_MULTI` | `multinode lb cert quota lbrecovery instancerecovery storagefault` |
|
||||
| `E2E_SUITES_MULTI` | `multinode lb cert quota kvquorum lbrecovery instancerecovery storagefault` |
|
||||
| `E2E_SUITES_NIGHTLY_SINGLE` | `single cert iam` |
|
||||
| `E2E_SUITES_NIGHTLY_MULTI` | `multinode cert lb` |
|
||||
|
||||
The nightly permutation sets are deliberately narrower than the full ones: those cells have a ~35 minute budget and exist to prove every install / network / host-OS combination boots and serves. `eks`, `ecs`, `rds`, `storagefault`, `instancerecovery` and `lbrecovery` each get a dedicated cell instead.
|
||||
The nightly permutation sets are deliberately narrower than the full ones: those cells have a ~35 minute budget and exist to prove every install / network / host-OS combination boots and serves. `eks`, `ecs`, `rds`, `storagefault`, `instancerecovery`, `lbrecovery` and `kvquorum` each get a dedicated cell instead.
|
||||
|
||||
**`multinode` carries the KV replication sweep as well**, which is why the thing `kvquorum` proves is still checked in every multi-node permutation cell even though the suite itself is not. The read-only half costs a NATS connection and a stream listing; only the node-loss half needs a cell of its own.
|
||||
|
||||
## The suites
|
||||
|
||||
@@ -27,6 +29,7 @@ The largest suite, 25 top-level tests. Notable entry points:
|
||||
- `TestInstanceMetadata` — IMDS from inside a guest.
|
||||
- `TestStopStart`, `TestEarlyRebootLiveness`, `TestGuestChurnDurability`, `TestSnapshotLifecycle`, `TestSnapshotBackedLaunch`, `TestCreateImage`, `TestVolumeLifecycle`, `TestSpotInstanceLifecycle`, `TestLaunchTemplateBoot`.
|
||||
- `TestNegativeErrorPaths` — the `8a`–`8h` AWS error-shape cases (malformed AMI ID, invalid instance type, volume in use, detach-root forbidden, and so on).
|
||||
- **`TestKVReplication`** — the single-node half of the KV replication policy, and it is not the multi-node assertion with a smaller number in it. On one node one replica per bucket is the correct answer rather than the defect, so what it proves is the opposite: that nothing declared a replica count this deployment cannot place, that `spx admin kv replicas` exits 0 and reports `WANT 1`, and that `--repair` run twice against a cluster with nothing to repair changes no bucket's replica count and leaves every bucket serving. The multi-node suites cannot cover any of that, because on three nodes the unplaceable count is the right one. It lives here rather than in its own suite so it runs in every single-node nightly cell, across all three host and network permutations.
|
||||
|
||||
### `natuplink` — routed NAT (`external_mode = "nat"`), one node or three
|
||||
|
||||
@@ -42,6 +45,22 @@ Two nightly cells run it: `nat-single` (cell 19) and `nat-multi` (cell 30). Cell
|
||||
|
||||
`VPCSetup`, `SpansMultipleNodes`, `SpreadPlacement`, `EveryRunningInstanceReported`, `BastionSSH`, and its own NAT Gateway lane: `PreNATIsolation`, `NATGatewayInternet`, `NATCleanupOrdering`.
|
||||
|
||||
**`TestMultinodeJetStreamReplicas` is a whole-cluster sweep, not a named list, and it lives here on purpose.** It audits every KV bucket the cluster has and fails any that is replicated across fewer nodes than the cluster has, capped at JetStream's ceiling of five. Naming buckets would only ever cover the ones somebody remembered — a replica count is set where a bucket is created, and there is a bucket-creating call in most services — so the sweep is the assertion and the three daemon-owned buckets are named only so that a run enumerating nothing still fails. Because `multinode` is in `E2E_SUITES_NIGHTLY_MULTI`, this runs in every multi-node permutation cell, and no cell can be added that quietly opts out of it.
|
||||
|
||||
### `kvquorum` — the cluster's own state survives losing a node (multi-node, needs three)
|
||||
|
||||
**The control plane keeps its state in JetStream KV buckets, and this is the suite that proves those buckets are actually replicated.** A bucket on one replica lives on one JetStream-chosen server recorded in no config, so losing that node makes it return `nats: no responders available for request` from *every* node at once — a cluster-wide outage caused by a single-node event. Leader leases are the worst case: acquiring one is a write, so a lease bucket without a quorum cannot be acquired by anyone, and every reconciler sharing it stops cluster-wide at exactly the moment a node has failed and there is repair work to do.
|
||||
|
||||
Three entry points, in the order they run:
|
||||
|
||||
- **`TestKVBucketsAreReplicatedAcrossTheCluster`** is read-only and runs first, so a cluster that is already wrong is reported as wrong rather than as a failure to survive a node loss. It sweeps every bucket, then separately requires the leader-lease buckets to be among those checked — they were the worst case of the defect, and a regression reaching only them would otherwise be one line in a long list rather than the headline.
|
||||
- **`TestKVReplicasCommandReportsHealthy`** runs `spx admin kv replicas --json` on **every** node, not one. The command reads cluster-global Raft metadata, so all nodes must give the same answer; a node that disagrees is reporting on a cluster it is not fully part of, which is worth knowing before a deployment is gated on its exit status.
|
||||
- **`TestKVBucketsSurviveANodeLoss`** produces the fault rather than inspecting for it. It takes away **the node leading the most buckets** — the worst one to lose, and the one a test picking arbitrarily would usually miss — then requires every survivor to still answer for every bucket, brings the node back, and requires it to carry its share again.
|
||||
|
||||
**Configured, placed and working replicas are three different facts, and all three are checked.** A stream can carry the right replica count while its Raft group is short of peers, or hold the right peers while one of them is offline or still catching up, and either reads as healthy right up until the node it is really on goes away. The audit reports the leader and a count of peers NATS itself calls current and reachable, and the assertion is against that count rather than against the configured one. The last assertion is that no bucket came back with *fewer* replicas than it had: a cluster that "repaired" itself by lowering a count would pass every other assertion here and have quietly traded the durability this suite is about for a green run.
|
||||
|
||||
**Its own suite and its own cell, for lbrecovery's reasons.** It takes a node away, so anything sharing the environment reports that outage as its own failure, and the question it answers — "is our internal state actually replicated" — has to read as itself rather than as one more red subtest somewhere else. Three nodes is the floor: on two, a majority is two, so losing either leaves no quorum for any stream and there is nothing to assert beyond "Raft needs a majority", which is true of NATS and not a property of ours.
|
||||
|
||||
### `instancerecovery` — a guest whose node goes away (multi-node)
|
||||
|
||||
**Asserted from both sides, and the pair is the point.** `InstanceAutoRecovery` takes a node down and requires the guest to come back on exactly one survivor, identified by the qemu process rather than by what the API says, then brings the node back and requires it to drop the local copy instead of relaunching it. `InstanceRecoveryRefusesStorageFault` freezes predastore cluster-wide first, so the guest is paused by `werror=stop` when its node goes away, and requires that **nothing moves** — a store that refuses every node cannot be fixed by moving a guest, and the relaunch would cost it the request QEMU is holding. A reconciler that cannot tell a host failure from a storage failure passes the first and fails the second. Neither turns recovery on, because there is no setting for it: both wait for every node to report the reconciler active on a cluster nothing configured, so a build that reintroduced a switch fails here.
|
||||
|
||||
@@ -23,10 +23,10 @@ require (
|
||||
github.com/klauspost/cpuid/v2 v2.4.0
|
||||
github.com/miekg/dns v1.1.73
|
||||
github.com/minio/highwayhash v1.0.4
|
||||
github.com/mulgadc/bluebottle v1.20.1-0.20260928004821-163f43196afd
|
||||
github.com/mulgadc/northstar v1.20.1-0.20260917032839-357bbc94201d
|
||||
github.com/mulgadc/predastore v1.20.1-0.20260918222809-019dd349cf1f
|
||||
github.com/mulgadc/viperblock v1.20.0
|
||||
github.com/mulgadc/bluebottle v1.21.1-0.20261001020941-036da4967e1c
|
||||
github.com/mulgadc/northstar v1.21.0
|
||||
github.com/mulgadc/predastore v1.21.0
|
||||
github.com/mulgadc/viperblock v1.21.0
|
||||
github.com/nats-io/nats-server/v2 v2.15.0
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
github.com/opencontainers/runtime-spec v1.3.0
|
||||
@@ -60,16 +60,15 @@ require (
|
||||
github.com/Microsoft/go-winio v0.6.3-0.20251027160822-ad3df93bed29 // indirect
|
||||
github.com/Microsoft/hcsshim v0.15.0-rc.4 // indirect
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect
|
||||
github.com/armon/go-metrics v0.4.1 // indirect
|
||||
github.com/atotto/clipboard v0.1.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.113.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.113.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0 // indirect
|
||||
@@ -125,12 +124,12 @@ require (
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect
|
||||
github.com/hashicorp/go-hclog v1.6.3 // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
|
||||
github.com/hashicorp/go-metrics v0.5.4 // indirect
|
||||
github.com/hashicorp/go-metrics v0.7.0 // indirect
|
||||
github.com/hashicorp/go-msgpack/v2 v2.1.5 // indirect
|
||||
github.com/hashicorp/golang-lru v1.0.2 // indirect
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
|
||||
github.com/hashicorp/raft v1.7.3 // indirect
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.3.1 // indirect
|
||||
github.com/hashicorp/raft v1.8.0 // indirect
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.4.2 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
@@ -141,8 +140,8 @@ require (
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/lithammer/fuzzysearch v1.1.8 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-colorable v0.1.15 // indirect
|
||||
github.com/mattn/go-isatty v0.0.24 // indirect
|
||||
github.com/mattn/go-localereader v0.0.1 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.21 // indirect
|
||||
github.com/mdlayher/packet v1.1.2 // indirect
|
||||
@@ -166,10 +165,10 @@ require (
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/prometheus/client_golang v1.24.1 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.70.1 // indirect
|
||||
github.com/prometheus/client_model v0.6.3 // indirect
|
||||
github.com/prometheus/common v0.71.0 // indirect
|
||||
github.com/prometheus/procfs v0.21.1 // indirect
|
||||
github.com/quic-go/quic-go v0.62.0 // indirect
|
||||
github.com/quic-go/quic-go v0.63.0 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/sagikazarmark/locafero v0.12.0 // indirect
|
||||
github.com/sirupsen/logrus v1.10.2 // indirect
|
||||
|
||||
@@ -6,10 +6,8 @@ atomicgo.dev/keyboard v0.2.9 h1:tOsIid3nlPLZ3lwgG8KZMp/SFmr7P0ssEN5JUsm78K8=
|
||||
atomicgo.dev/keyboard v0.2.9/go.mod h1:BC4w9g00XkxH/f1HXhW2sXmJFOCWbKn9xrOunSFtExQ=
|
||||
atomicgo.dev/schedule v0.1.0 h1:nTthAbhZS5YZmgYbb2+DH8uQIZcTlIrd4eYr3UQxEjs=
|
||||
atomicgo.dev/schedule v0.1.0/go.mod h1:xeUa3oAkiuHYh8bKiQBRojqAMq3PXXbJujjb0hw8pEU=
|
||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
|
||||
github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
|
||||
github.com/MarvinJWendt/testza v0.1.0/go.mod h1:7AxNvlfeHP7Z/hDQ5JtE3OKYT3XFUeLCDE2DQninSqs=
|
||||
github.com/MarvinJWendt/testza v0.2.1/go.mod h1:God7bhG8n6uQxwdScay+gjm9/LnO4D3kkcZX4hv9Rp8=
|
||||
github.com/MarvinJWendt/testza v0.2.8/go.mod h1:nwIcjmr0Zz+Rcwfh3/4UhBp7ePKVhuBExvZqnKYWlII=
|
||||
@@ -23,11 +21,6 @@ github.com/Microsoft/go-winio v0.6.3-0.20251027160822-ad3df93bed29 h1:0kQAzHq8vL
|
||||
github.com/Microsoft/go-winio v0.6.3-0.20251027160822-ad3df93bed29/go.mod h1:ZWa7ssZJT30CCDGJ7fk/2SBTq9BIQrrVjrcss0UW2s0=
|
||||
github.com/Microsoft/hcsshim v0.15.0-rc.4 h1:aZFX4LH0S20Lgjq0wG61StIClj7im4yzrxIClkaR8Z8=
|
||||
github.com/Microsoft/hcsshim v0.15.0-rc.4/go.mod h1:BA9CBztgu4h/6Jsvo1O1M4qjWw09PoYpaEYgexPE578=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA=
|
||||
@@ -55,14 +48,14 @@ github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3 h1:fuSCw4Z2qfRCztMPO3GXJNSiEp6W
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3/go.mod h1:6SxcHheD1pPR5+kWm1wGvjlL/YqUsh267sAfEmN4K7A=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 h1:bAdDl/HkGCcGPoe25ToSHEw23VIxt6CT5fLcg111BKg=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19/go.mod h1:KaUzbLxv4CeSxh6ZCl9B4m7CuFenS8kUEaDs+f/DQr4=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.3 h1:BHKCSX4QXERe8So8rbWqaM7owqOmDJxATXgJwGng22A=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.3/go.mod h1:GqWeeKfYfezihA2KfFL9l7ohEdZWe1tuFWh3GfyNSnE=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.4 h1:7TXEkbDzy4BkYYTfTVjKcADTfkDnvHxDwcF1pJG1yF0=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.4/go.mod h1:GqWeeKfYfezihA2KfFL9l7ohEdZWe1tuFWh3GfyNSnE=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3 h1:bON1rJf67TSTDCKg816AAIE4xSTtoo9tl0XRkO72R+I=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3/go.mod h1:c5BBpjJcQXpfeq9iASyVKA3T6vX6B6LEXY4mL/gklDY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.3 h1:L8vIOxylma91TcR96NFTEC07G3JDwSl+CvK2b+IODms=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.3/go.mod h1:fmPIZQzTExYuBNWFyi1P7IoDjvskgphXqK1yObMzusM=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.113.0 h1:0slwIjBv1sEigSUW4EfTtNw9mbHl0PlOUPX9Y1C/eLE=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.113.0/go.mod h1:/uA+2Qj4jd5qBWagVC1AyzzDFXVK997E7U04w7Kw0wI=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.113.3 h1:IDIchqG5C/o/JdprtYn1NirCi7iUx7XQ8+WwZ6odFX8=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.113.3/go.mod h1:iJ69H4lkK6a9zQ+L5i9pDERMok5Jvts0iZaMjWEi/78=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.0 h1:ZD5qFpWcaOKdTuhBi431pIDkCgrMkMlMT6jlpSPoIRI=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.0/go.mod h1:8Nuuf+tR346PjJ3MvZPh9pekbLiLQFWJhzMXfwy7alA=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.0 h1:JGeeBcMlhg1xtOXYpeCaTQBZObtXMPQCUqBcmr65NRA=
|
||||
@@ -71,14 +64,10 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0 h1:obhahQXDEdVEv8y5bTKXR30L
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0/go.mod h1:6twZZ/aXHNy1vXUO8koUbp++MYzMASkOgEBdkbJYmO0=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.0 h1:Zpnqa6XtrNzXZnwbdCqHOXpXhMsa01ql/pcRQ1sb4hk=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.0/go.mod h1:/8JRcdTt//hG0Q4BTmGbuOplT7ABe+5rdtqUHqXvYIM=
|
||||
github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ=
|
||||
github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/aws/smithy-go v1.28.2 h1:myhcykQcatTul2B/zITjDk203G7t0awUAs1hVry5Bvg=
|
||||
github.com/aws/smithy-go v1.28.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
|
||||
@@ -93,7 +82,6 @@ github.com/cenkalti/hub v1.0.2 h1:Nqv9TNaA9boeO2wQFW8o87BY3zKthtnzXmWGmJqhAV8=
|
||||
github.com/cenkalti/hub v1.0.2/go.mod h1:8LAFAZcCasb83vfxatMUnZHRoQcffho2ELpHb+kaTJU=
|
||||
github.com/cenkalti/rpc2 v1.0.5 h1:T6l4SS3ja3eaJfRyZrn7Oco/PSx/pr3YK5cjCgLVLTk=
|
||||
github.com/cenkalti/rpc2 v1.0.5/go.mod h1:2yfU5b86vOr16+iY1jN3MvT6Kxc9Nf8j5iZWwUf7iaw=
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/charmbracelet/bubbles v1.0.0 h1:12J8/ak/uCZEMQ6KU7pcfwceyjLlWsDLAxB5fXonfvc=
|
||||
@@ -110,8 +98,6 @@ github.com/charmbracelet/x/cellbuf v0.0.15 h1:ur3pZy0o6z/R7EylET877CBxaiE1Sp1GMx
|
||||
github.com/charmbracelet/x/cellbuf v0.0.15/go.mod h1:J1YVbR7MUuEGIFPCaaZ96KDl5NoS0DAWkskup+mOY+Q=
|
||||
github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk=
|
||||
github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI=
|
||||
github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
|
||||
github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
|
||||
github.com/clipperhouse/displaywidth v0.9.0 h1:Qb4KOhYwRiN3viMv1v/3cTBlz3AcAZX3+y9OLhMtAtA=
|
||||
github.com/clipperhouse/displaywidth v0.9.0/go.mod h1:aCAAqTlh4GIVkhQnJpbL0T/WfcrJXHcj8C0yjYcjOZA=
|
||||
github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs=
|
||||
@@ -179,12 +165,6 @@ github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9
|
||||
github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
|
||||
github.com/go-chi/chi/v5 v5.3.2 h1:5YQkICvTCSZ25hoRsyJazN0scjzKGiu4VAUc7H1o1nY=
|
||||
github.com/go-chi/chi/v5 v5.3.2/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
|
||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
|
||||
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
|
||||
github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
@@ -198,40 +178,22 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
|
||||
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
|
||||
github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w=
|
||||
github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/gofrs/flock v0.10.0 h1:SHMXenfaB03KbroETaCMtbBg3Yn29v4w1r+tgy4ff4k=
|
||||
github.com/gofrs/flock v0.10.0/go.mod h1:FirDy1Ing0mI2+kB6wk+vyyAH+e6xiE+EYA0jnzV9jc=
|
||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
|
||||
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
|
||||
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
|
||||
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
|
||||
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs=
|
||||
github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
|
||||
github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786 h1:rcv+Ippz6RAtvaGgKxc+8FQIpxHgsF+HBzPyYL2cyVU=
|
||||
github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786/go.mod h1:apVn/GCasLZUVpAJ6oWAuyP7Ne7CEsQbTnc0plM3m+o=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
@@ -244,19 +206,16 @@ github.com/gookit/color v1.6.0 h1:JjJXBTk1ETNyqyilJhkTXJYYigHG24TM9Xa2M1xAhRA=
|
||||
github.com/gookit/color v1.6.0/go.mod h1:9ACFc7/1IpHGBW8RwuDm/0YEnhg3dwwXpoMsmtyHfjs=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
|
||||
github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
|
||||
github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
|
||||
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
|
||||
github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc=
|
||||
github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
|
||||
github.com/hashicorp/go-metrics v0.5.4 h1:8mmPiIJkTPPEbAiV97IxdAGNdRdaWwVap1BU6elejKY=
|
||||
github.com/hashicorp/go-metrics v0.5.4/go.mod h1:CG5yz4NZ/AI/aQt9Ucm/vdBnbh7fvmv4lxZ350i+QQI=
|
||||
github.com/hashicorp/go-metrics v0.7.0 h1:lLWieZTcbzZT+rY0zrqKbyryXG8RIajdUjmM0+R79eg=
|
||||
github.com/hashicorp/go-metrics v0.7.0/go.mod h1:8T/Es8FPTfQvY7azBPGyrwXwwg7mbA9/TmQ1/lWfxb4=
|
||||
github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI=
|
||||
github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
|
||||
github.com/hashicorp/go-msgpack/v2 v2.1.5 h1:Ue879bPnutj/hXfmUk6s/jtIK90XxgiUIcXRl656T44=
|
||||
github.com/hashicorp/go-msgpack/v2 v2.1.5/go.mod h1:bjCsRXpZ7NsJdk45PoCQnzRGDaK8TKm5ZnDI/9y3J4M=
|
||||
github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
|
||||
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
|
||||
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
@@ -265,12 +224,12 @@ github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iP
|
||||
github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/hashicorp/raft v1.7.3 h1:DxpEqZJysHN0wK+fviai5mFcSYsCkNpFUl1xpAW8Rbo=
|
||||
github.com/hashicorp/raft v1.7.3/go.mod h1:DfvCGFxpAUPE0L4Uc8JLlTPtc3GzSbdH0MTJCLgnmJQ=
|
||||
github.com/hashicorp/raft v1.8.0 h1:YbfecBcuTar/LNFEDfVTpqu9Aw+MczTk7MYczvy+62k=
|
||||
github.com/hashicorp/raft v1.8.0/go.mod h1:agL5fncrpEsbxr5P5KOd2srskDwPY18opjXN5x0661s=
|
||||
github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702 h1:RLKEcCuKcZ+qp2VlaaZsYZfLOmIiuJNpEi48Rl8u9cQ=
|
||||
github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702/go.mod h1:nTakvJ4XYq45UXtn0DbwR4aU9ZdjlnIenpbs6Cd+FM0=
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.3.1 h1:ackhdCNPKblmOhjEU9+4lHSJYFkJd6Jqyvj6eW9pwkc=
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.3.1/go.mod h1:n4S+g43dXF1tqDT+yzcXHhXM6y7MrlUd3TTwGRcUvQE=
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.4.2 h1:r2RRgZ6ajT+VH8/9yC5mfoKAlt3KrYiUDhfbUSmaoTs=
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.4.2/go.mod h1:+wvKK1thWEqM7amY1MXjtQpkm+b9KNUXks0fcus5ZGU=
|
||||
github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714 h1:/jC7qQFrv8CrSJVmaolDVOxTfS9kc36uB6H40kdbQq8=
|
||||
github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714/go.mod h1:2Goc3h8EklBH5mspfHFxBnEoURQCGzQQH1ga9Myjvis=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
@@ -292,13 +251,6 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC
|
||||
github.com/josharian/native v1.0.1-0.20221213033349-c1e37c09b531/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
|
||||
github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA=
|
||||
github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
|
||||
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
|
||||
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
@@ -308,9 +260,6 @@ github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV91
|
||||
github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
|
||||
github.com/klauspost/reedsolomon v1.14.2 h1:SafJYwpBBQBI6amHUygcjxZjXeN2HpiENHQDwuPWCCQ=
|
||||
github.com/klauspost/reedsolomon v1.14.2/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
@@ -326,18 +275,17 @@ github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQ
|
||||
github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
||||
github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
|
||||
github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
|
||||
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
|
||||
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
||||
github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY=
|
||||
github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
||||
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
|
||||
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
|
||||
github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
|
||||
github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
|
||||
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/mattn/go-runewidth v0.0.21 h1:jJKAZiQH+2mIinzCJIaIG9Be1+0NR+5sz/lYEEjdM8w=
|
||||
github.com/mattn/go-runewidth v0.0.21/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/mdlayher/packet v1.1.2 h1:3Up1NG6LZrsgDVn6X4L9Ge/iyRyxFEFD9o6Pr3Q1nQY=
|
||||
github.com/mdlayher/packet v1.1.2/go.mod h1:GEu1+n9sG5VtiRE4SydOmX5GTwyyYlteZiFU+x0kew4=
|
||||
github.com/mdlayher/socket v0.6.0 h1:ScZPaAGyO1icQnbFrhPM8mnXyMu9qukC1K4ZoM2IQKU=
|
||||
@@ -358,30 +306,22 @@ github.com/moby/sys/user v0.4.1 h1:RgjRlaDKi/Xmyrz4t8lyzXT6v2ooFeO/7xtchmhVWE0=
|
||||
github.com/moby/sys/user v0.4.1/go.mod h1:E9QsW5WRe1kUAf7kW8hXKwu1uhsZEAdPLYHYSDudF4Y=
|
||||
github.com/moby/sys/userns v0.2.1 h1:4OvdM7BcPkASbuouHsbW3aeMJSFlYDldBRnXVZhaRk8=
|
||||
github.com/moby/sys/userns v0.2.1/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI=
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo=
|
||||
github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA=
|
||||
github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
|
||||
github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
|
||||
github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
|
||||
github.com/mulgadc/bluebottle v1.20.1-0.20260916040751-5a5ca6f0fb33 h1:Rw3mnRF2YrhVZgHpCsOi6La8cOu/V3xnhB000iWcdr8=
|
||||
github.com/mulgadc/bluebottle v1.20.1-0.20260916040751-5a5ca6f0fb33/go.mod h1:4rjLL3B0xZBPmxacaXPkw6sPOZZgyuCDBFmPSOC81yE=
|
||||
github.com/mulgadc/bluebottle v1.20.1-0.20260928004821-163f43196afd h1:Ffa6OIUqHLUNn+NM4Ij4kGOLm7DEeicM16+Sn/Kulm8=
|
||||
github.com/mulgadc/bluebottle v1.20.1-0.20260928004821-163f43196afd/go.mod h1:LH0MuQcFxet0dDEISlZsuULIYj592eCN2Fv39A6EH5o=
|
||||
github.com/mulgadc/northstar v1.20.1-0.20260917032839-357bbc94201d h1:xYdO+KXrn+/aBLva3yuRly9MnKrSNSouGYleITgkoXE=
|
||||
github.com/mulgadc/northstar v1.20.1-0.20260917032839-357bbc94201d/go.mod h1:3GB49VUE9rA2idp88se5jAMv3ql1Oq9kIcT29WAZwDU=
|
||||
github.com/mulgadc/predastore v1.20.1-0.20260918222809-019dd349cf1f h1:P7echVc7fCYiU2CS/L0dLAf0i/6eWcjDqqf7AnFKpDg=
|
||||
github.com/mulgadc/predastore v1.20.1-0.20260918222809-019dd349cf1f/go.mod h1:r1idTXiSBAq1GtIIVpRFjlplXzWm0iS8hviF33K85us=
|
||||
github.com/mulgadc/viperblock v1.20.0 h1:g17NkHQcXGSXBGp20UcAPrklzTGXdyERXdaldo1aKM4=
|
||||
github.com/mulgadc/viperblock v1.20.0/go.mod h1:q+Zs0C1pKfJVj8oALXZrZl33Sd+vPX0yc8ZroFtJBxM=
|
||||
github.com/mulgadc/bluebottle v1.21.1-0.20261001020941-036da4967e1c h1:oRb9PlvooMenQjKU9dD76/bxUOvJaYi6taxm3famoj4=
|
||||
github.com/mulgadc/bluebottle v1.21.1-0.20261001020941-036da4967e1c/go.mod h1:LH0MuQcFxet0dDEISlZsuULIYj592eCN2Fv39A6EH5o=
|
||||
github.com/mulgadc/northstar v1.21.0 h1:LE/5kiT3kBelCg02cXa3b7awhaYkGHsOXeJ4nHRLNaQ=
|
||||
github.com/mulgadc/northstar v1.21.0/go.mod h1:O446cApNMB6qebqieBhDZfCIGVWX++v3QdOz+wIGhUk=
|
||||
github.com/mulgadc/predastore v1.21.0 h1:pAm4fFWf0vnlwRgnoL2oJ+KSCw8mtjT2rQoBHFmnjzk=
|
||||
github.com/mulgadc/predastore v1.21.0/go.mod h1:icCfKGnyqDaJ/Jlbp2aYeo2qwrKHZfxmPHDYK1MM7wc=
|
||||
github.com/mulgadc/viperblock v1.21.0 h1:RbQ/GpHCg/uoPffYioSKqzqyoz22Uc7wpQ7XY7etLg0=
|
||||
github.com/mulgadc/viperblock v1.21.0/go.mod h1:n0wCOGV58Lt2IVPXEvpbHDkTITlXtFw+2CMrQXgbJ7s=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc=
|
||||
github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc=
|
||||
github.com/nats-io/nats-server/v2 v2.15.0 h1:M99yf0y05rTr46/qc/Is6ZAowI58Ryp2SjufLCUeVJc=
|
||||
@@ -402,43 +342,22 @@ github.com/oracle/oci-go-sdk/v65 v65.126.0 h1:RuV0MEcLOOgNOBadYbbkUQriCK4Gm5348F
|
||||
github.com/oracle/oci-go-sdk/v65 v65.126.0/go.mod h1:Pzy+BpgkDesvGZXEHgslwhIYobHCPHg6wRta1mWnlqQ=
|
||||
github.com/ovn-kubernetes/libovsdb v0.8.1 h1:M2J8bcJt5mXCom0HqzfEtuHkT80CTSQRcYG7acT8gf4=
|
||||
github.com/ovn-kubernetes/libovsdb v0.8.1/go.mod h1:ZlnHLzagmLOSvyd9qfxBIZp6wOSOw0IsRsc+6lNUGbU=
|
||||
github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
|
||||
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY=
|
||||
github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
|
||||
github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
|
||||
github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
|
||||
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
|
||||
github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
|
||||
github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU=
|
||||
github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
|
||||
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
|
||||
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
||||
github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
|
||||
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
|
||||
github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
|
||||
github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY=
|
||||
github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc=
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
||||
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
|
||||
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
|
||||
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
|
||||
github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo=
|
||||
github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM=
|
||||
github.com/prometheus/common v0.71.0 h1:9KDAKb7Mj3HEVKyFCK6Dc/HIwlBzZIN2l7/lrHl3KK8=
|
||||
github.com/prometheus/common v0.71.0/go.mod h1:CLJ5H8TEsGX8bl31BdMkfhIZ+QmZ9tBPPotUxUbfcmk=
|
||||
github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI=
|
||||
github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
|
||||
github.com/pterm/pterm v0.12.27/go.mod h1:PhQ89w4i95rhgE+xedAoqous6K9X+r6aSOI2eFF7DZI=
|
||||
@@ -452,8 +371,8 @@ github.com/pterm/pterm v0.12.83 h1:ie+YmGmA727VuhxBlyGr74Ks+7McV6kT99IB8EU80aA=
|
||||
github.com/pterm/pterm v0.12.83/go.mod h1:xlgc6bFWyJIMtmLJvGim+L7jhSReilOlOnodeIYe4Tk=
|
||||
github.com/quic-go/go-ossfuzz-seeds v0.1.0 h1:APacT+iIaNF6fd8AGEiN3bT/Jtkd2jz4v4TzM7MFjy0=
|
||||
github.com/quic-go/go-ossfuzz-seeds v0.1.0/go.mod h1:3IOHRbJIc+L6YKMwfDtJAM9Vj9k0YY4muhuyUYk5tbk=
|
||||
github.com/quic-go/quic-go v0.62.0 h1:ZHDjCk5OacATwGvs8PWE97CTvX7AqZiVoW7++ZOXTf8=
|
||||
github.com/quic-go/quic-go v0.62.0/go.mod h1:RAro2j2yN9a9EiPACLHT9IB2NXCvGQmmo/alT0yYI0w=
|
||||
github.com/quic-go/quic-go v0.63.0 h1:LIFGHI4PFUhhw2dDD1ARHdCff143ffMHwZtbnbuJ78A=
|
||||
github.com/quic-go/quic-go v0.63.0/go.mod h1:RAro2j2yN9a9EiPACLHT9IB2NXCvGQmmo/alT0yYI0w=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
@@ -464,9 +383,6 @@ github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88ee
|
||||
github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI=
|
||||
github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
|
||||
github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
|
||||
github.com/sirupsen/logrus v1.10.2 h1:G2SED73/qrAu6YwbdxOD6peLkCBI3z7L+ykJFTXJBBo=
|
||||
github.com/sirupsen/logrus v1.10.2/go.mod h1:SLEg8TqYulVKKfIGHldVp2K2aYz2DKSVBq4g/H5bR7Q=
|
||||
github.com/sony/gobreaker/v2 v2.4.0 h1:g2KJRW1Ubty3+ZOcSEUN7K+REQJdN6yo6XvaML+jptg=
|
||||
@@ -483,13 +399,11 @@ github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3A
|
||||
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
|
||||
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4=
|
||||
github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
@@ -505,7 +419,6 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8
|
||||
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||
github.com/tidwall/btree v1.8.1 h1:27ehoXvm5AG/g+1VxLS1SD3vRhp/H7LuEfwNvddEdmA=
|
||||
github.com/tidwall/btree v1.8.1/go.mod h1:jBbTdUWhSZClZWoDg54VnvV7/54modSOzDN7VXftj1A=
|
||||
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
|
||||
github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923 h1:tHNk7XK9GkmKUR6Gh8gVBKXc2MVSZ4G/NnWLtzw4gNA=
|
||||
github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923/go.mod h1:eLL9Nub3yfAho7qB0MzZizFhTU2QkLeoVsWdHtDW264=
|
||||
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs=
|
||||
@@ -563,9 +476,7 @@ go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
@@ -577,44 +488,23 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
|
||||
golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
|
||||
golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
|
||||
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
@@ -627,7 +517,6 @@ golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
@@ -641,7 +530,6 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE=
|
||||
golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
@@ -663,36 +551,23 @@ golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGN
|
||||
golang.org/x/vuln v1.6.0 h1:FeMO9Rm/HwyduOztbvKcOw+zvDEPr4I4aQNSfevFcKY=
|
||||
golang.org/x/vuln v1.6.0/go.mod h1:bWlG2493/sjR7ksvicBgMrznH3eYQEyK8ifUYBrqUbg=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
|
||||
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260825221802-da73d73af1c5 h1:izFU9hz7aeLI/Mi1J0991ae+xcwRLr7hTqWnB/9aIIU=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260825221802-da73d73af1c5/go.mod h1:3LhxRw4YYkf+ylAfgaY9JlVLFKhokkCV8duhLLe7+t0=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260825221802-da73d73af1c5 h1:1VUiZAXyC+zmiFYi+WLtBzr68Cj8wOofHjjrA/kkizc=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260825221802-da73d73af1c5/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA=
|
||||
google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU=
|
||||
google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
|
||||
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
|
||||
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
|
||||
google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw=
|
||||
gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@@ -77,8 +77,8 @@ func TestEC2ErrorCatalogMetadataAndClasses(t *testing.T) {
|
||||
if catalog.metadata.VerifiedOn == "" {
|
||||
t.Fatal("verifiedOn is empty")
|
||||
}
|
||||
if len(catalog.codes) != 112 {
|
||||
t.Fatalf("catalog code count = %d, want 112", len(catalog.codes))
|
||||
if len(catalog.codes) != 114 {
|
||||
t.Fatalf("catalog code count = %d, want 114", len(catalog.codes))
|
||||
}
|
||||
if catalog.codes["InvalidParameterValue"] != ec2ErrorClassClient {
|
||||
t.Fatal("InvalidParameterValue is not classified as client")
|
||||
|
||||
@@ -48,7 +48,14 @@ func ValidateErrorResponse(service Service, operationName string, body []byte) (
|
||||
// they are not treated as proof that an operation-specific code conforms.
|
||||
func isUnmodelledCommonError(service Service, code string) bool {
|
||||
switch service {
|
||||
case IAM, ElasticLoadBalancingV2:
|
||||
case IAM:
|
||||
return isUnmodelledQueryError(code)
|
||||
case ElasticLoadBalancingV2:
|
||||
// ELBv2's common errors list the parameter validation codes too.
|
||||
switch code {
|
||||
case "InvalidParameterCombination", "InvalidParameterValue", "MissingParameter", "ValidationError":
|
||||
return true
|
||||
}
|
||||
return isUnmodelledQueryError(code)
|
||||
case STS:
|
||||
// STS also uses these two runtime input errors even though the model
|
||||
@@ -107,9 +114,7 @@ func (m *Model) decodeErrorCode(body []byte) (string, error) {
|
||||
if separator := strings.LastIndex(code, "#"); separator >= 0 {
|
||||
code = code[separator+1:]
|
||||
}
|
||||
if separator := strings.Index(code, ":"); separator >= 0 {
|
||||
code = code[:separator]
|
||||
}
|
||||
code, _, _ = strings.Cut(code, ":")
|
||||
if code == "" {
|
||||
return "", fmt.Errorf("JSON error envelope has no code")
|
||||
}
|
||||
|
||||
@@ -97,6 +97,12 @@ func TestValidateErrorResponseClassifiesCommonErrorsAsUnmodelled(t *testing.T) {
|
||||
operation: "AssumeRole",
|
||||
body: `<ErrorResponse><Error><Code>ValidationError</Code></Error></ErrorResponse>`,
|
||||
},
|
||||
{
|
||||
name: "ELBv2 parameter validation",
|
||||
service: ElasticLoadBalancingV2,
|
||||
operation: "AddTags",
|
||||
body: `<ErrorResponse><Error><Code>InvalidParameterValue</Code></Error></ErrorResponse>`,
|
||||
},
|
||||
{
|
||||
name: "JSON authorization",
|
||||
service: ECS,
|
||||
|
||||
@@ -131,12 +131,7 @@ var modelCache sync.Map
|
||||
|
||||
// Services returns the supported service identifiers in stable order.
|
||||
func Services() []Service {
|
||||
services := make([]Service, 0, len(modelFiles))
|
||||
for service := range modelFiles {
|
||||
services = append(services, service)
|
||||
}
|
||||
slices.Sort(services)
|
||||
return services
|
||||
return slices.Sorted(maps.Keys(modelFiles))
|
||||
}
|
||||
|
||||
// Load parses and indexes the embedded model for service. Each service is
|
||||
|
||||
@@ -68,6 +68,7 @@
|
||||
"InvalidLaunchTemplateName.AlreadyExistsException",
|
||||
"InvalidLaunchTemplateName.MalformedException",
|
||||
"InvalidLaunchTemplateName.NotFoundException",
|
||||
"InvalidMaxResults",
|
||||
"InvalidNatGatewayID.NotFound",
|
||||
"InvalidNetworkInterface.InUse",
|
||||
"InvalidNetworkInterface.NotFound",
|
||||
@@ -77,6 +78,7 @@
|
||||
"InvalidPlacementGroup.Duplicate",
|
||||
"InvalidPlacementGroup.InUse",
|
||||
"InvalidPlacementGroup.Unknown",
|
||||
"InvalidRequest",
|
||||
"InvalidRoute.NotFound",
|
||||
"InvalidRouteTableID.NotFound",
|
||||
"InvalidSecurityGroupRuleId.Malformed",
|
||||
|
||||
@@ -4,8 +4,9 @@ import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"regexp"
|
||||
"sort"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
@@ -253,12 +254,7 @@ func validateFrontmatter(owner string, page PageMetadata) error {
|
||||
// key rather than by first use, so adding an operation cannot renumber a note
|
||||
// that is already published.
|
||||
func noteOrder(notes map[string]string) []string {
|
||||
keys := make([]string, 0, len(notes))
|
||||
for key := range notes {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
return slices.Sorted(maps.Keys(notes))
|
||||
}
|
||||
|
||||
// checkNotes rejects a note nothing references and a reference to a note that
|
||||
|
||||
@@ -51,8 +51,24 @@ func newPatternSampler(pattern string) (*patternSampler, error) {
|
||||
return sampler, err
|
||||
}
|
||||
|
||||
// re2Equivalents rewrite model patterns that use lookahead, which Go regexp
|
||||
// lacks, as patterns matching the same strings.
|
||||
var re2Equivalents = map[string]string{
|
||||
// ACM DomainNameString: labels start and end alphanumeric, the last has two
|
||||
// or more characters, with an optional leading wildcard label.
|
||||
`^(\*\.)?(((?!-)[A-Za-z0-9-]{0,62}[A-Za-z0-9])\.)+((?!-)[A-Za-z0-9-]{1,62}[A-Za-z0-9])$`: `^(\*\.)?([A-Za-z0-9]([A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.)+[A-Za-z0-9][A-Za-z0-9-]{0,61}[A-Za-z0-9]$`,
|
||||
}
|
||||
|
||||
// re2Pattern spells a model pattern in Go regexp syntax.
|
||||
func re2Pattern(pattern string) string {
|
||||
if equivalent, ok := re2Equivalents[pattern]; ok {
|
||||
return equivalent
|
||||
}
|
||||
return javaEscape.ReplaceAllString(pattern, `\x{$1}`)
|
||||
}
|
||||
|
||||
func compilePatternSampler(pattern string) (*patternSampler, error) {
|
||||
pattern = javaEscape.ReplaceAllString(pattern, `\x{$1}`)
|
||||
pattern = re2Pattern(pattern)
|
||||
re, err := regexp.Compile(pattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -65,22 +65,33 @@ func GenerateRequests(service Service, operationName string, options RequestOpti
|
||||
if !ok {
|
||||
return RequestPlan{}, fmt.Errorf("awsmodel: %s operation %q is not modelled", service, operationName)
|
||||
}
|
||||
if service == ACM {
|
||||
if _, err := importMaterial(); err != nil {
|
||||
return RequestPlan{}, err
|
||||
}
|
||||
}
|
||||
declared := model.operationErrorCodes(operation)
|
||||
if operation.Input == nil {
|
||||
return RequestPlan{Cases: []RequestCase{{Operation: operationName, Input: map[string]any{}}}, DeclaredErrors: declared}, nil
|
||||
}
|
||||
|
||||
first := model.newRequestGenerator(0, options)
|
||||
// A create or delete names a resource of its own, so it neither collides
|
||||
// with a fixture nor removes one.
|
||||
if strings.HasPrefix(operationName, "Create") || strings.HasPrefix(operationName, "Delete") {
|
||||
options.Fixtures = nil
|
||||
}
|
||||
first := model.newRequestGenerator(operationName, 0, options)
|
||||
full, ok := first.structure(operation.Input.Shape, nil, 0)
|
||||
plan := RequestPlan{Skipped: first.skipped, Unbroken: first.unbroken, DeclaredErrors: declared}
|
||||
if !ok {
|
||||
plan.Skipped = append(plan.Skipped, Skip{Path: "$", Reason: "a required member cannot be generated"})
|
||||
return plan, nil
|
||||
}
|
||||
input := model.shapes[operation.Input.Shape]
|
||||
// Conditionally required members join every request but are never omitted.
|
||||
required := append(slices.Clone(model.shapes[operation.Input.Shape].Required), conditionallyRequired[service][operationName]...)
|
||||
optional := make([]string, 0, len(full))
|
||||
for _, member := range slices.Sorted(maps.Keys(full)) {
|
||||
if !slices.Contains(input.Required, member) {
|
||||
if !slices.Contains(required, member) {
|
||||
optional = append(optional, member)
|
||||
}
|
||||
}
|
||||
@@ -90,7 +101,7 @@ func GenerateRequests(service Service, operationName string, options RequestOpti
|
||||
// what an earlier case created.
|
||||
salt := 0
|
||||
generate := func() (map[string]any, []constraintSite, error) {
|
||||
generator := model.newRequestGenerator(salt, options)
|
||||
generator := model.newRequestGenerator(operationName, salt, options)
|
||||
salt++
|
||||
value, _ := generator.structure(operation.Input.Shape, nil, 0)
|
||||
if len(generator.sites) != len(first.sites) {
|
||||
@@ -111,7 +122,7 @@ func GenerateRequests(service Service, operationName string, options RequestOpti
|
||||
Operation: operationName,
|
||||
Member: member,
|
||||
Path: memberPathOf(member),
|
||||
Input: keepMembers(value, input.Required, member),
|
||||
Input: keepMembers(value, required, member),
|
||||
})
|
||||
}
|
||||
for index := range first.sites {
|
||||
@@ -121,10 +132,10 @@ func GenerateRequests(service Service, operationName string, options RequestOpti
|
||||
}
|
||||
site := sites[index]
|
||||
member, _ := site.path[0].(string)
|
||||
if slices.Contains(input.Required, member) {
|
||||
if slices.Contains(required, member) {
|
||||
member = ""
|
||||
}
|
||||
broken, err := site.apply(keepMembers(value, input.Required, member))
|
||||
broken, err := site.apply(keepMembers(value, required, member))
|
||||
if err != nil {
|
||||
return RequestPlan{}, err
|
||||
}
|
||||
@@ -269,6 +280,7 @@ var generatedTimestamp = time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC)
|
||||
|
||||
type requestGenerator struct {
|
||||
model *Model
|
||||
operation string
|
||||
options RequestOptions
|
||||
salt string
|
||||
saltNumber int
|
||||
@@ -278,8 +290,8 @@ type requestGenerator struct {
|
||||
visiting map[string]bool
|
||||
}
|
||||
|
||||
func (m *Model) newRequestGenerator(salt int, options RequestOptions) *requestGenerator {
|
||||
return &requestGenerator{model: m, options: options, salt: saltLetters(salt), saltNumber: salt, visiting: map[string]bool{}}
|
||||
func (m *Model) newRequestGenerator(operation string, salt int, options RequestOptions) *requestGenerator {
|
||||
return &requestGenerator{model: m, operation: operation, options: options, salt: saltLetters(salt), saltNumber: salt, visiting: map[string]bool{}}
|
||||
}
|
||||
|
||||
// saltLetters spells n in base 26 with lower-case letters, which fit more
|
||||
@@ -462,6 +474,9 @@ func (g *requestGenerator) mapValue(shape *Shape, path []pathStep, depth int) (a
|
||||
func (g *requestGenerator) stringValue(shape *Shape, ref ShapeRef, path []pathStep) (any, bool) {
|
||||
if len(shape.Enum) > 0 {
|
||||
g.site(path, constraintSite{constraint: ConstraintEnum, detail: "value not in enum", replacement: invalidEnumValue(shape.Enum)})
|
||||
if value, ok := g.enumHint(path); ok && slices.Contains(shape.Enum, value) {
|
||||
return value, true
|
||||
}
|
||||
return shape.Enum[0], true
|
||||
}
|
||||
|
||||
@@ -583,7 +598,10 @@ func invalidEnumValue(values []string) string {
|
||||
}
|
||||
|
||||
func (g *requestGenerator) integer(shape *Shape, path []pathStep) int64 {
|
||||
value := int64(1)
|
||||
value, ok := g.integerHint(path)
|
||||
if !ok {
|
||||
value = 1
|
||||
}
|
||||
if shape.Min != nil {
|
||||
value = max(value, int64(math.Ceil(*shape.Min)))
|
||||
}
|
||||
@@ -630,5 +648,8 @@ func (g *requestGenerator) blob(shape *Shape, path []pathStep) []byte {
|
||||
g.leaveUnbroken(path, fmt.Sprintf("max length %g bytes is above the generated limit", *shape.Max))
|
||||
}
|
||||
}
|
||||
if value, ok := g.blobHint(path); ok && len(value) >= minLength && len(value) <= maxLength {
|
||||
return value
|
||||
}
|
||||
return []byte(fitLength("spx"+g.salt, minLength, maxLength))
|
||||
}
|
||||
|
||||
@@ -1,17 +1,29 @@
|
||||
package awsmodel
|
||||
|
||||
import (
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// RequestOptions places generated ARNs in the account and region the
|
||||
// requests are sent as.
|
||||
// requests are sent as. AccessKeyID is the key the requests are signed with.
|
||||
// Fixtures maps a member to an existing resource it can name.
|
||||
type RequestOptions struct {
|
||||
AccountID string
|
||||
Region string
|
||||
AccountID string
|
||||
Region string
|
||||
AccessKeyID string
|
||||
Fixtures map[string]string
|
||||
}
|
||||
|
||||
// Models constrain most identifiers only by length, so "spxa" is a model-valid
|
||||
@@ -56,11 +68,71 @@ var ec2IDPrefixes = map[string]string{
|
||||
|
||||
var ec2IDMember = regexp.MustCompile(`^(.*?)Ids?$`)
|
||||
|
||||
// conditionallyRequired names members the model marks optional that the
|
||||
// operation still needs in its base request. The services refuse their absence
|
||||
// with a code the judge cannot tell from a real finding.
|
||||
var conditionallyRequired = map[Service]map[string][]string{
|
||||
EC2: {"CreateCapacityReservation": {"AvailabilityZone"}},
|
||||
ECS: {
|
||||
"CreateCapacityProvider": {"autoScalingGroupProvider"},
|
||||
"CreateService": {"taskDefinition"},
|
||||
},
|
||||
RDS: {
|
||||
"CreateDBInstance": {"AllocatedStorage", "MasterUsername", "MasterUserPassword"},
|
||||
"RestoreDBInstanceFromDBSnapshot": {"DBSnapshotIdentifier"},
|
||||
},
|
||||
}
|
||||
|
||||
const (
|
||||
// imageManifest is well formed but names blobs no repository holds.
|
||||
imageManifest = `{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json","config":{"mediaType":"application/vnd.docker.container.image.v1+json","size":2,"digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000"},"layers":[]}`
|
||||
lifecyclePolicy = `{"rules":[{"rulePriority":1,"description":"expire untagged","selection":{"tagStatus":"untagged","countType":"sinceImagePushed","countUnit":"days","countNumber":14},"action":{"type":"expire"}}]}`
|
||||
repositoryPolicy = `{"Version":"2012-10-17","Statement":[{"Sid":"pull","Effect":"Allow","Principal":{"AWS":"arn:aws:iam::%s:root"},"Action":"ecr:BatchGetImage"}]}`
|
||||
|
||||
policyDocument = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"*"}]}`
|
||||
trustPolicyDocument = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"ec2.amazonaws.com"},"Action":"sts:AssumeRole"}]}`
|
||||
)
|
||||
|
||||
// ec2TaggedTypes is the resource type each EC2 create tags through its
|
||||
// TagSpecifications. AWS refuses any other type, so the enum's first value
|
||||
// would make an otherwise valid request invalid.
|
||||
var ec2TaggedTypes = map[string]string{
|
||||
"AllocateAddress": "elastic-ip",
|
||||
"AuthorizeSecurityGroupEgress": "security-group-rule",
|
||||
"AuthorizeSecurityGroupIngress": "security-group-rule",
|
||||
"CopyImage": "image",
|
||||
"CopySnapshot": "snapshot",
|
||||
"CreateCapacityReservation": "capacity-reservation",
|
||||
"CreateEgressOnlyInternetGateway": "egress-only-internet-gateway",
|
||||
"CreateImage": "image",
|
||||
"CreateInternetGateway": "internet-gateway",
|
||||
"CreateKeyPair": "key-pair",
|
||||
"CreateLaunchTemplate": "launch-template",
|
||||
"CreateNatGateway": "natgateway",
|
||||
"CreateNetworkInterface": "network-interface",
|
||||
"CreatePlacementGroup": "placement-group",
|
||||
"CreateRouteTable": "route-table",
|
||||
"CreateSecurityGroup": "security-group",
|
||||
"CreateSnapshot": "snapshot",
|
||||
"CreateSubnet": "subnet",
|
||||
"CreateVolume": "volume",
|
||||
"CreateVpc": "vpc",
|
||||
"ImportKeyPair": "key-pair",
|
||||
"RegisterImage": "image",
|
||||
"RequestSpotInstances": "spot-instances-request",
|
||||
"RunInstances": "instance",
|
||||
}
|
||||
|
||||
// enumHint returns the enum value a valid request needs at path, where the
|
||||
// first value would not do.
|
||||
func (g *requestGenerator) enumHint(path []pathStep) (string, bool) {
|
||||
if g.model.service != EC2 || len(path) == 0 || path[0] != "TagSpecifications" || memberName(path) != "ResourceType" {
|
||||
return "", false
|
||||
}
|
||||
value, ok := ec2TaggedTypes[g.operation]
|
||||
return value, ok
|
||||
}
|
||||
|
||||
// hint returns a value of the right form for the member at path, if one is
|
||||
// known. salt keeps names distinct between request cases.
|
||||
func (g *requestGenerator) hint(path []pathStep) (string, bool) {
|
||||
@@ -69,11 +141,20 @@ func (g *requestGenerator) hint(path []pathStep) (string, bool) {
|
||||
return "", false
|
||||
}
|
||||
options := g.options
|
||||
if fixture, ok := options.Fixtures[member]; ok {
|
||||
return fixture, true
|
||||
}
|
||||
suffix := fmt.Sprintf("%017x", g.saltNumber)
|
||||
name := "spx" + g.salt
|
||||
|
||||
switch g.model.service {
|
||||
case EC2:
|
||||
switch member {
|
||||
case "AvailabilityZone":
|
||||
return options.Region + "a", true
|
||||
case "SourceRegion":
|
||||
return options.Region, true
|
||||
}
|
||||
if match := ec2IDMember.FindStringSubmatch(capitalize(member)); match != nil {
|
||||
if prefix, ok := ec2IDPrefixes[match[1]]; ok {
|
||||
return prefix + "-" + suffix, true
|
||||
@@ -90,6 +171,10 @@ func (g *requestGenerator) hint(path []pathStep) (string, bool) {
|
||||
return base + "listener/app/" + name + "/" + suffix[1:] + "/" + suffix[1:], true
|
||||
case "RuleArn", "RuleArns":
|
||||
return base + "listener-rule/app/" + name + "/" + suffix[1:] + "/" + suffix[1:] + "/" + suffix[1:], true
|
||||
case "ProtocolVersion":
|
||||
return "HTTP1", true
|
||||
case "HttpCode":
|
||||
return "200", true
|
||||
}
|
||||
case RDS:
|
||||
switch member {
|
||||
@@ -101,6 +186,8 @@ func (g *requestGenerator) hint(path []pathStep) (string, bool) {
|
||||
return "db.t3.medium", true
|
||||
case "DBParameterGroupFamily":
|
||||
return "postgres18", true
|
||||
case "MasterUserPassword":
|
||||
return "spx-conformance-" + g.salt, true
|
||||
case "SnapshotType":
|
||||
return "manual", true
|
||||
case "Source":
|
||||
@@ -113,13 +200,40 @@ func (g *requestGenerator) hint(path []pathStep) (string, bool) {
|
||||
case "taskDefinition":
|
||||
return name + ":1", true
|
||||
}
|
||||
case ACM:
|
||||
// The integration tenant CA permits example.com.
|
||||
switch member {
|
||||
case "DomainName", "SubjectAlternativeNames", "ValidationDomain":
|
||||
return name + ".example.com", true
|
||||
}
|
||||
case ECR:
|
||||
if member == "resourceArn" {
|
||||
switch member {
|
||||
case "resourceArn":
|
||||
return fmt.Sprintf("arn:aws:ecr:%s:%s:repository/%s", options.Region, options.AccountID, name), true
|
||||
case "imageManifest":
|
||||
return imageManifest, true
|
||||
case "lifecyclePolicyText":
|
||||
return lifecyclePolicy, true
|
||||
case "policyText":
|
||||
return fmt.Sprintf(repositoryPolicy, options.AccountID), true
|
||||
}
|
||||
case EKS:
|
||||
if member == "resourceArn" {
|
||||
switch member {
|
||||
case "resourceArn":
|
||||
return fmt.Sprintf("arn:aws:eks:%s:%s:cluster/%s", options.Region, options.AccountID, name), true
|
||||
case "policyArn":
|
||||
return "arn:aws:eks::aws:cluster-access-policy/AmazonEKSViewPolicy", true
|
||||
case "addonName":
|
||||
return "aws-ebs-csi-driver", true
|
||||
}
|
||||
// An access entry's type; a nested type names an identity provider's.
|
||||
if member == "type" && len(path) == 1 {
|
||||
return "STANDARD", true
|
||||
}
|
||||
case STS:
|
||||
// GetAccessKeyInfo answers only for a key that exists.
|
||||
if member == "AccessKeyId" && options.AccessKeyID != "" {
|
||||
return options.AccessKeyID, true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -142,6 +256,103 @@ func (g *requestGenerator) hint(path []pathStep) (string, bool) {
|
||||
return "", false
|
||||
}
|
||||
|
||||
// integerHint returns a value AWS accepts for an integer member whose range
|
||||
// the model leaves out, if one is known.
|
||||
func (g *requestGenerator) integerHint(path []pathStep) (int64, bool) {
|
||||
if g.model.service == RDS && memberName(path) == "AllocatedStorage" {
|
||||
// The smallest gp2/gp3 allocation PostgreSQL, the Engine hint, accepts.
|
||||
return 20, true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// blobHint returns a value AWS accepts for a blob member, if one is known.
|
||||
func (g *requestGenerator) blobHint(path []pathStep) ([]byte, bool) {
|
||||
if g.model.service != ACM {
|
||||
return nil, false
|
||||
}
|
||||
// GenerateRequests has already surfaced a generation error.
|
||||
material, err := importMaterial()
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
switch memberName(path) {
|
||||
case "Certificate":
|
||||
return material.certificate, true
|
||||
case "PrivateKey":
|
||||
return material.privateKey, true
|
||||
case "CertificateChain":
|
||||
return material.chain, true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// certificateMaterial is a PEM leaf certificate, its key and the CA that
|
||||
// signed it, as ImportCertificate takes them.
|
||||
type certificateMaterial struct {
|
||||
certificate, privateKey, chain []byte
|
||||
}
|
||||
|
||||
// importMaterial is generated once per run; minting a key per request would
|
||||
// dominate the ACM cases.
|
||||
var importMaterial = sync.OnceValues(func() (certificateMaterial, error) {
|
||||
now := time.Now().UTC()
|
||||
caKey, caDER, err := newCertificate(&x509.Certificate{
|
||||
SerialNumber: big.NewInt(1),
|
||||
Subject: pkix.Name{CommonName: "Spinifex Conformance CA"},
|
||||
NotBefore: now.Add(-time.Hour),
|
||||
NotAfter: now.AddDate(1, 0, 0),
|
||||
KeyUsage: x509.KeyUsageCertSign,
|
||||
BasicConstraintsValid: true,
|
||||
IsCA: true,
|
||||
}, nil, nil)
|
||||
if err != nil {
|
||||
return certificateMaterial{}, err
|
||||
}
|
||||
ca, err := x509.ParseCertificate(caDER)
|
||||
if err != nil {
|
||||
return certificateMaterial{}, fmt.Errorf("awsmodel: parse conformance CA: %w", err)
|
||||
}
|
||||
leafKey, leafDER, err := newCertificate(&x509.Certificate{
|
||||
SerialNumber: big.NewInt(2),
|
||||
Subject: pkix.Name{CommonName: "spx.example.com"},
|
||||
DNSNames: []string{"spx.example.com"},
|
||||
NotBefore: now.Add(-time.Hour),
|
||||
NotAfter: now.AddDate(1, 0, 0),
|
||||
KeyUsage: x509.KeyUsageDigitalSignature,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
}, ca, caKey)
|
||||
if err != nil {
|
||||
return certificateMaterial{}, err
|
||||
}
|
||||
keyDER, err := x509.MarshalPKCS8PrivateKey(leafKey)
|
||||
if err != nil {
|
||||
return certificateMaterial{}, fmt.Errorf("awsmodel: marshal conformance key: %w", err)
|
||||
}
|
||||
return certificateMaterial{
|
||||
certificate: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: leafDER}),
|
||||
privateKey: pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}),
|
||||
chain: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER}),
|
||||
}, nil
|
||||
})
|
||||
|
||||
// newCertificate creates a P-256 key and a certificate for it from template,
|
||||
// signed by parent's key, or self-signed when parent is nil.
|
||||
func newCertificate(template, parent *x509.Certificate, parentKey *ecdsa.PrivateKey) (*ecdsa.PrivateKey, []byte, error) {
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("awsmodel: generate conformance key: %w", err)
|
||||
}
|
||||
if parent == nil {
|
||||
parent, parentKey = template, key
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, template, parent, &key.PublicKey, parentKey)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("awsmodel: create conformance certificate: %w", err)
|
||||
}
|
||||
return key, der, nil
|
||||
}
|
||||
|
||||
// memberName is the structure member a value at path belongs to, looking
|
||||
// through list indices.
|
||||
func memberName(path []pathStep) string {
|
||||
|
||||
@@ -4,6 +4,8 @@ package awsmodel
|
||||
//shapes, and the unexported pattern sampler and salt helpers.
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"fmt"
|
||||
"maps"
|
||||
"regexp"
|
||||
@@ -17,7 +19,7 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var testRequestOptions = RequestOptions{AccountID: "123456789012", Region: "ap-southeast-2"}
|
||||
var testRequestOptions = RequestOptions{AccountID: "123456789012", Region: "ap-southeast-2", AccessKeyID: "AKIAIOSFODNN7EXAMPLE"}
|
||||
|
||||
// constraintViolation is one model constraint an input breaks, found by
|
||||
// walking the input against the model independently of the generator.
|
||||
@@ -99,7 +101,7 @@ func testPattern(pattern string) *regexp.Regexp {
|
||||
if re, ok := testPatterns.Load(pattern); ok {
|
||||
return re.(*regexp.Regexp)
|
||||
}
|
||||
re := regexp.MustCompile(javaEscape.ReplaceAllString(pattern, `\x{$1}`))
|
||||
re := regexp.MustCompile(re2Pattern(pattern))
|
||||
testPatterns.Store(pattern, re)
|
||||
return re
|
||||
}
|
||||
@@ -136,7 +138,7 @@ func TestGeneratedRequestsBreakOnlyTheirConstraint(t *testing.T) {
|
||||
var violations []constraintViolation
|
||||
model.inputViolations(operation.Input.Shape, "$", request.Input, &violations)
|
||||
for member := range request.Input {
|
||||
if !slices.Contains(input.Required, member) {
|
||||
if !slices.Contains(input.Required, member) && !slices.Contains(conditionallyRequired[service][operationName], member) {
|
||||
require.Equal(t, request.Member, member, "%s %s %s carries an unrelated optional member", operationName, request.Constraint, request.Path)
|
||||
}
|
||||
}
|
||||
@@ -180,6 +182,65 @@ func TestGenerateRequestsIsolatesEachOptionalMember(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateRequestsSeedsConditionallyRequiredMembers(t *testing.T) {
|
||||
for service, operations := range conditionallyRequired {
|
||||
model, err := Load(service)
|
||||
require.NoError(t, err)
|
||||
for operationName, members := range operations {
|
||||
operation, ok := model.Operation(operationName)
|
||||
require.True(t, ok, "%s %s is not modelled", service, operationName)
|
||||
input := model.shapes[operation.Input.Shape]
|
||||
for _, member := range members {
|
||||
require.Contains(t, input.Members, member, "%s %s", service, operationName)
|
||||
require.NotContains(t, input.Required, member, "%s %s now models %s as required", service, operationName, member)
|
||||
}
|
||||
|
||||
plan, err := GenerateRequests(service, operationName, testRequestOptions)
|
||||
require.NoError(t, err)
|
||||
for _, request := range plan.Cases {
|
||||
for _, member := range members {
|
||||
require.NotEqual(t, member, request.Member, "%s %s", service, operationName)
|
||||
require.Contains(t, request.Input, member, "%s %s %s %s", service, operationName, request.Constraint, request.Path)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
plan, err := GenerateRequests(RDS, "CreateDBInstance", testRequestOptions)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int64(20), plan.Cases[0].Input["AllocatedStorage"])
|
||||
plan, err = GenerateRequests(EC2, "CreateCapacityReservation", testRequestOptions)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "ap-southeast-2a", plan.Cases[0].Input["AvailabilityZone"])
|
||||
|
||||
// A constraint inside a seeded member is judged against the base request.
|
||||
plan, err = GenerateRequests(ECS, "CreateCapacityProvider", testRequestOptions)
|
||||
require.NoError(t, err)
|
||||
nested := 0
|
||||
for _, request := range plan.Cases {
|
||||
if strings.HasPrefix(request.Path, "$.autoScalingGroupProvider.") {
|
||||
nested++
|
||||
require.Empty(t, request.Member, request.Path)
|
||||
}
|
||||
}
|
||||
require.NotZero(t, nested)
|
||||
}
|
||||
|
||||
func TestGenerateRequestsTargetsFixturesOutsideCreateAndDelete(t *testing.T) {
|
||||
options := testRequestOptions
|
||||
options.Fixtures = map[string]string{"RoleName": "fixture"}
|
||||
for operation, wantFixture := range map[string]bool{"TagRole": true, "CreateRole": false, "DeleteRole": false} {
|
||||
plan, err := GenerateRequests(IAM, operation, options)
|
||||
require.NoError(t, err)
|
||||
for _, request := range plan.Cases {
|
||||
if request.Path == "$.RoleName" {
|
||||
continue
|
||||
}
|
||||
require.Equal(t, wantFixture, request.Input["RoleName"] == "fixture", "%s %s %s", operation, request.Constraint, request.Path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateRequestsAttributesRejections(t *testing.T) {
|
||||
plan, err := GenerateRequests(IAM, "CreateRole", testRequestOptions)
|
||||
require.NoError(t, err)
|
||||
@@ -278,6 +339,13 @@ func TestGenerateRequestsUsesResourceHints(t *testing.T) {
|
||||
{RDS, "CreateDBInstance", "Engine", regexp.MustCompile(`^postgres$`)},
|
||||
{ECR, "TagResource", "resourceArn", regexp.MustCompile(`^arn:aws:ecr:ap-southeast-2:123456789012:repository/`)},
|
||||
{EKS, "TagResource", "resourceArn", regexp.MustCompile(`^arn:aws:eks:ap-southeast-2:123456789012:cluster/`)},
|
||||
{EKS, "AssociateAccessPolicy", "policyArn", regexp.MustCompile(`^arn:aws:eks::aws:cluster-access-policy/AmazonEKSViewPolicy$`)},
|
||||
{EKS, "CreateAddon", "addonName", regexp.MustCompile(`^aws-ebs-csi-driver$`)},
|
||||
{EKS, "CreateAccessEntry", "type", regexp.MustCompile(`^STANDARD$`)},
|
||||
{ElasticLoadBalancingV2, "CreateTargetGroup", "ProtocolVersion", regexp.MustCompile(`^HTTP1$`)},
|
||||
{EC2, "CopyImage", "SourceRegion", regexp.MustCompile(`^ap-southeast-2$`)},
|
||||
{STS, "GetAccessKeyInfo", "AccessKeyId", regexp.MustCompile(`^AKIAIOSFODNN7EXAMPLE$`)},
|
||||
{IAM, "DeleteAccessKey", "AccessKeyId", regexp.MustCompile(`^AKIA[0-9A-F]{16}$`)},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.operation+"."+test.member, func(t *testing.T) {
|
||||
@@ -302,6 +370,80 @@ func TestGenerateRequestsUsesResourceHints(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateRequestsHintsNestedMembers(t *testing.T) {
|
||||
plan, err := GenerateRequests(ElasticLoadBalancingV2, "CreateTargetGroup", testRequestOptions)
|
||||
require.NoError(t, err)
|
||||
for _, request := range plan.Cases {
|
||||
if request.Acceptance() && request.Member == "Matcher" {
|
||||
require.Equal(t, "200", request.Input["Matcher"].(map[string]any)["HttpCode"])
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("no acceptance request sets Matcher")
|
||||
}
|
||||
|
||||
// AWS refuses a tag resource type the operation does not create, so a valid
|
||||
// request tags the operation's own type rather than the enum's first value.
|
||||
func TestGenerateRequestsTagsTheOperationsOwnResourceType(t *testing.T) {
|
||||
for operation, want := range map[string]string{"CreateVpc": "vpc", "CreateLaunchTemplate": "launch-template"} {
|
||||
t.Run(operation, func(t *testing.T) {
|
||||
plan, err := GenerateRequests(EC2, operation, testRequestOptions)
|
||||
require.NoError(t, err)
|
||||
for _, request := range plan.Cases {
|
||||
if request.Acceptance() && request.Member == "TagSpecifications" {
|
||||
spec := request.Input["TagSpecifications"].([]any)[0].(map[string]any)
|
||||
require.Equal(t, want, spec["ResourceType"])
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("no acceptance request sets TagSpecifications")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateRequestsImportsAVerifiableCertificate(t *testing.T) {
|
||||
plan, err := GenerateRequests(ACM, "ImportCertificate", testRequestOptions)
|
||||
require.NoError(t, err)
|
||||
var input map[string]any
|
||||
for _, request := range plan.Cases {
|
||||
if request.Acceptance() && request.Member == "CertificateChain" {
|
||||
input = request.Input
|
||||
}
|
||||
}
|
||||
require.NotNil(t, input, "no acceptance request sets CertificateChain")
|
||||
|
||||
pair, err := tls.X509KeyPair(input["Certificate"].([]byte), input["PrivateKey"].([]byte))
|
||||
require.NoError(t, err)
|
||||
leaf, err := x509.ParseCertificate(pair.Certificate[0])
|
||||
require.NoError(t, err)
|
||||
roots := x509.NewCertPool()
|
||||
require.True(t, roots.AppendCertsFromPEM(input["CertificateChain"].([]byte)))
|
||||
_, err = leaf.Verify(x509.VerifyOptions{Roots: roots, DNSName: "spx.example.com"})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestRE2EquivalentsMatchTheModelledSemantics(t *testing.T) {
|
||||
modelled := map[string]bool{}
|
||||
for _, service := range Services() {
|
||||
model, err := Load(service)
|
||||
require.NoError(t, err)
|
||||
for _, shape := range model.shapes {
|
||||
modelled[shape.Pattern] = true
|
||||
}
|
||||
}
|
||||
for pattern := range re2Equivalents {
|
||||
require.True(t, modelled[pattern], "no model uses %s", pattern)
|
||||
}
|
||||
|
||||
domain := testPattern(`^(\*\.)?(((?!-)[A-Za-z0-9-]{0,62}[A-Za-z0-9])\.)+((?!-)[A-Za-z0-9-]{1,62}[A-Za-z0-9])$`)
|
||||
for _, name := range []string{"example.com", "*.example.com", "a.b.co", "x-y.example.com", "a." + strings.Repeat("b", 63)} {
|
||||
require.True(t, domain.MatchString(name), name)
|
||||
}
|
||||
for _, name := range []string{"example", "-a.com", "a-.com", "a.c", "a.-com", "a.com-", "a..com", "*.com.*", strings.Repeat("a", 64) + ".com"} {
|
||||
require.False(t, domain.MatchString(name), name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPatternSamplerMatchesWholeValue(t *testing.T) {
|
||||
tests := []struct {
|
||||
pattern string
|
||||
|
||||
@@ -135,9 +135,7 @@ func DecodeRequestResult(service Service, status int, header http.Header, body [
|
||||
}
|
||||
|
||||
func trimErrorCode(code string) string {
|
||||
if separator := strings.Index(code, ":"); separator >= 0 {
|
||||
code = code[:separator]
|
||||
}
|
||||
code, _, _ = strings.Cut(code, ":")
|
||||
if separator := strings.LastIndex(code, "#"); separator >= 0 {
|
||||
code = code[separator+1:]
|
||||
}
|
||||
|
||||
@@ -109,11 +109,9 @@ func GatewayHTTPClient(caPath string) (*http.Client, error) {
|
||||
// HostFromImage returns the host[:port] portion of an image ref or URL. It strips
|
||||
// any scheme and trims at the first path separator.
|
||||
func HostFromImage(image string) string {
|
||||
if i := strings.Index(image, "://"); i >= 0 {
|
||||
image = image[i+3:]
|
||||
if _, rest, ok := strings.Cut(image, "://"); ok {
|
||||
image = rest
|
||||
}
|
||||
if i := strings.IndexByte(image, '/'); i >= 0 {
|
||||
image = image[:i]
|
||||
}
|
||||
return image
|
||||
host, _, _ := strings.Cut(image, "/")
|
||||
return host
|
||||
}
|
||||
|
||||
@@ -12,7 +12,9 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"math"
|
||||
"slices"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -503,10 +505,5 @@ func Summary(report *Report) string {
|
||||
}
|
||||
|
||||
func sortedOps(ops map[string]OpStats) []string {
|
||||
names := make([]string, 0, len(ops))
|
||||
for name := range ops {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names
|
||||
return slices.Sorted(maps.Keys(ops))
|
||||
}
|
||||
|
||||
@@ -3,7 +3,8 @@ package loadgen
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
@@ -102,12 +103,7 @@ var DefaultOps = []string{"DescribeInstances", "DescribeVpcs", "DescribeVolumes"
|
||||
|
||||
// OpNames lists every operation the registry knows, for the usage text.
|
||||
func OpNames() []string {
|
||||
names := make([]string, 0, len(registry))
|
||||
for name := range registry {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names
|
||||
return slices.Sorted(maps.Keys(registry))
|
||||
}
|
||||
|
||||
// ResolveOps turns names into operations, rejecting an unknown one rather than
|
||||
@@ -135,22 +131,16 @@ func ResolveOps(names []string) ([]Op, error) {
|
||||
// NeedsVPC reports whether any selected operation acts on a tenant resource,
|
||||
// so the caller knows whether to pay for discovery before the run starts.
|
||||
func NeedsVPC(ops []Op) bool {
|
||||
for _, op := range ops {
|
||||
if op.Name == "CreateTags" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
return slices.ContainsFunc(ops, func(op Op) bool {
|
||||
return op.Name == "CreateTags"
|
||||
})
|
||||
}
|
||||
|
||||
// NeedsVolume reports whether any selected operation needs a volume id.
|
||||
func NeedsVolume(ops []Op) bool {
|
||||
for _, op := range ops {
|
||||
if op.Name == "DescribeVolumesByID" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
return slices.ContainsFunc(ops, func(op Op) bool {
|
||||
return op.Name == "DescribeVolumesByID"
|
||||
})
|
||||
}
|
||||
|
||||
// ResolveVolume finds one volume for the target to ask about by id. A tenant
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/clustersize"
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
@@ -73,6 +74,11 @@ func StartTestJetStream(t *testing.T) (*server.Server, *nats.Conn, jetstream.Jet
|
||||
|
||||
js, err := jetstream.New(nc)
|
||||
require.NoError(t, err)
|
||||
|
||||
// One embedded server is a one-node cluster, and bucket creation refuses to
|
||||
// guess. A test that wants a different count calls RedeclareForTest, because
|
||||
// declaring is write-once and this has already done it.
|
||||
clustersize.DeclareForTest(t, 1)
|
||||
return ns, nc, js
|
||||
}
|
||||
|
||||
@@ -95,6 +101,7 @@ var vpcdStubTopics = []string{
|
||||
"vpc.update-port-sgs",
|
||||
"vpc.create-port",
|
||||
"vpc.delete-port",
|
||||
"vpc.delete-nat",
|
||||
}
|
||||
|
||||
// vpcdStubRegistry holds the per-conn response map. Per-conn so concurrent tests with separate conns don't interfere.
|
||||
|
||||
@@ -7,6 +7,7 @@ package recordingstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
@@ -77,10 +78,7 @@ func (s *Store) TouchedPrefix(prefix string) bool {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, listed := range s.ListPrefixes() {
|
||||
if strings.HasPrefix(listed, prefix) || strings.HasPrefix(prefix, listed) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
return slices.ContainsFunc(s.ListPrefixes(), func(listed string) bool {
|
||||
return strings.HasPrefix(listed, prefix) || strings.HasPrefix(prefix, listed)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@ func TestNewClusterEngineCoversEveryStage(t *testing.T) {
|
||||
|
||||
masterKey, err := handlers_iam.GenerateMasterKey()
|
||||
require.NoError(t, err)
|
||||
svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey, 1)
|
||||
svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey)
|
||||
require.NoError(t, err)
|
||||
|
||||
engine, err := NewClusterEngine(t.Context(), nc, 1, svc, newFakeBuckets())
|
||||
@@ -48,7 +48,7 @@ func TestClusterEnginePrechecks(t *testing.T) {
|
||||
|
||||
masterKey, err := handlers_iam.GenerateMasterKey()
|
||||
require.NoError(t, err)
|
||||
svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey, 1)
|
||||
svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey)
|
||||
require.NoError(t, err)
|
||||
|
||||
engine, err := NewClusterEngine(t.Context(), nc, 1, svc, newFakeBuckets())
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"slices"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
@@ -270,12 +271,9 @@ func (e *Engine) drainPoll() time.Duration {
|
||||
}
|
||||
|
||||
func containsResource(resources []Resource, want Resource) bool {
|
||||
for _, resource := range resources {
|
||||
if resource.Kind == want.Kind && resource.ID == want.ID {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
return slices.ContainsFunc(resources, func(resource Resource) bool {
|
||||
return resource.Kind == want.Kind && resource.ID == want.ID
|
||||
})
|
||||
}
|
||||
|
||||
// SortReapers orders reapers by stage while preserving the caller's order
|
||||
|
||||
@@ -3,6 +3,7 @@ package accountteardown
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -664,12 +665,9 @@ func isAlreadyGone(err error) bool {
|
||||
return true
|
||||
}
|
||||
message := err.Error()
|
||||
for _, marker := range notFoundMarkers {
|
||||
if strings.Contains(message, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
return slices.ContainsFunc(notFoundMarkers, func(marker string) bool {
|
||||
return strings.Contains(message, marker)
|
||||
})
|
||||
}
|
||||
|
||||
// isNotAssociated reports the benign case of disassociating something that was
|
||||
|
||||
@@ -398,7 +398,7 @@ func TestPolicyReaperDeletesAPolicyWithSeveralVersions(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
masterKey, err := handlers_iam.GenerateMasterKey()
|
||||
require.NoError(t, err)
|
||||
svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey, 1)
|
||||
svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey)
|
||||
require.NoError(t, err)
|
||||
|
||||
const accountID = "000000000042"
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"fmt"
|
||||
"net"
|
||||
"os/exec"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/mulgadc/bluebottle/pkg/safecast"
|
||||
@@ -190,10 +191,7 @@ func isVirtualInterface(name string, defaultRouteDev string) bool {
|
||||
"docker", "br-", "veth", "ovs", "virbr",
|
||||
"tun", "tap", "tailscale", "wg", "lo",
|
||||
}
|
||||
for _, p := range prefixes {
|
||||
if strings.HasPrefix(name, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
return slices.ContainsFunc(prefixes, func(p string) bool {
|
||||
return strings.HasPrefix(name, p)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/utils"
|
||||
@@ -86,12 +87,9 @@ func CheckHostAt(root string) []Result {
|
||||
|
||||
// HasProblem reports whether any result is not OK.
|
||||
func HasProblem(results []Result) bool {
|
||||
for _, r := range results {
|
||||
if r.Status != OK {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
return slices.ContainsFunc(results, func(r Result) bool {
|
||||
return r.Status != OK
|
||||
})
|
||||
}
|
||||
|
||||
func checkHelper(root, path string) Result {
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
// Package exoscale is the narrow slice of the Exoscale API the external
|
||||
// allocator needs, driven through the exo CLI rather than the Go SDK.
|
||||
//
|
||||
// The CLI keeps a heavy dependency out of the build while this is a proof of
|
||||
// concept, and it means every call Spinifex makes is one an operator can repeat
|
||||
// by hand with the same config file. Nothing outside this package knows a
|
||||
// process is being run, so moving to the SDK later changes only this file.
|
||||
package exoscale
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DefaultBinary is where the exo release package installs the CLI.
|
||||
const DefaultBinary = "/usr/bin/exo"
|
||||
|
||||
// DefaultConfigFile holds the node's scoped API key, in the exo CLI's own
|
||||
// format, readable by the daemon's service user only.
|
||||
const DefaultConfigFile = "/etc/spinifex/exoscale/exoscale.toml"
|
||||
|
||||
// DefaultCallTimeout bounds one CLI invocation. Allocation sits inside the AWS
|
||||
// client's 60s read timeout and makes two calls, so each gets well under half.
|
||||
const DefaultCallTimeout = 25 * time.Second
|
||||
|
||||
var (
|
||||
// ErrNotFound means the EIP or instance does not exist in the zone.
|
||||
ErrNotFound = errors.New("exoscale: resource not found")
|
||||
// ErrQuotaExceeded means the organisation's EIP quota is used up.
|
||||
ErrQuotaExceeded = errors.New("exoscale: quota exceeded")
|
||||
)
|
||||
|
||||
// ElasticIP is one Exoscale Elastic IP as the allocator sees it.
|
||||
type ElasticIP struct {
|
||||
ID string
|
||||
Address netip.Addr
|
||||
Description string
|
||||
Zone string
|
||||
}
|
||||
|
||||
// Client is the surface the allocator uses. Expressed in this package's types
|
||||
// so the fake in fake.go is a map, not a mock of a process.
|
||||
type Client interface {
|
||||
// CreateElasticIP creates an unattached manual IPv4 EIP.
|
||||
CreateElasticIP(ctx context.Context, description string) (ElasticIP, error)
|
||||
// AttachElasticIP attaches eipID to instanceID.
|
||||
AttachElasticIP(ctx context.Context, instanceID, eipID string) error
|
||||
// DetachElasticIP detaches eipID from instanceID, keeping the address.
|
||||
DetachElasticIP(ctx context.Context, instanceID, eipID string) error
|
||||
// DeleteElasticIP releases the address back to Exoscale.
|
||||
DeleteElasticIP(ctx context.Context, eipID string) error
|
||||
// ListElasticIPs returns every EIP in the zone, ours or not.
|
||||
ListElasticIPs(ctx context.Context) ([]ElasticIP, error)
|
||||
// Version reports the CLI version, proving the binary runs at all.
|
||||
Version(ctx context.Context) (string, error)
|
||||
}
|
||||
|
||||
// runFunc executes the CLI. Swapped in tests so no process is started.
|
||||
type runFunc func(ctx context.Context, binary string, args []string) (stdout, stderr []byte, err error)
|
||||
|
||||
// CLIConfig selects the binary, credentials and zone.
|
||||
type CLIConfig struct {
|
||||
Binary string
|
||||
ConfigFile string
|
||||
Account string
|
||||
Zone string
|
||||
CallTimeout time.Duration
|
||||
}
|
||||
|
||||
// cliClient implements Client by running exo with JSON output.
|
||||
type cliClient struct {
|
||||
cfg CLIConfig
|
||||
run runFunc
|
||||
}
|
||||
|
||||
var _ Client = (*cliClient)(nil)
|
||||
|
||||
// NewCLIClient builds a client over the exo binary. It does not run anything;
|
||||
// call Version to prove the binary and config are usable.
|
||||
func NewCLIClient(cfg CLIConfig) (Client, error) {
|
||||
if cfg.Zone == "" {
|
||||
return nil, errors.New("exoscale: zone is required")
|
||||
}
|
||||
if cfg.Binary == "" {
|
||||
cfg.Binary = DefaultBinary
|
||||
}
|
||||
if cfg.ConfigFile == "" {
|
||||
cfg.ConfigFile = DefaultConfigFile
|
||||
}
|
||||
if cfg.CallTimeout <= 0 {
|
||||
cfg.CallTimeout = DefaultCallTimeout
|
||||
}
|
||||
return &cliClient{cfg: cfg, run: execRun}, nil
|
||||
}
|
||||
|
||||
// eipJSON is the shape of `exo compute elastic-ip create|show|list -O json`.
|
||||
type eipJSON struct {
|
||||
ID string `json:"id"`
|
||||
IPAddress string `json:"ip_address"`
|
||||
Description string `json:"description"`
|
||||
Zone string `json:"zone"`
|
||||
}
|
||||
|
||||
func (e eipJSON) toElasticIP() (ElasticIP, error) {
|
||||
if e.ID == "" {
|
||||
return ElasticIP{}, errors.New("exoscale: elastic IP with no id in exo output")
|
||||
}
|
||||
addr, err := netip.ParseAddr(e.IPAddress)
|
||||
if err != nil {
|
||||
return ElasticIP{}, fmt.Errorf("exoscale: elastic IP %s has unparseable address %q: %w", e.ID, e.IPAddress, err)
|
||||
}
|
||||
return ElasticIP{ID: e.ID, Address: addr, Description: e.Description, Zone: e.Zone}, nil
|
||||
}
|
||||
|
||||
func (c *cliClient) CreateElasticIP(ctx context.Context, description string) (ElasticIP, error) {
|
||||
out, err := c.exo(ctx, "compute", "elastic-ip", "create", "--description", description, "-z", c.cfg.Zone)
|
||||
if err != nil {
|
||||
return ElasticIP{}, err
|
||||
}
|
||||
var e eipJSON
|
||||
if err := json.Unmarshal(out, &e); err != nil {
|
||||
return ElasticIP{}, fmt.Errorf("exoscale: decode created elastic IP: %w", err)
|
||||
}
|
||||
return e.toElasticIP()
|
||||
}
|
||||
|
||||
func (c *cliClient) AttachElasticIP(ctx context.Context, instanceID, eipID string) error {
|
||||
_, err := c.exo(ctx, "compute", "instance", "elastic-ip", "attach", instanceID, eipID, "-z", c.cfg.Zone)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *cliClient) DetachElasticIP(ctx context.Context, instanceID, eipID string) error {
|
||||
_, err := c.exo(ctx, "compute", "instance", "elastic-ip", "detach", instanceID, eipID, "-z", c.cfg.Zone)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *cliClient) DeleteElasticIP(ctx context.Context, eipID string) error {
|
||||
_, err := c.exo(ctx, "compute", "elastic-ip", "delete", "--force", eipID, "-z", c.cfg.Zone)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *cliClient) ListElasticIPs(ctx context.Context) ([]ElasticIP, error) {
|
||||
out, err := c.exo(ctx, "compute", "elastic-ip", "list", "-z", c.cfg.Zone)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var raw []eipJSON
|
||||
if err := json.Unmarshal(out, &raw); err != nil {
|
||||
return nil, fmt.Errorf("exoscale: decode elastic IP list: %w", err)
|
||||
}
|
||||
eips := make([]ElasticIP, 0, len(raw))
|
||||
for _, e := range raw {
|
||||
// The list spans zones on some CLI versions; only ours is ours to judge.
|
||||
if e.Zone != "" && e.Zone != c.cfg.Zone {
|
||||
continue
|
||||
}
|
||||
eip, err := e.toElasticIP()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
eips = append(eips, eip)
|
||||
}
|
||||
return eips, nil
|
||||
}
|
||||
|
||||
func (c *cliClient) Version(ctx context.Context) (string, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, c.cfg.CallTimeout)
|
||||
defer cancel()
|
||||
stdout, stderr, err := c.run(ctx, c.cfg.Binary, []string{"version"})
|
||||
if err != nil {
|
||||
return "", classify(err, stderr)
|
||||
}
|
||||
return strings.TrimSpace(string(stdout)), nil
|
||||
}
|
||||
|
||||
// exo runs one command with JSON output against the configured account.
|
||||
func (c *cliClient) exo(ctx context.Context, args ...string) ([]byte, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, c.cfg.CallTimeout)
|
||||
defer cancel()
|
||||
full := []string{"--config", c.cfg.ConfigFile, "--output-format", "json", "--quiet"}
|
||||
if c.cfg.Account != "" {
|
||||
full = append(full, "--use-account", c.cfg.Account)
|
||||
}
|
||||
full = append(full, args...)
|
||||
stdout, stderr, err := c.run(ctx, c.cfg.Binary, full)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("exo %s: %w", strings.Join(args, " "), classify(err, stderr))
|
||||
}
|
||||
return stdout, nil
|
||||
}
|
||||
|
||||
// classify maps the CLI's stderr onto the sentinels the allocator acts on. The
|
||||
// texts are the ones exo 1.101 prints; anything else keeps its own message.
|
||||
func classify(runErr error, stderr []byte) error {
|
||||
msg := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(string(stderr)), "error:"))
|
||||
switch {
|
||||
case strings.Contains(msg, "resource not found"):
|
||||
return fmt.Errorf("%w: %s", ErrNotFound, msg)
|
||||
case strings.Contains(msg, "has been exceeded"):
|
||||
return fmt.Errorf("%w: %s", ErrQuotaExceeded, msg)
|
||||
case msg != "":
|
||||
return fmt.Errorf("%s: %w", msg, runErr)
|
||||
default:
|
||||
return runErr
|
||||
}
|
||||
}
|
||||
|
||||
// credentialEnv are the variables exo would read in preference to --config. A
|
||||
// daemon started with them in its environment would act as a different key
|
||||
// from the one the operator provisioned, so they are dropped.
|
||||
var credentialEnv = []string{"EXOSCALE_API_KEY=", "EXOSCALE_API_SECRET=", "EXOSCALE_ACCOUNT=", "EXOSCALE_CONFIG="}
|
||||
|
||||
func execRun(ctx context.Context, binary string, args []string) ([]byte, []byte, error) {
|
||||
cmd := exec.CommandContext(ctx, binary, args...)
|
||||
cmd.Env = scrubEnv(os.Environ())
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
err := cmd.Run()
|
||||
return stdout.Bytes(), stderr.Bytes(), err
|
||||
}
|
||||
|
||||
func scrubEnv(env []string) []string {
|
||||
out := env[:0:0]
|
||||
for _, kv := range env {
|
||||
drop := false
|
||||
for _, prefix := range credentialEnv {
|
||||
if strings.HasPrefix(kv, prefix) {
|
||||
drop = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !drop {
|
||||
out = append(out, kv)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package exoscale
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// scripted replays captured exo output and records the argv it was given.
|
||||
type scripted struct {
|
||||
stdout, stderr []byte
|
||||
err error
|
||||
args []string
|
||||
}
|
||||
|
||||
func (s *scripted) run(_ context.Context, _ string, args []string) ([]byte, []byte, error) {
|
||||
s.args = args
|
||||
return s.stdout, s.stderr, s.err
|
||||
}
|
||||
|
||||
func testdata(t *testing.T, name string) []byte {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(filepath.Join("testdata", name))
|
||||
require.NoError(t, err)
|
||||
return b
|
||||
}
|
||||
|
||||
func newScripted(t *testing.T, s *scripted) *cliClient {
|
||||
t.Helper()
|
||||
c, err := NewCLIClient(CLIConfig{Zone: "de-fra-1", Account: "spx"})
|
||||
require.NoError(t, err)
|
||||
cc := c.(*cliClient)
|
||||
cc.run = s.run
|
||||
return cc
|
||||
}
|
||||
|
||||
func TestCreateElasticIP_DecodesCapturedOutput(t *testing.T) {
|
||||
s := &scripted{stdout: testdata(t, "create.json")}
|
||||
e, err := newScripted(t, s).CreateElasticIP(context.Background(), "spinifex:x:eipalloc-1")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "9d86594b-1bb0-4908-ae28-5870f961aaef", e.ID)
|
||||
assert.Equal(t, "194.182.171.24", e.Address.String())
|
||||
assert.Equal(t, "spinifex:capture:1", e.Description)
|
||||
|
||||
// Credentials come only from the named config file and account, and the
|
||||
// description travels as one argv element however it is spelled.
|
||||
assert.Equal(t, []string{
|
||||
"--config", DefaultConfigFile, "--output-format", "json", "--quiet", "--use-account", "spx",
|
||||
"compute", "elastic-ip", "create", "--description", "spinifex:x:eipalloc-1", "-z", "de-fra-1",
|
||||
}, s.args)
|
||||
}
|
||||
|
||||
func TestListElasticIPs_DecodesCapturedOutput(t *testing.T) {
|
||||
s := &scripted{stdout: testdata(t, "list.json")}
|
||||
eips, err := newScripted(t, s).ListElasticIPs(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Len(t, eips, 5)
|
||||
assert.Equal(t, "mulga1", eips[0].Description)
|
||||
assert.Equal(t, "89.145.162.53", eips[0].Address.String())
|
||||
}
|
||||
|
||||
func TestListElasticIPs_SkipsOtherZones(t *testing.T) {
|
||||
s := &scripted{stdout: []byte(`[{"id":"a","ip_address":"1.2.3.4","zone":"ch-gva-2"},{"id":"b","ip_address":"1.2.3.5","zone":"de-fra-1"}]`)}
|
||||
eips, err := newScripted(t, s).ListElasticIPs(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Len(t, eips, 1)
|
||||
assert.Equal(t, "b", eips[0].ID)
|
||||
}
|
||||
|
||||
func TestCreateElasticIP_RejectsOutputWithoutAddress(t *testing.T) {
|
||||
s := &scripted{stdout: []byte(`{"id":"a","ip_address":""}`)}
|
||||
_, err := newScripted(t, s).CreateElasticIP(context.Background(), "d")
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "unparseable address")
|
||||
}
|
||||
|
||||
func TestClassify_CapturedErrors(t *testing.T) {
|
||||
exitErr := errors.New("exit status 1")
|
||||
cases := []struct {
|
||||
file string
|
||||
want error
|
||||
}{
|
||||
{"quota.stderr", ErrQuotaExceeded},
|
||||
{"notfound.stderr", ErrNotFound},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.file, func(t *testing.T) {
|
||||
s := &scripted{stderr: testdata(t, tc.file), err: exitErr}
|
||||
_, err := newScripted(t, s).CreateElasticIP(context.Background(), "d")
|
||||
require.ErrorIs(t, err, tc.want)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassify_UnknownErrorKeepsMessageAndCause(t *testing.T) {
|
||||
exitErr := errors.New("exit status 1")
|
||||
s := &scripted{stderr: []byte("error: invalid API key\n"), err: exitErr}
|
||||
err := newScripted(t, s).DeleteElasticIP(context.Background(), "a")
|
||||
require.ErrorIs(t, err, exitErr)
|
||||
assert.NotErrorIs(t, err, ErrNotFound)
|
||||
assert.Contains(t, err.Error(), "invalid API key")
|
||||
}
|
||||
|
||||
func TestScrubEnv_DropsCredentialVariables(t *testing.T) {
|
||||
got := scrubEnv([]string{"PATH=/usr/bin", "EXOSCALE_API_KEY=k", "EXOSCALE_API_SECRET=s", "EXOSCALE_CONFIG=/x", "HOME=/h"})
|
||||
assert.Equal(t, []string{"PATH=/usr/bin", "HOME=/h"}, got)
|
||||
}
|
||||
|
||||
// A binary that exits non-zero is reported, not mistaken for empty output.
|
||||
func TestExecRun_RealProcessFailure(t *testing.T) {
|
||||
sh, err := exec.LookPath("sh")
|
||||
require.NoError(t, err)
|
||||
c, err := NewCLIClient(CLIConfig{Zone: "de-fra-1", Binary: sh})
|
||||
require.NoError(t, err)
|
||||
_, err = c.Version(context.Background())
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
// The instance comes before the EIP in both commands. Swapped, exo reports the
|
||||
// instance as not found, which Release would read as already detached.
|
||||
func TestAttachDetach_PassInstanceThenEIP(t *testing.T) {
|
||||
s := &scripted{}
|
||||
c := newScripted(t, s)
|
||||
require.NoError(t, c.AttachElasticIP(context.Background(), "inst-1", "eip-1"))
|
||||
assert.Equal(t, []string{"compute", "instance", "elastic-ip", "attach", "inst-1", "eip-1", "-z", "de-fra-1"}, s.args[7:])
|
||||
require.NoError(t, c.DetachElasticIP(context.Background(), "inst-1", "eip-1"))
|
||||
assert.Equal(t, []string{"compute", "instance", "elastic-ip", "detach", "inst-1", "eip-1", "-z", "de-fra-1"}, s.args[7:])
|
||||
}
|
||||
|
||||
func TestDetach_NotFoundIsTheSentinel(t *testing.T) {
|
||||
s := &scripted{stderr: testdata(t, "notfound.stderr"), err: errors.New("exit status 1")}
|
||||
err := newScripted(t, s).DetachElasticIP(context.Background(), "inst-1", "eip-1")
|
||||
assert.ErrorIs(t, err, ErrNotFound)
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package exoscale
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"slices"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Fake is an in-memory Client for tests. It enforces the organisation quota
|
||||
// and the attach relationship, which are the two behaviours the allocator's
|
||||
// error and rollback paths depend on.
|
||||
type Fake struct {
|
||||
mu sync.Mutex
|
||||
zone string
|
||||
quota int
|
||||
next netip.Addr
|
||||
seq int
|
||||
eips map[string]ElasticIP
|
||||
attached map[string]string // eip ID -> instance ID
|
||||
|
||||
// Fail, when set for an operation name ("create", "attach", "detach",
|
||||
// "delete", "list"), makes that operation return the error once.
|
||||
Fail map[string]error
|
||||
// Calls records operation names in order.
|
||||
Calls []string
|
||||
}
|
||||
|
||||
var _ Client = (*Fake)(nil)
|
||||
|
||||
// NewFake returns a fake with the given quota; zero means unlimited.
|
||||
func NewFake(zone string, quota int) *Fake {
|
||||
return &Fake{
|
||||
zone: zone,
|
||||
quota: quota,
|
||||
next: netip.MustParseAddr("194.182.160.1"),
|
||||
eips: map[string]ElasticIP{},
|
||||
attached: map[string]string{},
|
||||
Fail: map[string]error{},
|
||||
}
|
||||
}
|
||||
|
||||
// Seed adds an EIP the allocator did not create, such as an operator's own.
|
||||
func (f *Fake) Seed(e ElasticIP) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.eips[e.ID] = e
|
||||
}
|
||||
|
||||
// AttachedTo reports which instance an EIP is attached to, if any.
|
||||
func (f *Fake) AttachedTo(eipID string) (string, bool) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
inst, ok := f.attached[eipID]
|
||||
return inst, ok
|
||||
}
|
||||
|
||||
// IDs returns every EIP ID, sorted.
|
||||
func (f *Fake) IDs() []string {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
ids := make([]string, 0, len(f.eips))
|
||||
for id := range f.eips {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
slices.Sort(ids)
|
||||
return ids
|
||||
}
|
||||
|
||||
func (f *Fake) fail(op string) error {
|
||||
f.Calls = append(f.Calls, op)
|
||||
if err, ok := f.Fail[op]; ok {
|
||||
delete(f.Fail, op)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Fake) CreateElasticIP(_ context.Context, description string) (ElasticIP, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if err := f.fail("create"); err != nil {
|
||||
return ElasticIP{}, err
|
||||
}
|
||||
if f.quota > 0 && len(f.eips) >= f.quota {
|
||||
return ElasticIP{}, fmt.Errorf("%w: Conflict: Usage of resource 'eip' has been exceeded", ErrQuotaExceeded)
|
||||
}
|
||||
f.seq++
|
||||
e := ElasticIP{
|
||||
ID: fmt.Sprintf("00000000-0000-4000-8000-%012d", f.seq),
|
||||
Address: f.next,
|
||||
Description: description,
|
||||
Zone: f.zone,
|
||||
}
|
||||
f.next = f.next.Next()
|
||||
f.eips[e.ID] = e
|
||||
return e, nil
|
||||
}
|
||||
|
||||
func (f *Fake) AttachElasticIP(_ context.Context, instanceID, eipID string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if err := f.fail("attach"); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, ok := f.eips[eipID]; !ok {
|
||||
return fmt.Errorf("%w: elastic IP %s", ErrNotFound, eipID)
|
||||
}
|
||||
f.attached[eipID] = instanceID
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Fake) DetachElasticIP(_ context.Context, instanceID, eipID string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if err := f.fail("detach"); err != nil {
|
||||
return err
|
||||
}
|
||||
if f.attached[eipID] != instanceID {
|
||||
return fmt.Errorf("%w: elastic IP %s is not attached to %s", ErrNotFound, eipID, instanceID)
|
||||
}
|
||||
delete(f.attached, eipID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Fake) DeleteElasticIP(_ context.Context, eipID string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if err := f.fail("delete"); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, ok := f.eips[eipID]; !ok {
|
||||
return fmt.Errorf("%w: elastic IP %s", ErrNotFound, eipID)
|
||||
}
|
||||
delete(f.eips, eipID)
|
||||
delete(f.attached, eipID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Fake) ListElasticIPs(context.Context) ([]ElasticIP, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if err := f.fail("list"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]ElasticIP, 0, len(f.eips))
|
||||
for _, e := range f.eips {
|
||||
out = append(out, e)
|
||||
}
|
||||
slices.SortFunc(out, func(a, b ElasticIP) int { return a.Address.Compare(b.Address) })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (f *Fake) Version(context.Context) (string, error) { return "exo fake", nil }
|
||||
@@ -0,0 +1,68 @@
|
||||
package exoscale_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/netip"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/cloud/exoscale"
|
||||
)
|
||||
|
||||
// The Fake is what every exonet test is measured against, so these pin the
|
||||
// behaviours of the real CLI that the allocator's rollback and release paths
|
||||
// depend on.
|
||||
|
||||
// The quota is per organisation, so an EIP Spinifex did not make still counts.
|
||||
func TestFakeQuotaCountsEveryEIPAndUsesTheSentinel(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
f := exoscale.NewFake("de-fra-1", 2)
|
||||
f.Seed(exoscale.ElasticIP{ID: "operator", Address: netip.MustParseAddr("89.145.162.53")})
|
||||
|
||||
_, err := f.CreateElasticIP(ctx, "spinifex:a")
|
||||
require.NoError(t, err)
|
||||
_, err = f.CreateElasticIP(ctx, "spinifex:b")
|
||||
require.ErrorIs(t, err, exoscale.ErrQuotaExceeded)
|
||||
}
|
||||
|
||||
func TestFakeGoneEIPIsNotFoundOnEveryCall(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
f := exoscale.NewFake("de-fra-1", 0)
|
||||
e, err := f.CreateElasticIP(ctx, "spinifex:a")
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, f.AttachElasticIP(ctx, "inst-1", e.ID))
|
||||
require.NoError(t, f.DeleteElasticIP(ctx, e.ID))
|
||||
|
||||
_, attached := f.AttachedTo(e.ID)
|
||||
assert.False(t, attached, "deleting an EIP detaches it")
|
||||
require.ErrorIs(t, f.AttachElasticIP(ctx, "inst-1", e.ID), exoscale.ErrNotFound)
|
||||
require.ErrorIs(t, f.DetachElasticIP(ctx, "inst-1", e.ID), exoscale.ErrNotFound)
|
||||
require.ErrorIs(t, f.DeleteElasticIP(ctx, e.ID), exoscale.ErrNotFound)
|
||||
}
|
||||
|
||||
func TestFakeDetachFromTheWrongInstanceIsNotFound(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
f := exoscale.NewFake("de-fra-1", 0)
|
||||
e, err := f.CreateElasticIP(ctx, "spinifex:a")
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, f.AttachElasticIP(ctx, "inst-1", e.ID))
|
||||
|
||||
require.ErrorIs(t, f.DetachElasticIP(ctx, "inst-2", e.ID), exoscale.ErrNotFound)
|
||||
inst, attached := f.AttachedTo(e.ID)
|
||||
assert.True(t, attached)
|
||||
assert.Equal(t, "inst-1", inst)
|
||||
}
|
||||
|
||||
func TestFakeFailIsOneShot(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
f := exoscale.NewFake("de-fra-1", 0)
|
||||
f.Fail["list"] = assert.AnError
|
||||
|
||||
_, err := f.ListElasticIPs(ctx)
|
||||
require.ErrorIs(t, err, assert.AnError)
|
||||
_, err = f.ListElasticIPs(ctx)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"list", "list"}, f.Calls)
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"id":"9d86594b-1bb0-4908-ae28-5870f961aaef","ip_address":"194.182.171.24","address_family":"inet4","cidr":"194.182.171.24/32","description":"spinifex:capture:1","zone":"de-fra-1","type":"manual","reverse_dns":"","instances":null}
|
||||
+1
@@ -0,0 +1 @@
|
||||
[{"id":"94ca392b-621e-4989-a7f8-6f690d5abcb9","ip_address":"89.145.162.53","description":"mulga1","type":"Manual","zone":"de-fra-1"},{"id":"49381b16-0521-41d0-9ce6-6bd55d1cb254","ip_address":"194.182.169.155","description":"spinifex:capture:3","type":"Manual","zone":"de-fra-1"},{"id":"4b6e412e-93a1-489c-8a87-ef3c4081def9","ip_address":"194.182.170.227","description":"spinifex:capture:2","type":"Manual","zone":"de-fra-1"},{"id":"ab4cb90f-e2dc-48ea-ab49-8e3a82d2f2b6","ip_address":"194.182.170.246","description":"spinifex:capture:4","type":"Manual","zone":"de-fra-1"},{"id":"9d86594b-1bb0-4908-ae28-5870f961aaef","ip_address":"194.182.171.24","description":"spinifex:capture:1","type":"Manual","zone":"de-fra-1"}]
|
||||
@@ -0,0 +1 @@
|
||||
error: resource not found in zone "de-fra-1"
|
||||
@@ -0,0 +1 @@
|
||||
error: Conflict: Usage of resource 'eip' has been exceeded.
|
||||
@@ -0,0 +1,157 @@
|
||||
// Package clustersize holds one fact and the one number derived from it: how
|
||||
// many nodes this process believes the cluster has, and therefore how many
|
||||
// nodes each of its JetStream streams is replicated across.
|
||||
//
|
||||
// It is its own package because the fact is declared in one place,
|
||||
// config.LoadConfig, and read in another, kvutil, and those sit either side of
|
||||
// packages that would otherwise import each other.
|
||||
//
|
||||
// The number matters because a stream on one replica lives on one
|
||||
// JetStream-chosen server recorded in no config. Losing that node makes the
|
||||
// stream unreadable and unwritable from every node at once, so a single-node
|
||||
// event becomes a cluster-wide outage.
|
||||
package clustersize
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// MaxReplicas is JetStream's own ceiling on stream replicas, so a cluster
|
||||
// larger than this replicates to five nodes and no further. Asking for more is
|
||||
// refused by the server, which is why the count is clamped rather than passed
|
||||
// through.
|
||||
const MaxReplicas = 5
|
||||
|
||||
// MinHANodes is the smallest cluster in which losing a node is survivable. Below
|
||||
// it no layer of the stack has a majority to keep: OVN runs standalone, the
|
||||
// storage metadata quorum has nothing to lose, and a KV bucket has no replica
|
||||
// count that both survives a loss and accepts writes.
|
||||
const MinHANodes = 3
|
||||
|
||||
// ErrUndeclared is returned by Replicas in a process that never loaded a
|
||||
// cluster config. It is an error rather than a fallback to one replica because
|
||||
// one replica is the failure this package exists to prevent, and a fallback
|
||||
// would reinstate it in exactly the processes nobody checked.
|
||||
var ErrUndeclared = errors.New("cluster size was never declared, so the stream replica count is unknown")
|
||||
|
||||
// nodes is the declared node count. Zero means undeclared.
|
||||
var nodes atomic.Int64
|
||||
|
||||
// Declare records the cluster's node count for this process. It is called from
|
||||
// config.LoadConfig, so every process that can reach a cluster's NATS has
|
||||
// already declared it before any handler creates a bucket.
|
||||
//
|
||||
// The first declaration wins. Repeating the same count is a no-op, because two
|
||||
// call sites legitimately read it from the same config; a different count is an
|
||||
// error rather than an overwrite, since a process that disagrees with itself
|
||||
// about how many nodes exist would silently pick whichever ran last.
|
||||
func Declare(n int) error {
|
||||
n = max(n, 0)
|
||||
if nodes.CompareAndSwap(0, int64(n)) {
|
||||
return nil
|
||||
}
|
||||
if have := Nodes(); have != n {
|
||||
return fmt.Errorf("cluster size was already declared as %d nodes and cannot be redeclared as %d", have, n)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ResetForTest clears the declaration so a test can declare a different size.
|
||||
// It panics outside a test binary: production has exactly one cluster size for
|
||||
// the life of the process, and a reset available there would undo that.
|
||||
func ResetForTest() {
|
||||
if !testing.Testing() {
|
||||
//nolint:forbidigo // Returning an error would let a caller ignore it and carry on with the count cleared.
|
||||
panic("clustersize.ResetForTest called outside a test binary")
|
||||
}
|
||||
nodes.Store(0)
|
||||
}
|
||||
|
||||
// DeclareForTest declares n for the rest of the test binary, tolerating a fixture
|
||||
// that already declared the same count.
|
||||
//
|
||||
// It is for fixtures rather than for tests. Standing up a one-node embedded server
|
||||
// says something about the binary, not about one test, and several tests running
|
||||
// in parallel share that one server — so restoring the previous count when any one
|
||||
// of them finishes would leave its siblings undeclared mid-run.
|
||||
func DeclareForTest(tb testing.TB, n int) {
|
||||
tb.Helper()
|
||||
if err := Declare(n); err != nil {
|
||||
tb.Fatalf("declare cluster size %d: %v", n, err)
|
||||
}
|
||||
}
|
||||
|
||||
// RedeclareForTest declares n for the duration of one test, whatever was declared
|
||||
// before it, and puts the previous count back afterwards.
|
||||
//
|
||||
// Declaring is write-once, so a test needing a different size than its fixture set
|
||||
// up cannot simply call Declare again. It must not run in parallel with anything
|
||||
// else that reads the count, because there is one count per process and this
|
||||
// changes it.
|
||||
func RedeclareForTest(tb testing.TB, n int) {
|
||||
tb.Helper()
|
||||
was := Nodes()
|
||||
ResetForTest()
|
||||
if err := Declare(n); err != nil {
|
||||
tb.Fatalf("declare cluster size %d: %v", n, err)
|
||||
}
|
||||
tb.Cleanup(func() {
|
||||
ResetForTest()
|
||||
if was > 0 {
|
||||
_ = Declare(was)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Nodes returns the declared node count, or 0 when it was never declared.
|
||||
func Nodes() int {
|
||||
return int(nodes.Load())
|
||||
}
|
||||
|
||||
// Replicas is the replica count every stream is created with: the largest odd
|
||||
// number of nodes this cluster can place a replica on, capped at JetStream's
|
||||
// ceiling of five.
|
||||
//
|
||||
// Odd, because a majority is what a raft group needs and an even count buys
|
||||
// none. Four replicas store a fourth full copy of every write and still survive
|
||||
// only one loss, exactly like three; two survive nothing, because losing either
|
||||
// member leaves no majority to accept a write. So four nodes replicate three
|
||||
// ways and five replicate five ways, which is also the most any cluster does
|
||||
// however large it grows.
|
||||
//
|
||||
// There is no argument and no override. A replica count is a property of the
|
||||
// cluster, not of the stream or of the caller, and every stream here is read or
|
||||
// written by a node other than the one that created it.
|
||||
func Replicas() (int, error) {
|
||||
n := Nodes()
|
||||
if n < 1 {
|
||||
return 0, ErrUndeclared
|
||||
}
|
||||
return ReplicasFor(n), nil
|
||||
}
|
||||
|
||||
// ReplicasFor is Replicas for a node count the caller already holds, so code that
|
||||
// knows the cluster's size without having declared it — a test harness reading a
|
||||
// cluster it did not build — gets its answer from the same rule rather than a
|
||||
// second copy of it.
|
||||
func ReplicasFor(nodes int) int {
|
||||
r := min(max(nodes, 1), MaxReplicas)
|
||||
if r%2 == 0 {
|
||||
r--
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// Permanent reports whether err is this package's own refusal rather than a
|
||||
// transient cluster condition.
|
||||
//
|
||||
// The retry loops that wait for JetStream quorum on a cold start must stop on
|
||||
// one: no amount of waiting declares a cluster size, so a loop that cannot tell
|
||||
// the two apart spends its whole budget on a misconfiguration and then reports
|
||||
// a quorum problem that was never the cause.
|
||||
func Permanent(err error) bool {
|
||||
return errors.Is(err, ErrUndeclared)
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
package clustersize_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/clustersize"
|
||||
)
|
||||
|
||||
func TestDeclareAndNodes(t *testing.T) {
|
||||
t.Cleanup(clustersize.ResetForTest)
|
||||
clustersize.ResetForTest()
|
||||
|
||||
if got := clustersize.Nodes(); got != 0 {
|
||||
t.Fatalf("undeclared cluster size = %d, want 0", got)
|
||||
}
|
||||
if err := clustersize.Declare(3); err != nil {
|
||||
t.Fatalf("Declare(3): %v", err)
|
||||
}
|
||||
if got := clustersize.Nodes(); got != 3 {
|
||||
t.Fatalf("Declare(3) = %d, want 3", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDeclareIsWriteOnce covers the three ways a second declaration can arrive.
|
||||
// Repeating the same count is how two call sites reading the same config behave
|
||||
// and must be accepted; a different count is a process disagreeing with itself
|
||||
// about how many nodes exist, which must not be resolved by whichever ran last.
|
||||
func TestDeclareIsWriteOnce(t *testing.T) {
|
||||
t.Cleanup(clustersize.ResetForTest)
|
||||
clustersize.ResetForTest()
|
||||
|
||||
if err := clustersize.Declare(3); err != nil {
|
||||
t.Fatalf("first Declare(3): %v", err)
|
||||
}
|
||||
if err := clustersize.Declare(3); err != nil {
|
||||
t.Fatalf("repeating Declare(3) must be accepted, got %v", err)
|
||||
}
|
||||
err := clustersize.Declare(5)
|
||||
if err == nil {
|
||||
t.Fatal("Declare(5) after Declare(3) returned no error")
|
||||
}
|
||||
if got := clustersize.Nodes(); got != 3 {
|
||||
t.Fatalf("a rejected redeclaration changed the count to %d, want 3 kept", got)
|
||||
}
|
||||
// The message has to name both counts, because the failure it reports is a
|
||||
// disagreement and neither number alone identifies it.
|
||||
for _, want := range []string{"3", "5"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("error %q does not name the count %s", err, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestDeclareRejectsANegativeCount: a negative node count is not a smaller
|
||||
// cluster, it is a broken config, and storing it as-is would read back as a
|
||||
// count rather than as undeclared.
|
||||
func TestDeclareRejectsANegativeCount(t *testing.T) {
|
||||
t.Cleanup(clustersize.ResetForTest)
|
||||
clustersize.ResetForTest()
|
||||
|
||||
if err := clustersize.Declare(-2); err != nil {
|
||||
t.Fatalf("Declare(-2): %v", err)
|
||||
}
|
||||
if got := clustersize.Nodes(); got != 0 {
|
||||
t.Fatalf("Declare(-2) = %d, want 0 (undeclared)", got)
|
||||
}
|
||||
if _, err := clustersize.Replicas(); !errors.Is(err, clustersize.ErrUndeclared) {
|
||||
t.Fatalf("Replicas() after Declare(-2) = %v, want ErrUndeclared", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReplicas covers the policy and the fail-closed answer together: a bucket is
|
||||
// replicated across the largest odd number of nodes the cluster can place, capped
|
||||
// at five, and a process that never declared a size gets an error rather than a
|
||||
// guess at one.
|
||||
//
|
||||
// The even counts are the point of the table. Four replicas cost a fourth full
|
||||
// copy of every write and survive exactly one loss, the same as three, and two
|
||||
// survive nothing at all because losing either member leaves no majority.
|
||||
func TestReplicas(t *testing.T) {
|
||||
t.Cleanup(clustersize.ResetForTest)
|
||||
|
||||
for _, tc := range []struct{ nodes, want int }{
|
||||
{1, 1}, {2, 1}, {3, 3}, {4, 3}, {5, 5}, {6, 5}, {10, 5}, {64, 5},
|
||||
} {
|
||||
clustersize.ResetForTest()
|
||||
if err := clustersize.Declare(tc.nodes); err != nil {
|
||||
t.Fatalf("Declare(%d): %v", tc.nodes, err)
|
||||
}
|
||||
got, err := clustersize.Replicas()
|
||||
if err != nil {
|
||||
t.Fatalf("Replicas() with %d nodes: %v", tc.nodes, err)
|
||||
}
|
||||
if got != tc.want {
|
||||
t.Errorf("Replicas() with %d nodes = %d, want %d", tc.nodes, got, tc.want)
|
||||
}
|
||||
if got%2 == 0 {
|
||||
t.Errorf("Replicas() with %d nodes = %d, which is even", tc.nodes, got)
|
||||
}
|
||||
if fromCount := clustersize.ReplicasFor(tc.nodes); fromCount != got {
|
||||
t.Errorf("ReplicasFor(%d) = %d but Replicas() = %d; the two must be one rule",
|
||||
tc.nodes, fromCount, got)
|
||||
}
|
||||
}
|
||||
|
||||
clustersize.ResetForTest()
|
||||
if _, err := clustersize.Replicas(); !errors.Is(err, clustersize.ErrUndeclared) {
|
||||
t.Fatalf("Replicas() with an undeclared cluster size = %v, want ErrUndeclared", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReplicasForNeverReturnsZero: callers that hold a node count use it without
|
||||
// an error return, so a count below one has to resolve to the single replica a
|
||||
// lone server can hold rather than to none.
|
||||
func TestReplicasForNeverReturnsZero(t *testing.T) {
|
||||
for _, nodes := range []int{-1, 0} {
|
||||
if got := clustersize.ReplicasFor(nodes); got != 1 {
|
||||
t.Errorf("ReplicasFor(%d) = %d, want 1", nodes, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestPermanent separates this package's own refusal from a cluster that has yet
|
||||
// to form. The retry loops waiting for JetStream quorum stop on the first and
|
||||
// keep waiting through the second, so confusing them spends a whole startup
|
||||
// budget on a misconfiguration.
|
||||
func TestPermanent(t *testing.T) {
|
||||
if !clustersize.Permanent(clustersize.ErrUndeclared) {
|
||||
t.Error("Permanent(ErrUndeclared) = false")
|
||||
}
|
||||
wrapped := errors.Join(errors.New("create KV bucket x"), clustersize.ErrUndeclared)
|
||||
if !clustersize.Permanent(wrapped) {
|
||||
t.Error("Permanent() did not see through a wrapped ErrUndeclared")
|
||||
}
|
||||
if clustersize.Permanent(errors.New("no suitable peers for placement")) {
|
||||
t.Error("Permanent() treated a placement failure as permanent")
|
||||
}
|
||||
if clustersize.Permanent(nil) {
|
||||
t.Error("Permanent(nil) = true")
|
||||
}
|
||||
}
|
||||
@@ -5,9 +5,11 @@ import (
|
||||
"log/slog"
|
||||
"net/netip"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/clustersize"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
@@ -65,7 +67,7 @@ type AWSConfig struct {
|
||||
// ExternalPool defines a range of routable IPs that Spinifex manages for public subnets.
|
||||
type ExternalPool struct {
|
||||
Name string `mapstructure:"name"` // Pool identifier (e.g., "wan", "dc1-primary")
|
||||
Source string `mapstructure:"source"` // IP source: "static" (default), "dhcp" or "oci"
|
||||
Source string `mapstructure:"source"` // IP source: "static" (default), "dhcp", "oci" or "exoscale"
|
||||
BindBridge string `mapstructure:"bind_bridge"` // Linux bridge for DHCP DORA (source=dhcp only)
|
||||
DHCPMAC string `mapstructure:"dhcp_mac"` // DHCP client MAC strategy: "derived" (default) or "interface" (source=dhcp only)
|
||||
RangeStart string `mapstructure:"range_start"` // First IP in range (static source only)
|
||||
@@ -92,6 +94,20 @@ type ExternalPool struct {
|
||||
OCIPublicIPPool string `mapstructure:"oci_public_ip_pool"` // Public IP pool OCID for BYOIP (optional)
|
||||
OCIConfigFile string `mapstructure:"oci_config_file"` // API-key config file (optional; defaults to ~/.oci/config)
|
||||
OCIConfigProfile string `mapstructure:"oci_config_profile"` // Profile within that file (optional; defaults to DEFAULT)
|
||||
|
||||
// Exoscale names the zone and the instance its Elastic IPs attach to
|
||||
// (source=exoscale only). Calls go through the exo CLI, so the instance ID
|
||||
// is configured rather than read from metadata the IMDS endpoint shadows.
|
||||
ExoscaleZone string `mapstructure:"exoscale_zone"` // Zone the EIPs are created in, e.g. "de-fra-1"
|
||||
ExoscaleInstanceID string `mapstructure:"exoscale_instance_id"` // Exoscale instance UUID of this node
|
||||
ExoscaleConfigFile string `mapstructure:"exoscale_config_file"` // exo CLI config holding the API key (optional; see exoscale.DefaultConfigFile)
|
||||
ExoscaleAccount string `mapstructure:"exoscale_account"` // Account name within that file (optional; the file's default)
|
||||
ExoscaleBinary string `mapstructure:"exoscale_binary"` // Absolute path to exo (optional; see exoscale.DefaultBinary)
|
||||
}
|
||||
|
||||
// hasExoscaleKeys reports whether any exoscale_* key is set.
|
||||
func (p ExternalPool) hasExoscaleKeys() bool {
|
||||
return p.ExoscaleZone != "" || p.ExoscaleInstanceID != "" || p.ExoscaleConfigFile != "" || p.ExoscaleAccount != "" || p.ExoscaleBinary != ""
|
||||
}
|
||||
|
||||
// DefaultUnderlayMTU is the standard Ethernet payload, and the assumption a
|
||||
@@ -605,6 +621,24 @@ func LoadConfig(configPath string) (*ClusterConfig, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Every process that creates a KV bucket loads the cluster config first, so
|
||||
// this is the one place the node count can be declared without a service
|
||||
// being able to forget to. Buckets refuse to be created until it has been.
|
||||
//
|
||||
// A conflicting second load is reported rather than refused, because this
|
||||
// function also reads config files that are not this process's own cluster —
|
||||
// an installer inspecting a peer's, a tool rendering one. The first
|
||||
// declaration stands either way, so nothing that already created a bucket can
|
||||
// have the count changed underneath it.
|
||||
if err := clustersize.Declare(len(config.Nodes)); err != nil {
|
||||
slog.Warn("Ignoring a cluster size that disagrees with the one already declared; the first one stands",
|
||||
"loaded_nodes", len(config.Nodes), "declared_nodes", clustersize.Nodes(), "error", err)
|
||||
}
|
||||
if len(config.Nodes) > 1 && len(config.Nodes) < clustersize.MinHANodes {
|
||||
slog.Warn("This cluster has too few nodes to survive losing one, so its state is replicated to a single node",
|
||||
"nodes", len(config.Nodes), "ha_minimum", clustersize.MinHANodes)
|
||||
}
|
||||
|
||||
return &config, nil
|
||||
}
|
||||
|
||||
@@ -657,6 +691,9 @@ func validateClusterConfig(cc *ClusterConfig) error {
|
||||
if p.Source != "oci" && (p.OCICompartmentID != "" || p.OCIVNICID != "" || p.OCIVNICIface != "" || p.OCISubnetID != "" || p.OCIPublicIPPool != "" || p.OCIConfigFile != "" || p.OCIConfigProfile != "") {
|
||||
return fmt.Errorf("config: [[network.external_pools]] %q: oci_* keys are only valid with source=\"oci\"", p.Name)
|
||||
}
|
||||
if p.Source != "exoscale" && p.hasExoscaleKeys() {
|
||||
return fmt.Errorf("config: [[network.external_pools]] %q: exoscale_* keys are only valid with source=\"exoscale\"", p.Name)
|
||||
}
|
||||
switch p.DHCPMAC {
|
||||
case "", "derived", "interface":
|
||||
default:
|
||||
@@ -712,8 +749,30 @@ func validateClusterConfig(cc *ClusterConfig) error {
|
||||
return fmt.Errorf("config: [[network.external_pools]] %q: source=\"oci\" requires [network] external_mode = \"nat\" (routed); OCI VNICs accept only their own MAC, so pool mode's per-VPC gateway MACs are dropped by the provider", p.Name)
|
||||
}
|
||||
continue
|
||||
case "exoscale":
|
||||
if p.ExoscaleZone == "" || p.ExoscaleInstanceID == "" {
|
||||
return fmt.Errorf("config: [[network.external_pools]] %q: source=\"exoscale\" requires exoscale_zone and exoscale_instance_id", p.Name)
|
||||
}
|
||||
if p.ExoscaleBinary != "" && !filepath.IsAbs(p.ExoscaleBinary) {
|
||||
return fmt.Errorf("config: [[network.external_pools]] %q: exoscale_binary must be an absolute path, got %q", p.Name, p.ExoscaleBinary)
|
||||
}
|
||||
if p.BindBridge != "" || p.DHCPMAC != "" {
|
||||
return fmt.Errorf("config: [[network.external_pools]] %q: bind_bridge/dhcp_mac are only valid with source=\"dhcp\"", p.Name)
|
||||
}
|
||||
if p.RangeStart != "" || p.RangeEnd != "" || p.Gateway != "" || p.GatewayIP != "" {
|
||||
return fmt.Errorf("config: [[network.external_pools]] %q: range_start/range_end/gateway/gateway_ip not allowed with source=\"exoscale\" (Exoscale picks the address)", p.Name)
|
||||
}
|
||||
if p.GwLrpRangeStart != "" || p.GwLrpRangeEnd != "" {
|
||||
return fmt.Errorf("config: [[network.external_pools]] %q: gw_lrp_range_start/gw_lrp_range_end not allowed with source=\"exoscale\" (an EIP per VPC gateway would exhaust the per-organisation quota)", p.Name)
|
||||
}
|
||||
// An Exoscale instance answers ARP only from its own MAC and drops
|
||||
// unknown MACs, so pool mode's per-VPC gateway MACs never reach the wire.
|
||||
if cc.Network.ExternalMode != "nat" {
|
||||
return fmt.Errorf("config: [[network.external_pools]] %q: source=\"exoscale\" requires [network] external_mode = \"nat\" (routed); Exoscale instances accept only their own MAC, so pool mode's per-VPC gateway MACs are dropped by the provider", p.Name)
|
||||
}
|
||||
continue
|
||||
default:
|
||||
return fmt.Errorf("config: [[network.external_pools]] %q: source=%q unsupported; use \"static\", \"dhcp\" or \"oci\"", p.Name, p.Source)
|
||||
return fmt.Errorf("config: [[network.external_pools]] %q: source=%q unsupported; use \"static\", \"dhcp\", \"oci\" or \"exoscale\"", p.Name, p.Source)
|
||||
}
|
||||
if p.RangeStart == "" || p.RangeEnd == "" {
|
||||
continue
|
||||
|
||||
@@ -1332,3 +1332,45 @@ func TestLoadConfig_UnknownSourceErrorNamesOCI(t *testing.T) {
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), `"oci"`)
|
||||
}
|
||||
|
||||
func TestLoadConfig_NetworkPoolExoscaleSourceAccepted(t *testing.T) {
|
||||
cfg, err := loadOCIPool(t, "nat", `
|
||||
source = "exoscale"
|
||||
exoscale_zone = "de-fra-1"
|
||||
exoscale_instance_id = "0f1e2d3c-0000-4000-8000-000000000001"
|
||||
exoscale_binary = "/usr/local/bin/exo"
|
||||
`)
|
||||
require.NoError(t, err)
|
||||
p := cfg.Network.ExternalPools[0]
|
||||
assert.Equal(t, "exoscale", p.Source)
|
||||
assert.Equal(t, "de-fra-1", p.ExoscaleZone)
|
||||
assert.Equal(t, "0f1e2d3c-0000-4000-8000-000000000001", p.ExoscaleInstanceID)
|
||||
}
|
||||
|
||||
func TestLoadConfig_NetworkPoolExoscaleRejections(t *testing.T) {
|
||||
const base = `
|
||||
source = "exoscale"
|
||||
exoscale_zone = "de-fra-1"
|
||||
exoscale_instance_id = "0f1e2d3c-0000-4000-8000-000000000001"
|
||||
`
|
||||
cases := []struct {
|
||||
name, mode, body, want string
|
||||
}{
|
||||
{"pool mode", "pool", base, `requires [network] external_mode = "nat"`},
|
||||
{"no zone", "nat", "source = \"exoscale\"\nexoscale_instance_id = \"i\"\n", "requires exoscale_zone and exoscale_instance_id"},
|
||||
{"no instance", "nat", "source = \"exoscale\"\nexoscale_zone = \"de-fra-1\"\n", "requires exoscale_zone and exoscale_instance_id"},
|
||||
{"range", "nat", base + "range_start = \"1.2.3.4\"\nrange_end = \"1.2.3.4\"\n", "Exoscale picks the address"},
|
||||
{"gateway", "nat", base + "gateway = \"1.2.3.1\"\n", "Exoscale picks the address"},
|
||||
{"gw lrp range", "nat", base + "gw_lrp_range_start = \"1.2.3.4\"\ngw_lrp_range_end = \"1.2.3.5\"\n", "per-organisation quota"},
|
||||
{"relative binary", "nat", base + "exoscale_binary = \"exo\"\n", "absolute path"},
|
||||
{"oci keys", "nat", base + "oci_compartment_id = \"c\"\n", `oci_* keys are only valid with source="oci"`},
|
||||
{"exoscale keys on static", "nat", "exoscale_zone = \"de-fra-1\"\n", `exoscale_* keys are only valid with source="exoscale"`},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := loadOCIPool(t, tc.mode, tc.body)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), tc.want)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,11 +22,7 @@ func (d *Daemon) buildBedrockLaunchDeps() handlers_bedrock.LaunchDeps {
|
||||
if err != nil {
|
||||
slog.Warn("Bedrock: jetstream init failed; serving VMs cannot resolve staged weights", "err", err)
|
||||
} else {
|
||||
clusterSize := 1
|
||||
if d.clusterConfig != nil {
|
||||
clusterSize = len(d.clusterConfig.Nodes)
|
||||
}
|
||||
weights = gateway_bedrock.NewWeightsStore(js, clusterSize)
|
||||
weights = gateway_bedrock.NewWeightsStore(js)
|
||||
}
|
||||
|
||||
return handlers_bedrock.LaunchDeps{
|
||||
|
||||
@@ -92,7 +92,10 @@ func TestWaitForClusterReady_PredastoreBecomesReady(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
d := &Daemon{
|
||||
config: &config.Config{Predastore: config.PredastoreConfig{Host: closedPort(t)}},
|
||||
config: &config.Config{
|
||||
Predastore: config.PredastoreConfig{Host: closedPort(t)},
|
||||
NATS: config.NATSConfig{CACert: writeServerCA(t, srv)},
|
||||
},
|
||||
natsConn: nc,
|
||||
}
|
||||
|
||||
|
||||
+91
-36
@@ -17,6 +17,7 @@ import (
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -36,6 +37,7 @@ import (
|
||||
operatorv1 "github.com/mulgadc/spinifex/contracts/operator/v1"
|
||||
"github.com/mulgadc/spinifex/spinifex/admin"
|
||||
"github.com/mulgadc/spinifex/spinifex/bootstrap/preflight"
|
||||
"github.com/mulgadc/spinifex/spinifex/clustersize"
|
||||
"github.com/mulgadc/spinifex/spinifex/config"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ec2/instancetypes"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
@@ -69,6 +71,7 @@ import (
|
||||
handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
|
||||
"github.com/mulgadc/spinifex/spinifex/network/external"
|
||||
"github.com/mulgadc/spinifex/spinifex/network/external/dhcp"
|
||||
"github.com/mulgadc/spinifex/spinifex/network/external/exonet"
|
||||
"github.com/mulgadc/spinifex/spinifex/network/external/ocinet"
|
||||
"github.com/mulgadc/spinifex/spinifex/network/host"
|
||||
"github.com/mulgadc/spinifex/spinifex/objectstore"
|
||||
@@ -1460,6 +1463,12 @@ func (d *Daemon) externalPoolConfigs() (pools []external.ExternalPoolConfig, any
|
||||
OCIPublicIPPool: p.OCIPublicIPPool,
|
||||
OCIConfigFile: p.OCIConfigFile,
|
||||
OCIConfigProfile: p.OCIConfigProfile,
|
||||
|
||||
ExoscaleZone: p.ExoscaleZone,
|
||||
ExoscaleInstanceID: p.ExoscaleInstanceID,
|
||||
ExoscaleConfigFile: p.ExoscaleConfigFile,
|
||||
ExoscaleAccount: p.ExoscaleAccount,
|
||||
ExoscaleBinary: p.ExoscaleBinary,
|
||||
})
|
||||
if p.Source == "dhcp" {
|
||||
anyDHCP = true
|
||||
@@ -1511,6 +1520,31 @@ func (d *Daemon) installOCIAllocators(ipam *handlers_ec2_vpc.ExternalIPAM, js je
|
||||
return nil
|
||||
}
|
||||
|
||||
// installExoscaleAllocators builds one allocator per source="exoscale" pool
|
||||
// and reconciles it before it serves. As with OCI, Allocate creates the EIP
|
||||
// before recording it, and this pass is the only thing that finds one a crash
|
||||
// left behind — on a five-EIP quota, one leak is a fifth of the capacity.
|
||||
func (d *Daemon) installExoscaleAllocators(ipam *handlers_ec2_vpc.ExternalIPAM, js jetstream.JetStream) error {
|
||||
for _, p := range ipam.PoolsWithSource(external.SourceExoscale) {
|
||||
alloc, err := exonet.FromPoolConfig(d.ctx, js, p)
|
||||
if err != nil {
|
||||
return fmt.Errorf("build Exoscale allocator for pool %q: %w", p.Name, err)
|
||||
}
|
||||
if err := ipam.InstallAllocator(p.Name, alloc); err != nil {
|
||||
return err
|
||||
}
|
||||
res, err := alloc.Reconcile(d.ctx)
|
||||
if err != nil {
|
||||
slog.Error("Exoscale allocator reconcile failed; leaked elastic IPs may be billing",
|
||||
"pool", p.Name, "err", err)
|
||||
continue
|
||||
}
|
||||
slog.Info("Exoscale allocator ready", "pool", p.Name,
|
||||
"collected", len(res.Collected), "stale_bindings", len(res.Stale), "skipped", res.Skipped)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ociPool pairs a pool's allocator with its name, so one pass can cover every
|
||||
// pool and reach a single verdict about each guest. Per-pool loops could not: a
|
||||
// guest is impaired or it is not, and two loops writing that answer from half the
|
||||
@@ -1670,14 +1704,18 @@ func (d *Daemon) startCluster() error {
|
||||
return fmt.Errorf("connect NATS: %w", err)
|
||||
}
|
||||
|
||||
if err := d.initJetStream(); err != nil {
|
||||
return fmt.Errorf("initialize JetStream: %w", err)
|
||||
// Declared before JetStream, not after: initJetStream creates buckets, and
|
||||
// bucket creation refuses until the count is known. A daemon assembled in a
|
||||
// test may never have gone through config.LoadConfig, which is where a real
|
||||
// process declares it from this same source.
|
||||
if d.clusterConfig != nil {
|
||||
if err := clustersize.Declare(len(d.clusterConfig.Nodes)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Set the default KV replica count before any handler creates a bucket, so
|
||||
// lazily-created buckets are born at cluster-size replication instead of R1.
|
||||
if d.clusterConfig != nil {
|
||||
kvutil.SetDefaultKVReplicas(len(d.clusterConfig.Nodes))
|
||||
if err := d.initJetStream(); err != nil {
|
||||
return fmt.Errorf("initialize JetStream: %w", err)
|
||||
}
|
||||
|
||||
// Remove the obsolete spinifex-dhcp-leases bucket (idempotent).
|
||||
@@ -1751,6 +1789,9 @@ func (d *Daemon) startCluster() error {
|
||||
return fmt.Errorf("failed to get tags KV bucket: %w", err)
|
||||
}
|
||||
d.tagsService = handlers_ec2_tags.NewTagsServiceImpl(d.config, tagsKV)
|
||||
// Key pairs keep their creation tags in their own metadata; project them so
|
||||
// describe-tags sees them, and clear them when the key pair is deleted.
|
||||
d.keyService.SetCentralTagStore(d.tagsService)
|
||||
|
||||
d.eigwService, err = initServiceWithRetry("EIGW service", func() (*handlers_ec2_eigw.EgressOnlyIGWServiceImpl, error) {
|
||||
return handlers_ec2_eigw.NewEgressOnlyIGWServiceImplWithNATS(d.ctx, d.config, d.natsConn)
|
||||
@@ -1801,6 +1842,12 @@ func (d *Daemon) startCluster() error {
|
||||
// describe-tags agrees with each resource's own describe from birth and
|
||||
// stops answering for it once it is deleted.
|
||||
d.vpcService.SetCentralTagStore(d.tagsService)
|
||||
// Name interfaces as their instances are named, so the two agree.
|
||||
region, internalDomain := d.config.Region, handlers_dns.ResolveInternalDomain(d.config)
|
||||
d.vpcService.SetPrivateDNSNamer(func(privateIP string) string {
|
||||
_, private := handlers_dns.EC2DNSNames(region, "", internalDomain, "", privateIP)
|
||||
return private
|
||||
})
|
||||
|
||||
d.routeTableService, err = initServiceWithRetry("RouteTable service", func() (*handlers_ec2_routetable.RouteTableServiceImpl, error) {
|
||||
return handlers_ec2_routetable.NewRouteTableServiceImplWithNATS(d.ctx, d.config, d.natsConn)
|
||||
@@ -1841,6 +1888,9 @@ func (d *Daemon) startCluster() error {
|
||||
if ociErr := d.installOCIAllocators(ipam, js); ociErr != nil {
|
||||
return nil, ociErr
|
||||
}
|
||||
if exoErr := d.installExoscaleAllocators(ipam, js); exoErr != nil {
|
||||
return nil, exoErr
|
||||
}
|
||||
return ipam, nil
|
||||
})
|
||||
if err != nil {
|
||||
@@ -2359,11 +2409,7 @@ func (d *Daemon) nodeRunningVMs() ([]*vm.VM, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
vms := make([]*vm.VM, 0, len(running))
|
||||
for _, v := range running {
|
||||
vms = append(vms, v)
|
||||
}
|
||||
return vms, nil
|
||||
return slices.Collect(maps.Values(running)), nil
|
||||
}
|
||||
|
||||
// connectNATS connects to NATS with infinite retry (cap 60s backoff). Tests
|
||||
@@ -2400,7 +2446,7 @@ func (d *Daemon) initJetStream() error {
|
||||
for {
|
||||
attempt++
|
||||
var err error
|
||||
d.jsManager, err = NewJetStreamManager(d.natsConn, 1)
|
||||
d.jsManager, err = NewJetStreamManager(d.natsConn)
|
||||
if err == nil {
|
||||
err = d.jsManager.InitKVBucket()
|
||||
}
|
||||
@@ -2415,10 +2461,20 @@ func (d *Daemon) initJetStream() error {
|
||||
|
||||
if err == nil {
|
||||
d.jsManager.SetSyncObserver(d)
|
||||
slog.Info("JetStream KV stores initialized successfully", "replicas", 1, "attempts", attempt, "elapsed_ms", otelsetup.Millis(time.Since(start)))
|
||||
// Replicas cannot be undeclared here: bucket creation refuses
|
||||
// before it, so reaching this line means it was declared.
|
||||
replicas, _ := clustersize.Replicas()
|
||||
slog.Info("JetStream KV stores initialized successfully", "replicas", replicas, "attempts", attempt, "elapsed_ms", otelsetup.Millis(time.Since(start)))
|
||||
break
|
||||
}
|
||||
|
||||
// A misdeclared cluster size is not something quorum arrives and fixes,
|
||||
// so waiting for it spends the whole budget and then reports a quorum
|
||||
// problem that was never the cause.
|
||||
if clustersize.Permanent(err) {
|
||||
return fmt.Errorf("initialize JetStream: %w", err)
|
||||
}
|
||||
|
||||
elapsed := time.Since(start)
|
||||
if elapsed >= maxWait {
|
||||
return fmt.Errorf("failed to initialize JetStream after %s (%d attempts): %w", elapsed.Round(time.Second), attempt, err)
|
||||
@@ -2434,15 +2490,20 @@ func (d *Daemon) initJetStream() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// upgradeJetStreamReplicas bumps KV_* stream replication to match the cluster
|
||||
// size. Runs after all buckets are created and the cluster is ready.
|
||||
// upgradeJetStreamReplicas raises every KV bucket to the cluster's replica
|
||||
// count. Runs after all buckets are created and the cluster is ready, which is
|
||||
// when a bucket created by whichever node got there first can be repaired.
|
||||
func (d *Daemon) upgradeJetStreamReplicas() {
|
||||
clusterSize := len(d.clusterConfig.Nodes)
|
||||
if clusterSize <= 1 || d.jsManager == nil {
|
||||
if d.jsManager == nil || d.jsManager.js == nil {
|
||||
return
|
||||
}
|
||||
if err := d.jsManager.UpdateReplicas(clusterSize); err != nil {
|
||||
slog.Warn("Failed to upgrade JetStream replicas", "targetReplicas", clusterSize, "error", err)
|
||||
raised, err := kvutil.RaiseAllBucketReplicas(d.ctx, d.jsManager.js)
|
||||
if err != nil {
|
||||
slog.Warn("Failed to raise KV bucket replicas to the cluster's node count", "error", err)
|
||||
return
|
||||
}
|
||||
if raised > 0 {
|
||||
slog.Info("Raised KV buckets to the cluster's replica count", "buckets", raised)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2551,15 +2612,6 @@ func (d *Daemon) checkViperblockReady() bool {
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// predastoreReadinessClient is hoisted to package scope so the 2s readiness
|
||||
// poll loop doesn't build a fresh transport every call. Verification is off
|
||||
// because the probe authenticates nothing and reads no data.
|
||||
var predastoreReadinessClient = &http.Client{
|
||||
Transport: &http.Transport{
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec // readiness probe only, no data exchanged
|
||||
},
|
||||
}
|
||||
|
||||
// checkPredastoreReady reports whether predastore is serving HTTPS, not just
|
||||
// listening. Any response counts as ready, including the 401/403 an S3
|
||||
// endpoint returns for this deliberately unsigned request.
|
||||
@@ -2572,11 +2624,18 @@ func (d *Daemon) checkPredastoreReady() bool {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
|
||||
rootCAs, err := loadClusterTrustRoot(d)
|
||||
if err != nil {
|
||||
slog.Warn("predastore readiness probe: could not load cluster CA", "err", err)
|
||||
return false
|
||||
}
|
||||
client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: rootCAs}}}
|
||||
defer client.CloseIdleConnections()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+host+"/", nil)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
resp, err := predastoreReadinessClient.Do(req)
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
@@ -3525,13 +3584,9 @@ func probeGPU() gpuProbeResult {
|
||||
|
||||
// MIG-enabled GPUs are capable without vfio-pci: the NVIDIA driver owns
|
||||
// isolation via the mdev subsystem, so vfio-pci is not required.
|
||||
hasMIG := false
|
||||
for _, d := range r.Devices {
|
||||
if d.MIGEnabled {
|
||||
hasMIG = true
|
||||
break
|
||||
}
|
||||
}
|
||||
hasMIG := slices.ContainsFunc(r.Devices, func(d gpu.GPUDevice) bool {
|
||||
return d.MIGEnabled
|
||||
})
|
||||
r.Capable = len(r.Devices) > 0 && ((r.IOMMUActive && r.VFIOPresent) || hasMIG)
|
||||
return r
|
||||
}
|
||||
|
||||
@@ -483,7 +483,7 @@ var roleHTTPClient = &http.Client{Timeout: 500 * time.Millisecond}
|
||||
|
||||
// fetchNATSRole queries a NATS /varz endpoint and returns "leader", "follower", or "".
|
||||
func fetchNATSRole(url string, client *http.Client) string {
|
||||
resp, err := client.Get(url) //nolint:noctx // internal monitoring call
|
||||
resp, err := client.Get(url)
|
||||
if err != nil {
|
||||
slog.Debug("Failed to query NATS monitoring", "err", err)
|
||||
return ""
|
||||
|
||||
@@ -135,7 +135,7 @@ func createFullTestDaemonWithJetStream(t *testing.T, natsURL string) *Daemon {
|
||||
daemon := createFullTestDaemon(t, natsURL)
|
||||
|
||||
var err error
|
||||
daemon.jsManager, err = NewJetStreamManager(daemon.natsConn, 1)
|
||||
daemon.jsManager, err = NewJetStreamManager(daemon.natsConn)
|
||||
require.NoError(t, err)
|
||||
err = daemon.jsManager.InitKVBucket()
|
||||
require.NoError(t, err)
|
||||
@@ -611,7 +611,7 @@ func runInstancesAndCheckENISGs(t *testing.T, mutator func(input *ec2.RunInstanc
|
||||
subnetID := *subnetOut.Subnet.SubnetId
|
||||
|
||||
sg1Out, err := daemon.vpcService.CreateSecurityGroup(t.Context(), &ec2.CreateSecurityGroupInput{
|
||||
GroupName: aws.String("sg-prop-1"),
|
||||
GroupName: aws.String("grp-prop-1"),
|
||||
Description: aws.String("test"),
|
||||
VpcId: aws.String(vpcID),
|
||||
}, testAccountID)
|
||||
@@ -619,7 +619,7 @@ func runInstancesAndCheckENISGs(t *testing.T, mutator func(input *ec2.RunInstanc
|
||||
sg1 = *sg1Out.GroupId
|
||||
|
||||
sg2Out, err := daemon.vpcService.CreateSecurityGroup(t.Context(), &ec2.CreateSecurityGroupInput{
|
||||
GroupName: aws.String("sg-prop-2"),
|
||||
GroupName: aws.String("grp-prop-2"),
|
||||
Description: aws.String("test"),
|
||||
VpcId: aws.String(vpcID),
|
||||
}, testAccountID)
|
||||
@@ -2648,7 +2648,7 @@ func TestDelegateHandlers_EIGW(t *testing.T) {
|
||||
topic: "ec2.test.DeleteEgressOnlyIGW",
|
||||
handler: asMsgHandler(handleNATSRequest(daemon.node, daemon.eigwService.DeleteEgressOnlyInternetGateway)),
|
||||
input: &ec2.DeleteEgressOnlyInternetGatewayInput{EgressOnlyInternetGatewayId: aws.String("eigw-nonexistent")},
|
||||
expectedCode: awserrors.ErrorInvalidEgressOnlyInternetGatewayIdNotFound,
|
||||
expectedCode: awserrors.ErrorInvalidGatewayIDNotFound,
|
||||
},
|
||||
{
|
||||
name: "DescribeEgressOnlyInternetGateways",
|
||||
|
||||
@@ -24,7 +24,7 @@ func newTestMgmtJSM(t *testing.T) *JetStreamManager {
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
if err != nil {
|
||||
t.Fatalf("new JetStreamManager: %v", err)
|
||||
}
|
||||
@@ -552,7 +552,7 @@ func TestMgmtIPAllocator_Allocate_RefusedWhenKVUnhealthy(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -799,6 +799,7 @@ func TestLaunchSystemInstance_NATFailureRollsBackPublicIP(t *testing.T) {
|
||||
// rule vpcd may have committed after the AddNAT timeout window.
|
||||
deleteNATCh := make(chan map[string]string, 1)
|
||||
delSub, err := d.natsConn.Subscribe("vpc.delete-nat", func(msg *nats.Msg) {
|
||||
_ = msg.Respond([]byte(`{"success":true}`))
|
||||
var p map[string]string
|
||||
_ = json.Unmarshal(msg.Data, &p)
|
||||
select {
|
||||
|
||||
@@ -742,7 +742,7 @@ func TestDaemon_BootAllocation(t *testing.T) {
|
||||
defer nc.Close()
|
||||
|
||||
daemon.natsConn = nc
|
||||
daemon.jsManager, err = NewJetStreamManager(nc, 1)
|
||||
daemon.jsManager, err = NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
err = daemon.jsManager.InitKVBucket()
|
||||
require.NoError(t, err)
|
||||
@@ -1694,7 +1694,7 @@ func TestTerminatedTeardownReaper_SelfHealsFailedVolumeTeardown(t *testing.T) {
|
||||
|
||||
// A real terminated-instance KV, so Sweep can list/update/(not yet)purge
|
||||
// the record exactly as production does.
|
||||
jsManager, err := NewJetStreamManager(daemon.natsConn, 1)
|
||||
jsManager, err := NewJetStreamManager(daemon.natsConn)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsManager.InitTerminatedInstanceBucket())
|
||||
daemon.stateStore = newStateStoreAdapter(jsManager, daemon.persistState)
|
||||
@@ -4547,7 +4547,7 @@ func newEBSProviderTestDaemon(t *testing.T, provider string) *Daemon {
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(nc.Close)
|
||||
|
||||
jsManager, err := NewJetStreamManager(nc, 1)
|
||||
jsManager, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -45,9 +45,8 @@ func (d *Daemon) instanceStateWatchBuckets(context.Context) ([]*kvstore.Bucket,
|
||||
return nil, nil
|
||||
}
|
||||
return []*kvstore.Bucket{kvstore.NewBucket(d.jsManager.js, kvstore.Config{
|
||||
Name: InstanceStateBucket,
|
||||
History: 1,
|
||||
Replicas: d.jsManager.replicas,
|
||||
Name: InstanceStateBucket,
|
||||
History: 1,
|
||||
})}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -26,10 +26,10 @@ func dnsTestDaemon(t *testing.T) (*Daemon, func(id, publicIP, privateIP string,
|
||||
t.Helper()
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
store := kvstore.New[vm.InstanceRecord](js, kvstore.Config{
|
||||
Name: InstanceStateBucket, History: 1, Replicas: 1,
|
||||
Name: InstanceStateBucket, History: 1,
|
||||
})
|
||||
d := &Daemon{
|
||||
jsManager: &JetStreamManager{js: js, records: store, replicas: 1},
|
||||
jsManager: &JetStreamManager{js: js, records: store},
|
||||
vmMgr: vm.NewManager(),
|
||||
config: &config.Config{Region: "ap-southeast-2"},
|
||||
dnsBaseDomain: "spx3.net",
|
||||
|
||||
@@ -62,7 +62,7 @@ func TestDNSWatchBuckets_ZeroValueDaemonIsNilSafe(t *testing.T) {
|
||||
|
||||
func TestInstanceStateWatchBuckets_ReturnsSharedBucket(t *testing.T) {
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
d := &Daemon{jsManager: &JetStreamManager{js: js, replicas: 1}}
|
||||
d := &Daemon{jsManager: &JetStreamManager{js: js}}
|
||||
|
||||
buckets, err := d.instanceStateWatchBuckets(t.Context())
|
||||
require.NoError(t, err)
|
||||
@@ -111,7 +111,7 @@ func TestEksWatchBuckets_NilGuards(t *testing.T) {
|
||||
func TestEksWatchBuckets_DelegatesToAccountWatchBuckets(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
js := testutil.NewJetStream(t, nc)
|
||||
_, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), js, "111111111111", 1)
|
||||
_, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), js, "111111111111")
|
||||
require.NoError(t, err)
|
||||
|
||||
d := &Daemon{eksService: &handlers_eks.EKSServiceImpl{}, natsConn: nc}
|
||||
|
||||
@@ -43,16 +43,12 @@ func (d *Daemon) buildECSServiceDeps() handlers_ecs.Deps {
|
||||
return deps
|
||||
}
|
||||
|
||||
clusterSize := 1
|
||||
if d.clusterConfig != nil {
|
||||
clusterSize = len(d.clusterConfig.Nodes)
|
||||
}
|
||||
// Retried, because the constructor runs a KV migration that needs JetStream
|
||||
// to have responders. A daemon that started first got one attempt, failed
|
||||
// it, and had capacity provisioning off for the life of the process with no
|
||||
// symptom but a 500 from ProvisionCapacity.
|
||||
iamSvc, iamErr := initServiceWithRetry("ECS IAM service", func() (*handlers_iam.IAMServiceImpl, error) {
|
||||
return handlers_iam.NewIAMServiceImpl(d.ctx, d.natsConn, masterKey, clusterSize)
|
||||
return handlers_iam.NewIAMServiceImpl(d.ctx, d.natsConn, masterKey)
|
||||
})
|
||||
if iamErr != nil {
|
||||
// Off rather than fatal: ECS CRUD, EKS and EC2 do not need this, and a
|
||||
|
||||
@@ -36,11 +36,7 @@ func (d *Daemon) systemRoleEnsurer() handlers_iam.SystemInstanceRoleEnsurer {
|
||||
"err", err)
|
||||
return nil
|
||||
}
|
||||
clusterSize := 1
|
||||
if d.clusterConfig != nil {
|
||||
clusterSize = len(d.clusterConfig.Nodes)
|
||||
}
|
||||
iamSvc, iamErr := handlers_iam.NewIAMServiceImpl(d.ctx, d.natsConn, masterKey, clusterSize)
|
||||
iamSvc, iamErr := handlers_iam.NewIAMServiceImpl(d.ctx, d.natsConn, masterKey)
|
||||
if iamErr != nil {
|
||||
slog.Warn("System role ensurer: IAM service init failed (retried on next launch); system VMs fall back to baked static creds",
|
||||
"err", iamErr)
|
||||
@@ -61,10 +57,8 @@ func (d *Daemon) buildEKSServiceDeps() handlers_eks.EKSServiceDeps {
|
||||
}
|
||||
|
||||
internalSuffix := ""
|
||||
clusterSize := 1
|
||||
if d.clusterConfig != nil {
|
||||
internalSuffix = d.clusterConfig.AWS.ServicesDomain
|
||||
clusterSize = len(d.clusterConfig.Nodes)
|
||||
}
|
||||
|
||||
gatewayCA := ""
|
||||
@@ -84,7 +78,6 @@ func (d *Daemon) buildEKSServiceDeps() handlers_eks.EKSServiceDeps {
|
||||
GatewayBaseURL: d.resolveGatewayBaseURL(),
|
||||
Region: d.config.Region,
|
||||
HolderID: d.node,
|
||||
ClusterSize: clusterSize,
|
||||
InternalSuffix: internalSuffix,
|
||||
SystemGatewayURL: d.resolveSystemGatewayBaseURL(),
|
||||
SystemAccessKey: d.config.Predastore.AccessKey,
|
||||
|
||||
@@ -165,7 +165,7 @@ func computePredastoreHealth(ctx context.Context, d *Daemon) string {
|
||||
return probeLocalGate(ctx, d, cfg, hostID)
|
||||
}
|
||||
|
||||
rootCAs, err := loadPredastoreTrustRoot(d)
|
||||
rootCAs, err := loadClusterTrustRoot(d)
|
||||
if err != nil {
|
||||
slog.Warn("predastore health probe: could not load cluster CA", "err", err)
|
||||
return predastoreHealthUnreachable
|
||||
@@ -332,11 +332,11 @@ func predastoreConfigPath(daemonConfigPath string) string {
|
||||
return filepath.Join(filepath.Dir(daemonConfigPath), "predastore", "predastore.toml")
|
||||
}
|
||||
|
||||
// loadPredastoreTrustRoot returns the cluster CA pool predastore's meta
|
||||
// replicas are issued from — the same CA (d.config.NATS.CACert, conventionally
|
||||
// /etc/spinifex/ca.pem) the daemon already trusts NATS peers with.
|
||||
func loadPredastoreTrustRoot(d *Daemon) (*x509.CertPool, error) {
|
||||
if d.config.NATS.CACert == "" {
|
||||
// loadClusterTrustRoot returns the cluster CA pool (d.config.NATS.CACert,
|
||||
// conventionally /etc/spinifex/ca.pem) that issues every node's server cert:
|
||||
// daemon peers, predastore replicas and NATS alike.
|
||||
func loadClusterTrustRoot(d *Daemon) (*x509.CertPool, error) {
|
||||
if d.config == nil || d.config.NATS.CACert == "" {
|
||||
return nil, fmt.Errorf("cluster CA not configured (nats.cacert)")
|
||||
}
|
||||
return utils.LoadCertPool(d.config.NATS.CACert)
|
||||
|
||||
@@ -103,7 +103,7 @@ func TestHeartbeatKVContract(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitClusterStateBucket())
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ func newRecordManagerInPackage(t *testing.T) *JetStreamManager {
|
||||
t.Helper()
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
|
||||
m, err := NewJetStreamManager(nc, 1)
|
||||
m, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, m.InitKVBucket())
|
||||
require.NoError(t, m.InitTerminatedInstanceBucket())
|
||||
|
||||
@@ -46,7 +46,7 @@ func TestRekey_MovesInstanceStateRecordsOntoTheDot(t *testing.T) {
|
||||
kv := seedSlashRecord(t, nc, daemon.InstanceStateBucket, 3,
|
||||
&vm.VM{ID: "i-1", InstanceType: "t3.nano", LastNode: "node-1"})
|
||||
|
||||
m, err := daemon.NewJetStreamManager(nc, 1)
|
||||
m, err := daemon.NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, m.InitKVBucket())
|
||||
|
||||
@@ -64,7 +64,7 @@ func TestRekey_MovesTerminatedRecordsOntoTheDot(t *testing.T) {
|
||||
kv := seedSlashRecord(t, nc, daemon.TerminatedInstanceBucket, 2,
|
||||
&vm.VM{ID: "i-1", InstanceType: "t3.nano", LastNode: "node-1"})
|
||||
|
||||
m, err := daemon.NewJetStreamManager(nc, 1)
|
||||
m, err := daemon.NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, m.InitTerminatedInstanceBucket())
|
||||
|
||||
|
||||
@@ -29,7 +29,7 @@ func newRecordManagerConn(t *testing.T) (*daemon.JetStreamManager, *nats.Conn) {
|
||||
t.Helper()
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
|
||||
m, err := daemon.NewJetStreamManager(nc, 1)
|
||||
m, err := daemon.NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, m.InitKVBucket())
|
||||
require.NoError(t, m.InitTerminatedInstanceBucket())
|
||||
@@ -259,7 +259,7 @@ func TestInstanceRecord_CarriesDeletionTimestamp(t *testing.T) {
|
||||
// store: Tier 1 boot runs before cluster KV exists.
|
||||
func TestInstanceRecordAccessors_ReportAnUnopenedBucket(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
m, err := daemon.NewJetStreamManager(nc, 1)
|
||||
m, err := daemon.NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Error(t, m.WriteInstanceRecord("i-1", testRecord("i-1")))
|
||||
|
||||
@@ -223,7 +223,7 @@ func TestInstanceStateMigration_CopiesStoppedInstancesForward(t *testing.T) {
|
||||
seedLegacyBucket(t, nc, daemon.InstanceStateBucket, daemon.StoppedInstancePrefix,
|
||||
&vm.VM{ID: "i-1", InstanceType: "t3.nano"})
|
||||
|
||||
m, err := daemon.NewJetStreamManager(nc, 1)
|
||||
m, err := daemon.NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, m.InitKVBucket())
|
||||
|
||||
@@ -247,7 +247,7 @@ func TestInstanceStateMigration_RunsEveryStepFromTheOldestVersion(t *testing.T)
|
||||
_, err = kv.Put(context.Background(), daemon.InstanceStatePrefix+"node-1", blob)
|
||||
require.NoError(t, err)
|
||||
|
||||
m, err := daemon.NewJetStreamManager(nc, 1)
|
||||
m, err := daemon.NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, m.InitKVBucket())
|
||||
|
||||
@@ -277,7 +277,7 @@ func TestInstanceStateMigration_DoesNotOverwriteAFresherRecord(t *testing.T) {
|
||||
_, err = kv.Put(context.Background(), daemon.StoppedInstancePrefix+"i-2", untouched)
|
||||
require.NoError(t, err)
|
||||
|
||||
m, err := daemon.NewJetStreamManager(nc, 1)
|
||||
m, err := daemon.NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, m.InitKVBucket())
|
||||
|
||||
@@ -297,12 +297,12 @@ func TestInstanceStateMigration_IsSafeToRunTwice(t *testing.T) {
|
||||
kv := seedLegacyBucket(t, nc, daemon.InstanceStateBucket, daemon.StoppedInstancePrefix,
|
||||
&vm.VM{ID: "i-1", InstanceType: "t3.nano"})
|
||||
|
||||
first, err := daemon.NewJetStreamManager(nc, 1)
|
||||
first, err := daemon.NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, first.InitKVBucket())
|
||||
|
||||
require.NoError(t, kvutil.WriteVersion(context.Background(), kv, 1))
|
||||
second, err := daemon.NewJetStreamManager(nc, 1)
|
||||
second, err := daemon.NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, second.InitKVBucket())
|
||||
|
||||
@@ -317,7 +317,7 @@ func TestTerminatedInstanceMigration_CopiesTerminatedInstancesForward(t *testing
|
||||
seedLegacyBucket(t, nc, daemon.TerminatedInstanceBucket, daemon.TerminatedInstancePrefix,
|
||||
&vm.VM{ID: "i-1", InstanceType: "t3.nano"})
|
||||
|
||||
m, err := daemon.NewJetStreamManager(nc, 1)
|
||||
m, err := daemon.NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, m.InitTerminatedInstanceBucket())
|
||||
|
||||
|
||||
@@ -522,7 +522,7 @@ func (d *Daemon) newLiveness() *instancecache.Liveness {
|
||||
if d.jsManager == nil || d.jsManager.js == nil {
|
||||
return nil
|
||||
}
|
||||
return instancecache.NewLiveness(d.jsManager.js, clusterStateConfig(d.jsManager.replicas))
|
||||
return instancecache.NewLiveness(d.jsManager.js, clusterStateConfig())
|
||||
}
|
||||
|
||||
// forgetSupersededInstances drops any local copy of an instance the records say
|
||||
|
||||
@@ -234,7 +234,7 @@ func TestWriteRunningSet_SeedsFromWhatIsAlreadyThere(t *testing.T) {
|
||||
writeRunningSet(t, first, runningVM("i-1", "t3.nano"))
|
||||
before := recordRevision(t, nc, "i-1")
|
||||
|
||||
second, err := daemon.NewJetStreamManager(nc, 1)
|
||||
second, err := daemon.NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, second.InitKVBucket())
|
||||
writeRunningSet(t, second, runningVM("i-1", "t3.nano"))
|
||||
@@ -293,7 +293,7 @@ func TestInstanceStateMigration_SplitsTheNodeBlob(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
seedNodeBlobBucket(t, nc, localNode, runningSet(runningVM("i-1", "t3.nano"), runningVM("i-2", "t3.micro")))
|
||||
|
||||
m, err := daemon.NewJetStreamManager(nc, 1)
|
||||
m, err := daemon.NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, m.InitKVBucket())
|
||||
|
||||
@@ -314,7 +314,7 @@ func TestInstanceStateMigration_LeavesTheNodeBlobIntact(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
kv := seedNodeBlobBucket(t, nc, localNode, runningSet(runningVM("i-1", "t3.nano")))
|
||||
|
||||
m, err := daemon.NewJetStreamManager(nc, 1)
|
||||
m, err := daemon.NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, m.InitKVBucket())
|
||||
writeRunningSet(t, m, runningVM("i-1", "t3.nano"))
|
||||
@@ -330,7 +330,7 @@ func TestInstanceStateMigration_SeedsAPresenceMarkerPerNode(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
kv := seedNodeBlobBucket(t, nc, localNode, runningSet(runningVM("i-1", "t3.nano")))
|
||||
|
||||
m, err := daemon.NewJetStreamManager(nc, 1)
|
||||
m, err := daemon.NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, m.InitKVBucket())
|
||||
|
||||
@@ -359,7 +359,7 @@ func TestInstanceStateMigration_SplitsEveryNodesBlob(t *testing.T) {
|
||||
_, err = kv.Put(context.Background(), daemon.InstanceStatePrefix+"node-2", data)
|
||||
require.NoError(t, err)
|
||||
|
||||
m, err := daemon.NewJetStreamManager(nc, 1)
|
||||
m, err := daemon.NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, m.InitKVBucket())
|
||||
|
||||
@@ -383,7 +383,7 @@ func TestInstanceStateMigration_CarriesOwnershipOffTheBlobKey(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
seedNodeBlobBucket(t, nc, localNode, runningSet(unownedVM("i-1", "t3.nano")))
|
||||
|
||||
m, err := daemon.NewJetStreamManager(nc, 1)
|
||||
m, err := daemon.NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, m.InitKVBucket())
|
||||
|
||||
@@ -401,7 +401,7 @@ func TestInstanceStateMigration_ANodeRecoversAnUnownedRunningSet(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
seedNodeBlobBucket(t, nc, localNode, runningSet(unownedVM("i-1", "t3.nano"), unownedVM("i-2", "t3.micro")))
|
||||
|
||||
m, err := daemon.NewJetStreamManager(nc, 1)
|
||||
m, err := daemon.NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, m.InitKVBucket())
|
||||
|
||||
@@ -419,7 +419,7 @@ func TestInstanceStateMigration_LeavesAnOwnedRecordAlone(t *testing.T) {
|
||||
moved := &vm.VM{ID: "i-1", InstanceType: "t3.nano", Status: vm.StateRunning, LastNode: "node-2"}
|
||||
seedNodeBlobBucket(t, nc, localNode, runningSet(moved))
|
||||
|
||||
m, err := daemon.NewJetStreamManager(nc, 1)
|
||||
m, err := daemon.NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, m.InitKVBucket())
|
||||
|
||||
@@ -433,13 +433,13 @@ func TestInstanceStateMigration_OwnershipStampIsSafeToRunTwice(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
kv := seedNodeBlobBucket(t, nc, localNode, runningSet(unownedVM("i-1", "t3.nano")))
|
||||
|
||||
first, err := daemon.NewJetStreamManager(nc, 1)
|
||||
first, err := daemon.NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, first.InitKVBucket())
|
||||
before := recordRevision(t, nc, "i-1")
|
||||
|
||||
require.NoError(t, kvutil.WriteVersion(context.Background(), kv, 4))
|
||||
second, err := daemon.NewJetStreamManager(nc, 1)
|
||||
second, err := daemon.NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, second.InitKVBucket())
|
||||
|
||||
@@ -450,13 +450,13 @@ func TestInstanceStateMigration_BlobSplitIsSafeToRunTwice(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
kv := seedNodeBlobBucket(t, nc, localNode, runningSet(runningVM("i-1", "t3.nano")))
|
||||
|
||||
first, err := daemon.NewJetStreamManager(nc, 1)
|
||||
first, err := daemon.NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, first.InitKVBucket())
|
||||
before := recordRevision(t, nc, "i-1")
|
||||
|
||||
require.NoError(t, kvutil.WriteVersion(context.Background(), kv, 2))
|
||||
second, err := daemon.NewJetStreamManager(nc, 1)
|
||||
second, err := daemon.NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, second.InitKVBucket())
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ func newTestIPSecBarrier(t *testing.T) (*KVIPSecBarrier, *JetStreamManager) {
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(nc.Close)
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitClusterStateBucket())
|
||||
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
@@ -83,7 +82,6 @@ type JetStreamManager struct {
|
||||
records *kvstore.Store[vm.InstanceRecord] // i.<id>, instance-state bucket
|
||||
termRecords *kvstore.Store[vm.InstanceRecord] // i.<id>, terminated bucket
|
||||
clusterKV jetstream.KeyValue // spinifex-cluster-state
|
||||
replicas int
|
||||
obs KVSyncObserver
|
||||
running runningSetState
|
||||
}
|
||||
@@ -117,28 +115,23 @@ func (m *JetStreamManager) KVHealthy() bool {
|
||||
}
|
||||
|
||||
// NewJetStreamManager creates a new JetStreamManager from a NATS connection.
|
||||
// replicas specifies the number of replicas for the KV bucket (typically matches cluster node count).
|
||||
func NewJetStreamManager(nc *nats.Conn, replicas int) (*JetStreamManager, error) {
|
||||
func NewJetStreamManager(nc *nats.Conn) (*JetStreamManager, error) {
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &JetStreamManager{
|
||||
js: js,
|
||||
replicas: replicas,
|
||||
}, nil
|
||||
return &JetStreamManager{js: js}, nil
|
||||
}
|
||||
|
||||
// instanceStateConfig describes the instance-state bucket. Extracted so the
|
||||
// bucket can be rebuilt against a different JetStream client without the
|
||||
// description drifting from the one InitKVBucket creates.
|
||||
func instanceStateConfig(replicas int) kvstore.Config {
|
||||
func instanceStateConfig() kvstore.Config {
|
||||
return kvstore.Config{
|
||||
Name: InstanceStateBucket,
|
||||
Description: "Spinifex instance state storage",
|
||||
History: 1,
|
||||
Replicas: replicas,
|
||||
// The owning node republishes its record on its next write, so an
|
||||
// emptied bucket costs a sync rather than the records themselves.
|
||||
RecreateIfMissing: true,
|
||||
@@ -150,12 +143,11 @@ func instanceStateConfig(replicas int) kvstore.Config {
|
||||
}
|
||||
|
||||
// terminatedInstanceConfig describes the terminated-instances bucket.
|
||||
func terminatedInstanceConfig(replicas int) kvstore.Config {
|
||||
func terminatedInstanceConfig() kvstore.Config {
|
||||
return kvstore.Config{
|
||||
Name: TerminatedInstanceBucket,
|
||||
Description: "Terminated instances (auto-expire after 1 hour)",
|
||||
History: 1,
|
||||
Replicas: replicas,
|
||||
TTL: 1 * time.Hour,
|
||||
RecreateIfMissing: true,
|
||||
OnOpen: func(ctx context.Context, kv jetstream.KeyValue) error {
|
||||
@@ -176,7 +168,7 @@ func (m *JetStreamManager) setInstanceStateBucket(b *kvstore.Bucket) {
|
||||
|
||||
// InitKVBucket initializes the KV bucket, creating it if it doesn't exist.
|
||||
func (m *JetStreamManager) InitKVBucket() error {
|
||||
m.setInstanceStateBucket(kvstore.NewBucket(m.js, instanceStateConfig(m.replicas)))
|
||||
m.setInstanceStateBucket(kvstore.NewBucket(m.js, instanceStateConfig()))
|
||||
|
||||
// Opened eagerly: a bucket that cannot be created must fail startup here,
|
||||
// not on the first write, and Tier 1 boot must not reach cluster KV at all.
|
||||
@@ -185,12 +177,11 @@ func (m *JetStreamManager) InitKVBucket() error {
|
||||
}
|
||||
|
||||
// clusterStateConfig describes the cluster-state bucket.
|
||||
func clusterStateConfig(replicas int) kvstore.Config {
|
||||
func clusterStateConfig() kvstore.Config {
|
||||
return kvstore.Config{
|
||||
Name: ClusterStateBucket,
|
||||
Description: "Spinifex cluster state (heartbeats, shutdown markers, service maps)",
|
||||
History: 1,
|
||||
Replicas: replicas,
|
||||
TTL: 1 * time.Hour,
|
||||
OnOpen: func(ctx context.Context, kv jetstream.KeyValue) error {
|
||||
return migrate.DefaultRegistry.RunKV(ctx, ClusterStateBucket, kv, ClusterStateBucketVersion)
|
||||
@@ -204,7 +195,7 @@ func clusterStateConfig(replicas int) kvstore.Config {
|
||||
// The handle is kept rather than the Bucket: this one holds several unrelated
|
||||
// record types under one namespace, and its callers still read it as raw keys.
|
||||
func (m *JetStreamManager) InitClusterStateBucket() error {
|
||||
kv, err := kvstore.NewBucket(m.js, clusterStateConfig(m.replicas)).KV(context.Background())
|
||||
kv, err := kvstore.NewBucket(m.js, clusterStateConfig()).KV(context.Background())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -215,7 +206,7 @@ func (m *JetStreamManager) InitClusterStateBucket() error {
|
||||
// InitTerminatedInstanceBucket initializes the terminated-instances KV bucket with a 1-hour TTL.
|
||||
// JetStream automatically purges keys after 1 hour, matching AWS behavior for terminated instances.
|
||||
func (m *JetStreamManager) InitTerminatedInstanceBucket() error {
|
||||
m.term = kvstore.New[vm.VM](m.js, terminatedInstanceConfig(m.replicas))
|
||||
m.term = kvstore.New[vm.VM](m.js, terminatedInstanceConfig())
|
||||
// A second view over the same handle, for the same reason the instance-state
|
||||
// bucket carries several.
|
||||
m.termRecords = kvstore.On[vm.InstanceRecord](m.term.Bucket)
|
||||
@@ -521,63 +512,6 @@ func (m *JetStreamManager) DeleteState(nodeID string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateReplicas updates the replica count for ALL JetStream KV buckets.
|
||||
// It iterates over every KV_* stream and bumps replicas to match the cluster size.
|
||||
// This ensures service buckets (IAM, VPC, IGW, etc.) are replicated alongside daemon buckets.
|
||||
// This should be called when new nodes join the cluster.
|
||||
func (m *JetStreamManager) UpdateReplicas(newReplicas int) error {
|
||||
if m.js == nil {
|
||||
return errors.New("JetStream context not initialized")
|
||||
}
|
||||
|
||||
m.replicas = newReplicas
|
||||
|
||||
ctx := context.Background()
|
||||
// Iterate all streams and update any KV-backed stream (prefixed "KV_")
|
||||
updated := 0
|
||||
lister := m.js.StreamNames(ctx)
|
||||
for name := range lister.Name() {
|
||||
if !strings.HasPrefix(name, "KV_") {
|
||||
continue
|
||||
}
|
||||
|
||||
stream, err := m.js.Stream(ctx, name)
|
||||
if err != nil {
|
||||
slog.Warn("Failed to open stream", "stream", name, "error", err)
|
||||
continue
|
||||
}
|
||||
info, err := stream.Info(ctx)
|
||||
if err != nil {
|
||||
slog.Warn("Failed to get stream info", "stream", name, "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if info.Config.Replicas >= newReplicas {
|
||||
continue
|
||||
}
|
||||
|
||||
oldReplicas := info.Config.Replicas
|
||||
info.Config.Replicas = newReplicas
|
||||
if _, err := m.js.UpdateStream(ctx, info.Config); err != nil {
|
||||
slog.Warn("Failed to update KV bucket replicas", "stream", name, "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
bucket := strings.TrimPrefix(name, "KV_")
|
||||
slog.Info("Updated KV bucket replicas", "bucket", bucket, "oldReplicas", oldReplicas, "newReplicas", newReplicas)
|
||||
updated++
|
||||
}
|
||||
if err := lister.Err(); err != nil {
|
||||
return fmt.Errorf("list JetStream streams: %w", err)
|
||||
}
|
||||
|
||||
if updated > 0 {
|
||||
slog.Info("KV replication update complete", "bucketsUpdated", updated, "replicas", newReplicas)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteStoppedInstance writes a stopped instance to the shared KV store,
|
||||
// replacing any existing record for instanceID. Uses CAS internally (bounded
|
||||
// retry) so a write racing a concurrent update is detected and retried
|
||||
|
||||
@@ -40,7 +40,7 @@ func TestJetStreamManager_WriteAndLoadState(t *testing.T) {
|
||||
defer nc.Close()
|
||||
|
||||
// Create JetStreamManager
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err, "Failed to create JetStreamManager")
|
||||
|
||||
// Initialize the KV bucket
|
||||
@@ -90,7 +90,7 @@ func TestJetStreamManager_LoadState_KeyNotFound(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = jsm.InitKVBucket()
|
||||
@@ -112,7 +112,7 @@ func TestJetStreamManager_BucketCreation(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
|
||||
// InitKVBucket should create the bucket
|
||||
@@ -134,7 +134,7 @@ func TestJetStreamManager_BucketReconnection(t *testing.T) {
|
||||
nc1, err := nats.Connect(natsURL)
|
||||
require.NoError(t, err)
|
||||
|
||||
jsm1, err := NewJetStreamManager(nc1, 1)
|
||||
jsm1, err := NewJetStreamManager(nc1)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = jsm1.InitKVBucket()
|
||||
@@ -157,7 +157,7 @@ func TestJetStreamManager_BucketReconnection(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc2.Close()
|
||||
|
||||
jsm2, err := NewJetStreamManager(nc2, 1)
|
||||
jsm2, err := NewJetStreamManager(nc2)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = jsm2.InitKVBucket()
|
||||
@@ -180,7 +180,7 @@ func TestJetStreamManager_DeleteState(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = jsm.InitKVBucket()
|
||||
@@ -222,7 +222,7 @@ func TestJetStreamManager_DeleteState_NonExistent(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = jsm.InitKVBucket()
|
||||
@@ -241,7 +241,7 @@ func TestJetStreamManager_WriteState_UpdateExisting(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = jsm.InitKVBucket()
|
||||
@@ -289,7 +289,7 @@ func TestJetStreamManager_MultipleNodes(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = jsm.InitKVBucket()
|
||||
@@ -337,7 +337,7 @@ func TestJetStreamManager_KVNotInitialized(t *testing.T) {
|
||||
defer nc.Close()
|
||||
|
||||
// Create JetStreamManager but don't call InitKVBucket
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
|
||||
testInstances := make(map[string]*vm.VM)
|
||||
@@ -351,51 +351,6 @@ func TestJetStreamManager_KVNotInitialized(t *testing.T) {
|
||||
assert.Error(t, err, "DeleteState should error when KV not initialized")
|
||||
}
|
||||
|
||||
// TestJetStreamManager_UpdateReplicas tests updating replica count for the KV bucket.
|
||||
func TestJetStreamManager_UpdateReplicas(t *testing.T) {
|
||||
natsURL := sharedJSNATSURL
|
||||
|
||||
nc, err := nats.Connect(natsURL)
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
// Create with 1 replica (typical for single node startup)
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = jsm.InitKVBucket()
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify initial replica count
|
||||
js, _ := jetstream.New(nc)
|
||||
stream, err := js.Stream(t.Context(), "KV_"+InstanceStateBucket)
|
||||
require.NoError(t, err)
|
||||
streamInfo, err := stream.Info(t.Context())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, streamInfo.Config.Replicas, "Should start with 1 replica")
|
||||
|
||||
// Try to update to same replica count (should be a no-op)
|
||||
err = jsm.UpdateReplicas(1)
|
||||
assert.NoError(t, err, "Updating to same replica count should succeed")
|
||||
|
||||
// Note: Increasing replicas beyond 1 requires additional NATS servers in the cluster,
|
||||
// which we don't have in the test environment. In a single-node test server,
|
||||
// attempting to increase replicas will fail with "insufficient resources" error.
|
||||
// This test verifies the basic functionality works.
|
||||
}
|
||||
|
||||
// TestJetStreamManager_UpdateReplicas_NoInit tests UpdateReplicas when JS not initialized.
|
||||
func TestJetStreamManager_UpdateReplicas_NoInit(t *testing.T) {
|
||||
// Test with nil JetStream context
|
||||
jsm := &JetStreamManager{
|
||||
js: nil,
|
||||
replicas: 1,
|
||||
}
|
||||
|
||||
err := jsm.UpdateReplicas(3)
|
||||
assert.Error(t, err, "UpdateReplicas should error when JetStream not initialized")
|
||||
}
|
||||
|
||||
// --- Stopped instance KV tests ---
|
||||
|
||||
// TestJetStreamManager_WriteAndLoadStoppedInstance tests round-trip write and load of a stopped instance.
|
||||
@@ -404,7 +359,7 @@ func TestJetStreamManager_WriteAndLoadStoppedInstance(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
err = jsm.InitKVBucket()
|
||||
require.NoError(t, err)
|
||||
@@ -439,7 +394,7 @@ func TestJetStreamManager_LoadStoppedInstance_NotFound(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
err = jsm.InitKVBucket()
|
||||
require.NoError(t, err)
|
||||
@@ -455,7 +410,7 @@ func TestJetStreamManager_DeleteStoppedInstance(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
err = jsm.InitKVBucket()
|
||||
require.NoError(t, err)
|
||||
@@ -489,7 +444,7 @@ func TestJetStreamManager_ClaimStoppedInstance_HappyPath(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitKVBucket())
|
||||
|
||||
@@ -517,7 +472,7 @@ func TestJetStreamManager_ClaimStoppedInstance_NotFound(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitKVBucket())
|
||||
|
||||
@@ -537,7 +492,7 @@ func TestJetStreamManager_ClaimStoppedInstance_ConcurrentClaim(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitKVBucket())
|
||||
|
||||
@@ -588,7 +543,7 @@ func TestJetStreamManager_UpdateStoppedInstance_ConflictRetry(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitKVBucket())
|
||||
|
||||
@@ -631,7 +586,7 @@ func TestJetStreamManager_UpdateStoppedInstance_NotFound(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitKVBucket())
|
||||
|
||||
@@ -652,7 +607,7 @@ func TestJetStreamManager_UpdateStoppedInstance_NoResurrectAfterClaim(t *testing
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitKVBucket())
|
||||
|
||||
@@ -696,7 +651,7 @@ func TestJetStreamManager_ListStoppedInstances(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
err = jsm.InitKVBucket()
|
||||
require.NoError(t, err)
|
||||
@@ -738,7 +693,7 @@ func TestJetStreamManager_StoppedInstances_NoInterference(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
err = jsm.InitKVBucket()
|
||||
require.NoError(t, err)
|
||||
@@ -778,7 +733,7 @@ func TestJetStreamManager_StoppedInstance_KVNotInitialized(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
// Don't call InitKVBucket
|
||||
|
||||
@@ -803,7 +758,7 @@ func TestJetStreamManager_WriteServiceManifest(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
err = jsm.InitClusterStateBucket()
|
||||
require.NoError(t, err)
|
||||
@@ -840,7 +795,7 @@ func TestJetStreamManager_WriteServiceManifest_EmptyServices(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
err = jsm.InitClusterStateBucket()
|
||||
require.NoError(t, err)
|
||||
@@ -867,7 +822,7 @@ func TestJetStreamManager_WriteServiceManifest_ClusterKVNotInitialized(t *testin
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
// Don't call InitClusterStateBucket
|
||||
|
||||
@@ -895,7 +850,7 @@ func TestJetStreamManager_WriteState_RecoverAfterStreamLost(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
err = jsm.InitKVBucket()
|
||||
require.NoError(t, err)
|
||||
@@ -922,7 +877,7 @@ func TestJetStreamManager_LoadState_RecoverAfterStreamLost(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
err = jsm.InitKVBucket()
|
||||
require.NoError(t, err)
|
||||
@@ -941,7 +896,7 @@ func TestJetStreamManager_DeleteState_RecoverAfterStreamLost(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
err = jsm.InitKVBucket()
|
||||
require.NoError(t, err)
|
||||
@@ -958,7 +913,7 @@ func TestJetStreamManager_WriteStoppedInstance_RecoverAfterStreamLost(t *testing
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
err = jsm.InitKVBucket()
|
||||
require.NoError(t, err)
|
||||
@@ -984,7 +939,7 @@ func TestJetStreamManager_LoadStoppedInstance_RecoverAfterStreamLost(t *testing.
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
err = jsm.InitKVBucket()
|
||||
require.NoError(t, err)
|
||||
@@ -1002,7 +957,7 @@ func TestJetStreamManager_DeleteStoppedInstance_RecoverAfterStreamLost(t *testin
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
err = jsm.InitKVBucket()
|
||||
require.NoError(t, err)
|
||||
@@ -1018,7 +973,7 @@ func TestJetStreamManager_ListStoppedInstances_RecoverAfterStreamLost(t *testing
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
err = jsm.InitKVBucket()
|
||||
require.NoError(t, err)
|
||||
@@ -1055,12 +1010,12 @@ func swapToNonJSContext(t *testing.T, jsm *JetStreamManager) {
|
||||
if jsm.stateB != nil {
|
||||
kv, err := jsm.stateB.KV(t.Context())
|
||||
require.NoError(t, err)
|
||||
jsm.setInstanceStateBucket(kvstore.NewOpenBucket(js, kv, instanceStateConfig(jsm.replicas)))
|
||||
jsm.setInstanceStateBucket(kvstore.NewOpenBucket(js, kv, instanceStateConfig()))
|
||||
}
|
||||
if jsm.term != nil {
|
||||
kv, err := jsm.term.KV(t.Context())
|
||||
require.NoError(t, err)
|
||||
jsm.term = kvstore.Over[vm.VM](js, kv, terminatedInstanceConfig(jsm.replicas))
|
||||
jsm.term = kvstore.Over[vm.VM](js, kv, terminatedInstanceConfig())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1069,7 +1024,7 @@ func TestJetStreamManager_WriteState_RecoveryFailure(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
err = jsm.InitKVBucket()
|
||||
require.NoError(t, err)
|
||||
@@ -1087,7 +1042,7 @@ func TestJetStreamManager_LoadState_RecoveryFailure(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
err = jsm.InitKVBucket()
|
||||
require.NoError(t, err)
|
||||
@@ -1105,7 +1060,7 @@ func TestJetStreamManager_DeleteState_RecoveryFailure(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
err = jsm.InitKVBucket()
|
||||
require.NoError(t, err)
|
||||
@@ -1122,7 +1077,7 @@ func TestJetStreamManager_WriteStoppedInstance_RecoveryFailure(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
err = jsm.InitKVBucket()
|
||||
require.NoError(t, err)
|
||||
@@ -1140,7 +1095,7 @@ func TestJetStreamManager_LoadStoppedInstance_RecoveryFailure(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
err = jsm.InitKVBucket()
|
||||
require.NoError(t, err)
|
||||
@@ -1158,7 +1113,7 @@ func TestJetStreamManager_DeleteStoppedInstance_RecoveryFailure(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
err = jsm.InitKVBucket()
|
||||
require.NoError(t, err)
|
||||
@@ -1175,7 +1130,7 @@ func TestJetStreamManager_ListStoppedInstances_RecoveryFailure(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
err = jsm.InitKVBucket()
|
||||
require.NoError(t, err)
|
||||
@@ -1195,7 +1150,7 @@ func TestJetStreamManager_WriteAndLoadTerminatedInstance(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitTerminatedInstanceBucket())
|
||||
|
||||
@@ -1225,7 +1180,7 @@ func TestJetStreamManager_LoadTerminatedInstance_NotFound(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitTerminatedInstanceBucket())
|
||||
|
||||
@@ -1239,7 +1194,7 @@ func TestJetStreamManager_DeleteTerminatedInstance(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitTerminatedInstanceBucket())
|
||||
|
||||
@@ -1268,7 +1223,7 @@ func TestJetStreamManager_ListTerminatedInstances(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitTerminatedInstanceBucket())
|
||||
|
||||
@@ -1304,7 +1259,7 @@ func TestJetStreamManager_WriteLoadDeleteTerminatedInstance_RoundTrip(t *testing
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitTerminatedInstanceBucket())
|
||||
|
||||
@@ -1336,7 +1291,7 @@ func TestJetStreamManager_TerminatedInstance_KVNotInitialized(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
// Don't call InitTerminatedInstanceBucket
|
||||
|
||||
@@ -1365,7 +1320,7 @@ func TestJetStreamManager_UpdateTerminatedInstance_ConflictRetry(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitTerminatedInstanceBucket())
|
||||
|
||||
@@ -1414,7 +1369,7 @@ func TestJetStreamManager_UpdateTerminatedInstance_NotFound(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitTerminatedInstanceBucket())
|
||||
|
||||
@@ -1432,7 +1387,7 @@ func TestJetStreamManager_WriteStoppedInstance_OverwritesConcurrentValue(t *test
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitKVBucket())
|
||||
|
||||
@@ -1470,7 +1425,7 @@ func TestJetStreamManager_WriteTerminatedInstance_RecoverAfterStreamLost(t *test
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitTerminatedInstanceBucket())
|
||||
|
||||
@@ -1493,7 +1448,7 @@ func TestJetStreamManager_LoadTerminatedInstance_RecoverAfterStreamLost(t *testi
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitTerminatedInstanceBucket())
|
||||
|
||||
@@ -1509,7 +1464,7 @@ func TestJetStreamManager_DeleteTerminatedInstance_RecoverAfterStreamLost(t *tes
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitTerminatedInstanceBucket())
|
||||
|
||||
@@ -1524,7 +1479,7 @@ func TestJetStreamManager_ListTerminatedInstances_RecoverAfterStreamLost(t *test
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitTerminatedInstanceBucket())
|
||||
|
||||
@@ -1540,7 +1495,7 @@ func TestJetStreamManager_InitBuckets_WritesVersion(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, jsm.InitKVBucket())
|
||||
@@ -1570,7 +1525,7 @@ func TestJetStreamManager_ListStoppedInstances_SkipsVersionKey(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitKVBucket())
|
||||
|
||||
@@ -1585,7 +1540,7 @@ func TestJetStreamManager_ListTerminatedInstances_SkipsVersionKey(t *testing.T)
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitTerminatedInstanceBucket())
|
||||
|
||||
@@ -1632,7 +1587,7 @@ func TestJetStreamManager_BestEffort_Success_NotifiesObserver(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitKVBucket())
|
||||
|
||||
@@ -1651,7 +1606,7 @@ func TestJetStreamManager_BestEffort_PutError_NotifiesObserver(t *testing.T) {
|
||||
nc, err := nats.Connect(sharedJSNATSURL)
|
||||
require.NoError(t, err)
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitKVBucket())
|
||||
|
||||
@@ -1675,7 +1630,7 @@ func TestJetStreamManager_BestEffort_NilObserver_NoPanic(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitKVBucket())
|
||||
|
||||
@@ -1696,7 +1651,7 @@ func TestJetStreamManager_UpdateMgmtIPAM_ConflictRetry(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitClusterStateBucket())
|
||||
|
||||
@@ -1746,7 +1701,7 @@ func TestJetStreamManager_UpdateMgmtIPAM_NotFound(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitClusterStateBucket())
|
||||
|
||||
@@ -1761,7 +1716,7 @@ func TestJetStreamManager_UpdateMgmtIPAM_ClusterKVNotInitialized(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
// Don't call InitClusterStateBucket
|
||||
|
||||
@@ -1781,7 +1736,7 @@ func TestJetStreamManager_UpdateMgmtIPAM_ClusterKVNotInitialized(t *testing.T) {
|
||||
func TestJetStreamManager_ListStoppedInstances_FailsOnAnUndecodableRecord(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitKVBucket())
|
||||
|
||||
@@ -1806,7 +1761,7 @@ func TestJetStreamManager_UpdateStoppedInstance_AbsentKeyIsNotFound(t *testing.T
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitKVBucket())
|
||||
require.NoError(t, jsm.InitTerminatedInstanceBucket())
|
||||
@@ -1842,7 +1797,7 @@ func TestJetStreamManager_LoadState_UnversionedRecordStillReads(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitKVBucket())
|
||||
|
||||
@@ -1867,7 +1822,7 @@ func TestJetStreamManager_LoadState_FutureRecordIsRefused(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitKVBucket())
|
||||
|
||||
@@ -1887,7 +1842,7 @@ func TestJetStreamManager_WriteState_StampsSchemaVersion(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, jsm.InitKVBucket())
|
||||
|
||||
|
||||
@@ -55,10 +55,17 @@ func (d *Daemon) monitorPeerReachability() {
|
||||
return
|
||||
}
|
||||
|
||||
// Without the CA every probe would fail, flipping the node to partitioned;
|
||||
// leave peersReachable at its initial true instead.
|
||||
rootCAs, err := loadClusterTrustRoot(d)
|
||||
if err != nil {
|
||||
slog.Error("peer reachability probe disabled: could not load cluster CA", "err", err)
|
||||
return
|
||||
}
|
||||
client := &http.Client{
|
||||
Timeout: peerProbeTimeout,
|
||||
Transport: &http.Transport{
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec // self-signed per-node certs (matches harness DaemonClient)
|
||||
TLSClientConfig: &tls.Config{RootCAs: rootCAs},
|
||||
},
|
||||
}
|
||||
defer client.CloseIdleConnections()
|
||||
|
||||
@@ -2,11 +2,20 @@ package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/pem"
|
||||
"math/big"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -20,8 +29,9 @@ import (
|
||||
// /health responses, returning the cluster config the daemon-under-test should
|
||||
// see plus a handle to flip each peer up or down at runtime.
|
||||
type peerHealthFixture struct {
|
||||
peers []*peerStub
|
||||
cfg *config.ClusterConfig
|
||||
peers []*peerStub
|
||||
cfg *config.ClusterConfig
|
||||
caPath string
|
||||
}
|
||||
|
||||
type peerStub struct {
|
||||
@@ -61,6 +71,7 @@ func newPeerHealthFixture(t *testing.T, peerCount int) *peerHealthFixture {
|
||||
Daemon: config.DaemonConfig{Host: "0.0.0.0:" + port},
|
||||
}
|
||||
f.peers = append(f.peers, stub)
|
||||
f.caPath = writeServerCA(t, stub.srv)
|
||||
}
|
||||
f.cfg = &config.ClusterConfig{Node: "node-self", Nodes: nodes}
|
||||
return f
|
||||
@@ -76,6 +87,40 @@ func daemonForPeerHealth(t *testing.T, cfg *config.ClusterConfig) (*Daemon, cont
|
||||
return d, cancel
|
||||
}
|
||||
|
||||
// writeServerCA writes srv's self-signed cert as a PEM CA file, standing in for
|
||||
// the cluster CA. Every httptest TLS server shares that one cert.
|
||||
func writeServerCA(t *testing.T, srv *httptest.Server) string {
|
||||
t.Helper()
|
||||
return writePEMCert(t, srv.Certificate().Raw)
|
||||
}
|
||||
|
||||
// writeUntrustedCA writes a freshly generated self-signed CA that signed none
|
||||
// of the httptest servers' certs.
|
||||
func writeUntrustedCA(t *testing.T) string {
|
||||
t.Helper()
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
require.NoError(t, err)
|
||||
tmpl := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(1),
|
||||
Subject: pkix.Name{CommonName: "untrusted test CA"},
|
||||
NotBefore: time.Now().Add(-time.Hour),
|
||||
NotAfter: time.Now().Add(time.Hour),
|
||||
IsCA: true,
|
||||
BasicConstraintsValid: true,
|
||||
KeyUsage: x509.KeyUsageCertSign,
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
|
||||
require.NoError(t, err)
|
||||
return writePEMCert(t, der)
|
||||
}
|
||||
|
||||
func writePEMCert(t *testing.T, der []byte) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "ca.pem")
|
||||
require.NoError(t, os.WriteFile(path, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600))
|
||||
return path
|
||||
}
|
||||
|
||||
func TestPeerCount_SingleNode(t *testing.T) {
|
||||
cfg := &config.ClusterConfig{
|
||||
Node: "only",
|
||||
@@ -194,6 +239,7 @@ func TestMonitorPeerReachability_EndToEnd(t *testing.T) {
|
||||
}
|
||||
f := newPeerHealthFixture(t, 2)
|
||||
d, cancel := daemonForPeerHealth(t, f.cfg)
|
||||
d.config = &config.Config{NATS: config.NATSConfig{CACert: f.caPath}}
|
||||
|
||||
for _, p := range f.peers {
|
||||
p.setHealthy(false)
|
||||
@@ -211,3 +257,47 @@ func TestMonitorPeerReachability_EndToEnd(t *testing.T) {
|
||||
peerProbeInterval+peerProbeTimeout+time.Second, 50*time.Millisecond,
|
||||
"peersReachable should flip true when any peer comes back")
|
||||
}
|
||||
|
||||
// TestMonitorPeerReachability_UntrustedPeerCert asserts a peer whose cert the
|
||||
// cluster CA did not sign counts as unreachable, even while serving 200.
|
||||
func TestMonitorPeerReachability_UntrustedPeerCert(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping ticker-driven peer probe test in short mode")
|
||||
}
|
||||
f := newPeerHealthFixture(t, 1)
|
||||
d, cancel := daemonForPeerHealth(t, f.cfg)
|
||||
d.config = &config.Config{NATS: config.NATSConfig{CACert: writeUntrustedCA(t)}}
|
||||
d.peersReachable.Store(true)
|
||||
|
||||
go d.monitorPeerReachability()
|
||||
defer cancel()
|
||||
|
||||
require.Eventually(t, func() bool { return !d.peersReachable.Load() },
|
||||
peerProbeTimeout+time.Second, 50*time.Millisecond,
|
||||
"a peer presenting an untrusted cert must not count as reachable")
|
||||
}
|
||||
|
||||
// TestMonitorPeerReachability_NoCAKeepsPeersReachable asserts a node that
|
||||
// cannot load the cluster CA stops probing rather than declaring every peer
|
||||
// unreachable and dropping out of cluster mode.
|
||||
func TestMonitorPeerReachability_NoCAKeepsPeersReachable(t *testing.T) {
|
||||
f := newPeerHealthFixture(t, 2)
|
||||
for _, p := range f.peers {
|
||||
p.setHealthy(false)
|
||||
}
|
||||
d, _ := daemonForPeerHealth(t, f.cfg)
|
||||
d.config = &config.Config{}
|
||||
d.peersReachable.Store(true)
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
d.monitorPeerReachability()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("monitorPeerReachability kept running without a cluster CA")
|
||||
}
|
||||
assert.True(t, d.peersReachable.Load())
|
||||
}
|
||||
|
||||
@@ -26,10 +26,46 @@ func TestCheckPredastoreReady_HTTPSServing(t *testing.T) {
|
||||
u, err := url.Parse(srv.URL)
|
||||
require.NoError(t, err)
|
||||
|
||||
d := &Daemon{config: &config.Config{Predastore: config.PredastoreConfig{Host: u.Host}}}
|
||||
d := &Daemon{config: &config.Config{
|
||||
Predastore: config.PredastoreConfig{Host: u.Host},
|
||||
NATS: config.NATSConfig{CACert: writeServerCA(t, srv)},
|
||||
}}
|
||||
assert.True(t, d.checkPredastoreReady())
|
||||
}
|
||||
|
||||
// TestCheckPredastoreReady_UntrustedCert asserts an endpoint whose cert the
|
||||
// cluster CA did not sign is not ready, even though it answers HTTPS.
|
||||
func TestCheckPredastoreReady_UntrustedCert(t *testing.T) {
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
u, err := url.Parse(srv.URL)
|
||||
require.NoError(t, err)
|
||||
|
||||
d := &Daemon{config: &config.Config{
|
||||
Predastore: config.PredastoreConfig{Host: u.Host},
|
||||
NATS: config.NATSConfig{CACert: writeUntrustedCA(t)},
|
||||
}}
|
||||
assert.False(t, d.checkPredastoreReady())
|
||||
}
|
||||
|
||||
// TestCheckPredastoreReady_NoClusterCA asserts a node without a configured
|
||||
// cluster CA fails closed instead of probing unverified.
|
||||
func TestCheckPredastoreReady_NoClusterCA(t *testing.T) {
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
u, err := url.Parse(srv.URL)
|
||||
require.NoError(t, err)
|
||||
|
||||
d := &Daemon{config: &config.Config{Predastore: config.PredastoreConfig{Host: u.Host}}}
|
||||
assert.False(t, d.checkPredastoreReady())
|
||||
}
|
||||
|
||||
// TestCheckPredastoreReady_ClosedPort asserts a port nothing is listening on
|
||||
// is reported not-ready, rather than panicking or failing open.
|
||||
func TestCheckPredastoreReady_ClosedPort(t *testing.T) {
|
||||
|
||||
@@ -31,7 +31,7 @@ func TestClusterShutdownStateKVRoundTrip(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer nc.Close()
|
||||
|
||||
jsm, err := NewJetStreamManager(nc, 1)
|
||||
jsm, err := NewJetStreamManager(nc)
|
||||
require.NoError(t, err)
|
||||
err = jsm.InitClusterStateBucket()
|
||||
require.NoError(t, err)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user