diff --git a/scripts/images/eks-node/k3s.initd b/scripts/images/eks-node/k3s.initd index b48ea792e..8a9049dc5 100755 --- a/scripts/images/eks-node/k3s.initd +++ b/scripts/images/eks-node/k3s.initd @@ -1,9 +1,7 @@ #!/sbin/openrc-run -# K3s server control plane. v1 = single-node embedded SQLite (kine) datastore; -# the per-cluster config.yaml omits cluster-init so k3s uses SQLite, not etcd -# (etcd's per-commit fsync stalls on the viperblock root volume). Data lives at -# /var/lib/rancher/k3s; subsequent starts reuse it. Multi-server HA needs etcd -# and is deferred. +# K3s server control plane. The per-cluster config.yaml sets cluster-init, so +# k3s uses the embedded etcd datastore (required for multi-server HA). Data +# lives at /var/lib/rancher/k3s; subsequent starts reuse it. description="K3s Kubernetes server (Mulga EKS control plane)" command="/usr/local/bin/k3s" @@ -13,8 +11,8 @@ pidfile="/run/k3s.pid" output_log="/var/log/k3s.log" error_log="/var/log/k3s.log" -# Generous start timeout — K3s bootstraps containerd, initialises the SQLite -# datastore, pulls system images. First boot can take 2-3 minutes on cold cache. +# Generous start timeout — K3s bootstraps containerd, initialises the embedded +# etcd datastore, pulls system images. First boot can take 2-3 minutes on cold cache. start_stop_daemon_args="--wait 5000" depend() { diff --git a/scripts/images/eks-node/manifest.conf b/scripts/images/eks-node/manifest.conf index 6d1e9d13d..a4c7aaab7 100644 --- a/scripts/images/eks-node/manifest.conf +++ b/scripts/images/eks-node/manifest.conf @@ -3,7 +3,7 @@ IMAGE_DESCRIPTION="Unified EKS K3s node VM — Alpine + K3s v1.32.5+k3s1 + eks-t DISTRO="alpine" ALPINE_VERSION="3.21.7" -# 4G — server superset (K3s server binary + SQLite datastore working set under +# 4G — server superset (K3s server binary + embedded-etcd datastore working set under # /var/lib/rancher/k3s on the root disk). Agents use the same image; the ~10MB # of unused server binary + webhook is dwarfed by the headroom for node images. IMAGE_SIZE="4G" diff --git a/scripts/images/eks-node/setup.sh b/scripts/images/eks-node/setup.sh index d0dc65088..9fe3e3390 100755 --- a/scripts/images/eks-node/setup.sh +++ b/scripts/images/eks-node/setup.sh @@ -72,7 +72,9 @@ mkdir -p /etc/rancher/k3s cat > /etc/rancher/k3s/config.yaml.skel <<'EOF' # Populated at first boot by cloud-init user-data via k3s-first-boot.sh. # Fields documented at https://docs.k3s.io/installation/configuration. -# No cluster-init: single-node v1 uses the embedded SQLite datastore, not etcd. +# cluster-init selects the embedded etcd datastore. cloud-init write_files +# normally overrides this skeleton; kept consistent as the boot fallback. +cluster-init: true disable: - traefik - servicelb diff --git a/spinifex/handlers/ec2/instance/helpers_test.go b/spinifex/handlers/ec2/instance/helpers_test.go index 782272d7d..11687ee04 100644 --- a/spinifex/handlers/ec2/instance/helpers_test.go +++ b/spinifex/handlers/ec2/instance/helpers_test.go @@ -10,24 +10,24 @@ import ( // --- generateNetworkConfig --- func TestGenerateNetworkConfig_BothEmpty(t *testing.T) { - cfg := generateNetworkConfig("", "", "", "", nil) + cfg := generateNetworkConfig("", "", "", "", nil, true) assert.Equal(t, cloudInitNetworkConfigWildcard, cfg) } func TestGenerateNetworkConfig_OneEmpty(t *testing.T) { - cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "", "", "", nil) + cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "", "", "", nil, true) assert.Contains(t, cfg, "vpc0:", "eniMAC alone should produce per-interface config") assert.NotContains(t, cfg, "dev0:", "no dev NIC without devMAC") // The IMDS on-link route rides vpc0 even with no dev NIC / extra ENIs — the // common production shape — so guard it here too, not only on the multi-NIC path. assert.Contains(t, cfg, "to: 169.254.169.254/32", "vpc0 must carry the IMDS on-link route") - cfg = generateNetworkConfig("", "02:00:00:dd:ee:ff", "", "", nil) + cfg = generateNetworkConfig("", "02:00:00:dd:ee:ff", "", "", nil, true) assert.Equal(t, cloudInitNetworkConfigWildcard, cfg, "should fall back to wildcard if eniMAC empty") } func TestGenerateNetworkConfig_DualNIC(t *testing.T) { - cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:00:00:dd:ee:ff", "", "", nil) + cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:00:00:dd:ee:ff", "", "", nil, true) assert.Contains(t, cfg, "version: 2") assert.Contains(t, cfg, `macaddress: "02:00:00:aa:bb:cc"`) assert.Contains(t, cfg, `macaddress: "02:00:00:dd:ee:ff"`) @@ -39,7 +39,7 @@ func TestGenerateNetworkConfig_DualNIC(t *testing.T) { } func TestGenerateNetworkConfig_TripleNIC(t *testing.T) { - cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:de:00:dd:ee:ff", "02:a0:00:11:22:33", "10.15.8.101", nil) + cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:de:00:dd:ee:ff", "02:a0:00:11:22:33", "10.15.8.101", nil, true) assert.Contains(t, cfg, "version: 2") assert.Contains(t, cfg, `macaddress: "02:00:00:aa:bb:cc"`) assert.Contains(t, cfg, `macaddress: "02:de:00:dd:ee:ff"`) @@ -53,7 +53,7 @@ func TestGenerateNetworkConfig_TripleNIC(t *testing.T) { func TestGenerateNetworkConfig_MgmtWithoutDev(t *testing.T) { // System instances: eniMAC + mgmtMAC, no devMAC — should get per-interface config with mgmt NIC - cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "", "02:a0:00:11:22:33", "10.15.8.101", nil) + cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "", "02:a0:00:11:22:33", "10.15.8.101", nil, true) assert.Contains(t, cfg, "vpc0:") assert.NotContains(t, cfg, "dev0:", "no dev NIC without devMAC") assert.Contains(t, cfg, "mgmt0:") @@ -62,12 +62,12 @@ func TestGenerateNetworkConfig_MgmtWithoutDev(t *testing.T) { } func TestGenerateNetworkConfig_MgmtMACWithoutIP(t *testing.T) { - cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:de:00:dd:ee:ff", "02:a0:00:11:22:33", "", nil) + cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:de:00:dd:ee:ff", "02:a0:00:11:22:33", "", nil, true) assert.NotContains(t, cfg, "mgmt0:", "mgmt NIC should not appear without IP") } func TestGenerateNetworkConfig_MgmtIPWithoutMAC(t *testing.T) { - cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:de:00:dd:ee:ff", "", "10.15.8.101", nil) + cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:de:00:dd:ee:ff", "", "10.15.8.101", nil, true) assert.NotContains(t, cfg, "mgmt0:", "mgmt NIC should not appear without MAC") } @@ -75,7 +75,7 @@ func TestGenerateNetworkConfig_IMDSOnLinkRoute(t *testing.T) { // The primary VPC NIC carries an on-link route to the IMDS metadata IP so the // guest ARPs 169.254.169.254 directly; the per-subnet localport answers. cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:00:00:dd:ee:ff", "", "", - []string{"02:00:00:bb:bb:bb"}) + []string{"02:00:00:bb:bb:bb"}, true) assert.Contains(t, cfg, "to: 169.254.169.254/32") assert.Contains(t, cfg, "scope: link") // Only the primary NIC (vpc0) gets it — not extra VPC NICs or the dev NIC, @@ -83,5 +83,41 @@ func TestGenerateNetworkConfig_IMDSOnLinkRoute(t *testing.T) { assert.Equal(t, 1, strings.Count(cfg, "169.254.169.254/32")) } +func TestGenerateNetworkConfig_IMDSRouteOmittedWhenDisabled(t *testing.T) { + // Alpine's eni renderer crashes on a gateway-less route, so the route is + // omitted from network-config and delivered out-of-band (see + // buildAlpineCloudInit). emitIMDSRoute=false must drop it entirely. + cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:00:00:dd:ee:ff", "", "", nil, false) + assert.Contains(t, cfg, "vpc0:") + assert.NotContains(t, cfg, "169.254.169.254/32", "route must not render when emitIMDSRoute=false") + assert.NotContains(t, cfg, " routes:", "no vpc0 routes block when the only route is the IMDS one") +} + +// selectNetworkConfigForFamily must omit the IMDS route for Alpine (eni renderer +// can't handle the gateway-less route) but keep it for everyone else. +func TestSelectNetworkConfigForFamily_AlpineOmitsIMDSRoute(t *testing.T) { + alpine := selectNetworkConfigForFamily("alpine", "02:00:00:aa:bb:cc", "02:00:00:dd:ee:ff", "", "", nil) + assert.NotContains(t, alpine, "169.254.169.254/32", "Alpine network-config must not carry the IMDS route") + + deb := selectNetworkConfigForFamily("debian", "02:00:00:aa:bb:cc", "02:00:00:dd:ee:ff", "", "", nil) + assert.Contains(t, deb, "169.254.169.254/32", "non-Alpine families keep the netplan IMDS route") +} + +// buildAlpineCloudInit delivers the IMDS on-link route via a persistent +// /etc/local.d script + OpenRC local service, since the eni renderer can't. +func TestBuildAlpineCloudInit(t *testing.T) { + wf, rc := buildAlpineCloudInit("02:00:00:aa:bb:cc") + assert.Contains(t, wf, "/etc/local.d/imds-onlink-route.start") + assert.Contains(t, wf, "ip route show default") + assert.Contains(t, wf, `ip route replace 169.254.169.254/32 dev "$dev" scope link`) + assert.Contains(t, rc, "rc-update") + assert.Contains(t, rc, "local") + + // No VPC NIC → nothing to add. + wf, rc = buildAlpineCloudInit("") + assert.Empty(t, wf) + assert.Empty(t, rc) +} + // Route for multi-node LB mgmt traffic is handled via the fw_cfg netcfg // blob delivered to the LB microVM, not in the network-config. diff --git a/spinifex/handlers/ec2/instance/service_impl.go b/spinifex/handlers/ec2/instance/service_impl.go index 370b7e5f3..80e881d10 100644 --- a/spinifex/handlers/ec2/instance/service_impl.go +++ b/spinifex/handlers/ec2/instance/service_impl.go @@ -64,19 +64,21 @@ ca_certs: {{.UserDataCloudConfig}} {{end}} -{{if or .RHELWriteFiles .UserDataScript}} +{{if or .RHELWriteFiles .AlpineWriteFiles .UserDataScript}} write_files: {{- if .RHELWriteFiles}} -{{.RHELWriteFiles}}{{end}}{{if .UserDataScript}} +{{.RHELWriteFiles}}{{end}}{{if .AlpineWriteFiles}} +{{.AlpineWriteFiles}}{{end}}{{if .UserDataScript}} - path: /tmp/cloud-init-startup.sh permissions: '0755' content: | {{.UserDataScript}}{{end}} {{end}} -{{if or .RHELRunCmd .UserDataScript}} +{{if or .RHELRunCmd .AlpineRunCmd .UserDataScript}} runcmd: {{- if .RHELRunCmd}} -{{.RHELRunCmd}}{{end}}{{if .UserDataScript}} +{{.RHELRunCmd}}{{end}}{{if .AlpineRunCmd}} +{{.AlpineRunCmd}}{{end}}{{if .UserDataScript}} - [ "/bin/bash", "/tmp/cloud-init-startup.sh" ] {{end}} {{end}} @@ -118,7 +120,11 @@ func selectNetworkConfigForFamily(distroFamily, eniMAC, devMAC, mgmtMAC, mgmtIP if distroFamily == "rhel" { return cloudInitNetworkConfigDisabled } - return generateNetworkConfig(eniMAC, devMAC, mgmtMAC, mgmtIP, extraENIMACs) + // Alpine renders via cloud-init's eni backend, which cannot emit a + // gateway-less route; the IMDS on-link route is delivered via + // buildAlpineCloudInit instead. Debian/Ubuntu render via netplan, which + // handles the on-link route natively. + return generateNetworkConfig(eniMAC, devMAC, mgmtMAC, mgmtIP, extraENIMACs, distroFamily != "alpine") } // generateNetworkConfig produces the cloud-init network-config for the instance. @@ -142,7 +148,7 @@ func selectNetworkConfigForFamily(distroFamily, eniMAC, devMAC, mgmtMAC, mgmtIP // route is reached via the primary ENI, matching AWS; it is not delivered via // DHCP option 121 (which would force the default route into option 121 per // RFC 3442) and the default gateway stays option 3. -func generateNetworkConfig(eniMAC, devMAC, mgmtMAC, mgmtIP string, extraENIMACs []string) string { +func generateNetworkConfig(eniMAC, devMAC, mgmtMAC, mgmtIP string, extraENIMACs []string, emitIMDSRoute bool) string { if eniMAC == "" { return cloudInitNetworkConfigWildcard } @@ -154,10 +160,17 @@ func generateNetworkConfig(eniMAC, devMAC, mgmtMAC, mgmtIP string, extraENIMACs macaddress: "%s" dhcp4: true dhcp-identifier: mac - routes: +`, eniMAC) + // The gateway-less IMDS on-link route renders only under netplan. cloud-init's + // eni renderer (Alpine) crashes on a route with no gateway, taking the whole + // network-config down with it, so eni-rendered guests get this route via + // buildAlpineCloudInit instead (emitIMDSRoute=false). + if emitIMDSRoute { + cfg += fmt.Sprintf(` routes: - to: %s/32 scope: link -`, eniMAC, handlers_imds.MetaDataServerIP) +`, handlers_imds.MetaDataServerIP) + } for i, mac := range extraENIMACs { if mac == "" { @@ -220,6 +233,12 @@ type CloudInitData struct { // + nmcli reload/up) needed to load and bring up the keyfiles. Empty on // non-RHEL guests. RHELRunCmd string + // AlpineWriteFiles / AlpineRunCmd carry the IMDS on-link route delivery for + // Alpine guests (a persistent /etc/local.d script + OpenRC `local` enable), + // since the eni renderer cannot emit the gateway-less route in the + // network-config. Empty on non-Alpine guests and on non-VPC instances. + AlpineWriteFiles string + AlpineRunCmd string } // buildRHELCloudInit produces the cloud-init write_files (NM keyfiles) and @@ -322,6 +341,43 @@ func appendRHELStaticKeyfile(b *strings.Builder, name, mac, ip string) { b.WriteString(" method=disabled\n") } +// buildAlpineCloudInit produces the cloud-init write_files + runcmd blocks that +// deliver the IMDS on-link route on Alpine guests. Returns empty strings when +// eniMAC is empty (wildcard / non-VPC path). +// +// Alpine renders networking via cloud-init's eni backend, which crashes on a +// gateway-less route (it assumes every route has a next hop), so the route is +// omitted from the netplan network-config (see generateNetworkConfig) and added +// here instead. A persistent /etc/local.d script keeps the on-link route present +// across reboots — runcmd runs once per instance, so the OpenRC `local` service +// re-applies it on every boot. cloud-init's eni renderer renames the primary +// VPC NIC to vpc0 via a 70-persistent-net.rules udev rule, so the device name is +// stable. `ip route replace` is idempotent. +func buildAlpineCloudInit(eniMAC string) (writeFiles, runCmd string) { + if eniMAC == "" { + return "", "" + } + + var wf strings.Builder + wf.WriteString(" - path: /etc/local.d/imds-onlink-route.start\n") + wf.WriteString(" owner: root:root\n") + wf.WriteString(" permissions: '0755'\n") + wf.WriteString(" content: |\n") + wf.WriteString(" #!/bin/sh\n") + // Resolve the primary VPC NIC via its default route, not by name: the + // persistent-net udev rename to vpc0 doesn't apply on the Alpine AMI, so the + // kernel name is eth0 at first boot. The mgmt NIC (eth1) carries no default + // route, so this always picks the VPC egress NIC — the one IMDS rides. + wf.WriteString(" dev=$(ip route show default | awk '{print $5; exit}')\n") + fmt.Fprintf(&wf, " [ -n \"$dev\" ] && ip route replace %s/32 dev \"$dev\" scope link\n", handlers_imds.MetaDataServerIP) + + var rc strings.Builder + rc.WriteString(" - [ rc-update, add, local, default ]\n") + rc.WriteString(" - [ rc-service, local, start ]\n") + + return strings.TrimRight(wf.String(), "\n"), strings.TrimRight(rc.String(), "\n") +} + type CloudInitMetaData struct { InstanceID string Hostname string @@ -1871,7 +1927,8 @@ func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, i sudoGroup := "sudo" var rhelWriteFiles, rhelRunCmd string - if distroFamily == "rhel" { + switch distroFamily { + case "rhel": sudoGroup = "wheel" rhelWriteFiles, rhelRunCmd = buildRHELCloudInit(instance.ENIMac, instance.DevMAC, instance.MgmtMAC, instance.MgmtIP, extraMACs) } @@ -1909,6 +1966,15 @@ func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, i } } + // Alpine's eni renderer can't emit the gateway-less IMDS route, so deliver it + // out-of-band via a local.d script. Only when the caller didn't supply its own + // #cloud-config: that payload owns the write_files/runcmd keys (e.g. the EKS + // k3s server VM, which bakes the same route into its own block), and a second + // write_files key here would collide — last key wins, silently dropping a block. + if distroFamily == "alpine" && userData.UserDataCloudConfig == "" { + userData.AlpineWriteFiles, userData.AlpineRunCmd = buildAlpineCloudInit(instance.ENIMac) + } + var buf bytes.Buffer t := template.Must(template.New("cloud-init").Parse(cloudInitUserDataTemplate)) @@ -2155,6 +2221,16 @@ func (s *InstanceServiceImpl) DescribeInstances(input *ec2.DescribeInstancesInpu if !IsInstanceVisible(accountID, instance.AccountID) { continue } + // Platform-managed system VMs (LB HAProxy, EKS control plane) carry a + // ManagedBy tag and must stay out of customer EC2 listings, the same + // way they're filtered from the UI Nodes page. LB VMs are owned by the + // system account so IsInstanceVisible already hides them, but the EKS + // control-plane VM is owned by the customer account (its ENI lives in + // the customer VPC), so guard on ManagedBy here. Root/operator callers + // still see system instances. + if instance.ManagedBy != "" && accountID != utils.GlobalAccountID { + continue + } if len(instanceIDFilter) > 0 && !instanceIDFilter[instance.ID] { continue } diff --git a/spinifex/handlers/ec2/instance/service_impl_test.go b/spinifex/handlers/ec2/instance/service_impl_test.go index b2da93053..fe26bbe19 100644 --- a/spinifex/handlers/ec2/instance/service_impl_test.go +++ b/spinifex/handlers/ec2/instance/service_impl_test.go @@ -15,6 +15,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/config" "github.com/mulgadc/spinifex/spinifex/gpu" + "github.com/mulgadc/spinifex/spinifex/tags" spxtypes "github.com/mulgadc/spinifex/spinifex/types" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/mulgadc/spinifex/spinifex/vm" @@ -662,7 +663,7 @@ func TestParseVolumeParams_PartialEbs(t *testing.T) { func TestCloudInitNetworkConfigWildcard(t *testing.T) { // No MACs → wildcard config (non-VPC or VPC without DEV_NETWORKING) - cfg := generateNetworkConfig("", "", "", "", nil) + cfg := generateNetworkConfig("", "", "", "", nil, true) assert.Contains(t, cfg, "version: 2") assert.Contains(t, cfg, "dhcp4: true") assert.Contains(t, cfg, "dhcp-identifier: mac") @@ -674,7 +675,7 @@ func TestCloudInitNetworkConfigDualNIC(t *testing.T) { eniMAC := "02:00:00:61:ef:c2" devMAC := "02:de:00:60:83:0d" - cfg := generateNetworkConfig(eniMAC, devMAC, "", "", nil) + cfg := generateNetworkConfig(eniMAC, devMAC, "", "", nil, true) // Both MACs present in config assert.Contains(t, cfg, eniMAC) @@ -694,12 +695,12 @@ func TestCloudInitNetworkConfigDualNIC(t *testing.T) { func TestCloudInitNetworkConfigPartialMAC(t *testing.T) { // Only ENI MAC (VPC without dev) → per-interface config with VPC NIC only - cfg := generateNetworkConfig("02:00:00:61:ef:c2", "", "", "", nil) + cfg := generateNetworkConfig("02:00:00:61:ef:c2", "", "", "", nil, true) assert.Contains(t, cfg, "vpc0:") assert.NotContains(t, cfg, "dev0:") // Only dev MAC (shouldn't happen, but defensive) → wildcard - cfg = generateNetworkConfig("", "02:de:00:60:83:0d", "", "", nil) + cfg = generateNetworkConfig("", "02:de:00:60:83:0d", "", "", nil, true) assert.Contains(t, cfg, `name: "e*"`) assert.NotContains(t, cfg, "use-routes") } @@ -710,7 +711,7 @@ func TestCloudInitNetworkConfigMultiVPCNICs(t *testing.T) { "02:00:00:bb:bb:bb", "02:00:00:cc:cc:cc", } - cfg := generateNetworkConfig("02:00:00:aa:aa:aa", "", "", "", extras) + cfg := generateNetworkConfig("02:00:00:aa:aa:aa", "", "", "", extras, true) assert.Contains(t, cfg, "vpc0:") assert.Contains(t, cfg, "vpc1:") @@ -729,7 +730,7 @@ func TestCloudInitNetworkConfigMultiVPCNICs(t *testing.T) { func TestCloudInitNetworkConfigEmptyExtraMACSkipped(t *testing.T) { // Empty strings inside the extras slice are ignored rather than producing // a malformed ethernets block. - cfg := generateNetworkConfig("02:00:00:aa:aa:aa", "", "", "", []string{""}) + cfg := generateNetworkConfig("02:00:00:aa:aa:aa", "", "", "", []string{""}, true) assert.Contains(t, cfg, "vpc0:") assert.NotContains(t, cfg, "vpc1:") } @@ -1081,6 +1082,70 @@ func TestDescribeInstances_AccountFilteringHidesOtherTenant(t *testing.T) { assert.Empty(t, out.Reservations) } +// EKS control-plane VMs are owned by the customer account (their ENI lives in +// the customer VPC) but are platform-managed system instances, so they must be +// hidden from the owning customer's DescribeInstances the same way LB VMs are. +func TestDescribeInstances_HidesManagedSystemVMFromCustomer(t *testing.T) { + owner := "111122223333" + v := &vm.VM{ + ID: "i-ekscp", + AccountID: owner, + ManagedBy: tags.ManagedByEKS, + Reservation: &ec2.Reservation{ + ReservationId: aws.String("r-ekscp"), + OwnerId: aws.String(owner), + }, + Instance: &ec2.Instance{InstanceId: aws.String("i-ekscp")}, + } + svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{v.ID: v})} + + out, err := svc.DescribeInstances(&ec2.DescribeInstancesInput{}, owner) + require.NoError(t, err) + assert.Empty(t, out.Reservations, "managed system VM must not appear in customer listing") +} + +// Root/operator callers still see managed system VMs. +func TestDescribeInstances_RootSeesManagedSystemVM(t *testing.T) { + v := &vm.VM{ + ID: "i-lb", + AccountID: utils.GlobalAccountID, + ManagedBy: tags.ManagedByELBv2, + Reservation: &ec2.Reservation{ + ReservationId: aws.String("r-lb"), + OwnerId: aws.String(utils.GlobalAccountID), + }, + Instance: &ec2.Instance{InstanceId: aws.String("i-lb")}, + } + svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{v.ID: v})} + + out, err := svc.DescribeInstances(&ec2.DescribeInstancesInput{}, utils.GlobalAccountID) + require.NoError(t, err) + require.Len(t, out.Reservations, 1) + assert.Equal(t, "i-lb", *out.Reservations[0].Instances[0].InstanceId) +} + +// A customer's own unmanaged instance (no ManagedBy) stays visible — confirms +// the exclusion keys on ManagedBy, not on account scope. Future EKS worker +// nodes (customer-owned, no ManagedBy tag) follow this path. +func TestDescribeInstances_CustomerWorkloadStaysVisible(t *testing.T) { + owner := "111122223333" + v := &vm.VM{ + ID: "i-worker", + AccountID: owner, + Reservation: &ec2.Reservation{ + ReservationId: aws.String("r-worker"), + OwnerId: aws.String(owner), + }, + Instance: &ec2.Instance{InstanceId: aws.String("i-worker")}, + } + svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{v.ID: v})} + + out, err := svc.DescribeInstances(&ec2.DescribeInstancesInput{}, owner) + require.NoError(t, err) + require.Len(t, out.Reservations, 1) + assert.Equal(t, "i-worker", *out.Reservations[0].Instances[0].InstanceId) +} + func TestDescribeInstances_MalformedID(t *testing.T) { svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{})} input := &ec2.DescribeInstancesInput{ diff --git a/spinifex/handlers/eks/k3s_server_vm.go b/spinifex/handlers/eks/k3s_server_vm.go index 794f251e2..05f0a7ae1 100644 --- a/spinifex/handlers/eks/k3s_server_vm.go +++ b/spinifex/handlers/eks/k3s_server_vm.go @@ -15,6 +15,11 @@ import ( "github.com/mulgadc/spinifex/spinifex/tags" ) +// imdsServerIP is the link-local IMDS address (mirror of +// handlers/imds.MetaDataServerIP, duplicated to avoid an import cycle: +// imds → sts → eks). +const imdsServerIP = "169.254.169.254" + // ErrEKSServerAMINotFound is returned by the launcher when no AMI carrying the // EKS managed-by tag exists in the account. CreateCluster maps it to a precise // client error: it signals an operator/config gap (the eks-server image was @@ -366,11 +371,10 @@ func buildK3sUserData(in K3sServerInput) string { "EKS_OIDC_ISSUER=" + in.OIDCIssuer, }, "\n") - // Omitting cluster-init selects the embedded SQLite (kine) datastore, not - // embedded etcd. etcd's per-commit fsync stalls past apiserver's 5s deadline - // on the viperblock-backed root volume, so it never reaches /healthz; kine - // tolerates the deferred-durability block path. Trade-off: no multi-server - // HA (that needs etcd) — acceptable for single-node v1. + // cluster-init selects the embedded etcd datastore (required for multi-server + // HA). etcd-expose-metrics surfaces etcd's own wal_fsync_duration_seconds and + // backend_commit_duration_seconds on 127.0.0.1:2381/metrics so control-plane + // commit latency is measurable directly, not inferred. // // anonymous-auth=true: k3s hardens it off by default, which makes the // apiserver answer 401 to an unauthenticated /healthz. The cluster reconciler @@ -378,6 +382,8 @@ func buildK3sUserData(in K3sServerInput) string { // reachable; the default RBAC binds only the health/version non-resource // paths to system:unauthenticated, so this exposes nothing else. k3sConfig := strings.Join([]string{ + "cluster-init: true", + "etcd-expose-metrics: true", "tls-san:", " - " + in.NLBDNS, "kube-apiserver-arg:", @@ -395,6 +401,23 @@ func buildK3sUserData(in K3sServerInput) string { {Path: k3sOIDCPublicKeyPath, Perms: "0644", Body: strings.TrimRight(in.OIDCPublicKeyPEM, "\n")}, {Path: k3sConfigPath, Perms: "0644", Body: k3sConfig}, {Path: k3sNATSCAPath, Perms: "0644", Body: strings.TrimRight(in.NATSCACert, "\n")}, + // IMDS on-link route. Alpine's cloud-init eni renderer crashes on a + // gateway-less route in network-config, so it's delivered out-of-band: + // a persistent local.d script (re-applied every boot by the OpenRC + // `local` service, enabled via runcmd below) ARPs 169.254.169.254 + // directly on the VPC subnet. This block owns the only write_files key, + // so it must carry the route itself — appending a second write_files via + // the generic cloud-init template would collide (last key wins, silently + // dropping the k3s config). The device is resolved via the default route + // (the VPC egress NIC), not a name: the persistent-net rename to vpc0 + // doesn't apply on the Alpine AMI, so the kernel name is eth0 at boot. + { + Path: "/etc/local.d/imds-onlink-route.start", + Perms: "0755", + Body: "#!/bin/sh\n" + + "dev=$(ip route show default | awk '{print $5; exit}')\n" + + "[ -n \"$dev\" ] && ip route replace " + imdsServerIP + "/32 dev \"$dev\" scope link", + }, } var buf strings.Builder @@ -424,5 +447,12 @@ func buildK3sUserData(in K3sServerInput) string { buf.WriteString("\n") } } + + // Enable + start the OpenRC `local` service so the IMDS route script runs on + // first boot and is re-applied on every subsequent boot. + buf.WriteString("runcmd:\n") + buf.WriteString(" - [ rc-update, add, local, default ]\n") + buf.WriteString(" - [ rc-service, local, start ]\n") + return buf.String() } diff --git a/spinifex/handlers/eks/k3s_server_vm_test.go b/spinifex/handlers/eks/k3s_server_vm_test.go index bed201978..cddbbe9fe 100644 --- a/spinifex/handlers/eks/k3s_server_vm_test.go +++ b/spinifex/handlers/eks/k3s_server_vm_test.go @@ -301,6 +301,34 @@ func TestLaunchK3sServerVM_UserDataContainsAllArtifacts(t *testing.T) { assert.Contains(t, udata, "path: "+k3sNATSCAPath) assert.Contains(t, udata, "-----BEGIN CERTIFICATE-----") assert.Contains(t, udata, "FAKECA") + + // IMDS on-link route delivered out-of-band (Alpine's eni renderer can't emit + // the gateway-less route in network-config). It rides this payload's own + // write_files/runcmd, enabled via the OpenRC local service. + assert.Contains(t, udata, "path: /etc/local.d/imds-onlink-route.start") + assert.Contains(t, udata, "ip route show default") + assert.Contains(t, udata, `ip route replace 169.254.169.254/32 dev "$dev" scope link`) + assert.Contains(t, udata, "runcmd:") + assert.Contains(t, udata, "[ rc-update, add, local, default ]") + + // Exactly one write_files / runcmd key — a second would collide and silently + // drop a block under yaml.safe_load (last key wins). + assert.Equal(t, 1, strings.Count(udata, "\nwrite_files:")) + assert.Equal(t, 1, strings.Count(udata, "\nruncmd:")) +} + +func TestLaunchK3sServerVM_UsesEmbeddedEtcd(t *testing.T) { + vpc, inst, ami := &fakeK3sVPC{}, &fakeK3sInst{}, &fakeK3sAMI{} + + _, err := LaunchK3sServerVM(vpc, inst, ami, validK3sInput()) + require.NoError(t, err) + require.Len(t, inst.launchCalls, 1) + + udata := inst.launchCalls[0].UserData + // cluster-init selects the embedded etcd datastore (not SQLite/kine); + // etcd-expose-metrics surfaces wal_fsync/backend_commit on 127.0.0.1:2381. + assert.Contains(t, udata, "cluster-init: true") + assert.Contains(t, udata, "etcd-expose-metrics: true") } func TestLaunchK3sServerVM_RunInstancesEmptyReservationRollsBack(t *testing.T) { diff --git a/spinifex/handlers/eks/service_impl.go b/spinifex/handlers/eks/service_impl.go index 943f5a9d4..0c35e70a1 100644 --- a/spinifex/handlers/eks/service_impl.go +++ b/spinifex/handlers/eks/service_impl.go @@ -243,6 +243,16 @@ func (s *EKSServiceImpl) missingOrchestrationDeps() []string { // --- Cluster --- +// logCreateErr records the precise cause of a CreateCluster step failure at +// ERROR. The request surface collapses these wrapped errors to an opaque +// ServerInternal ("An internal error has occurred... contact AWS re:Post"), so +// without this the wrapped error never reaches the operator. Returns the error +// wrapped with the stage for the caller to surface. +func logCreateErr(name, accountID, stage string, err error) error { + slog.Error("CreateCluster: "+stage+" failed", "cluster", name, "accountID", accountID, "err", err) + return fmt.Errorf("%s: %w", stage, err) +} + func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID, callerPrincipalARN string) (*eks.CreateClusterOutput, error) { if err := s.requireOrchestrationDeps("CreateCluster"); err != nil { return nil, err @@ -255,11 +265,11 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID, js, err := s.deps.NATSConn.JetStream() if err != nil { - return nil, fmt.Errorf("jetstream: %w", err) + return nil, logCreateErr(name, accountID, "jetstream", err) } acctKV, err := GetOrCreateAccountBucket(js, accountID) if err != nil { - return nil, fmt.Errorf("get account bucket: %w", err) + return nil, logCreateErr(name, accountID, "get account bucket", err) } if existing, err := GetClusterMeta(acctKV, name); err == nil { @@ -277,15 +287,20 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID, slog.Info("CreateCluster: reclaiming FAILED cluster before recreate", "name", name, "accountID", accountID) if perr := s.purgeClusterInfra(accountID, name, existing, acctKV); perr != nil { - return nil, fmt.Errorf("reclaim failed cluster %s: %w", name, perr) + return nil, logCreateErr(name, accountID, "reclaim failed cluster", perr) } } else if !errors.Is(err, ErrClusterNotFound) { - return nil, fmt.Errorf("preflight get meta: %w", err) + return nil, logCreateErr(name, accountID, "preflight get meta", err) } vpcID, err := s.deps.VPCSubnet.GetSubnetVPC(accountID, subnetIDs[0]) if err != nil { - return nil, fmt.Errorf("resolve subnet VPC: %w", err) + // Subnet is a caller-supplied parameter — a resolve failure is a client + // fault (bad/foreign subnet), not an internal one. Surface it as such + // instead of collapsing to a retryable ServerInternal the SDK retries. + slog.Error("CreateCluster: resolve subnet VPC failed", + "cluster", name, "accountID", accountID, "subnet", subnetIDs[0], "err", err) + return nil, errors.New(awserrors.ErrorInvalidParameterValue) } region := s.deps.Region @@ -304,20 +319,20 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID, CreatedAt: time.Now().UTC(), } if err := PutClusterMeta(acctKV, meta); err != nil { - return nil, err + return nil, logCreateErr(name, accountID, "persist initial meta", err) } cpSG, ngSG, err := EnsureClusterSGs(s.deps.VPCSG, accountID, name, vpcID) if err != nil { s.markFailed(acctKV, name) - return nil, fmt.Errorf("ensure cluster SGs: %w", err) + return nil, logCreateErr(name, accountID, "ensure cluster SGs", err) } meta.ResourcesVpcConfig.SecurityGroupIds = []string{cpSG, ngSG} nlb, err := EnsureClusterNLB(s.deps.NLB, accountID, name, subnetIDs) if err != nil { s.markFailed(acctKV, name) - return nil, fmt.Errorf("ensure cluster NLB: %w", err) + return nil, logCreateErr(name, accountID, "ensure cluster NLB", err) } meta.Endpoint = "https://" + net.JoinHostPort(nlb.DNSName, strconv.FormatInt(clusterNLBListenPort, 10)) meta.NLBArn = nlb.LoadBalancerArn @@ -330,25 +345,25 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID, // reclaim them. if err := PutClusterMeta(acctKV, meta); err != nil { s.markFailed(acctKV, name) - return nil, fmt.Errorf("persist NLB arns: %w", err) + return nil, logCreateErr(name, accountID, "persist NLB arns", err) } oidcIssuer, err := ClusterOIDCIssuer(s.deps.GatewayBaseURL, region, accountID, name) if err != nil { s.markFailed(acctKV, name) - return nil, fmt.Errorf("build OIDC issuer: %w", err) + return nil, logCreateErr(name, accountID, "build OIDC issuer", err) } meta.OIDCIssuer = oidcIssuer privPEM, _, err := GenerateClusterOIDCKeypair(acctKV, name, s.deps.MasterKey) if err != nil { s.markFailed(acctKV, name) - return nil, fmt.Errorf("generate OIDC keypair: %w", err) + return nil, logCreateErr(name, accountID, "generate OIDC keypair", err) } pubPEM, err := PublicKeyPEMFromPrivate(privPEM) if err != nil { s.markFailed(acctKV, name) - return nil, fmt.Errorf("derive OIDC public key: %w", err) + return nil, logCreateErr(name, accountID, "derive OIDC public key", err) } k3sOut, err := LaunchK3sServerVM(s.deps.VPCK3s, s.deps.Instance, s.deps.Image, K3sServerInput{ @@ -387,7 +402,7 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID, egressIP, egressAllocID, err := s.allocateClusterEgressIP(accountID, name) if err != nil { s.markFailed(acctKV, name) - return nil, fmt.Errorf("allocate cluster egress IP: %w", err) + return nil, logCreateErr(name, accountID, "allocate cluster egress IP", err) } meta.EgressEIPAllocationID = egressAllocID meta.EgressEIPPublicIP = egressIP @@ -402,16 +417,17 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID, // register (or anything after) leaves them recoverable by DeleteCluster. if err := PutClusterMeta(acctKV, meta); err != nil { s.markFailed(acctKV, name) - return nil, fmt.Errorf("persist control-plane ids: %w", err) + return nil, logCreateErr(name, accountID, "persist control-plane ids", err) } if err := RegisterClusterTarget(s.deps.NLB, accountID, nlb.TargetGroupArn, k3sOut.ENIIP); err != nil { s.markFailed(acctKV, name) - return nil, fmt.Errorf("register NLB target: %w", err) + return nil, logCreateErr(name, accountID, "register NLB target", err) } if err := PutClusterMeta(acctKV, meta); err != nil { - return nil, err + s.markFailed(acctKV, name) + return nil, logCreateErr(name, accountID, "persist final meta", err) } // bootstrapClusterCreatorAdminPermissions (default true): grant the caller diff --git a/spinifex/handlers/eks/service_impl_test.go b/spinifex/handlers/eks/service_impl_test.go index b6dfe9a00..99d990b41 100644 --- a/spinifex/handlers/eks/service_impl_test.go +++ b/spinifex/handlers/eks/service_impl_test.go @@ -64,6 +64,17 @@ func TestValidateCreateClusterInput_RejectsConfigMapAuthMode(t *testing.T) { require.EqualError(t, err, awserrors.ErrorInvalidParameter) } +// The API_AND_CONFIG_MAP hybrid still enables the unsupported aws-auth ConfigMap +// path, so it must be rejected the same as plain CONFIG_MAP — the sibling test +// only covers the CONFIG_MAP value. +func TestValidateCreateClusterInput_RejectsAPIAndConfigMapAuthMode(t *testing.T) { + in := createInput("alpha") + in.AccessConfig = &eks.CreateAccessConfigRequest{ + AuthenticationMode: aws.String(eks.AuthenticationModeApiAndConfigMap), + } + require.EqualError(t, validateCreateClusterInput(in), awserrors.ErrorInvalidParameter) +} + func TestValidateCreateClusterInput_AcceptsAPIAuthMode(t *testing.T) { in := createInput("alpha") in.AccessConfig = &eks.CreateAccessConfigRequest{ @@ -92,6 +103,16 @@ func TestDescribeCluster_NotFoundWithFullDeps(t *testing.T) { require.EqualError(t, err, awserrors.ErrorEKSResourceNotFound) } +// DeleteCluster on an absent cluster must reach the KV lookup with full deps +// wired (the shim path short-circuits to ServiceUnavailable before the meta +// read) and surface ResourceNotFoundException, not a teardown of nothing. +func TestDeleteCluster_NotFoundWithFullDeps(t *testing.T) { + f := newEKSServiceFixture(t) + + _, err := f.svc.DeleteCluster(deleteInput("ghost"), testAccountID) + require.EqualError(t, err, awserrors.ErrorEKSResourceNotFound) +} + // Security guarantee: the OIDC signing key must be zeroized BEFORE infra // teardown, so a teardown failure (which leaves the meta for retry) can never // leave recoverable key material behind. Force VM terminate to fail and assert diff --git a/spinifex/handlers/sysinstance/launcher.go b/spinifex/handlers/sysinstance/launcher.go index 7b20ee677..7e6cb1f52 100644 --- a/spinifex/handlers/sysinstance/launcher.go +++ b/spinifex/handlers/sysinstance/launcher.go @@ -1,6 +1,9 @@ // Package sysinstance holds the boot-agnostic contract for launching -// system-managed VMs. A system instance is owned by the system account and -// hidden from customer DescribeInstances calls. Two boot styles are supported: +// system-managed VMs. System instances are hidden from customer +// DescribeInstances calls — BootDirect VMs (LB) via system-account ownership, +// BootAMI VMs (EKS control plane, owned by the customer account so their ENI +// can live in the customer VPC) via their ManagedBy tag. Two boot styles are +// supported: // // - BootDirect: a direct-boot microVM (bundled vmlinuz+initramfs, per-VM // config delivered via QEMU fw_cfg blobs; no AMI, volume, or cloud-init).