fix(eks): embedded etcd datastore + Alpine IMDS-route fix + CP-VM visibility (#299)

* fix: switch EKS datastore to embedded etcd + fix Alpine cloud-init IMDS route Flip the K3s control-plane datastore from SQLite/kine back to embedded etcd (cluster-init: true, etcd-expose-metrics: true). The earlier abandonment premise — that etcd's per-commit fsync stalls past the apiserver's 5s deadline on the viperblock-backed root volume — was wrong: etcd inits fine on an unchanged viperblock (apiserver ready 25-30s, clusters reach ACTIVE). The real blocker was a network-config regression, fixed here. Alpine's cloud-init eni renderer crashes (KeyError: 'gateway') on the gateway-less IMDS on-link route, which failed the whole network-config render so the static mgmt NIC never came up (no NATS bootstrap, no /healthz). Fixes: - generateNetworkConfig: add emitIMDSRoute bool; selectNetworkConfigForFamily passes false for Alpine so the eni renderer never sees the route. - Deliver the IMDS route out-of-band via a persistent /etc/local.d script + OpenRC local service. Resolve the VPC NIC via its default route at runtime (ip route show default), not a hardcoded name: the persistent-net rename to vpc0 doesn't apply on the Alpine AMI, so the NIC is eth0 at first boot. - EKS path bakes the same route into buildK3sUserData's own write_files/runcmd; the generic template Alpine block fires only when no #cloud-config user-data, avoiding a duplicate write_files key (yaml.safe_load keeps last, silently dropping a block). - eks-node image: config.yaml.skel adds cluster-init; comments updated. * fix: surface CreateCluster step failures at ERROR + map subnet faults; add auth-mode/delete test cases CreateCluster wrapped its step failures in bare fmt.Errorf that the request surface collapses to an opaque ServerInternal ("contact AWS re:Post"), so the real cause never reached the operator. Add logCreateErr to log each step's precise cause at ERROR before returning, and map a subnet-resolve failure (caller-supplied parameter) to InvalidParameterValue instead of a retryable ServerInternal the SDK pointlessly retries. (mulga-siv-165.4) Tests (mulga-siv-165.15): reject authenticationMode=API_AND_CONFIG_MAP (only the CONFIG_MAP value was covered); DeleteCluster on a nonexistent cluster via the full-deps path returns ResourceNotFound (the shim path short-circuits to ServiceUnavailable before the meta lookup). * fix: hide EKS control-plane VMs from customer DescribeInstances EKS K3s control-plane VMs are owned by the customer account (their ENI lives in the customer VPC for the NLB target), so unlike LB microVMs (owned by the system account) they leaked into customer ec2:DescribeInstances. The spinifex:managed-by tag only filtered the UI Nodes page, not the EC2 API. Add a caller-aware exclusion: skip any ManagedBy-tagged VM unless the caller is root. Preserves operator visibility, matches LB behaviour, and leaves future customer-owned worker nodes (no ManagedBy) visible. * docs: correct stale SQLite datastore comments in eks-node image (now embedded etcd)
2026-10-22 04:07:04 +00:00 · 2026-06-05 17:55:31 +10:00
parent 6be3ccbb23
commit f9c08130ba
11 changed files with 331 additions and 56 deletions
@@ -1,9 +1,7 @@
 #!/sbin/openrc-run
-# K3s server control plane. v1 = single-node embedded SQLite (kine) datastore;
-# the per-cluster config.yaml omits cluster-init so k3s uses SQLite, not etcd
-# (etcd's per-commit fsync stalls on the viperblock root volume). Data lives at
-# /var/lib/rancher/k3s; subsequent starts reuse it. Multi-server HA needs etcd
-# and is deferred.
+# K3s server control plane. The per-cluster config.yaml sets cluster-init, so
+# k3s uses the embedded etcd datastore (required for multi-server HA). Data
+# lives at /var/lib/rancher/k3s; subsequent starts reuse it.

 description="K3s Kubernetes server (Mulga EKS control plane)"
 command="/usr/local/bin/k3s"
@@ -13,8 +11,8 @@ pidfile="/run/k3s.pid"
 output_log="/var/log/k3s.log"
 error_log="/var/log/k3s.log"

-# Generous start timeout — K3s bootstraps containerd, initialises the SQLite
-# datastore, pulls system images. First boot can take 2-3 minutes on cold cache.
+# Generous start timeout — K3s bootstraps containerd, initialises the embedded
+# etcd datastore, pulls system images. First boot can take 2-3 minutes on cold cache.
 start_stop_daemon_args="--wait 5000"

 depend() {
@@ -3,7 +3,7 @@ IMAGE_DESCRIPTION="Unified EKS K3s node VM — Alpine + K3s v1.32.5+k3s1 + eks-t
 DISTRO="alpine"
 ALPINE_VERSION="3.21.7"

-# 4G — server superset (K3s server binary + SQLite datastore working set under
+# 4G — server superset (K3s server binary + embedded-etcd datastore working set under
 # /var/lib/rancher/k3s on the root disk). Agents use the same image; the ~10MB
 # of unused server binary + webhook is dwarfed by the headroom for node images.
 IMAGE_SIZE="4G"
@@ -72,7 +72,9 @@ mkdir -p /etc/rancher/k3s
 cat > /etc/rancher/k3s/config.yaml.skel <<'EOF'
 # Populated at first boot by cloud-init user-data via k3s-first-boot.sh.
 # Fields documented at https://docs.k3s.io/installation/configuration.
-# No cluster-init: single-node v1 uses the embedded SQLite datastore, not etcd.
+# cluster-init selects the embedded etcd datastore. cloud-init write_files
+# normally overrides this skeleton; kept consistent as the boot fallback.
+cluster-init: true
 disable:
  - traefik
  - servicelb
@@ -10,24 +10,24 @@ import (
 // --- generateNetworkConfig ---

 func TestGenerateNetworkConfig_BothEmpty(t *testing.T) {
-	cfg := generateNetworkConfig("", "", "", "", nil)
+	cfg := generateNetworkConfig("", "", "", "", nil, true)
 	assert.Equal(t, cloudInitNetworkConfigWildcard, cfg)
 }

 func TestGenerateNetworkConfig_OneEmpty(t *testing.T) {
-	cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "", "", "", nil)
+	cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "", "", "", nil, true)
 	assert.Contains(t, cfg, "vpc0:", "eniMAC alone should produce per-interface config")
 	assert.NotContains(t, cfg, "dev0:", "no dev NIC without devMAC")
 	// The IMDS on-link route rides vpc0 even with no dev NIC / extra ENIs — the
 	// common production shape — so guard it here too, not only on the multi-NIC path.
 	assert.Contains(t, cfg, "to: 169.254.169.254/32", "vpc0 must carry the IMDS on-link route")

-	cfg = generateNetworkConfig("", "02:00:00:dd:ee:ff", "", "", nil)
+	cfg = generateNetworkConfig("", "02:00:00:dd:ee:ff", "", "", nil, true)
 	assert.Equal(t, cloudInitNetworkConfigWildcard, cfg, "should fall back to wildcard if eniMAC empty")
 }

 func TestGenerateNetworkConfig_DualNIC(t *testing.T) {
-	cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:00:00:dd:ee:ff", "", "", nil)
+	cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:00:00:dd:ee:ff", "", "", nil, true)
 	assert.Contains(t, cfg, "version: 2")
 	assert.Contains(t, cfg, `macaddress: "02:00:00:aa:bb:cc"`)
 	assert.Contains(t, cfg, `macaddress: "02:00:00:dd:ee:ff"`)
@@ -39,7 +39,7 @@ func TestGenerateNetworkConfig_DualNIC(t *testing.T) {
 }

 func TestGenerateNetworkConfig_TripleNIC(t *testing.T) {
-	cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:de:00:dd:ee:ff", "02:a0:00:11:22:33", "10.15.8.101", nil)
+	cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:de:00:dd:ee:ff", "02:a0:00:11:22:33", "10.15.8.101", nil, true)
 	assert.Contains(t, cfg, "version: 2")
 	assert.Contains(t, cfg, `macaddress: "02:00:00:aa:bb:cc"`)
 	assert.Contains(t, cfg, `macaddress: "02:de:00:dd:ee:ff"`)
@@ -53,7 +53,7 @@ func TestGenerateNetworkConfig_TripleNIC(t *testing.T) {

 func TestGenerateNetworkConfig_MgmtWithoutDev(t *testing.T) {
 	// System instances: eniMAC + mgmtMAC, no devMAC — should get per-interface config with mgmt NIC
-	cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "", "02:a0:00:11:22:33", "10.15.8.101", nil)
+	cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "", "02:a0:00:11:22:33", "10.15.8.101", nil, true)
 	assert.Contains(t, cfg, "vpc0:")
 	assert.NotContains(t, cfg, "dev0:", "no dev NIC without devMAC")
 	assert.Contains(t, cfg, "mgmt0:")
@@ -62,12 +62,12 @@ func TestGenerateNetworkConfig_MgmtWithoutDev(t *testing.T) {
 }

 func TestGenerateNetworkConfig_MgmtMACWithoutIP(t *testing.T) {
-	cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:de:00:dd:ee:ff", "02:a0:00:11:22:33", "", nil)
+	cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:de:00:dd:ee:ff", "02:a0:00:11:22:33", "", nil, true)
 	assert.NotContains(t, cfg, "mgmt0:", "mgmt NIC should not appear without IP")
 }

 func TestGenerateNetworkConfig_MgmtIPWithoutMAC(t *testing.T) {
-	cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:de:00:dd:ee:ff", "", "10.15.8.101", nil)
+	cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:de:00:dd:ee:ff", "", "10.15.8.101", nil, true)
 	assert.NotContains(t, cfg, "mgmt0:", "mgmt NIC should not appear without MAC")
 }

@@ -75,7 +75,7 @@ func TestGenerateNetworkConfig_IMDSOnLinkRoute(t *testing.T) {
 	// The primary VPC NIC carries an on-link route to the IMDS metadata IP so the
 	// guest ARPs 169.254.169.254 directly; the per-subnet localport answers.
 	cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:00:00:dd:ee:ff", "", "",
-		[]string{"02:00:00:bb:bb:bb"})
+		[]string{"02:00:00:bb:bb:bb"}, true)
 	assert.Contains(t, cfg, "to: 169.254.169.254/32")
 	assert.Contains(t, cfg, "scope: link")
 	// Only the primary NIC (vpc0) gets it — not extra VPC NICs or the dev NIC,
@@ -83,5 +83,41 @@ func TestGenerateNetworkConfig_IMDSOnLinkRoute(t *testing.T) {
 	assert.Equal(t, 1, strings.Count(cfg, "169.254.169.254/32"))
 }

+func TestGenerateNetworkConfig_IMDSRouteOmittedWhenDisabled(t *testing.T) {
+	// Alpine's eni renderer crashes on a gateway-less route, so the route is
+	// omitted from network-config and delivered out-of-band (see
+	// buildAlpineCloudInit). emitIMDSRoute=false must drop it entirely.
+	cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:00:00:dd:ee:ff", "", "", nil, false)
+	assert.Contains(t, cfg, "vpc0:")
+	assert.NotContains(t, cfg, "169.254.169.254/32", "route must not render when emitIMDSRoute=false")
+	assert.NotContains(t, cfg, "      routes:", "no vpc0 routes block when the only route is the IMDS one")
+}
+
+// selectNetworkConfigForFamily must omit the IMDS route for Alpine (eni renderer
+// can't handle the gateway-less route) but keep it for everyone else.
+func TestSelectNetworkConfigForFamily_AlpineOmitsIMDSRoute(t *testing.T) {
+	alpine := selectNetworkConfigForFamily("alpine", "02:00:00:aa:bb:cc", "02:00:00:dd:ee:ff", "", "", nil)
+	assert.NotContains(t, alpine, "169.254.169.254/32", "Alpine network-config must not carry the IMDS route")
+
+	deb := selectNetworkConfigForFamily("debian", "02:00:00:aa:bb:cc", "02:00:00:dd:ee:ff", "", "", nil)
+	assert.Contains(t, deb, "169.254.169.254/32", "non-Alpine families keep the netplan IMDS route")
+}
+
+// buildAlpineCloudInit delivers the IMDS on-link route via a persistent
+// /etc/local.d script + OpenRC local service, since the eni renderer can't.
+func TestBuildAlpineCloudInit(t *testing.T) {
+	wf, rc := buildAlpineCloudInit("02:00:00:aa:bb:cc")
+	assert.Contains(t, wf, "/etc/local.d/imds-onlink-route.start")
+	assert.Contains(t, wf, "ip route show default")
+	assert.Contains(t, wf, `ip route replace 169.254.169.254/32 dev "$dev" scope link`)
+	assert.Contains(t, rc, "rc-update")
+	assert.Contains(t, rc, "local")
+
+	// No VPC NIC → nothing to add.
+	wf, rc = buildAlpineCloudInit("")
+	assert.Empty(t, wf)
+	assert.Empty(t, rc)
+}
+
 // Route for multi-node LB mgmt traffic is handled via the fw_cfg netcfg
 // blob delivered to the LB microVM, not in the network-config.
@@ -64,19 +64,21 @@ ca_certs:
 {{.UserDataCloudConfig}}
 {{end}}

-{{if or .RHELWriteFiles .UserDataScript}}
+{{if or .RHELWriteFiles .AlpineWriteFiles .UserDataScript}}
 write_files:
 {{- if .RHELWriteFiles}}
-{{.RHELWriteFiles}}{{end}}{{if .UserDataScript}}
+{{.RHELWriteFiles}}{{end}}{{if .AlpineWriteFiles}}
+{{.AlpineWriteFiles}}{{end}}{{if .UserDataScript}}
  - path: /tmp/cloud-init-startup.sh
    permissions: '0755'
    content: |
 {{.UserDataScript}}{{end}}
 {{end}}
-{{if or .RHELRunCmd .UserDataScript}}
+{{if or .RHELRunCmd .AlpineRunCmd .UserDataScript}}
 runcmd:
 {{- if .RHELRunCmd}}
-{{.RHELRunCmd}}{{end}}{{if .UserDataScript}}
+{{.RHELRunCmd}}{{end}}{{if .AlpineRunCmd}}
+{{.AlpineRunCmd}}{{end}}{{if .UserDataScript}}
  - [ "/bin/bash", "/tmp/cloud-init-startup.sh" ]
 {{end}}
 {{end}}
@@ -118,7 +120,11 @@ func selectNetworkConfigForFamily(distroFamily, eniMAC, devMAC, mgmtMAC, mgmtIP
 	if distroFamily == "rhel" {
 		return cloudInitNetworkConfigDisabled
 	}
-	return generateNetworkConfig(eniMAC, devMAC, mgmtMAC, mgmtIP, extraENIMACs)
+	// Alpine renders via cloud-init's eni backend, which cannot emit a
+	// gateway-less route; the IMDS on-link route is delivered via
+	// buildAlpineCloudInit instead. Debian/Ubuntu render via netplan, which
+	// handles the on-link route natively.
+	return generateNetworkConfig(eniMAC, devMAC, mgmtMAC, mgmtIP, extraENIMACs, distroFamily != "alpine")
 }

 // generateNetworkConfig produces the cloud-init network-config for the instance.
@@ -142,7 +148,7 @@ func selectNetworkConfigForFamily(distroFamily, eniMAC, devMAC, mgmtMAC, mgmtIP
 // route is reached via the primary ENI, matching AWS; it is not delivered via
 // DHCP option 121 (which would force the default route into option 121 per
 // RFC 3442) and the default gateway stays option 3.
-func generateNetworkConfig(eniMAC, devMAC, mgmtMAC, mgmtIP string, extraENIMACs []string) string {
+func generateNetworkConfig(eniMAC, devMAC, mgmtMAC, mgmtIP string, extraENIMACs []string, emitIMDSRoute bool) string {
 	if eniMAC == "" {
 		return cloudInitNetworkConfigWildcard
 	}
@@ -154,10 +160,17 @@ func generateNetworkConfig(eniMAC, devMAC, mgmtMAC, mgmtIP string, extraENIMACs
        macaddress: "%s"
      dhcp4: true
      dhcp-identifier: mac
-      routes:
+`, eniMAC)
+	// The gateway-less IMDS on-link route renders only under netplan. cloud-init's
+	// eni renderer (Alpine) crashes on a route with no gateway, taking the whole
+	// network-config down with it, so eni-rendered guests get this route via
+	// buildAlpineCloudInit instead (emitIMDSRoute=false).
+	if emitIMDSRoute {
+		cfg += fmt.Sprintf(`      routes:
        - to: %s/32
          scope: link
-`, eniMAC, handlers_imds.MetaDataServerIP)
+`, handlers_imds.MetaDataServerIP)
+	}

 	for i, mac := range extraENIMACs {
 		if mac == "" {
@@ -220,6 +233,12 @@ type CloudInitData struct {
 	// + nmcli reload/up) needed to load and bring up the keyfiles. Empty on
 	// non-RHEL guests.
 	RHELRunCmd string
+	// AlpineWriteFiles / AlpineRunCmd carry the IMDS on-link route delivery for
+	// Alpine guests (a persistent /etc/local.d script + OpenRC `local` enable),
+	// since the eni renderer cannot emit the gateway-less route in the
+	// network-config. Empty on non-Alpine guests and on non-VPC instances.
+	AlpineWriteFiles string
+	AlpineRunCmd     string
 }

 // buildRHELCloudInit produces the cloud-init write_files (NM keyfiles) and
@@ -322,6 +341,43 @@ func appendRHELStaticKeyfile(b *strings.Builder, name, mac, ip string) {
 	b.WriteString("      method=disabled\n")
 }

+// buildAlpineCloudInit produces the cloud-init write_files + runcmd blocks that
+// deliver the IMDS on-link route on Alpine guests. Returns empty strings when
+// eniMAC is empty (wildcard / non-VPC path).
+//
+// Alpine renders networking via cloud-init's eni backend, which crashes on a
+// gateway-less route (it assumes every route has a next hop), so the route is
+// omitted from the netplan network-config (see generateNetworkConfig) and added
+// here instead. A persistent /etc/local.d script keeps the on-link route present
+// across reboots — runcmd runs once per instance, so the OpenRC `local` service
+// re-applies it on every boot. cloud-init's eni renderer renames the primary
+// VPC NIC to vpc0 via a 70-persistent-net.rules udev rule, so the device name is
+// stable. `ip route replace` is idempotent.
+func buildAlpineCloudInit(eniMAC string) (writeFiles, runCmd string) {
+	if eniMAC == "" {
+		return "", ""
+	}
+
+	var wf strings.Builder
+	wf.WriteString("  - path: /etc/local.d/imds-onlink-route.start\n")
+	wf.WriteString("    owner: root:root\n")
+	wf.WriteString("    permissions: '0755'\n")
+	wf.WriteString("    content: |\n")
+	wf.WriteString("      #!/bin/sh\n")
+	// Resolve the primary VPC NIC via its default route, not by name: the
+	// persistent-net udev rename to vpc0 doesn't apply on the Alpine AMI, so the
+	// kernel name is eth0 at first boot. The mgmt NIC (eth1) carries no default
+	// route, so this always picks the VPC egress NIC — the one IMDS rides.
+	wf.WriteString("      dev=$(ip route show default | awk '{print $5; exit}')\n")
+	fmt.Fprintf(&wf, "      [ -n \"$dev\" ] && ip route replace %s/32 dev \"$dev\" scope link\n", handlers_imds.MetaDataServerIP)
+
+	var rc strings.Builder
+	rc.WriteString("  - [ rc-update, add, local, default ]\n")
+	rc.WriteString("  - [ rc-service, local, start ]\n")
+
+	return strings.TrimRight(wf.String(), "\n"), strings.TrimRight(rc.String(), "\n")
+}
+
 type CloudInitMetaData struct {
 	InstanceID string
 	Hostname   string
@@ -1871,7 +1927,8 @@ func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, i

 	sudoGroup := "sudo"
 	var rhelWriteFiles, rhelRunCmd string
-	if distroFamily == "rhel" {
+	switch distroFamily {
+	case "rhel":
 		sudoGroup = "wheel"
 		rhelWriteFiles, rhelRunCmd = buildRHELCloudInit(instance.ENIMac, instance.DevMAC, instance.MgmtMAC, instance.MgmtIP, extraMACs)
 	}
@@ -1909,6 +1966,15 @@ func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, i
 		}
 	}

+	// Alpine's eni renderer can't emit the gateway-less IMDS route, so deliver it
+	// out-of-band via a local.d script. Only when the caller didn't supply its own
+	// #cloud-config: that payload owns the write_files/runcmd keys (e.g. the EKS
+	// k3s server VM, which bakes the same route into its own block), and a second
+	// write_files key here would collide — last key wins, silently dropping a block.
+	if distroFamily == "alpine" && userData.UserDataCloudConfig == "" {
+		userData.AlpineWriteFiles, userData.AlpineRunCmd = buildAlpineCloudInit(instance.ENIMac)
+	}
+
 	var buf bytes.Buffer
 	t := template.Must(template.New("cloud-init").Parse(cloudInitUserDataTemplate))

@@ -2155,6 +2221,16 @@ func (s *InstanceServiceImpl) DescribeInstances(input *ec2.DescribeInstancesInpu
 			if !IsInstanceVisible(accountID, instance.AccountID) {
 				continue
 			}
+			// Platform-managed system VMs (LB HAProxy, EKS control plane) carry a
+			// ManagedBy tag and must stay out of customer EC2 listings, the same
+			// way they're filtered from the UI Nodes page. LB VMs are owned by the
+			// system account so IsInstanceVisible already hides them, but the EKS
+			// control-plane VM is owned by the customer account (its ENI lives in
+			// the customer VPC), so guard on ManagedBy here. Root/operator callers
+			// still see system instances.
+			if instance.ManagedBy != "" && accountID != utils.GlobalAccountID {
+				continue
+			}
 			if len(instanceIDFilter) > 0 && !instanceIDFilter[instance.ID] {
 				continue
 			}
@@ -15,6 +15,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/awserrors"
 	"github.com/mulgadc/spinifex/spinifex/config"
 	"github.com/mulgadc/spinifex/spinifex/gpu"
+	"github.com/mulgadc/spinifex/spinifex/tags"
 	spxtypes "github.com/mulgadc/spinifex/spinifex/types"
 	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/mulgadc/spinifex/spinifex/vm"
@@ -662,7 +663,7 @@ func TestParseVolumeParams_PartialEbs(t *testing.T) {

 func TestCloudInitNetworkConfigWildcard(t *testing.T) {
 	// No MACs → wildcard config (non-VPC or VPC without DEV_NETWORKING)
-	cfg := generateNetworkConfig("", "", "", "", nil)
+	cfg := generateNetworkConfig("", "", "", "", nil, true)
 	assert.Contains(t, cfg, "version: 2")
 	assert.Contains(t, cfg, "dhcp4: true")
 	assert.Contains(t, cfg, "dhcp-identifier: mac")
@@ -674,7 +675,7 @@ func TestCloudInitNetworkConfigDualNIC(t *testing.T) {
 	eniMAC := "02:00:00:61:ef:c2"
 	devMAC := "02:de:00:60:83:0d"

-	cfg := generateNetworkConfig(eniMAC, devMAC, "", "", nil)
+	cfg := generateNetworkConfig(eniMAC, devMAC, "", "", nil, true)

 	// Both MACs present in config
 	assert.Contains(t, cfg, eniMAC)
@@ -694,12 +695,12 @@ func TestCloudInitNetworkConfigDualNIC(t *testing.T) {

 func TestCloudInitNetworkConfigPartialMAC(t *testing.T) {
 	// Only ENI MAC (VPC without dev) → per-interface config with VPC NIC only
-	cfg := generateNetworkConfig("02:00:00:61:ef:c2", "", "", "", nil)
+	cfg := generateNetworkConfig("02:00:00:61:ef:c2", "", "", "", nil, true)
 	assert.Contains(t, cfg, "vpc0:")
 	assert.NotContains(t, cfg, "dev0:")

 	// Only dev MAC (shouldn't happen, but defensive) → wildcard
-	cfg = generateNetworkConfig("", "02:de:00:60:83:0d", "", "", nil)
+	cfg = generateNetworkConfig("", "02:de:00:60:83:0d", "", "", nil, true)
 	assert.Contains(t, cfg, `name: "e*"`)
 	assert.NotContains(t, cfg, "use-routes")
 }
@@ -710,7 +711,7 @@ func TestCloudInitNetworkConfigMultiVPCNICs(t *testing.T) {
 		"02:00:00:bb:bb:bb",
 		"02:00:00:cc:cc:cc",
 	}
-	cfg := generateNetworkConfig("02:00:00:aa:aa:aa", "", "", "", extras)
+	cfg := generateNetworkConfig("02:00:00:aa:aa:aa", "", "", "", extras, true)

 	assert.Contains(t, cfg, "vpc0:")
 	assert.Contains(t, cfg, "vpc1:")
@@ -729,7 +730,7 @@ func TestCloudInitNetworkConfigMultiVPCNICs(t *testing.T) {
 func TestCloudInitNetworkConfigEmptyExtraMACSkipped(t *testing.T) {
 	// Empty strings inside the extras slice are ignored rather than producing
 	// a malformed ethernets block.
-	cfg := generateNetworkConfig("02:00:00:aa:aa:aa", "", "", "", []string{""})
+	cfg := generateNetworkConfig("02:00:00:aa:aa:aa", "", "", "", []string{""}, true)
 	assert.Contains(t, cfg, "vpc0:")
 	assert.NotContains(t, cfg, "vpc1:")
 }
@@ -1081,6 +1082,70 @@ func TestDescribeInstances_AccountFilteringHidesOtherTenant(t *testing.T) {
 	assert.Empty(t, out.Reservations)
 }

+// EKS control-plane VMs are owned by the customer account (their ENI lives in
+// the customer VPC) but are platform-managed system instances, so they must be
+// hidden from the owning customer's DescribeInstances the same way LB VMs are.
+func TestDescribeInstances_HidesManagedSystemVMFromCustomer(t *testing.T) {
+	owner := "111122223333"
+	v := &vm.VM{
+		ID:        "i-ekscp",
+		AccountID: owner,
+		ManagedBy: tags.ManagedByEKS,
+		Reservation: &ec2.Reservation{
+			ReservationId: aws.String("r-ekscp"),
+			OwnerId:       aws.String(owner),
+		},
+		Instance: &ec2.Instance{InstanceId: aws.String("i-ekscp")},
+	}
+	svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{v.ID: v})}
+
+	out, err := svc.DescribeInstances(&ec2.DescribeInstancesInput{}, owner)
+	require.NoError(t, err)
+	assert.Empty(t, out.Reservations, "managed system VM must not appear in customer listing")
+}
+
+// Root/operator callers still see managed system VMs.
+func TestDescribeInstances_RootSeesManagedSystemVM(t *testing.T) {
+	v := &vm.VM{
+		ID:        "i-lb",
+		AccountID: utils.GlobalAccountID,
+		ManagedBy: tags.ManagedByELBv2,
+		Reservation: &ec2.Reservation{
+			ReservationId: aws.String("r-lb"),
+			OwnerId:       aws.String(utils.GlobalAccountID),
+		},
+		Instance: &ec2.Instance{InstanceId: aws.String("i-lb")},
+	}
+	svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{v.ID: v})}
+
+	out, err := svc.DescribeInstances(&ec2.DescribeInstancesInput{}, utils.GlobalAccountID)
+	require.NoError(t, err)
+	require.Len(t, out.Reservations, 1)
+	assert.Equal(t, "i-lb", *out.Reservations[0].Instances[0].InstanceId)
+}
+
+// A customer's own unmanaged instance (no ManagedBy) stays visible — confirms
+// the exclusion keys on ManagedBy, not on account scope. Future EKS worker
+// nodes (customer-owned, no ManagedBy tag) follow this path.
+func TestDescribeInstances_CustomerWorkloadStaysVisible(t *testing.T) {
+	owner := "111122223333"
+	v := &vm.VM{
+		ID:        "i-worker",
+		AccountID: owner,
+		Reservation: &ec2.Reservation{
+			ReservationId: aws.String("r-worker"),
+			OwnerId:       aws.String(owner),
+		},
+		Instance: &ec2.Instance{InstanceId: aws.String("i-worker")},
+	}
+	svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{v.ID: v})}
+
+	out, err := svc.DescribeInstances(&ec2.DescribeInstancesInput{}, owner)
+	require.NoError(t, err)
+	require.Len(t, out.Reservations, 1)
+	assert.Equal(t, "i-worker", *out.Reservations[0].Instances[0].InstanceId)
+}
+
 func TestDescribeInstances_MalformedID(t *testing.T) {
 	svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{})}
 	input := &ec2.DescribeInstancesInput{
@@ -15,6 +15,11 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/tags"
 )

+// imdsServerIP is the link-local IMDS address (mirror of
+// handlers/imds.MetaDataServerIP, duplicated to avoid an import cycle:
+// imds → sts → eks).
+const imdsServerIP = "169.254.169.254"
+
 // ErrEKSServerAMINotFound is returned by the launcher when no AMI carrying the
 // EKS managed-by tag exists in the account. CreateCluster maps it to a precise
 // client error: it signals an operator/config gap (the eks-server image was
@@ -366,11 +371,10 @@ func buildK3sUserData(in K3sServerInput) string {
 		"EKS_OIDC_ISSUER=" + in.OIDCIssuer,
 	}, "\n")

-	// Omitting cluster-init selects the embedded SQLite (kine) datastore, not
-	// embedded etcd. etcd's per-commit fsync stalls past apiserver's 5s deadline
-	// on the viperblock-backed root volume, so it never reaches /healthz; kine
-	// tolerates the deferred-durability block path. Trade-off: no multi-server
-	// HA (that needs etcd) — acceptable for single-node v1.
+	// cluster-init selects the embedded etcd datastore (required for multi-server
+	// HA). etcd-expose-metrics surfaces etcd's own wal_fsync_duration_seconds and
+	// backend_commit_duration_seconds on 127.0.0.1:2381/metrics so control-plane
+	// commit latency is measurable directly, not inferred.
 	//
 	// anonymous-auth=true: k3s hardens it off by default, which makes the
 	// apiserver answer 401 to an unauthenticated /healthz. The cluster reconciler
@@ -378,6 +382,8 @@ func buildK3sUserData(in K3sServerInput) string {
 	// reachable; the default RBAC binds only the health/version non-resource
 	// paths to system:unauthenticated, so this exposes nothing else.
 	k3sConfig := strings.Join([]string{
+		"cluster-init: true",
+		"etcd-expose-metrics: true",
 		"tls-san:",
 		"  - " + in.NLBDNS,
 		"kube-apiserver-arg:",
@@ -395,6 +401,23 @@ func buildK3sUserData(in K3sServerInput) string {
 		{Path: k3sOIDCPublicKeyPath, Perms: "0644", Body: strings.TrimRight(in.OIDCPublicKeyPEM, "\n")},
 		{Path: k3sConfigPath, Perms: "0644", Body: k3sConfig},
 		{Path: k3sNATSCAPath, Perms: "0644", Body: strings.TrimRight(in.NATSCACert, "\n")},
+		// IMDS on-link route. Alpine's cloud-init eni renderer crashes on a
+		// gateway-less route in network-config, so it's delivered out-of-band:
+		// a persistent local.d script (re-applied every boot by the OpenRC
+		// `local` service, enabled via runcmd below) ARPs 169.254.169.254
+		// directly on the VPC subnet. This block owns the only write_files key,
+		// so it must carry the route itself — appending a second write_files via
+		// the generic cloud-init template would collide (last key wins, silently
+		// dropping the k3s config). The device is resolved via the default route
+		// (the VPC egress NIC), not a name: the persistent-net rename to vpc0
+		// doesn't apply on the Alpine AMI, so the kernel name is eth0 at boot.
+		{
+			Path:  "/etc/local.d/imds-onlink-route.start",
+			Perms: "0755",
+			Body: "#!/bin/sh\n" +
+				"dev=$(ip route show default | awk '{print $5; exit}')\n" +
+				"[ -n \"$dev\" ] && ip route replace " + imdsServerIP + "/32 dev \"$dev\" scope link",
+		},
 	}

 	var buf strings.Builder
@@ -424,5 +447,12 @@ func buildK3sUserData(in K3sServerInput) string {
 			buf.WriteString("\n")
 		}
 	}
+
+	// Enable + start the OpenRC `local` service so the IMDS route script runs on
+	// first boot and is re-applied on every subsequent boot.
+	buf.WriteString("runcmd:\n")
+	buf.WriteString("  - [ rc-update, add, local, default ]\n")
+	buf.WriteString("  - [ rc-service, local, start ]\n")
+
 	return buf.String()
 }
@@ -301,6 +301,34 @@ func TestLaunchK3sServerVM_UserDataContainsAllArtifacts(t *testing.T) {
 	assert.Contains(t, udata, "path: "+k3sNATSCAPath)
 	assert.Contains(t, udata, "-----BEGIN CERTIFICATE-----")
 	assert.Contains(t, udata, "FAKECA")
+
+	// IMDS on-link route delivered out-of-band (Alpine's eni renderer can't emit
+	// the gateway-less route in network-config). It rides this payload's own
+	// write_files/runcmd, enabled via the OpenRC local service.
+	assert.Contains(t, udata, "path: /etc/local.d/imds-onlink-route.start")
+	assert.Contains(t, udata, "ip route show default")
+	assert.Contains(t, udata, `ip route replace 169.254.169.254/32 dev "$dev" scope link`)
+	assert.Contains(t, udata, "runcmd:")
+	assert.Contains(t, udata, "[ rc-update, add, local, default ]")
+
+	// Exactly one write_files / runcmd key — a second would collide and silently
+	// drop a block under yaml.safe_load (last key wins).
+	assert.Equal(t, 1, strings.Count(udata, "\nwrite_files:"))
+	assert.Equal(t, 1, strings.Count(udata, "\nruncmd:"))
+}
+
+func TestLaunchK3sServerVM_UsesEmbeddedEtcd(t *testing.T) {
+	vpc, inst, ami := &fakeK3sVPC{}, &fakeK3sInst{}, &fakeK3sAMI{}
+
+	_, err := LaunchK3sServerVM(vpc, inst, ami, validK3sInput())
+	require.NoError(t, err)
+	require.Len(t, inst.launchCalls, 1)
+
+	udata := inst.launchCalls[0].UserData
+	// cluster-init selects the embedded etcd datastore (not SQLite/kine);
+	// etcd-expose-metrics surfaces wal_fsync/backend_commit on 127.0.0.1:2381.
+	assert.Contains(t, udata, "cluster-init: true")
+	assert.Contains(t, udata, "etcd-expose-metrics: true")
 }

 func TestLaunchK3sServerVM_RunInstancesEmptyReservationRollsBack(t *testing.T) {
@@ -243,6 +243,16 @@ func (s *EKSServiceImpl) missingOrchestrationDeps() []string {

 // --- Cluster ---

+// logCreateErr records the precise cause of a CreateCluster step failure at
+// ERROR. The request surface collapses these wrapped errors to an opaque
+// ServerInternal ("An internal error has occurred... contact AWS re:Post"), so
+// without this the wrapped error never reaches the operator. Returns the error
+// wrapped with the stage for the caller to surface.
+func logCreateErr(name, accountID, stage string, err error) error {
+	slog.Error("CreateCluster: "+stage+" failed", "cluster", name, "accountID", accountID, "err", err)
+	return fmt.Errorf("%s: %w", stage, err)
+}
+
 func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID, callerPrincipalARN string) (*eks.CreateClusterOutput, error) {
 	if err := s.requireOrchestrationDeps("CreateCluster"); err != nil {
 		return nil, err
@@ -255,11 +265,11 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID,

 	js, err := s.deps.NATSConn.JetStream()
 	if err != nil {
-		return nil, fmt.Errorf("jetstream: %w", err)
+		return nil, logCreateErr(name, accountID, "jetstream", err)
 	}
 	acctKV, err := GetOrCreateAccountBucket(js, accountID)
 	if err != nil {
-		return nil, fmt.Errorf("get account bucket: %w", err)
+		return nil, logCreateErr(name, accountID, "get account bucket", err)
 	}

 	if existing, err := GetClusterMeta(acctKV, name); err == nil {
@@ -277,15 +287,20 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID,
 		slog.Info("CreateCluster: reclaiming FAILED cluster before recreate",
 			"name", name, "accountID", accountID)
 		if perr := s.purgeClusterInfra(accountID, name, existing, acctKV); perr != nil {
-			return nil, fmt.Errorf("reclaim failed cluster %s: %w", name, perr)
+			return nil, logCreateErr(name, accountID, "reclaim failed cluster", perr)
 		}
 	} else if !errors.Is(err, ErrClusterNotFound) {
-		return nil, fmt.Errorf("preflight get meta: %w", err)
+		return nil, logCreateErr(name, accountID, "preflight get meta", err)
 	}

 	vpcID, err := s.deps.VPCSubnet.GetSubnetVPC(accountID, subnetIDs[0])
 	if err != nil {
-		return nil, fmt.Errorf("resolve subnet VPC: %w", err)
+		// Subnet is a caller-supplied parameter — a resolve failure is a client
+		// fault (bad/foreign subnet), not an internal one. Surface it as such
+		// instead of collapsing to a retryable ServerInternal the SDK retries.
+		slog.Error("CreateCluster: resolve subnet VPC failed",
+			"cluster", name, "accountID", accountID, "subnet", subnetIDs[0], "err", err)
+		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
 	}

 	region := s.deps.Region
@@ -304,20 +319,20 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID,
 		CreatedAt: time.Now().UTC(),
 	}
 	if err := PutClusterMeta(acctKV, meta); err != nil {
-		return nil, err
+		return nil, logCreateErr(name, accountID, "persist initial meta", err)
 	}

 	cpSG, ngSG, err := EnsureClusterSGs(s.deps.VPCSG, accountID, name, vpcID)
 	if err != nil {
 		s.markFailed(acctKV, name)
-		return nil, fmt.Errorf("ensure cluster SGs: %w", err)
+		return nil, logCreateErr(name, accountID, "ensure cluster SGs", err)
 	}
 	meta.ResourcesVpcConfig.SecurityGroupIds = []string{cpSG, ngSG}

 	nlb, err := EnsureClusterNLB(s.deps.NLB, accountID, name, subnetIDs)
 	if err != nil {
 		s.markFailed(acctKV, name)
-		return nil, fmt.Errorf("ensure cluster NLB: %w", err)
+		return nil, logCreateErr(name, accountID, "ensure cluster NLB", err)
 	}
 	meta.Endpoint = "https://" + net.JoinHostPort(nlb.DNSName, strconv.FormatInt(clusterNLBListenPort, 10))
 	meta.NLBArn = nlb.LoadBalancerArn
@@ -330,25 +345,25 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID,
 	// reclaim them.
 	if err := PutClusterMeta(acctKV, meta); err != nil {
 		s.markFailed(acctKV, name)
-		return nil, fmt.Errorf("persist NLB arns: %w", err)
+		return nil, logCreateErr(name, accountID, "persist NLB arns", err)
 	}

 	oidcIssuer, err := ClusterOIDCIssuer(s.deps.GatewayBaseURL, region, accountID, name)
 	if err != nil {
 		s.markFailed(acctKV, name)
-		return nil, fmt.Errorf("build OIDC issuer: %w", err)
+		return nil, logCreateErr(name, accountID, "build OIDC issuer", err)
 	}
 	meta.OIDCIssuer = oidcIssuer

 	privPEM, _, err := GenerateClusterOIDCKeypair(acctKV, name, s.deps.MasterKey)
 	if err != nil {
 		s.markFailed(acctKV, name)
-		return nil, fmt.Errorf("generate OIDC keypair: %w", err)
+		return nil, logCreateErr(name, accountID, "generate OIDC keypair", err)
 	}
 	pubPEM, err := PublicKeyPEMFromPrivate(privPEM)
 	if err != nil {
 		s.markFailed(acctKV, name)
-		return nil, fmt.Errorf("derive OIDC public key: %w", err)
+		return nil, logCreateErr(name, accountID, "derive OIDC public key", err)
 	}

 	k3sOut, err := LaunchK3sServerVM(s.deps.VPCK3s, s.deps.Instance, s.deps.Image, K3sServerInput{
@@ -387,7 +402,7 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID,
 	egressIP, egressAllocID, err := s.allocateClusterEgressIP(accountID, name)
 	if err != nil {
 		s.markFailed(acctKV, name)
-		return nil, fmt.Errorf("allocate cluster egress IP: %w", err)
+		return nil, logCreateErr(name, accountID, "allocate cluster egress IP", err)
 	}
 	meta.EgressEIPAllocationID = egressAllocID
 	meta.EgressEIPPublicIP = egressIP
@@ -402,16 +417,17 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID,
 	// register (or anything after) leaves them recoverable by DeleteCluster.
 	if err := PutClusterMeta(acctKV, meta); err != nil {
 		s.markFailed(acctKV, name)
-		return nil, fmt.Errorf("persist control-plane ids: %w", err)
+		return nil, logCreateErr(name, accountID, "persist control-plane ids", err)
 	}

 	if err := RegisterClusterTarget(s.deps.NLB, accountID, nlb.TargetGroupArn, k3sOut.ENIIP); err != nil {
 		s.markFailed(acctKV, name)
-		return nil, fmt.Errorf("register NLB target: %w", err)
+		return nil, logCreateErr(name, accountID, "register NLB target", err)
 	}

 	if err := PutClusterMeta(acctKV, meta); err != nil {
-		return nil, err
+		s.markFailed(acctKV, name)
+		return nil, logCreateErr(name, accountID, "persist final meta", err)
 	}

 	// bootstrapClusterCreatorAdminPermissions (default true): grant the caller
@@ -64,6 +64,17 @@ func TestValidateCreateClusterInput_RejectsConfigMapAuthMode(t *testing.T) {
 	require.EqualError(t, err, awserrors.ErrorInvalidParameter)
 }

+// The API_AND_CONFIG_MAP hybrid still enables the unsupported aws-auth ConfigMap
+// path, so it must be rejected the same as plain CONFIG_MAP — the sibling test
+// only covers the CONFIG_MAP value.
+func TestValidateCreateClusterInput_RejectsAPIAndConfigMapAuthMode(t *testing.T) {
+	in := createInput("alpha")
+	in.AccessConfig = &eks.CreateAccessConfigRequest{
+		AuthenticationMode: aws.String(eks.AuthenticationModeApiAndConfigMap),
+	}
+	require.EqualError(t, validateCreateClusterInput(in), awserrors.ErrorInvalidParameter)
+}
+
 func TestValidateCreateClusterInput_AcceptsAPIAuthMode(t *testing.T) {
 	in := createInput("alpha")
 	in.AccessConfig = &eks.CreateAccessConfigRequest{
@@ -92,6 +103,16 @@ func TestDescribeCluster_NotFoundWithFullDeps(t *testing.T) {
 	require.EqualError(t, err, awserrors.ErrorEKSResourceNotFound)
 }

+// DeleteCluster on an absent cluster must reach the KV lookup with full deps
+// wired (the shim path short-circuits to ServiceUnavailable before the meta
+// read) and surface ResourceNotFoundException, not a teardown of nothing.
+func TestDeleteCluster_NotFoundWithFullDeps(t *testing.T) {
+	f := newEKSServiceFixture(t)
+
+	_, err := f.svc.DeleteCluster(deleteInput("ghost"), testAccountID)
+	require.EqualError(t, err, awserrors.ErrorEKSResourceNotFound)
+}
+
 // Security guarantee: the OIDC signing key must be zeroized BEFORE infra
 // teardown, so a teardown failure (which leaves the meta for retry) can never
 // leave recoverable key material behind. Force VM terminate to fail and assert
@@ -1,6 +1,9 @@
 // Package sysinstance holds the boot-agnostic contract for launching
-// system-managed VMs. A system instance is owned by the system account and
-// hidden from customer DescribeInstances calls. Two boot styles are supported:
+// system-managed VMs. System instances are hidden from customer
+// DescribeInstances calls — BootDirect VMs (LB) via system-account ownership,
+// BootAMI VMs (EKS control plane, owned by the customer account so their ENI
+// can live in the customer VPC) via their ManagedBy tag. Two boot styles are
+// supported:
 //
 //   - BootDirect: a direct-boot microVM (bundled vmlinuz+initramfs, per-VM
 //     config delivered via QEMU fw_cfg blobs; no AMI, volume, or cloud-init).