release: update go.mod for v1.12.1

Phase 8 of TestNATUplink checked the second VPC router's lr-nat-list with a
single non-polling assert; the igw-default-snat row commits a beat after the
gateway LRP appears, so the read raced the OVN commit and intermittently saw
an empty list (flaky CI red). Wrap it in EventuallyErr, matching the sibling
LRP and host-route polls in the same phase.

AddSNAT logged only on the idempotent-skip path, so the first install was
silent and RCA could not tell a late SNAT from a missing one. Emit an install
log keyed by router + vpc_cidr + external_ip.

.github/workflows/publish-eks-ami.yml

@@ -105,7 +105,13 @@ jobs:
           cache: false
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore
+        # Idempotent: the ci-single runner reuses one workspace across matrix
+        # legs and runs, and checkout never scrubs the root go.work. Remove any
+        # stale file so each leg re-inits cleanly instead of failing on
+        # "go.work already exists".
+        run: |
+          rm -f go.work go.work.sum
+          go work init ./spinifex ./viperblock ./predastore
 
       - name: Install build + upload tooling
         run: |

cmd/spinifex/cmd/admin.go

@@ -334,7 +334,6 @@ func init() {
 	adminInitCmd.Flags().String("external-gateway", "", "WAN gateway IP (auto-detected from default route)")
 	adminInitCmd.Flags().String("gateway-ip", "", "OVN gateway router's external IP for SNAT (default: pool range_start for pool mode, required for nat mode without DHCP)")
 	adminInitCmd.Flags().Int("external-prefix-len", 24, "External pool subnet prefix length (auto-detected)")
-	adminInitCmd.Flags().Bool("no-external", false, "Disable external networking (overlay-only, no internet for VMs)")
 	adminInitCmd.Flags().Bool("gpu-passthrough", false, "Enable VFIO GPU passthrough (sets gpu_passthrough = true in daemon config)")
 	adminInitCmd.Flags().Bool("ipsec", true, "Encrypt intra-AZ Geneve via OVN native IPsec (cluster-wide); disable only for trusted single-rack lab")
 
@@ -1018,7 +1017,6 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	externalGateway, _ := cmd.Flags().GetString("external-gateway")
 	externalPrefixLen, _ := cmd.Flags().GetInt("external-prefix-len")
 	gatewayIP, _ := cmd.Flags().GetString("gateway-ip")
-	noExternal, _ := cmd.Flags().GetBool("no-external")
 	gpuPassthrough, _ := cmd.Flags().GetBool("gpu-passthrough")
 	ipsecEnabled, _ := cmd.Flags().GetBool("ipsec")
 
@@ -1051,59 +1049,56 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	// Auto-detect network topology
 	var poolStart, poolEnd string
 	var detectedNet *admin.DetectedNetwork
-	if !noExternal {
-		detected, err := admin.DetectNetwork()
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "⚠️  Network auto-detection failed: %v\n", err)
-			fmt.Fprintf(os.Stderr, "   Use --no-external to skip, or specify flags manually.\n")
+	detected, err := admin.DetectNetwork()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "⚠️  Network auto-detection failed: %v\n", err)
+		fmt.Fprintf(os.Stderr, "   Use --external-mode=nat for outbound-only VMs on a non-bridgeable uplink, or specify --external-* flags manually.\n")
+	} else {
+		detectedNet = detected
+
+		// Print detected topology
+		fmt.Println("\n🔍 Detected network topology:")
+		fmt.Printf("  %-14s %-18s %-20s %-16s %s\n", "Interface", "IP", "Subnet", "Gateway", "Role")
+		for _, iface := range detected.Interfaces {
+			gw := "—"
+			if iface.Gateway != "" {
+				gw = iface.Gateway
+			}
+			fmt.Printf("  %-14s %-18s %-20s %-16s %s\n", iface.Name, iface.IP, iface.Subnet, gw, strings.ToUpper(iface.Role))
+		}
+		if detected.LANCount == 0 {
+			fmt.Println("\n  Mode: single-NIC (veth-bridged external)")
 		} else {
-			detectedNet = detected
+			fmt.Printf("\n  Mode: %d LAN + 1 WAN (veth-bridged external)\n", detected.LANCount)
+		}
 
-			// Print detected topology
-			fmt.Println("\n🔍 Detected network topology:")
-			fmt.Printf("  %-14s %-18s %-20s %-16s %s\n", "Interface", "IP", "Subnet", "Gateway", "Role")
-			for _, iface := range detected.Interfaces {
-				gw := "—"
-				if iface.Gateway != "" {
-					gw = iface.Gateway
-				}
-				fmt.Printf("  %-14s %-18s %-20s %-16s %s\n", iface.Name, iface.IP, iface.Subnet, gw, strings.ToUpper(iface.Role))
+		// Apply auto-detected values when flags not explicitly set
+		if detected.WAN != nil {
+			if externalIface == "" {
+				externalIface = detected.WAN.Name
 			}
-			if detected.LANCount == 0 {
-				fmt.Println("\n  Mode: single-NIC (veth-bridged external)")
-			} else {
-				fmt.Printf("\n  Mode: %d LAN + 1 WAN (veth-bridged external)\n", detected.LANCount)
+			if externalGateway == "" {
+				externalGateway = detected.WAN.Gateway
+			}
+			if !cmd.Flags().Changed("external-prefix-len") {
+				externalPrefixLen = detected.WAN.PrefixLen
 			}
 
-			// Apply auto-detected values when flags not explicitly set
-			if detected.WAN != nil {
-				if externalIface == "" {
-					externalIface = detected.WAN.Name
-				}
-				if externalGateway == "" {
-					externalGateway = detected.WAN.Gateway
-				}
-				if !cmd.Flags().Changed("external-prefix-len") {
-					externalPrefixLen = detected.WAN.PrefixLen
-				}
-
-				// Default mode: always "pool". Source defaults to "static"; if
-				// --external-pool is omitted the validator below will error with
-				// a SuggestPoolRange hint.
-				if externalMode == "" && !cmd.Flags().Changed("external-mode") {
-					if isNonBridgeableUplink(detected.WAN.Name) {
-						fmt.Fprintf(os.Stderr, "\n❌ Detected WAN interface %s cannot be bridged (WiFi/cellular/PPP).\n", detected.WAN.Name)
-						fmt.Fprintf(os.Stderr, "   Use routed NAT mode instead (outbound-only VM networking):\n")
-						fmt.Fprintf(os.Stderr, "     ./scripts/setup-ovn.sh --management --nat-uplink\n")
-						fmt.Fprintf(os.Stderr, "     spx admin init --external-mode=nat\n")
-						os.Exit(1)
-					}
-					externalMode = "pool"
+			// Default mode: always "pool". Source defaults to "static"; if
+			// --external-pool is omitted the validator below will error with
+			// a SuggestPoolRange hint.
+			if externalMode == "" && !cmd.Flags().Changed("external-mode") {
+				if isNonBridgeableUplink(detected.WAN.Name) {
+					fmt.Fprintf(os.Stderr, "\n❌ Detected WAN interface %s cannot be bridged (WiFi/cellular/PPP).\n", detected.WAN.Name)
+					fmt.Fprintf(os.Stderr, "   Use routed NAT mode instead (outbound-only VM networking):\n")
+					fmt.Fprintf(os.Stderr, "     ./scripts/setup-ovn.sh --management --nat-uplink\n")
+					fmt.Fprintf(os.Stderr, "     spx admin init --external-mode=nat\n")
+					os.Exit(1)
 				}
+				externalMode = "pool"
 			}
 		}
 	}
-
 	// Validate external networking flags
 	if externalMode != "" && externalMode != "pool" && externalMode != "nat" {
 		fmt.Fprintf(os.Stderr, "❌ Error: --external-mode must be 'pool', 'nat', or empty, got: %s\n", externalMode)
@@ -1209,8 +1204,8 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	// Resolve the off-host advertise IP. DetectNetwork may have been skipped
-	// when --no-external is set; detect lazily if we need the WAN IP.
+	// Resolve the off-host advertise IP. DetectNetwork may have failed earlier;
+	// detect lazily if we still need the WAN IP.
 	if advertiseFlag == "" && (bindIP == "0.0.0.0" || bindIP == "127.0.0.1") && detectedNet == nil {
 		if d, derr := admin.DetectNetwork(); derr == nil {
 			detectedNet = d

docs/COMMANDS.md

@@ -54,7 +54,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 
 | Command | Flags | Description |
 |---------|-------|-------------|
-| `spx admin init` | `--nodes`, `--node`, `--bind`, `--port`, `--region`, `--az`, `--cluster-name`, `--cluster-bind`, `--cluster-routes`, `--predastore-nodes`, `--services`, `--formation-timeout`, `--token-ttl`, `--force`, `--external-mode` (`pool` \| `nat` routed NAT for non-bridgeable uplinks, single-node; pair with `setup-ovn.sh --nat-uplink`; add `--external-pool start-end --external-gateway <ip>` or `--external-source=dhcp [--external-bind-bridge <iface>]` for a public pool with full EIP support; note: system instances (ECS/EKS/load-balancer agents) are unsupported in nat v1 — they require `pool` mode), `--no-external` | Generates root IAM credentials (AKIA-prefixed access key + secret) → creates master.key (AES-256, 32 bytes, 0600) → writes bootstrap.json (consumed on first start) → generates CA + server TLS certificates → generates join token (written to `join-token` file, displayed in join command) → creates NATS config with auth token → writes spinifex.toml, awsgw.toml, predastore.toml → configures AWS CLI `spx` profile → creates directory structure under `~/spinifex/`. `--force` on an already-initialized node is idempotent for crypto: it preserves the existing keys (master.key, predastore/viperblock encryption keys), system + admin credentials, and CA, and refreshes only the config files and the CA-signed server certificate (so a changed bind IP / SANs is picked up without breaking already-joined nodes or CA-baked AMIs). A genuine clean slate requires removing the data dirs first. |
+| `spx admin init` | `--nodes`, `--node`, `--bind`, `--port`, `--region`, `--az`, `--cluster-name`, `--cluster-bind`, `--cluster-routes`, `--predastore-nodes`, `--services`, `--formation-timeout`, `--token-ttl`, `--force`, `--external-mode` (`pool` \| `nat` routed NAT for non-bridgeable uplinks, single-node; pair with `setup-ovn.sh --nat-uplink`; add `--external-pool start-end --external-gateway <ip>` or `--external-source=dhcp [--external-bind-bridge <iface>]` for a public pool with full EIP support; note: system instances (ECS/EKS/load-balancer agents) are unsupported in nat v1 — they require `pool` mode) | Generates root IAM credentials (AKIA-prefixed access key + secret) → creates master.key (AES-256, 32 bytes, 0600) → writes bootstrap.json (consumed on first start) → generates CA + server TLS certificates → generates join token (written to `join-token` file, displayed in join command) → creates NATS config with auth token → writes spinifex.toml, awsgw.toml, predastore.toml → configures AWS CLI `spx` profile → creates directory structure under `~/spinifex/`. `--force` on an already-initialized node is idempotent for crypto: it preserves the existing keys (master.key, predastore/viperblock encryption keys), system + admin credentials, and CA, and refreshes only the config files and the CA-signed server certificate (so a changed bind IP / SANs is picked up without breaking already-joined nodes or CA-baked AMIs). A genuine clean slate requires removing the data dirs first. |
 | `spx admin join` | `--host` (required), `--node` (required), `--token` (required), `--bind`, `--port`, `--region`, `--az`, `--cluster-bind`, `--cluster-routes`, `--data-dir`, `--services` | Connects to leader node with join token (Authorization: Bearer header) → retrieves cluster configuration → configures local node to join cluster and participate in distributed operations |
 
 ### Version

go.mod

@@ -18,8 +18,8 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/insomniacslk/dhcp v0.0.0-20260407060928-11b94ed970f2
 	github.com/klauspost/cpuid/v2 v2.4.0
-	github.com/mulgadc/predastore v1.12.0
-	github.com/mulgadc/viperblock v1.12.0
+	github.com/mulgadc/predastore v1.12.1
+	github.com/mulgadc/viperblock v1.12.1
 	github.com/nats-io/nats-server/v2 v2.14.3
 	github.com/nats-io/nats.go v1.52.0
 	github.com/opencontainers/runtime-spec v1.3.0

go.sum

@@ -1315,10 +1315,10 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU
 github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
 github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
 github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
-github.com/mulgadc/predastore v1.12.0 h1:JmoF//cIr3DgJhGypNVVLtaBZGCVUDzOo6z63407x54=
-github.com/mulgadc/predastore v1.12.0/go.mod h1:kgTncuCaB11e3eCQeZF0ydEFG0biuOlrZ4DlHNzwvZY=
-github.com/mulgadc/viperblock v1.12.0 h1:1PYG7GQ1cBbHbLI7QB7IzVu6EeKhrtcAJ+VAvNczjs8=
-github.com/mulgadc/viperblock v1.12.0/go.mod h1:5MGciKEvqpwLv2pdjk9j7PR8NHtXPs4pJzJi4UJojeU=
+github.com/mulgadc/predastore v1.12.1 h1:mkJqmUbxfI3kBRmtBecjvBsRLFpnKARGWtmx4b9XKKg=
+github.com/mulgadc/predastore v1.12.1/go.mod h1:kgTncuCaB11e3eCQeZF0ydEFG0biuOlrZ4DlHNzwvZY=
+github.com/mulgadc/viperblock v1.12.1 h1:DWMbzPhIkYzdhOy4sn/D0/BJWJ5uhBoKWbVMfS7DMl4=
+github.com/mulgadc/viperblock v1.12.1/go.mod h1:5MGciKEvqpwLv2pdjk9j7PR8NHtXPs4pJzJi4UJojeU=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=

scripts/images/eks-node-gpu/k3s-agent.service

@@ -20,8 +20,9 @@ TasksMax=infinity
 LimitNOFILE=1048576
 LimitNPROC=infinity
 LimitCORE=infinity
-# Cloud-init drops K3S_URL/K3S_TOKEN/K3S_NODE_NAME/K3S_NODE_LABEL here at
-# RunInstances time per the Nodegroup bootstrap flow.
+# Cloud-init drops K3S_URL/K3S_TOKEN/K3S_NODE_NAME (plus SPINIFEX_NODEGROUP and,
+# for GPU workers, SPINIFEX_GPU_NODE) here at RunInstances time per the Nodegroup
+# bootstrap flow; mulga-eks-provider-id reads the SPINIFEX_* vars for node-labels.
 EnvironmentFile=-/etc/spinifex-eks/agent.env
 # Tee k3s (and its embedded kubelet's klog) stderr to the serial console as well
 # as journald: Ubuntu logs to journald only, which is off-box-invisible on a

scripts/images/eks-node-gpu/mulga-eks-provider-id.service

@@ -1,6 +1,13 @@
 [Unit]
 Description=Set K3s provider-id + topology labels from IMDS for EBS CSI
-After=network-online.target
+# After cloud-config.service so cloud-init has written /etc/spinifex-eks/agent.env
+# (write_files completes by the config stage): this script folds the nodegroup +
+# GPU node-labels (read from agent.env) into the same node-label list, so agent.env
+# must exist before it runs. NOT cloud-final.service: that unit is ordered after
+# multi-user.target and k3s-agent.service (ordered After this unit) is
+# WantedBy=multi-user.target, so chaining onto cloud-final forms an ordering cycle
+# that makes systemd skip both this unit and k3s-agent, so the worker never joins.
+After=network-online.target cloud-config.service
 Wants=network-online.target
 Before=k3s-agent.service
 

scripts/images/eks-node/addons/nvidia-device-plugin/0.17.4/10-daemonset.yaml

@@ -1,6 +1,7 @@
 # nvidia-device-plugin 0.17.4 — CDI-mode DaemonSet exposing nvidia.com/gpu
 # allocatable on GPU-tainted nodes. Gated to GPU nodes via nodeSelector on the
-# K3S_NODE_LABEL nvidia.com/gpu.present=true (nodegroup_userdata.go); inert
+# node-label nvidia.com/gpu.present=true (written by mulga-eks-provider-id from
+# the SPINIFEX_GPU_NODE agent.env signal); inert
 # elsewhere. --device-list-strategy=cdi-cri makes the plugin generate a CDI spec
 # for GPU workloads (containerd 2.x resolves the CRI-passed CDI devices; no
 # enable_cdi drop-in needed on k3s v1.32.5). The plugin's OWN container needs

scripts/images/eks-node/mulga-eks-provider-id.initd

@@ -2,14 +2,16 @@
 # mulga-eks-provider-id — boot oneshot that writes a K3s drop-in from IMDS
 # before K3s starts: the kubelet provider-id plus region/zone topology labels,
 # the node metadata the aws-ebs-csi-driver needs for NodeGetInfo + provisioning.
-# Ordered after local (which installs the IMDS on-link route) and before both
-# K3s roles. Role-agnostic: enabled on every node (server + agent).
+# Also folds the worker's nodegroup + GPU node-labels (from agent.env) into the
+# same node-label list, so it must run after cloud-init has written agent.env
+# (cloud-config/cloud-final stages, same as eks-node-role) and before the role
+# selector starts K3s. Role-agnostic: enabled on every node (server + agent).
 
 description="Set K3s provider-id + topology labels from IMDS for EBS CSI"
 
 depend() {
-    after net local
-    before k3s k3s-agent
+    after net local cloud-config cloud-final
+    before k3s k3s-agent eks-node-role
 }
 
 start() {

scripts/images/eks-node/mulga-eks-provider-id.sh

@@ -17,13 +17,22 @@ set -eu
 # family"). This drop-in supplies all four; region is derived from the AZ
 # (trailing zone letters stripped).
 #
-# K3s merges /etc/rancher/k3s/config.yaml.d/*.yaml over config.yaml; list keys
-# such as kubelet-arg are concatenated, so this adds the flag without touching
-# the cloud-init-written main config. Role-agnostic: runs on server + agent.
+# This drop-in is also the SINGLE writer of the node-label list. K3s replaces
+# (not merges) node-label across config.yaml.d files, so the last-sorted file
+# wins outright and any other node-label drop-in would silently wipe the rest.
+# The worker's nodegroup label (and, on GPU workers, the nvidia.com/gpu.present
+# label + NoSchedule taint) therefore ride here too, read from agent.env, so
+# they share the one list with the topology labels. On the K3s server there is
+# no agent.env, so only the topology labels are written — behavior unchanged.
+#
+# kubelet-arg lists ARE concatenated by k3s across drop-ins, so the provider-id
+# flag coexists with other files' kubelet-arg without touching the main config.
+# Role-agnostic: runs on server + agent.
 
 IMDS="http://169.254.169.254/latest"
 DROPIN_DIR=/etc/rancher/k3s/config.yaml.d
 DROPIN="${DROPIN_DIR}/10-provider-id.yaml"
+AGENT_ENV=/etc/spinifex-eks/agent.env
 LOGTAG="mulga-eks-provider-id"
 
 log() { echo "[${LOGTAG}] $*"; }
@@ -79,13 +88,32 @@ case "${itype}" in
     *) itype="m5.large" ;;
 esac
 
+# Fold the worker's nodegroup + GPU node-labels into the same list. agent.env is
+# written by cloud-init and absent on the server; source it only if present.
+nodegroup=""
+gpu_node=""
+if [ -f "${AGENT_ENV}" ]; then
+    # shellcheck disable=SC1090
+    . "${AGENT_ENV}"
+    nodegroup="${SPINIFEX_NODEGROUP:-}"
+    gpu_node="${SPINIFEX_GPU_NODE:-}"
+fi
+
 mkdir -p "${DROPIN_DIR}"
-cat > "${DROPIN}" <<EOF
-kubelet-arg:
-  - "provider-id=aws:///${az}/${iid}"
-node-label:
-  - "topology.kubernetes.io/region=${region}"
-  - "topology.kubernetes.io/zone=${az}"
-  - "node.kubernetes.io/instance-type=${itype}"
-EOF
-log "wrote provider-id aws:///${az}/${iid}, region=${region} zone=${az} type=${itype}"
+{
+    echo "kubelet-arg:"
+    echo "  - \"provider-id=aws:///${az}/${iid}\""
+    echo "node-label:"
+    echo "  - \"topology.kubernetes.io/region=${region}\""
+    echo "  - \"topology.kubernetes.io/zone=${az}\""
+    echo "  - \"node.kubernetes.io/instance-type=${itype}\""
+    if [ -n "${nodegroup}" ]; then
+        echo "  - \"eks.amazonaws.com/nodegroup=${nodegroup}\""
+    fi
+    if [ "${gpu_node}" = "true" ]; then
+        echo "  - \"nvidia.com/gpu.present=true\""
+        echo "node-taint:"
+        echo "  - \"nvidia.com/gpu=present:NoSchedule\""
+    fi
+} > "${DROPIN}"
+log "wrote provider-id aws:///${az}/${iid}, region=${region} zone=${az} type=${itype} nodegroup=${nodegroup:-<none>} gpu=${gpu_node:-false}"

scripts/publish-system-image.sh

@@ -67,7 +67,10 @@ if [[ -z "${IMAGE_NAME:-}" ]]; then
 fi
 
 # Mirror build-system-image.sh's derived paths so we reuse its cached raw.
-BUILD_DIR="/tmp/${IMAGE_NAME}-image-build"
+# Honor SYSTEM_IMAGE_BUILD_DIR exactly as the build script does — the workflow
+# points it off tmpfs so the 16G Ubuntu GPU raw fits, and the publish step must
+# look where the build actually wrote, not a hardcoded /tmp.
+BUILD_DIR="${SYSTEM_IMAGE_BUILD_DIR:-/tmp}/${IMAGE_NAME}-image-build"
 RAW_IMAGE="${BUILD_DIR}/${IMAGE_NAME}-${DISTRO}.raw"
 
 OBJECT_BASE="${AMI_NAME:-$IMAGE_NAME}-${ARCH}"

spinifex/admin/admin.go

@@ -21,6 +21,7 @@ import (
 	"text/template"
 	"time"
 
+	"github.com/mulgadc/spinifex/spinifex/config"
 	toml "github.com/pelletier/go-toml/v2"
 	"gopkg.in/ini.v1"
 )
@@ -219,7 +220,10 @@ func GenerateCertificatesIfNeeded(configDir string, force bool, bindIP string, a
 
 	if force || !FileExists(serverCertPath) || !FileExists(serverKeyPath) {
 		extraDNS := AWSGWServiceDNSNames(awsRegion, internalSuffix)
-		if err := GenerateSignedCert(serverCertPath, serverKeyPath, caCertPath, caKeyPath, []string{bindIP}, extraDNS); err != nil {
+		// Always pin the canonical mgmt-bridge IP; the control plane publishes to
+		// it regardless of whether br-mgmt is up when this cert is minted.
+		extraIPs := []string{bindIP, config.DefaultMgmtBridgeIP}
+		if err := GenerateSignedCert(serverCertPath, serverKeyPath, caCertPath, caKeyPath, extraIPs, extraDNS); err != nil {
 			fmt.Fprintf(os.Stderr, "Error generating server certificate: %v\n", err)
 			os.Exit(1)
 		}
@@ -247,7 +251,9 @@ func GenerateServerCertOnly(configDir string, bindIP, awsRegion, internalSuffix
 	}
 
 	extraDNS := AWSGWServiceDNSNames(awsRegion, internalSuffix)
-	return GenerateSignedCert(serverCertPath, serverKeyPath, caCertPath, caKeyPath, []string{bindIP}, extraDNS)
+	// Always pin the canonical mgmt-bridge IP (see GenerateCertificatesIfNeeded).
+	extraIPs := []string{bindIP, config.DefaultMgmtBridgeIP}
+	return GenerateSignedCert(serverCertPath, serverKeyPath, caCertPath, caKeyPath, extraIPs, extraDNS)
 }
 
 func CreateServiceDirectories(spxRoot string) {

spinifex/admin/admin_test.go

@@ -11,10 +11,28 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/mulgadc/spinifex/spinifex/config"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
+// certHasIP reports whether a PEM-encoded cert carries ip as an IP SAN.
+func certHasIP(t *testing.T, certPath, ip string) bool {
+	t.Helper()
+	certPEM, err := os.ReadFile(certPath)
+	require.NoError(t, err)
+	block, _ := pem.Decode(certPEM)
+	require.NotNil(t, block)
+	cert, err := x509.ParseCertificate(block.Bytes)
+	require.NoError(t, err)
+	for _, got := range cert.IPAddresses {
+		if got.Equal(net.ParseIP(ip)) {
+			return true
+		}
+	}
+	return false
+}
+
 // --- Key / Token generation ---
 
 func TestGenerateAWSAccessKey_Format(t *testing.T) {
@@ -672,6 +690,17 @@ func TestGenerateCertificatesIfNeeded(t *testing.T) {
 		_, err = srvCert.Verify(x509.VerifyOptions{Roots: pool, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}})
 		assert.NoError(t, err, "re-signed server cert must verify against preserved CA")
 	})
+
+	// The mgmt-bridge IP must be a SAN even though br-mgmt is not a live
+	// interface here (interface enumeration cannot discover it), mirroring a
+	// host where br-mgmt is down when the cert is minted. Without the explicit
+	// pin the control-plane publish to https://<mgmt-ip> would fail cert verify.
+	t.Run("MgmtBridgeIPAlwaysInSAN", func(t *testing.T) {
+		certDir := t.TempDir()
+		GenerateCertificatesIfNeeded(certDir, false, "10.0.0.5", "us-east-1", "spinifex.internal")
+		assert.True(t, certHasIP(t, filepath.Join(certDir, "server.pem"), config.DefaultMgmtBridgeIP),
+			"server cert must carry the canonical mgmt-bridge IP SAN regardless of br-mgmt state")
+	})
 }
 
 func TestGenerateServerCertOnly(t *testing.T) {
@@ -708,6 +737,10 @@ func TestGenerateServerCertOnly(t *testing.T) {
 		// AWS-parity ECR SANs must be present.
 		assert.Contains(t, cert.DNSNames, "ecr.us-east-1.spinifex.internal")
 		assert.Contains(t, cert.DNSNames, "*.dkr.ecr.us-east-1.spinifex.internal")
+
+		// Joining nodes must also pin the mgmt-bridge IP regardless of br-mgmt state.
+		assert.True(t, certHasIP(t, filepath.Join(dir, "server.pem"), config.DefaultMgmtBridgeIP),
+			"server cert must carry the canonical mgmt-bridge IP SAN")
 	})
 
 	t.Run("MissingCA", func(t *testing.T) {

spinifex/config/config.go

@@ -27,6 +27,13 @@ const (
 	DefaultAWSInternalSuffix = "spinifex.internal"
 )
 
+// DefaultMgmtBridgeIP is the canonical br-mgmt host address the control plane
+// advertises: the EKS gateway URL, predastore endpoint, and node-group userdata
+// all target it. Kept in sync with setup-ovn.sh MGMT_CIDR. Server certs must
+// always carry this as a SAN so publish succeeds even when br-mgmt happens to be
+// down at cert-generation time (interface enumeration would otherwise miss it).
+const DefaultMgmtBridgeIP = "10.15.8.1"
+
 // AWSConfig holds cluster-wide AWS-parity settings shared across services.
 // Region scopes the default AWS region; InternalSuffix is the internal DNS
 // suffix used to build service endpoints (e.g. ecr.{region}.{suffix}).

spinifex/handlers/eks/k3s_server_vm.go

@@ -72,7 +72,7 @@ const (
 	k3sSnapshotEnvPath = "/etc/spinifex-eks/etcd-snapshot.env"
 
 	// agentEnvPath is the env file the k3s-agent OpenRC service sources for
-	// K3S_URL/K3S_TOKEN/K3S_NODE_NAME/K3S_NODE_LABEL.
+	// K3S_URL/K3S_TOKEN/K3S_NODE_NAME.
 	agentEnvPath = "/etc/spinifex-eks/agent.env"
 
 	// k3sGatewayCAPath is the on-VM gateway TLS CA cert PEM path.

spinifex/handlers/eks/nodegroup_test.go

@@ -286,7 +286,7 @@ func TestBuildAgentUserData_Shape(t *testing.T) {
 	assert.Contains(t, ud, "K3S_URL=https://10.254.0.9:443")
 	assert.Contains(t, ud, "K3S_TOKEN=K10secret::server:xyz")
 	assert.Contains(t, ud, "K3S_NODE_NAME=c1-ng1-abc123de")
-	assert.Contains(t, ud, "eks.amazonaws.com/nodegroup=ng1")
+	assert.Contains(t, ud, "SPINIFEX_NODEGROUP=ng1")
 	assert.Contains(t, ud, agentEnvPath)
 	// IMDS is served at the host tap, so no in-guest on-link route is emitted.
 	assert.NotContains(t, ud, "169.254.169.254")

spinifex/handlers/eks/nodegroup_userdata.go

@@ -61,9 +61,12 @@ func buildAgentUserData(in agentUserDataInput) string {
 	}, "\n")
 
 	// The ECR credential provider reads /etc/spinifex-eks/agent.env, so the EKS_*
-	// settings live alongside the K3s join vars here. The nodegroup label rides a
-	// node-label config drop-in below, not K3S_NODE_LABEL (k3s has no such env).
-	agentBody := strings.Join([]string{
+	// settings live alongside the K3s join vars here. SPINIFEX_NODEGROUP (and
+	// SPINIFEX_GPU_NODE for GPU workers) are consumed by mulga-eks-provider-id.sh,
+	// the single writer of the k3s node-label list: k3s replaces (not merges) the
+	// node-label list across config.yaml.d files, so all node-labels must come
+	// from one drop-in or the last-sorted file silently wipes the rest.
+	agentLines := []string{
 		"K3S_URL=" + k3sURL,
 		"K3S_TOKEN=" + in.JoinToken,
 		"K3S_NODE_NAME=" + in.NodeName,
@@ -71,7 +74,12 @@ func buildAgentUserData(in agentUserDataInput) string {
 		"EKS_GATEWAY_CA=" + k3sGatewayCAPath,
 		"EKS_REGION=" + in.Region,
 		"EKS_ACCOUNT_ID=" + in.AccountID,
-	}, "\n")
+		"SPINIFEX_NODEGROUP=" + in.NodegroupName,
+	}
+	if in.GPUEnabled {
+		agentLines = append(agentLines, "SPINIFEX_GPU_NODE=true")
+	}
+	agentBody := strings.Join(agentLines, "\n")
 
 	// The mirror endpoint is dialed by IP when the gateway IP is known: the gateway
 	// cert carries an IP SAN, so TLS validates without DNS, and there is no internal
@@ -124,42 +132,20 @@ func buildAgentUserData(in agentUserDataInput) string {
 	)
 	credProviderConfig := strings.Join(credLines, "\n")
 
-	// The eks.amazonaws.com/nodegroup label must ride a k3s node-label config
-	// drop-in: k3s has no K3S_NODE_LABEL env, so agent.env cannot set it. The
-	// nodegroup readiness gate (waitWorkersReady) buckets Ready nodes by this
-	// label, so a worker without it never counts toward its nodegroup's ACTIVE
-	// transition. k3s concatenates node-label lists across config.yaml.d drop-ins.
-	nodegroupLabelDropin := strings.Join([]string{
-		"node-label:",
-		"  - \"eks.amazonaws.com/nodegroup=" + in.NodegroupName + "\"",
-	}, "\n")
-
+	// buildAgentUserData writes NO config.yaml.d node-label/node-taint drop-in:
+	// mulga-eks-provider-id.sh (AMI-baked) is the sole node-label writer. It reads
+	// SPINIFEX_NODEGROUP/SPINIFEX_GPU_NODE from agent.env and folds the nodegroup
+	// label (plus nvidia.com/gpu.present + the GPU NoSchedule taint) into the same
+	// 10-provider-id.yaml node-label list it writes for the EBS CSI topology labels.
 	files := []userDataFile{
 		{Path: k3sFirstBootEnvPath, Perms: "0600", Body: envBody},
 		{Path: agentEnvPath, Perms: "0600", Body: agentBody},
 		{Path: k3sGatewayCAPath, Perms: "0644", Body: strings.TrimRight(in.GatewayCACert, "\n")},
 		{Path: "/etc/rancher/k3s/registries.yaml", Perms: "0644", Body: registriesYAML},
 		{Path: "/etc/rancher/k3s/config.yaml.d/20-ecr-credential-provider.yaml", Perms: "0644", Body: credProviderDropin},
-		{Path: "/etc/rancher/k3s/config.yaml.d/15-nodegroup-label.yaml", Perms: "0644", Body: nodegroupLabelDropin},
 		{Path: "/etc/spinifex-eks/credential-provider-config.yaml", Perms: "0644", Body: credProviderConfig},
 	}
 
-	// GPU nodes advertise nvidia.com/gpu.present=true (so the device-plugin
-	// DaemonSet nodeSelector matches) and default to NoSchedule so ordinary
-	// workloads don't consume scarce GPU capacity. Both ride a separate
-	// node-label/node-taint drop-in; k3s merges it over the nodegroup-label one.
-	if in.GPUEnabled {
-		gpuDropin := strings.Join([]string{
-			"node-label:",
-			"  - \"nvidia.com/gpu.present=true\"",
-			"node-taint:",
-			"  - \"nvidia.com/gpu=present:NoSchedule\"",
-		}, "\n")
-		files = append(files, userDataFile{
-			Path: "/etc/rancher/k3s/config.yaml.d/20-gpu.yaml", Perms: "0644", Body: gpuDropin,
-		})
-	}
-
 	var buf strings.Builder
 	buf.WriteString("#cloud-config\n")
 

spinifex/handlers/eks/nodegroup_userdata_test.go

@@ -96,31 +96,36 @@ func baseTestInput() agentUserDataInput {
 	}
 }
 
-func TestBuildAgentUserData_GPUEnabledAddsLabelAndTaint(t *testing.T) {
+func TestBuildAgentUserData_GPUEnabledSignalsNodegroupAndGPU(t *testing.T) {
 	in := baseTestInput()
 	in.GPUEnabled = true
 	in.GPUVendor = "nvidia"
 
 	got := buildAgentUserData(in)
 
-	// The label must ride a node-label config drop-in, not K3S_NODE_LABEL (k3s
-	// has no such env var — it would be a no-op and the device-plugin
-	// nodeSelector would never match).
-	if !strings.Contains(got, "/etc/rancher/k3s/config.yaml.d/20-gpu.yaml") {
-		t.Errorf("expected GPU drop-in path, got:\n%s", got)
+	// buildAgentUserData writes NO node-label/node-taint config drop-in:
+	// mulga-eks-provider-id.sh is the single node-label writer (k3s replaces,
+	// not merges, node-label across config.yaml.d files). The worker signals its
+	// nodegroup + GPU role via agent.env; the AMI script folds the labels + taint
+	// into the one node-label list it already writes for the topology labels.
+	if strings.Contains(got, "config.yaml.d/10-nodegroup.yaml") {
+		t.Errorf("must not write a nodegroup node-label drop-in, got:\n%s", got)
 	}
-	if !strings.Contains(got, "node-label:") || !strings.Contains(got, "nvidia.com/gpu.present=true") {
-		t.Errorf("expected GPU node-label drop-in value, got:\n%s", got)
+	if strings.Contains(got, "node-label:") || strings.Contains(got, "node-taint:") {
+		t.Errorf("user-data must not carry any node-label/node-taint drop-in, got:\n%s", got)
 	}
-	if !strings.Contains(got, "nvidia.com/gpu=present:NoSchedule") {
-		t.Errorf("expected GPU taint value, got:\n%s", got)
+	if !strings.Contains(got, "SPINIFEX_NODEGROUP=ng-1") {
+		t.Errorf("expected SPINIFEX_NODEGROUP in agent.env, got:\n%s", got)
 	}
-	if strings.Contains(got, "K3S_NODE_LABEL=eks.amazonaws.com/nodegroup=ng-1,nvidia.com/gpu.present=true") {
-		t.Errorf("GPU label must not ride the no-op K3S_NODE_LABEL env, got:\n%s", got)
+	if !strings.Contains(got, "SPINIFEX_GPU_NODE=true") {
+		t.Errorf("expected SPINIFEX_GPU_NODE=true in agent.env for a GPU nodegroup, got:\n%s", got)
+	}
+	if strings.Contains(got, "K3S_NODE_LABEL") {
+		t.Errorf("label must not ride the no-op K3S_NODE_LABEL env, got:\n%s", got)
 	}
 }
 
-func TestBuildAgentUserData_NonGPUHasNoLabelOrTaint(t *testing.T) {
+func TestBuildAgentUserData_NonGPUSignalsNodegroupOnly(t *testing.T) {
 	in := baseTestInput()
 
 	got := buildAgentUserData(in)
@@ -128,17 +133,17 @@ func TestBuildAgentUserData_NonGPUHasNoLabelOrTaint(t *testing.T) {
 	if strings.Contains(got, "nvidia.com/gpu") {
 		t.Errorf("non-GPU nodegroup user-data must not reference nvidia.com/gpu, got:\n%s", got)
 	}
-	if strings.Contains(got, "20-gpu.yaml") {
-		t.Errorf("non-GPU nodegroup user-data must not carry the GPU drop-in, got:\n%s", got)
+	if strings.Contains(got, "SPINIFEX_GPU_NODE") {
+		t.Errorf("non-GPU nodegroup must not set SPINIFEX_GPU_NODE, got:\n%s", got)
 	}
-	// The nodegroup label must ride a node-label config drop-in, not the no-op
-	// K3S_NODE_LABEL env — waitWorkersReady buckets Ready nodes by this label,
-	// so a worker missing it never counts toward its nodegroup's ACTIVE gate.
-	if !strings.Contains(got, "/etc/rancher/k3s/config.yaml.d/15-nodegroup-label.yaml") {
-		t.Errorf("expected nodegroup-label drop-in path, got:\n%s", got)
+	if strings.Contains(got, "node-label:") || strings.Contains(got, "node-taint:") {
+		t.Errorf("user-data must not carry any node-label/node-taint drop-in, got:\n%s", got)
 	}
-	if !strings.Contains(got, `"eks.amazonaws.com/nodegroup=ng-1"`) {
-		t.Errorf("expected nodegroup label in the node-label drop-in, got:\n%s", got)
+	// The worker signals its nodegroup via agent.env; mulga-eks-provider-id.sh
+	// turns it into the eks.amazonaws.com/nodegroup label that gates the
+	// nodegroup reaching ACTIVE (waitWorkersReady buckets Ready nodes by it).
+	if !strings.Contains(got, "SPINIFEX_NODEGROUP=ng-1") {
+		t.Errorf("expected SPINIFEX_NODEGROUP in agent.env, got:\n%s", got)
 	}
 	if strings.Contains(got, "K3S_NODE_LABEL") {
 		t.Errorf("nodegroup label must not ride the no-op K3S_NODE_LABEL env, got:\n%s", got)

spinifex/network/policy/nat.go

@@ -289,6 +289,15 @@ func (m *natManager) AddEIP(ctx context.Context, eip EIPSpec) error {
 		slog.Info("policy: cleaned stale dnat_and_snat rules before AddEIP", "external_ip", eip.ExternalIP, "removed", removed)
 	}
 
+	// Scrub any predecessor row sharing this private IP under a different external
+	// IP: dnat_and_snat is 1:1 per private IP, and a stale row's SNAT half (keyed on
+	// logical_ip) blackholes the new EIP. Router-scoped — private IPs repeat per VPC.
+	if err := m.ovn.DeleteNAT(ctx, router, "dnat_and_snat", eip.LogicalIP); err != nil &&
+		!errors.Is(err, ovn.ErrNATNotFound) {
+		slog.Warn("policy: stale logical-IP NAT cleanup failed before AddEIP",
+			"logical_ip", eip.LogicalIP, "err", err)
+	}
+
 	if err := m.ovn.AddNAT(ctx, router, natRule); err != nil {
 		return fmt.Errorf("add dnat_and_snat %s -> %s on %s: %w", eip.LogicalIP, eip.ExternalIP, router, err)
 	}
@@ -570,6 +579,12 @@ func (m *natManager) AddSNAT(ctx context.Context, vpcID, vpcCIDR, externalIP str
 	if err := m.ovn.AddNAT(ctx, router, snatRule); err != nil {
 		return fmt.Errorf("add IGW snat %s -> %s on %s: %w", vpcCIDR, externalIP, router, err)
 	}
+
+	// Log the first install so RCA can pin the commit timestamp; without this
+	// only later reconcile passes emit "idempotent skip", masking whether a
+	// missing snat was programmed late or never programmed at all.
+	slog.Info("policy: AddSNAT installed",
+		"router", router, "vpc_cidr", vpcCIDR, "external_ip", externalIP)
 	return nil
 }
 

spinifex/network/policy/nat_test.go

@@ -380,6 +380,85 @@ func TestNATManager_AddEIP_IdempotentSkip_SameOwner_Centralized(t *testing.T) {
 	assert.Equal(t, 1, barrierCalls, "same-ENI re-add must not fire the barrier")
 }
 
+// A successor reusing a recycled private IP with a NEW external IP must leave
+// exactly one dnat_and_snat for that private IP, pointing at the new EIP. The
+// predecessor's old-EIP row survives its (lost) DeleteEIP and shares the private
+// IP; its SNAT half (keyed on logical_ip) would blackhole the successor's new EIP.
+// Sibling of the same-external-IP re-point (RePointsOnOwnerChange) and the
+// same-external/different-private scrub (RemovesStaleRuleOnOtherRouter).
+func TestNATManager_AddEIP_ScrubsPredecessorSharingPrivateIP(t *testing.T) {
+	const privateIP = "172.31.0.5"
+	for _, tc := range []struct {
+		name string
+		mode NATMode
+		mac  string
+	}{
+		{"centralized", NATModeCentralized, ""},
+		{"distributed", NATModeDistributed, "bb:bb:bb:bb:bb:bb"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			ctx := context.Background()
+			m := mock.New()
+			seedRouter(t, m, "vpc-1")
+			seedGatewayPort(t, m, "vpc-1", "aa:bb:cc:00:00:01")
+			nm, err := NewNATManager(m, tc.mode)
+			require.NoError(t, err)
+
+			// Predecessor terminated but its dnat_and_snat (old EIP -> private IP) survives.
+			require.NoError(t, nm.AddEIP(ctx, EIPSpec{
+				VPCID: "vpc-1", ExternalIP: "192.168.0.213", LogicalIP: privateIP,
+				PortName: "port-eni-old", MAC: tc.mac,
+			}))
+			// Successor recycles the private IP but binds a NEW EIP.
+			require.NoError(t, nm.AddEIP(ctx, EIPSpec{
+				VPCID: "vpc-1", ExternalIP: "192.168.0.214", LogicalIP: privateIP,
+				PortName: "port-eni-new", MAC: tc.mac,
+			}))
+
+			require.Equal(t, 1, countNAT(m, "dnat_and_snat", privateIP),
+				"exactly one dnat_and_snat may exist for a recycled private IP")
+			got := findNAT(m, "dnat_and_snat", privateIP)
+			require.NotNil(t, got)
+			assert.Equal(t, "192.168.0.214", got.ExternalIP,
+				"row must point at the successor's new EIP, not the predecessor's stale one")
+			assert.Equal(t, "port-eni-new", got.ExternalIDs["spinifex:logical_port"])
+		})
+	}
+}
+
+// The logical-IP scrub is router-scoped: private IPs repeat across VPCs, so a
+// same-private-IP row on another VPC's router is a legitimate separate EIP and
+// must not be touched.
+func TestNATManager_AddEIP_LeavesSamePrivateIPOnOtherRouterIntact(t *testing.T) {
+	ctx := context.Background()
+	m := mock.New()
+	seedRouter(t, m, "vpc-a")
+	seedRouter(t, m, "vpc-b")
+	nm, err := NewNATManager(m, NATModeDistributed)
+	require.NoError(t, err)
+
+	require.NoError(t, nm.AddEIP(ctx, EIPSpec{
+		VPCID: "vpc-a", ExternalIP: "192.168.0.10", LogicalIP: "172.31.0.5",
+		PortName: "port-a", MAC: "aa:aa:aa:aa:aa:aa",
+	}))
+	// A different instance in vpc-b that happens to reuse the same private IP.
+	require.NoError(t, nm.AddEIP(ctx, EIPSpec{
+		VPCID: "vpc-b", ExternalIP: "192.168.0.11", LogicalIP: "172.31.0.5",
+		PortName: "port-b", MAC: "bb:bb:bb:bb:bb:bb",
+	}))
+
+	byExt := map[string]*nbdb.NAT{}
+	for _, n := range m.NATs {
+		if n.Type == "dnat_and_snat" && n.LogicalIP == "172.31.0.5" {
+			byExt[n.ExternalIP] = n
+		}
+	}
+	assert.Len(t, byExt, 2,
+		"same private IP on a different VPC router is a separate EIP and must survive")
+	assert.Contains(t, byExt, "192.168.0.10", "vpc-a's row must survive")
+	assert.Contains(t, byExt, "192.168.0.11", "vpc-b's row must survive")
+}
+
 func TestNATManager_PruneOrphanEIPs(t *testing.T) {
 	ctx := context.Background()
 	m := mock.New()

tests/e2e/eks/eks_test.go

@@ -4,8 +4,10 @@ package eks
 
 import (
 	"encoding/base64"
+	"encoding/json"
 	"errors"
 	"fmt"
+	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -214,6 +216,11 @@ func TestEKS(t *testing.T) {
 		runIRSAWebIdentity(t, c, env, artifacts, fx)
 	})
 
+	t.Run("IRSAPod", func(t *testing.T) {
+		requireClusterReady(t, fx)
+		runIRSAPod(t, c, env, artifacts, fx)
+	})
+
 	t.Run("EBSCSIVolume", func(t *testing.T) {
 		requireClusterReady(t, fx)
 		runEBSCSIVolume(t, c, env, artifacts, fx)
@@ -237,47 +244,15 @@ func TestEKS(t *testing.T) {
 func runIRSAWebIdentity(t *testing.T, c *harness.AWSClient, env *harness.Env, artifacts string, fx *clusterFixture) {
 	require.NotNil(t, fx.Cluster.Identity, "ACTIVE cluster must expose Identity")
 	require.NotNil(t, fx.Cluster.Identity.Oidc, "ACTIVE cluster must expose Identity.Oidc")
-	issuer := aws.StringValue(fx.Cluster.Identity.Oidc.Issuer)
-	require.NotEmpty(t, issuer, "cluster OIDC issuer must be published")
-	t.Logf("OIDC issuer: %s", issuer)
 
-	// 1) Register the cluster's OIDC provider in the caller account so the STS
-	//    handler will accept tokens carrying this issuer.
-	oidcOut, err := c.IAM.CreateOpenIDConnectProvider(&iam.CreateOpenIDConnectProviderInput{
-		Url:            aws.String(issuer),
-		ClientIDList:   aws.StringSlice([]string{"sts.amazonaws.com"}),
-		ThumbprintList: aws.StringSlice([]string{"0000000000000000000000000000000000000000"}),
-	})
-	require.NoError(t, err, "create-open-id-connect-provider")
-	providerArn := aws.StringValue(oidcOut.OpenIDConnectProviderArn)
-	require.NotEmpty(t, providerArn, "OIDC provider ARN empty")
-	t.Cleanup(func() {
-		_, _ = c.IAM.DeleteOpenIDConnectProvider(&iam.DeleteOpenIDConnectProviderInput{
-			OpenIDConnectProviderArn: aws.String(providerArn),
-		})
-	})
+	// 1) Register the cluster's OIDC provider in the caller account and create a
+	//    role whose trust policy federates it, so the STS handler accepts
+	//    web-identity tokens carrying this issuer.
+	providerArn, roleArn, roleName := registerOIDCRole(t, c, fx, "irsa", "E2E IRSA web-identity role")
 	t.Logf("OIDC provider: %s", providerArn)
-
-	// 2) Create a role whose trust policy federates the OIDC provider. No
-	//    Condition block — the Federated principal + AssumeRoleWithWebIdentity
-	//    action are sufficient to grant, which keeps the test independent of the
-	//    condition-key issuer-prefix format.
-	roleName := fmt.Sprintf("%s-irsa", fx.ClusterName)
-	trustPolicy := fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Federated":%q},"Action":"sts:AssumeRoleWithWebIdentity"}]}`, providerArn)
-	roleOut, err := c.IAM.CreateRole(&iam.CreateRoleInput{
-		RoleName:                 aws.String(roleName),
-		AssumeRolePolicyDocument: aws.String(trustPolicy),
-		Description:              aws.String("E2E IRSA web-identity role"),
-	})
-	require.NoError(t, err, "create-role")
-	roleArn := aws.StringValue(roleOut.Role.Arn)
-	require.NotEmpty(t, roleArn, "role ARN empty")
-	t.Cleanup(func() {
-		_, _ = c.IAM.DeleteRole(&iam.DeleteRoleInput{RoleName: aws.String(roleName)})
-	})
 	t.Logf("role: %s", roleArn)
 
-	// 3) Mint a ServiceAccount token bound to sts.amazonaws.com. `kubectl create
+	// 2) Mint a ServiceAccount token bound to sts.amazonaws.com. `kubectl create
 	//    token` uses the TokenRequest API — no pod required. The token's iss is
 	//    the cluster OIDC issuer and aud includes sts.amazonaws.com (k3s is wired
 	//    with --service-account-issuer / --api-audiences at CreateCluster).
@@ -289,7 +264,7 @@ func runIRSAWebIdentity(t *testing.T, c *harness.AWSClient, env *harness.Env, ar
 	token := strings.TrimSpace(tokenOut)
 	require.Equal(t, 2, strings.Count(token, "."), "web-identity token must be a JWT (3 dot-separated parts)")
 
-	// 4) Exchange the token. AssumeRoleWithWebIdentity is anonymous (the SDK
+	// 3) Exchange the token. AssumeRoleWithWebIdentity is anonymous (the SDK
 	//    strips SigV4 for this op); the JWT is the identity.
 	const sessionName = "e2e-irsa"
 	var assumeOut *sts.AssumeRoleWithWebIdentityOutput
@@ -311,7 +286,7 @@ func runIRSAWebIdentity(t *testing.T, c *harness.AWSClient, env *harness.Env, ar
 	require.True(t, strings.HasPrefix(aws.StringValue(assumeOut.Credentials.AccessKeyId), "ASIA"),
 		"web-identity credentials must be temporary (ASIA…)")
 
-	// 5) The returned temporary credentials must be usable: GetCallerIdentity
+	// 4) The returned temporary credentials must be usable: GetCallerIdentity
 	//    must resolve to the assumed-role principal.
 	sessClient := harness.NewAWSClientWithSessionCreds(t, env,
 		aws.StringValue(assumeOut.Credentials.AccessKeyId),
@@ -325,6 +300,304 @@ func runIRSAWebIdentity(t *testing.T, c *harness.AWSClient, env *harness.Env, ar
 	t.Logf("GetCallerIdentity (web-identity creds): %s", aws.StringValue(ident.Arn))
 }
 
+// registerOIDCRole registers the cluster's OIDC provider in the caller account
+// and creates a role whose trust policy federates it — the shared setup every
+// IRSA scenario needs before a web-identity token can be exchanged. No
+// Condition block on the trust policy — the Federated principal +
+// AssumeRoleWithWebIdentity action are sufficient to grant, which keeps
+// callers independent of the condition-key issuer-prefix format. Both
+// resources are torn down via t.Cleanup (LIFO: role before provider).
+func registerOIDCRole(t *testing.T, c *harness.AWSClient, fx *clusterFixture, roleNameSuffix, description string) (providerArn, roleArn, roleName string) {
+	t.Helper()
+	require.NotNil(t, fx.Cluster.Identity, "ACTIVE cluster must expose Identity")
+	require.NotNil(t, fx.Cluster.Identity.Oidc, "ACTIVE cluster must expose Identity.Oidc")
+	issuer := aws.StringValue(fx.Cluster.Identity.Oidc.Issuer)
+	require.NotEmpty(t, issuer, "cluster OIDC issuer must be published")
+
+	oidcOut, err := c.IAM.CreateOpenIDConnectProvider(&iam.CreateOpenIDConnectProviderInput{
+		Url:            aws.String(issuer),
+		ClientIDList:   aws.StringSlice([]string{"sts.amazonaws.com"}),
+		ThumbprintList: aws.StringSlice([]string{"0000000000000000000000000000000000000000"}),
+	})
+	require.NoError(t, err, "create-open-id-connect-provider")
+	providerArn = aws.StringValue(oidcOut.OpenIDConnectProviderArn)
+	require.NotEmpty(t, providerArn, "OIDC provider ARN empty")
+	t.Cleanup(func() {
+		_, _ = c.IAM.DeleteOpenIDConnectProvider(&iam.DeleteOpenIDConnectProviderInput{
+			OpenIDConnectProviderArn: aws.String(providerArn),
+		})
+	})
+
+	roleName = fmt.Sprintf("%s-%s", fx.ClusterName, roleNameSuffix)
+	trustPolicy := fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Federated":%q},"Action":"sts:AssumeRoleWithWebIdentity"}]}`, providerArn)
+	roleOut, err := c.IAM.CreateRole(&iam.CreateRoleInput{
+		RoleName:                 aws.String(roleName),
+		AssumeRolePolicyDocument: aws.String(trustPolicy),
+		Description:              aws.String(description),
+	})
+	require.NoError(t, err, "create-role")
+	roleArn = aws.StringValue(roleOut.Role.Arn)
+	require.NotEmpty(t, roleArn, "role ARN empty")
+	t.Cleanup(func() {
+		_, _ = c.IAM.DeleteRole(&iam.DeleteRoleInput{RoleName: aws.String(roleName)})
+	})
+	return providerArn, roleArn, roleName
+}
+
+// runIRSAPod exercises the in-cluster IRSA path a real workload depends on:
+// spinifex ships no pod-identity webhook, so a pod must wire the projected SA
+// token + AWS_* env explicitly (mirroring
+// scripts/images/eks-node/mulga-eks-addon-sync.sh's {{IRSA_ENV}}/{{IRSA_VOLUME}}
+// blocks). The pod runs `aws sts get-caller-identity` against the awsgw STS
+// endpoint from inside the cluster; the returned ARN must resolve to the
+// assumed IRSA role, proving token mount + env wiring + in-cluster egress all
+// work together (unlike IRSAWebIdentity, which only proves the API exchange).
+func runIRSAPod(t *testing.T, c *harness.AWSClient, env *harness.Env, artifacts string, fx *clusterFixture) {
+	require.NotNil(t, fx.Cluster.Identity, "ACTIVE cluster must expose Identity")
+	require.NotNil(t, fx.Cluster.Identity.Oidc, "ACTIVE cluster must expose Identity.Oidc")
+	issuer := aws.StringValue(fx.Cluster.Identity.Oidc.Issuer)
+
+	_, roleArn, roleName := registerOIDCRole(t, c, fx, "irsa-pod", "E2E IRSA pod role")
+	t.Logf("IRSA pod role: %s", roleArn)
+
+	// awsgw enforces IAM on assumed-role STS calls, so the role needs a
+	// permission policy allowing GetCallerIdentity — an AWS-managed ARN is
+	// opaque and grants nothing, so use a customer-managed policy with an
+	// explicit allow (same pattern as the EBS CSI role below).
+	const stsPolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"sts:GetCallerIdentity","Resource":"*"}]}`
+	polOut, err := c.IAM.CreatePolicy(&iam.CreatePolicyInput{
+		PolicyName:     aws.String(roleName + "-policy"),
+		PolicyDocument: aws.String(stsPolicy),
+	})
+	require.NoError(t, err, "create-policy")
+	policyArn := aws.StringValue(polOut.Policy.Arn)
+	t.Cleanup(func() {
+		_, _ = c.IAM.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: aws.String(policyArn)})
+	})
+	_, err = c.IAM.AttachRolePolicy(&iam.AttachRolePolicyInput{
+		RoleName:  aws.String(roleName),
+		PolicyArn: aws.String(policyArn),
+	})
+	require.NoError(t, err, "attach-role-policy")
+	t.Cleanup(func() {
+		_, _ = c.IAM.DetachRolePolicy(&iam.DetachRolePolicyInput{
+			RoleName:  aws.String(roleName),
+			PolicyArn: aws.String(policyArn),
+		})
+	})
+
+	// gatewayBaseURL is the issuer with its /oidc/eks/<region>/<acct>/<cluster>
+	// path stripped — the customer-facing WAN advertise addr (:9999) in-cluster
+	// pods reach, exactly what the addon-sync agent injects as
+	// AWS_ENDPOINT_URL_STS.
+	u, err := url.Parse(issuer)
+	require.NoErrorf(t, err, "parse issuer URL %q", issuer)
+	require.NotEmpty(t, u.Scheme, "issuer URL must carry a scheme")
+	require.NotEmpty(t, u.Host, "issuer URL must carry a host")
+	gatewayBaseURL := u.Scheme + "://" + u.Host
+	t.Logf("gateway base URL: %s", gatewayBaseURL)
+
+	region := envOr("SPINIFEX_AWS_REGION", "ap-southeast-2")
+
+	kcPath := writeKubeconfig(t, artifacts, fx.Cluster)
+	kc := harness.NewKubectl(t, kcPath, getTokenEnv(t, env))
+
+	// The control-plane node carries CriticalAddonsOnly=true:NoExecute (EKS
+	// parity: k3s_server_vm.go taints it to keep user workloads off), so this
+	// pod needs a customer worker node like EBSCSIVolume — there is no
+	// untainted node to land on. A 1-node nodegroup is created and the pod is
+	// pinned to it via the eks.amazonaws.com/nodegroup label the worker
+	// registers under (never set on the control-plane node).
+	//
+	// CreateNodegroup's worker launch attaches an instance profile for the
+	// declared NodeRole (ensureNodeInstanceProfile in nodegroup.go), which
+	// requires the role to actually exist in IAM — real EKS has the same
+	// prerequisite (the customer creates the node instance role before
+	// CreateNodegroup). Standard EC2-service trust policy, no permission
+	// policy needed: the pod pulls from public.ecr.aws directly, never
+	// touching the node's own instance-profile credentials.
+	nodeRoleName := fx.ClusterName + "-node"
+	const ec2TrustPolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"ec2.amazonaws.com"},"Action":"sts:AssumeRole"}]}`
+	nodeRoleOut, err := c.IAM.CreateRole(&iam.CreateRoleInput{
+		RoleName:                 aws.String(nodeRoleName),
+		AssumeRolePolicyDocument: aws.String(ec2TrustPolicy),
+		Description:              aws.String("E2E IRSA pod nodegroup worker role"),
+	})
+	require.NoError(t, err, "create-node-role")
+	nodeRoleArn := aws.StringValue(nodeRoleOut.Role.Arn)
+	t.Cleanup(func() {
+		_, _ = c.IAM.DeleteRole(&iam.DeleteRoleInput{RoleName: aws.String(nodeRoleName)})
+	})
+
+	const nodegroup = "irsa-pod-e2e-ng"
+	harness.Phase(t, "Creating worker nodegroup %s", nodegroup)
+	// e2e:allow-create — the worker nodegroup is the subject under test (customer-space node the IRSA pod schedules onto).
+	_, err = c.EKS.CreateNodegroup(&eks.CreateNodegroupInput{
+		ClusterName:   aws.String(fx.ClusterName),
+		NodegroupName: aws.String(nodegroup),
+		Subnets:       aws.StringSlice([]string{fx.SubnetID}),
+		NodeRole:      aws.String(nodeRoleArn),
+		ScalingConfig: &eks.NodegroupScalingConfig{
+			MinSize:     aws.Int64(1),
+			MaxSize:     aws.Int64(1),
+			DesiredSize: aws.Int64(1),
+		},
+	})
+	require.NoError(t, err, "create-nodegroup")
+	t.Cleanup(func() {
+		_, _ = c.EKS.DeleteNodegroup(&eks.DeleteNodegroupInput{
+			ClusterName:   aws.String(fx.ClusterName),
+			NodegroupName: aws.String(nodegroup),
+		})
+	})
+	harness.EventuallyErr(t, func() error {
+		out, derr := c.EKS.DescribeNodegroup(&eks.DescribeNodegroupInput{
+			ClusterName:   aws.String(fx.ClusterName),
+			NodegroupName: aws.String(nodegroup),
+		})
+		if derr != nil {
+			return fmt.Errorf("describe-nodegroup: %w", derr)
+		}
+		if s := aws.StringValue(out.Nodegroup.Status); s != eks.NodegroupStatusActive {
+			return fmt.Errorf("nodegroup status %q, want ACTIVE", s)
+		}
+		return nil
+	}, 8*time.Minute, 10*time.Second)
+
+	// Deliver the awsgw CA into the pod via a ConfigMap — the same CA the host
+	// already trusts for awsgw (harness.ResolveCACert), mounted read-only.
+	caPath, err := harness.ResolveCACert(env)
+	require.NoError(t, err, "resolve gateway CA cert")
+	const caConfigMap = "irsa-pod-gateway-ca"
+	out, err := kc.Run(30*time.Second, "create", "configmap", caConfigMap,
+		"-n", "default", "--from-file=ca.pem="+caPath)
+	require.NoErrorf(t, err, "create gateway-ca configmap:\n%s", out)
+	t.Cleanup(func() {
+		_, _ = kc.Run(30*time.Second, "delete", "configmap", caConfigMap, "-n", "default", "--ignore-not-found")
+	})
+
+	const sa = "irsa-pod-e2e"
+	const podName = "irsa-pod-e2e"
+	nodeSelector := fmt.Sprintf("  nodeSelector:\n    eks.amazonaws.com/nodegroup: %s\n", nodegroup)
+	manifestPath := filepath.Join(artifacts, "irsa-pod.yaml")
+	require.NoError(t, os.WriteFile(manifestPath,
+		[]byte(irsaPodManifest(sa, podName, roleArn, gatewayBaseURL, region, caConfigMap, nodeSelector)), 0o600))
+	t.Cleanup(func() {
+		_, _ = kc.Run(60*time.Second, "delete", "-f", manifestPath, "--ignore-not-found", "--wait=false")
+	})
+
+	harness.OnFailure(t, func() {
+		dumps := map[string][]string{
+			"irsa-pod-describe.txt": {"-n", "default", "describe", "pod", podName},
+			"irsa-pod-events.txt":   {"-n", "default", "get", "events", "--sort-by", ".lastTimestamp"},
+			"irsa-pod-nodes.txt":    {"get", "nodes", "-o", "wide", "--show-labels"},
+		}
+		for name, args := range dumps {
+			out, _ := kc.Run(45*time.Second, args...)
+			harness.DumpFile(t, artifacts, name, []byte(out))
+		}
+		errOut, _ := kc.Run(30*time.Second, "-n", "default", "exec", podName, "--", "cat", "/tmp/err.txt")
+		harness.DumpFile(t, artifacts, "irsa-pod-err.txt", []byte(errOut))
+	})
+
+	harness.Phase(t, "Applying IRSA pod")
+	out, err = kc.Run(60*time.Second, "apply", "-f", manifestPath)
+	require.NoErrorf(t, err, "apply irsa pod:\n%s", out)
+
+	waitPodReady(t, kc, podName)
+
+	// aws sts get-caller-identity writes JSON to /tmp/out.json inside the pod;
+	// poll (not just wait-once) since network egress + the STS round trip can
+	// outlast the container's Ready flip (Ready only means the shell started).
+	var ident struct {
+		UserId  string `json:"UserId"`
+		Account string `json:"Account"`
+		Arn     string `json:"Arn"`
+	}
+	harness.EventuallyErr(t, func() error {
+		raw, err := kc.Run(30*time.Second, "-n", "default", "exec", podName, "--", "cat", "/tmp/out.json")
+		if err != nil {
+			return fmt.Errorf("exec cat out.json: %v\n%s", err, raw)
+		}
+		if jerr := json.Unmarshal([]byte(raw), &ident); jerr != nil {
+			return fmt.Errorf("parse out.json: %v\nraw: %s", jerr, raw)
+		}
+		return nil
+	}, 3*time.Minute, 5*time.Second)
+
+	t.Logf("in-cluster GetCallerIdentity: %+v", ident)
+	assert.Equal(t, fx.AccountID, ident.Account, "caller account must match")
+	assert.Contains(t, ident.Arn, "assumed-role/"+roleName,
+		"caller ARN must reflect the assumed IRSA role")
+}
+
+// irsaPodManifest renders a ServiceAccount (carrying the decorative
+// eks.amazonaws.com/role-arn annotation) plus a pod wired for IRSA exactly as
+// mulga-eks-addon-sync.sh's {{IRSA_ENV}}/{{IRSA_VOLUME}}/{{IRSA_VOLUME_MOUNT}}
+// blocks would: a projected SA token (audience sts.amazonaws.com) mounted at
+// the well-known eks.amazonaws.com path, the AWS_* env pointing the SDK at the
+// awsgw STS endpoint, and the gateway CA trusted via ConfigMap. extraSpec is
+// injected verbatim into the Pod spec (e.g. a nodeSelector), or "" for none.
+func irsaPodManifest(sa, pod, roleArn, gatewayBaseURL, region, caConfigMap, extraSpec string) string {
+	return fmt.Sprintf(`apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: %[1]s
+  namespace: default
+  annotations:
+    eks.amazonaws.com/role-arn: %[3]q
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: %[2]s
+  namespace: default
+spec:
+  serviceAccountName: %[1]s
+  restartPolicy: Never
+%[7]s  containers:
+  - name: aws-cli
+    image: public.ecr.aws/aws-cli/aws-cli:latest
+    command: ["sh", "-c", "aws sts get-caller-identity --output json >/tmp/out.json 2>/tmp/err.txt; sleep 3600"]
+    env:
+    - name: AWS_ROLE_ARN
+      value: %[3]q
+    - name: AWS_WEB_IDENTITY_TOKEN_FILE
+      value: /var/run/secrets/eks.amazonaws.com/serviceaccount/token
+    - name: AWS_STS_REGIONAL_ENDPOINTS
+      value: regional
+    - name: AWS_REGION
+      value: %[5]q
+    - name: AWS_DEFAULT_REGION
+      value: %[5]q
+    - name: AWS_ENDPOINT_URL_STS
+      value: %[4]q
+    - name: AWS_CA_BUNDLE
+      value: /etc/spinifex/gateway-ca/ca.pem
+    - name: SSL_CERT_FILE
+      value: /etc/spinifex/gateway-ca/ca.pem
+    volumeMounts:
+    - name: aws-iam-token
+      mountPath: /var/run/secrets/eks.amazonaws.com/serviceaccount
+      readOnly: true
+    - name: gateway-ca
+      mountPath: /etc/spinifex/gateway-ca
+      readOnly: true
+  volumes:
+  - name: aws-iam-token
+    projected:
+      defaultMode: 420
+      sources:
+      - serviceAccountToken:
+          audience: sts.amazonaws.com
+          expirationSeconds: 86400
+          path: token
+  - name: gateway-ca
+    configMap:
+      name: %[6]s
+`, sa, pod, roleArn, gatewayBaseURL, region, caConfigMap, extraSpec)
+}
+
 // runEBSCSIVolume exercises the full EBS CSI data path on the live cluster:
 // install the aws-ebs-csi-driver managed addon bound to an IRSA role, then
 // drive a default gp3 PVC -> Pod -> CreateVolume(Viperblock)/AttachVolume

tests/e2e/natuplink/natuplink_test.go

@@ -487,9 +487,17 @@ func phaseUniqueTransitIP(t *testing.T, fix *fixture, defaultGwIP string) {
 	assert.NotEqualf(t, defaultGwIP, gwIP,
 		"each VPC gateway LRP must get a unique transit IP (both got %s)", gwIP)
 
-	natList := harness.OvnNbctl(t, "lr-nat-list", "vpc-"+vpcID)
-	assert.Containsf(t, natList, secondVPCCIDR,
-		"second VPC router missing snat for %s\n%s", secondVPCCIDR, natList)
+	// The IGW default snat commits a beat after the gateway LRP appears, so
+	// poll lr-nat-list instead of reading it once — a single-shot check races
+	// the OVN commit and intermittently sees an empty list.
+	harness.Step(t, "second VPC transit snat programmed")
+	harness.EventuallyErr(t, func() error {
+		natList := harness.OvnNbctl(t, "lr-nat-list", "vpc-"+vpcID)
+		if !strings.Contains(natList, secondVPCCIDR) {
+			return fmt.Errorf("second VPC router missing snat for %s\n%s", secondVPCCIDR, natList)
+		}
+		return nil
+	}, 2*time.Minute, 3*time.Second)
 
 	harness.Step(t, "host ingress route for second VPC installed on attach")
 	harness.EventuallyErr(t, func() error {