From 2e397d6e97bd01b47423b4bb0309ca7abb6d1afe Mon Sep 17 00:00:00 2001
From: joshsiv-mulga <mrk@mulgadc.com>
Date: Tue, 8 Sep 2026 08:46:43 +1000
Subject: [PATCH] fix: keep the apiserver controlplane egress direct

The controlplane egress selector also carries the authentication
token-webhook dial, which targets 127.0.0.1:8443 on the control-plane VM.
Tunnelling it through konnectivity hands that dial to a randomly chosen
agent, where nothing is listening, so the authenticator returns an error
rather than a decision and every bearer-token request fails closed with a
bare 401.

It only surfaced once a second agent existed. The control-plane node runs
an agent itself, so while it is the only backend every tunnelled dial
lands back home and authentication works by luck. The apiserver's webhook
client also holds its connection for the 90s IdleConnTimeout, so there is
one backend selection per connection rather than per request.

TestEKS/IRSAPod hit both conditions every run: it builds a kubeconfig and
then waits ~105s for its nodegroup to go ACTIVE before issuing any
request, which both expires the pooled connection and brings up the
cluster's second agent.

Only the cluster egress needs the tunnel, for kubelet exec/logs/
portforward. This matches the upstream reference configuration.
---
 spinifex/handlers/eks/k3s_server_vm.go      | 32 ++++++++++-----------
 spinifex/handlers/eks/k3s_server_vm_test.go | 16 +++++------
 2 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/spinifex/handlers/eks/k3s_server_vm.go b/spinifex/handlers/eks/k3s_server_vm.go
index 86130d3a5..02f5936dc 100644
--- a/spinifex/handlers/eks/k3s_server_vm.go
+++ b/spinifex/handlers/eks/k3s_server_vm.go
@@ -472,25 +472,25 @@ func k3sServerJoinURL(ip string) string {
 // apiserver egress to the konnectivity-server UDS. v1beta1 is the schema
 // k3s/kube-apiserver accept for --egress-selector-config-file.
 func egressSelectorConfigYAML() string {
-	lines := []string{
+	// controlplane also carries the authentication token-webhook dial, which
+	// targets 127.0.0.1 on this node. Tunnelled, it reaches a random agent where
+	// nothing listens, and every bearer-token request fails closed with 401.
+	return strings.Join([]string{
 		"apiVersion: apiserver.k8s.io/v1beta1",
 		"kind: EgressSelectorConfiguration",
 		"egressSelections:",
-	}
-	// controlplane carries admission-webhook and aggregated-apiserver dials;
-	// cluster carries kubelet exec/logs/portforward. Both must ride the tunnel:
-	// an undefined selection falls back to a direct, unroutable dial from the CP.
-	for _, name := range []string{"controlplane", "cluster"} {
-		lines = append(lines,
-			"  - name: "+name,
-			"    connection:",
-			"      proxyProtocol: GRPC",
-			"      transport:",
-			"        uds:",
-			"          udsName: "+konnectivityUDSPath,
-		)
-	}
-	return strings.Join(lines, "\n")
+		"  - name: controlplane",
+		"    connection:",
+		"      proxyProtocol: Direct",
+		// cluster carries kubelet exec/logs/portforward, which must ride the
+		// tunnel: a direct dial to a node's kubelet is unroutable from the CP.
+		"  - name: cluster",
+		"    connection:",
+		"      proxyProtocol: GRPC",
+		"      transport:",
+		"        uds:",
+		"          udsName: " + konnectivityUDSPath,
+	}, "\n")
 }
 
 // dedupeNonEmpty returns the input with empty strings dropped and duplicates
diff --git a/spinifex/handlers/eks/k3s_server_vm_test.go b/spinifex/handlers/eks/k3s_server_vm_test.go
index 0914c65a7..e90371c66 100644
--- a/spinifex/handlers/eks/k3s_server_vm_test.go
+++ b/spinifex/handlers/eks/k3s_server_vm_test.go
@@ -440,18 +440,18 @@ func TestBuildK3sUserData_EgressSelectorDisabledWithKonnConfig(t *testing.T) {
 		"the EgressSelectorConfiguration file routes the cluster egress to the konn socket")
 }
 
-func TestEgressSelectorConfigYAML_TunnelsControlPlaneAndCluster(t *testing.T) {
+func TestEgressSelectorConfigYAML_TunnelsClusterOnly(t *testing.T) {
 	cfg := egressSelectorConfigYAML()
 
-	// Admission webhooks and aggregated apiservers dial via `controlplane`; only
-	// kubelet exec/logs/portforward use `cluster`. Omitting controlplane leaves
-	// webhook dials direct, which cannot route from the CP VPC to the service CIDR.
-	assert.Contains(t, cfg, "  - name: controlplane",
-		"webhook dials use the controlplane egress and must ride the konnectivity tunnel")
+	// The authentication token webhook listens on 127.0.0.1 on this node and is
+	// dialled through `controlplane`. Tunnelling it hands the dial to a random
+	// agent, where nothing listens, and every bearer token 401s.
+	assert.Contains(t, cfg, "  - name: controlplane\n    connection:\n      proxyProtocol: Direct",
+		"the controlplane egress must stay direct so the local token webhook is reachable")
 	assert.Contains(t, cfg, "  - name: cluster",
 		"kubelet exec/logs dials use the cluster egress")
-	assert.Equal(t, 2, strings.Count(cfg, "udsName: "+konnectivityUDSPath),
-		"both egress selections must terminate on the konnectivity socket")
+	assert.Equal(t, 1, strings.Count(cfg, "udsName: "+konnectivityUDSPath),
+		"only the cluster egress selection terminates on the konnectivity socket")
 }
 
 func TestBuildK3sUserData_KonnectivityEnv(t *testing.T) {