fix(eks): make aws eks get-token → kubectl auth work end-to-end (Sprint 6f) (#303)

* feat(eks): webhook observability — console log sink + per-request TokenReview decision

* fix(eks): wire token-webhook authenticator into generated k3s apiserver config

The generated /etc/rancher/k3s/config.yaml overrides the AMI-baked
config.yaml.skel, but its kube-apiserver-arg list omitted the
authentication-token-webhook-config-file arg. The running apiserver
therefore never loaded the token webhook, so bearer tokens minted by
'aws eks get-token' 401'd without the webhook ever being invoked.

* fix(eks): echo TokenReview audiences so api-audiences apiserver accepts the principal

With --api-audiences set, kube-apiserver sends spec.audiences in the
TokenReview and discards an authenticated=true response that does not
confirm a matching audience. The webhook returned an empty
status.audiences, so a valid IAM principal surfaced as a 401. Echo the
request's audiences on success.

* fix(eks): pin token-webhook version v1 so apiserver accepts the response

kube-apiserver --authentication-token-webhook-version defaults to
v1beta1, but the eks-token-webhook responds with an
authentication.k8s.io/v1 TokenReview. The apiserver decodes the response
against the expected GVK and rejects the mismatch, surfacing a valid
principal as a 401. Pin v1 to match the webhook.

cmd/eks-token-webhook/main.go

@@ -194,6 +194,12 @@ func (a *authenticator) handle(w http.ResponseWriter, r *http.Request) {
 	}
 
 	status := authenticate(review.Spec.Token, a.verify, a.lookup)
+	if status.Authenticated {
+		// Confirm the token for the audiences the apiserver asked about; an empty
+		// status.audiences is rejected when --api-audiences is configured.
+		status.Audiences = review.Spec.Audiences
+	}
+	slog.Info("TokenReview decision", "authenticated", status.Authenticated, "username", status.User.Username, "audiences", status.Audiences)
 
 	resp := tokenReview{
 		APIVersion: "authentication.k8s.io/v1",

cmd/eks-token-webhook/tokenreview.go

@@ -19,7 +19,12 @@ type tokenReviewSpec struct {
 type tokenReviewStatus struct {
 	Authenticated bool     `json:"authenticated"`
 	User          userInfo `json:"user,omitzero"`
-	Error         string   `json:"error,omitempty"`
+	// Audiences echoes the request's spec.audiences on success. When the
+	// apiserver runs with --api-audiences, it sends spec.audiences and discards
+	// an authenticated=true response that does not confirm a matching audience —
+	// so an empty Audiences here surfaces as a 401 despite a valid principal.
+	Audiences []string `json:"audiences,omitempty"`
+	Error     string   `json:"error,omitempty"`
 }
 
 type userInfo struct {

scripts/images/eks-node/eks-token-webhook.initd

@@ -11,8 +11,11 @@ command="/usr/local/bin/eks-token-webhook"
 command_args="--addr=127.0.0.1:8443"
 command_background="yes"
 pidfile="/run/eks-token-webhook.pid"
-output_log="/var/log/eks-token-webhook.log"
-error_log="/var/log/eks-token-webhook.log"
+# Headless appliance: no SSH. Route the webhook's stdout/stderr to the serial
+# console so its startup + per-request decisions land in the host-captured
+# console log (the only off-VM window into webhook health).
+output_log="/dev/console"
+error_log="/dev/console"
 
 # NATS creds + cluster identity come from the cloud-init-seeded first-boot env.
 # Sourced here and exported so start-stop-daemon hands them to the binary, which

spinifex/handlers/eks/k3s_server_vm.go

@@ -81,6 +81,13 @@ const (
 	// messages. Path matches k3s-first-boot.sh SPINIFEX_NATS_CA.
 	k3sNATSCAPath = "/etc/spinifex-eks/nats-ca.pem"
 
+	// k3sTokenWebhookKubeconfigPath is the apiserver token-webhook config the
+	// eks-token-webhook service (ordered `before k3s`) writes before the
+	// apiserver starts. Wired via the authentication-token-webhook-config-file
+	// apiserver arg so bearer tokens minted by `aws eks get-token` resolve
+	// through the webhook. Must match the webhook's EKS_WEBHOOK_KUBECONFIG default.
+	k3sTokenWebhookKubeconfigPath = "/etc/spinifex-eks/token-webhook.kubeconfig" //nolint:gosec // file path, not a credential
+
 	// k3sConfigPath is the K3s server config file cloud-init writes; K3s
 	// reads it at startup (overrides the AMI-baked config.yaml.skel).
 	k3sConfigPath = "/etc/rancher/k3s/config.yaml"
@@ -397,6 +404,11 @@ func buildK3sUserData(in K3sServerInput) string {
 		"  - service-account-issuer=" + in.OIDCIssuer,
 		"  - api-audiences=sts.amazonaws.com",
 		"  - anonymous-auth=true",
+		"  - authentication-token-webhook-config-file=" + k3sTokenWebhookKubeconfigPath,
+		"  - authentication-token-webhook-cache-ttl=5m",
+		// Pin v1 so the apiserver decodes the webhook's authentication.k8s.io/v1
+		// TokenReview response; the default v1beta1 rejects the GVK mismatch (401).
+		"  - authentication-token-webhook-version=v1",
 	}, "\n")
 
 	files := []userDataFile{

spinifex/handlers/eks/k3s_server_vm_test.go

@@ -297,6 +297,10 @@ func TestLaunchK3sServerVM_UserDataContainsAllArtifacts(t *testing.T) {
 	assert.NotContains(t, udata, "service-account-key-file="+k3sOIDCSigningKeyPath)
 	assert.Contains(t, udata, "service-account-issuer=https://oidc.spinifex.local/clusters/111122223333/alpha")
 	assert.Contains(t, udata, "api-audiences=sts.amazonaws.com")
+	// IAM bearer-token auth: the generated config overrides the AMI skel, so the
+	// token-webhook arg MUST be emitted here or `aws eks get-token` 401s with the
+	// webhook never invoked.
+	assert.Contains(t, udata, "authentication-token-webhook-config-file="+k3sTokenWebhookKubeconfigPath)
 
 	assert.Contains(t, udata, "path: "+k3sNATSCAPath)
 	assert.Contains(t, udata, "-----BEGIN CERTIFICATE-----")