test: cover the etcd member client and CLI against a TLS fake etcd

The real etcd client and the command had no tests, leaving the PR below the
diff coverage threshold. A fake etcd serving the Cluster and Maintenance APIs
over mutual TLS drives the client with the K3s certificate layout, and main
becomes a testable run function.

cmd/eks-etcd-member/main.go

@@ -0,0 +1,50 @@
+// eks-etcd-member removes a dead peer from the embedded etcd cluster of an EKS
+// K3s control-plane VM. A member that is already gone exits 0; any other
+// failure exits non-zero.
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"time"
+
+	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
+
+	"github.com/mulgadc/spinifex/spinifex/agents/eks/etcdmember"
+)
+
+func main() {
+	os.Exit(run(os.Args[1:], os.Stdout, os.Stderr))
+}
+
+func run(args []string, stdout, stderr io.Writer) int {
+	if len(args) < 1 || args[0] != "remove" {
+		fmt.Fprintln(stderr, "usage: eks-etcd-member remove --peer-ip 10.0.0.12 [--self-ip 10.0.0.10]")
+		return 2
+	}
+	fs := flag.NewFlagSet("remove", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	var cfg etcdmember.Config
+	endpoint := fs.String("endpoint", etcdmember.DefaultEndpoint, "etcd client endpoint")
+	certDir := fs.String("cert-dir", etcdmember.DefaultCertDir, "directory holding the K3s etcd client.crt, client.key and server-ca.crt")
+	timeout := fs.Duration("timeout", 30*time.Second, "overall deadline")
+	fs.StringVar(&cfg.PeerIP, "peer-ip", "", "peer URL host of the member to remove")
+	fs.StringVar(&cfg.SelfIP, "self-ip", "", "this host's own address; removal of it is refused")
+	if err := fs.Parse(args[1:]); err != nil {
+		return 2
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), *timeout)
+	defer cancel()
+
+	cli, err := etcdmember.Dial(*endpoint, *certDir, *timeout)
+	if err != nil {
+		fmt.Fprintln(stderr, "etcd member remove failed:", err)
+		return 1
+	}
+	defer func() { _ = cli.Close() }()
+	return etcdmember.Run(ctx, cli, cfg, stdout)
+}

cmd/eks-etcd-member/main_test.go

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/mulgadc/spinifex/internal/testkit/fakeetcd"
+	"github.com/stretchr/testify/assert"
+)
+
+var fakeMembers = []fakeetcd.Member{
+	{ID: 0xa1, Name: "cp-a", PeerURLs: []string{"https://10.0.0.10:2380"}},
+	{ID: 0xb2, Name: "cp-b", PeerURLs: []string{"https://10.0.0.12:2380"}},
+}
+
+func TestRunRequiresTheRemoveSubcommand(t *testing.T) {
+	t.Parallel()
+	for _, args := range [][]string{nil, {"list"}} {
+		var out, errOut bytes.Buffer
+		assert.Equal(t, 2, run(args, &out, &errOut))
+		assert.Contains(t, errOut.String(), "usage: eks-etcd-member remove")
+	}
+}
+
+func TestRunRejectsUnknownFlags(t *testing.T) {
+	t.Parallel()
+	var out, errOut bytes.Buffer
+	assert.Equal(t, 2, run([]string{"remove", "--bogus"}, &out, &errOut))
+}
+
+func TestRunFailsWhenEtcdCertificatesAreMissing(t *testing.T) {
+	t.Parallel()
+	var out, errOut bytes.Buffer
+	code := run([]string{"remove", "--peer-ip", "10.0.0.12", "--cert-dir", t.TempDir()}, &out, &errOut)
+	assert.Equal(t, 1, code)
+	assert.Contains(t, errOut.String(), "etcd member remove failed")
+}
+
+func TestRunRemovesTheNamedPeer(t *testing.T) {
+	t.Parallel()
+	srv := fakeetcd.Start(t, 0xa1, fakeMembers)
+	var out, errOut bytes.Buffer
+
+	code := run([]string{"remove", "--peer-ip", "10.0.0.12", "--self-ip", "10.0.0.10",
+		"--endpoint", srv.Endpoint, "--cert-dir", srv.CertDir}, &out, &errOut)
+
+	assert.Zero(t, code, errOut.String()+out.String())
+	assert.Contains(t, out.String(), "removed etcd member 10.0.0.12")
+	assert.Equal(t, []uint64{0xb2}, srv.Removed())
+}
+
+func TestRunTreatsAnAbsentPeerAsSuccess(t *testing.T) {
+	t.Parallel()
+	srv := fakeetcd.Start(t, 0xa1, fakeMembers)
+	var out, errOut bytes.Buffer
+
+	code := run([]string{"remove", "--peer-ip", "10.0.0.99",
+		"--endpoint", srv.Endpoint, "--cert-dir", srv.CertDir}, &out, &errOut)
+
+	assert.Zero(t, code)
+	assert.Contains(t, out.String(), "member already absent")
+	assert.Empty(t, srv.Removed())
+}

docs/admin/setting-up-your-cluster/README.md

@@ -106,7 +106,7 @@ done
 | `spinifex-rds-postgres` | RDS, `--engine postgres`         | Alpine 3.24.1 + PostgreSQL 18 + `rds-init`      |
 | `spinifex-rds-mariadb`  | RDS, `--engine mariadb`          | Alpine 3.24.1 + MariaDB 11.8 + `rds-init`       |
 | `spinifex-ecs-node`     | ECS container instances          | Alpine 3.21.7 + containerd + ecs-agent          |
-| `spinifex-eks-node`     | EKS control plane and nodegroups | Alpine 3.21.7 + K3s v1.32.5 + eks-token-webhook |
+| `spinifex-eks-node`     | EKS control plane and nodegroups | Alpine 3.21.7 + K3s v1.32.5 + eks-token-webhook + eks-etcd-member |
 
 All four boot **BIOS**, unlike the distro images, which boot UEFI. That is a property of how they are built and nothing you need to pass.
 

docs/containers/eks/README.md

@@ -35,6 +35,8 @@ resources:
 
 An EKS cluster on Spinifex has two parts: a **control plane** (the managed Kubernetes API server, provisioned on a Spinifex-managed VM and fronted by a network load balancer) and one or more **managed node groups** of worker VMs that run your pods. You provide the VPC, subnets, and IAM roles; Spinifex creates and reconciles the cluster, its load balancer, and its security groups.
 
+Control-plane hosts are not Kubernetes Nodes, as on AWS: `kubectl get nodes` lists only your node group workers, so a cluster with no node group lists none, and no pod runs on a control-plane host. Add-ons that used to tolerate the control-plane `CriticalAddonsOnly` taint, such as CoreDNS, metrics-server and konnectivity-agent, run on workers, so a Ready worker is required before they become available.
+
 Authentication uses **EKS access entries** (the API authentication mode) — IAM principals are mapped to Kubernetes RBAC groups, and `kubectl` authenticates with `aws eks get-token` exactly as it does on AWS.
 
 <p align="center">
@@ -172,6 +174,8 @@ aws eks update-kubeconfig --name demo
 kubectl get nodes
 ```
 
+The list shows the node group workers only. The control-plane VMs never appear as Nodes.
+
 ### 4. Expose a workload
 
 Workers join an auto-managed security group that only allows intra-cluster traffic. To reach a NodePort service, add one ingress rule to that SG (look it up by its deterministic name):
@@ -284,6 +288,12 @@ The workbooks form a ladder. **eks-quickstart** is a NodePort demo; **eks-https-
 
 :::
 
+### Upgrade notes
+
+**Control-plane hosts no longer register as Nodes.** `kubectl get nodes` shows workers only, so node counts drop by the number of control planes, for example 6 to 3 on a three-control-plane cluster with three workers. Add-ons that tolerated `CriticalAddonsOnly` now run on workers.
+
+**Existing clusters keep the old behaviour.** A cluster created before this change keeps registering its control planes as tainted Nodes until it is recreated. Recreate the cluster to adopt the new behaviour: automatic control-plane replacement does not currently work when the replacement reuses the failed member's IP, so it is not a supported way to upgrade.
+
 ## Troubleshooting
 
 **Cluster stuck in `CREATING`.** The control plane is still booting. Check `aws eks describe-cluster --name <name> --query 'cluster.health'` for reported issues. The control plane needs egress to bootstrap, so confirm its subnet has an Internet Gateway route.

docs/terraform-workbooks/eks-quickstart/README.md

@@ -153,6 +153,8 @@ kubectl get nodes
 kubectl get pods -o wide
 ```
 
+`kubectl get nodes` lists the worker nodes only, because control-plane hosts are not Kubernetes Nodes.
+
 `kubectl get pods -o wide` shows the demo pods and which node each landed on.
 
 ### Cleanup

docs/testing/e2e-coverage.md

@@ -124,7 +124,7 @@ An ALB whose targets all fail their health check keeps serving from them instead
 
 ### `eks`, `ecs`, `ecr` — container services
 
-`eks`: cluster create/delete, access entries, addon delivery, `GetToken`, kubeconfig artifacts, the EBS CSI volume path, an IRSA web-identity exchange under the stock AWS trust policy (`StringEquals` on the issuer host-path `:sub`/`:aud`, a mismatched `:sub` denied with `AccessDenied`, a `StringLike` `:sub` granted), and an IRSA pod wired with a projected SA token. Worker IMDS is covered too: a managed nodegroup worker reports IMDSv2 required with hop limit 2, a pod with no IRSA resolves to the node role through IMDS, and a nodegroup built from a launch template takes the template's metadata options. `ecs`: clusters, container instances, the three network modes (`awsvpc`, `bridge`, `host`), task-role credentials fetched from the container credentials endpoint, a service behind an ELB, and ENI release on terminate. `ecr`: control-plane CRUD plus a real OCI data plane — docker login/build/push/pull round trips, multi-layer images, and raw blob HTTP.
+`eks`: cluster create/delete, access entries, addon delivery, `GetToken`, kubeconfig artifacts, the EBS CSI volume path, an IRSA web-identity exchange under the stock AWS trust policy (`StringEquals` on the issuer host-path `:sub`/`:aud`, a mismatched `:sub` denied with `AccessDenied`, a `StringLike` `:sub` granted), and an IRSA pod wired with a projected SA token. Worker IMDS is covered too: a managed nodegroup worker reports IMDSv2 required with hop limit 2, a pod with no IRSA resolves to the node role through IMDS, and a nodegroup built from a launch template takes the template's metadata options. Control-plane isolation is covered as well: no control-plane host registers as a Node, a DaemonSet tolerating every taint runs on workers only, metrics-server is Ready, and logs work for every kube-system pod. `ecs`: clusters, container instances, the three network modes (`awsvpc`, `bridge`, `host`), task-role credentials fetched from the container credentials endpoint, a service behind an ELB, and ENI release on terminate. `ecr`: control-plane CRUD plus a real OCI data plane — docker login/build/push/pull round trips, multi-layer images, and raw blob HTTP.
 
 ### `rds` — managed databases
 

go.mod

@@ -43,6 +43,9 @@ require (
 	github.com/spf13/pflag v1.0.10
 	github.com/spf13/viper v1.21.0
 	github.com/stretchr/testify v1.12.1
+	go.etcd.io/etcd/api/v3 v3.5.35
+	go.etcd.io/etcd/client/pkg/v3 v3.5.35
+	go.etcd.io/etcd/client/v3 v3.5.35
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0
 	go.opentelemetry.io/otel v1.47.0
 	go.opentelemetry.io/otel/metric v1.47.0
@@ -57,6 +60,7 @@ require (
 	golang.org/x/term v0.47.0
 	golang.org/x/time v0.16.0
 	golang.org/x/tools v0.52.0
+	google.golang.org/grpc v1.83.2
 	gopkg.in/ini.v1 v1.67.3
 )
 
@@ -107,6 +111,8 @@ require (
 	github.com/containerd/plugin v1.1.0 // indirect
 	github.com/containerd/ttrpc v1.2.9 // indirect
 	github.com/containerd/typeurl/v2 v2.3.0 // indirect
+	github.com/coreos/go-semver v0.3.0 // indirect
+	github.com/coreos/go-systemd/v22 v22.7.0 // indirect
 	github.com/dgraph-io/badger/v4 v4.9.6 // indirect
 	github.com/dgraph-io/ristretto/v2 v2.4.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
@@ -122,6 +128,8 @@ require (
 	github.com/go-playground/validator/v10 v10.30.5 // indirect
 	github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
 	github.com/gofrs/flock v0.10.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/flatbuffers v25.12.19+incompatible // indirect
 	github.com/google/go-tpm v0.9.8 // indirect
 	github.com/google/uuid v1.6.0 // indirect
@@ -195,7 +203,10 @@ require (
 	go.opentelemetry.io/otel/log v1.47.0 // indirect
 	go.opentelemetry.io/otel/sdk/log v1.47.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.11.0 // indirect
+	go.uber.org/atomic v1.7.0 // indirect
+	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/nilaway v0.0.0-20260721205819-4a0f7653488a // indirect
+	go.uber.org/zap v1.17.0 // indirect
 	golang.org/x/exp/typeparams v0.0.0-20260611194520-c48552f49976 // indirect
 	golang.org/x/mod v0.42.0 // indirect
 	golang.org/x/net v0.61.0 // indirect
@@ -204,7 +215,6 @@ require (
 	golang.org/x/vuln v1.6.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirect
-	google.golang.org/grpc v1.83.2 // indirect
 	google.golang.org/protobuf v1.36.12 // indirect
 	tags.cncf.io/container-device-interface v1.1.1 // indirect
 	tags.cncf.io/container-device-interface/specs-go v1.1.1 // indirect

go.sum

@@ -128,6 +128,10 @@ github.com/containerd/ttrpc v1.2.9 h1:ha0ak962T0s3CA/RoZ6S6xiWZQF24GrBaEpiGX1uih
 github.com/containerd/ttrpc v1.2.9/go.mod h1:jjtQRwXm4DL3KsHKW8vDiUOV6wO0hi6IPhmJhxU7aEs=
 github.com/containerd/typeurl/v2 v2.3.0 h1:HZHPhRWo5XMy3QGQoPrUzbW/2ckwjfweHmOwlkIrPAQ=
 github.com/containerd/typeurl/v2 v2.3.0/go.mod h1:Qk+PAdUYArVj41TnGi6rJ+48RF0PkcTc4i/taoBcK0w=
+github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA=
+github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w=
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -177,6 +181,8 @@ github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPE
 github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/gofrs/flock v0.10.0 h1:SHMXenfaB03KbroETaCMtbBg3Yn29v4w1r+tgy4ff4k=
 github.com/gofrs/flock v0.10.0/go.mod h1:FirDy1Ing0mI2+kB6wk+vyyAH+e6xiE+EYA0jnzV9jc=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
 github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
@@ -243,6 +249,8 @@ github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtL
 github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
 github.com/jstemmer/go-junit-report/v2 v2.1.0 h1:X3+hPYlSczH9IMIpSC9CQSZA0L+BipYafciZUWHEmsc=
 github.com/jstemmer/go-junit-report/v2 v2.1.0/go.mod h1:mgHVr7VUo5Tn8OLVr1cKnLuEy0M92wdRntM99h7RkgQ=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4=
 github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
 github.com/klauspost/cpuid/v2 v2.4.1 h1:I9KxadAkvo+QjtcEE2f2tyfEvmIla/T0m+14PrZNl0U=
@@ -332,6 +340,7 @@ github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8
 github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
 github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -400,9 +409,17 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavM
 github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
 github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
 github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU=
 go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk=
+go.etcd.io/etcd/api/v3 v3.5.35 h1:lB//Av9mFrkljv94XDNAY+qiyQA9W6S/TTA2Uwx2jsI=
+go.etcd.io/etcd/api/v3 v3.5.35/go.mod h1:z1FGQolu/tpmQ2MPFwQNt7G3cXBiSkTevZBhfOtEArQ=
+go.etcd.io/etcd/client/pkg/v3 v3.5.35 h1:nUZutZyJVvsKddK6zddXMBxIhmVi0SzPMBRS0FnNM3o=
+go.etcd.io/etcd/client/pkg/v3 v3.5.35/go.mod h1:0MNjMMqajrOI0/l8OLFkJBhE5Vm6BAi7cerKlrO38hw=
+go.etcd.io/etcd/client/v3 v3.5.35 h1:dk0aSLjY5D1eOR0SNnPXWOWCVA9yIzJoB8a912qxmoI=
+go.etcd.io/etcd/client/v3 v3.5.35/go.mod h1:LjtCl6G5li7PgsGjoQIpYXz/OvSbI1pCxpSp1kqo9P4=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
 go.opentelemetry.io/contrib/bridges/otelslog v0.21.0 h1:/CdgE52i1OEPHhsGUqrggYNRaZy4v4YgVenGd52dRBI=
@@ -439,18 +456,26 @@ go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrF
 go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0=
 go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
 go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
+go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/goleak v1.3.1-0.20260602155401-5f791e3ef83b h1:FHYZT12VzXnb6P399kd1i9Ai7aLF+CFSUsOFylDt7Lk=
 go.uber.org/goleak v1.3.1-0.20260602155401-5f791e3ef83b/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y=
 go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU=
+go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/nilaway v0.0.0-20260721205819-4a0f7653488a h1:CsRd824LAzMmlpu9nYv2PHOqPQYv4Zjd4kGM+g0ahS0=
 go.uber.org/nilaway v0.0.0-20260721205819-4a0f7653488a/go.mod h1:i9Hi8GeZFLBtUCXP73ZNSHKZ4K2JtxT0pLmo3sbJzlo=
+go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U=
+go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
 go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
 go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
 go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
 go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
 golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
@@ -458,24 +483,33 @@ golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+h
 golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak=
 golang.org/x/exp/typeparams v0.0.0-20260611194520-c48552f49976 h1:GTD/WuaexTazIG/SxLOz4rEKZPDVilmVVC2nz4xhwfE=
 golang.org/x/exp/typeparams v0.0.0-20260611194520-c48552f49976/go.mod h1:PqrXSW65cXDZH0k4IeUbhmg/bcAZDbzNz3byBpKCsXo=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg=
 golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.61.0 h1:RR+6j/BTBCrggOLugxa+66R8FvoN6MHlzDlZIbN9YFw=
 golang.org/x/net v0.61.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
 golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -508,6 +542,8 @@ golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
 golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/tools v0.52.0 h1:ryxg4U63oaUYk7uOLTF37n43f4qgtGvqV4Ioit1BaEs=
@@ -519,6 +555,9 @@ golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGN
 golang.org/x/vuln v1.6.0 h1:FeMO9Rm/HwyduOztbvKcOw+zvDEPr4I4aQNSfevFcKY=
 golang.org/x/vuln v1.6.0/go.mod h1:bWlG2493/sjR7ksvicBgMrznH3eYQEyK8ifUYBrqUbg=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
 gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
 google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc h1:rZHRz0ogq4Pid1IKtA/ivHi8XONVF5tqUQXA2TtgpkE=
@@ -536,6 +575,7 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 pgregory.net/rapid v1.3.0 h1:vBvO0VSqti75J1jjYqpgPNBLKMd1+gxa9fYo7vk/Exc=
 pgregory.net/rapid v1.3.0/go.mod h1:dPlE4OBBxgXPqkP79flB6sJL1dx5azpI7HQ9MY9Z7uk=

internal/testkit/fakeetcd/fakeetcd.go

@@ -0,0 +1,217 @@
+// Package fakeetcd serves the etcd Cluster and Maintenance gRPC APIs over mutual
+// TLS with the file layout K3s uses, so a real etcd client can be driven in tests.
+package fakeetcd
+
+import (
+	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"errors"
+	"math/big"
+	"net"
+	"os"
+	"path/filepath"
+	"sync"
+	"testing"
+	"time"
+
+	pb "go.etcd.io/etcd/api/v3/etcdserverpb"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+)
+
+// Member is one entry of the fake cluster's member list.
+type Member struct {
+	ID       uint64
+	Name     string
+	PeerURLs []string
+}
+
+// Server is a running fake etcd.
+type Server struct {
+	// Endpoint is the https URL clients dial.
+	Endpoint string
+	// CertDir holds client.crt, client.key and server-ca.crt.
+	CertDir string
+
+	mu      sync.Mutex
+	members []Member
+	selfID  uint64
+	removed []uint64
+	failAll error
+}
+
+// Start serves members with selfID as the answering member. It stops with the test.
+func Start(t *testing.T, selfID uint64, members []Member) *Server {
+	t.Helper()
+	dir := t.TempDir()
+	caCert, caKey := newCA(t)
+	writePEM(t, filepath.Join(dir, "server-ca.crt"), "CERTIFICATE", caCert.Raw)
+	serverPair := issue(t, caCert, caKey, x509.ExtKeyUsageServerAuth)
+	clientPair := issue(t, caCert, caKey, x509.ExtKeyUsageClientAuth)
+	writeKeyPair(t, dir, "client", clientPair)
+
+	pool := x509.NewCertPool()
+	pool.AddCert(caCert)
+	lis, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	creds := credentials.NewTLS(&tls.Config{
+		Certificates: []tls.Certificate{serverPair.tls},
+		ClientCAs:    pool,
+		ClientAuth:   tls.RequireAndVerifyClientCert,
+		MinVersion:   tls.VersionTLS12,
+	})
+	s := &Server{Endpoint: "https://" + lis.Addr().String(), CertDir: dir, members: members, selfID: selfID}
+	gs := grpc.NewServer(grpc.Creds(creds))
+	pb.RegisterClusterServer(gs, &cluster{s})
+	pb.RegisterMaintenanceServer(gs, &maintenance{s: s})
+	go func() { _ = gs.Serve(lis) }()
+	t.Cleanup(gs.Stop)
+	return s
+}
+
+// Removed returns the member IDs removed so far.
+func (s *Server) Removed() []uint64 {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return append([]uint64(nil), s.removed...)
+}
+
+// FailWith makes every later call return err.
+func (s *Server) FailWith(err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.failAll = err
+}
+
+type cluster struct{ s *Server }
+
+func (c *cluster) MemberList(context.Context, *pb.MemberListRequest) (*pb.MemberListResponse, error) {
+	c.s.mu.Lock()
+	defer c.s.mu.Unlock()
+	if c.s.failAll != nil {
+		return nil, c.s.failAll
+	}
+	resp := &pb.MemberListResponse{Header: &pb.ResponseHeader{MemberId: c.s.selfID}}
+	for _, m := range c.s.members {
+		resp.Members = append(resp.Members, &pb.Member{ID: m.ID, Name: m.Name, PeerURLs: m.PeerURLs})
+	}
+	return resp, nil
+}
+
+func (c *cluster) MemberRemove(_ context.Context, req *pb.MemberRemoveRequest) (*pb.MemberRemoveResponse, error) {
+	c.s.mu.Lock()
+	defer c.s.mu.Unlock()
+	if c.s.failAll != nil {
+		return nil, c.s.failAll
+	}
+	c.s.removed = append(c.s.removed, req.ID)
+	return &pb.MemberRemoveResponse{Header: &pb.ResponseHeader{MemberId: c.s.selfID}}, nil
+}
+
+func (c *cluster) MemberAdd(context.Context, *pb.MemberAddRequest) (*pb.MemberAddResponse, error) {
+	return nil, errors.New("not implemented")
+}
+
+func (c *cluster) MemberUpdate(context.Context, *pb.MemberUpdateRequest) (*pb.MemberUpdateResponse, error) {
+	return nil, errors.New("not implemented")
+}
+
+func (c *cluster) MemberPromote(context.Context, *pb.MemberPromoteRequest) (*pb.MemberPromoteResponse, error) {
+	return nil, errors.New("not implemented")
+}
+
+type maintenance struct {
+	pb.UnimplementedMaintenanceServer
+
+	s *Server
+}
+
+func (m *maintenance) Status(context.Context, *pb.StatusRequest) (*pb.StatusResponse, error) {
+	m.s.mu.Lock()
+	defer m.s.mu.Unlock()
+	if m.s.failAll != nil {
+		return nil, m.s.failAll
+	}
+	return &pb.StatusResponse{Header: &pb.ResponseHeader{MemberId: m.s.selfID}}, nil
+}
+
+type pair struct {
+	tls tls.Certificate
+	der []byte
+	key *ecdsa.PrivateKey
+}
+
+func newCA(t *testing.T) (*x509.Certificate, *ecdsa.PrivateKey) {
+	t.Helper()
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatalf("ca key: %v", err)
+	}
+	tmpl := &x509.Certificate{
+		SerialNumber:          big.NewInt(1),
+		Subject:               pkix.Name{CommonName: "fake-etcd-ca"},
+		NotBefore:             time.Now().Add(-time.Hour),
+		NotAfter:              time.Now().Add(time.Hour),
+		IsCA:                  true,
+		BasicConstraintsValid: true,
+		KeyUsage:              x509.KeyUsageCertSign,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
+	if err != nil {
+		t.Fatalf("ca cert: %v", err)
+	}
+	cert, err := x509.ParseCertificate(der)
+	if err != nil {
+		t.Fatalf("parse ca: %v", err)
+	}
+	return cert, key
+}
+
+func issue(t *testing.T, ca *x509.Certificate, caKey *ecdsa.PrivateKey, usage x509.ExtKeyUsage) pair {
+	t.Helper()
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatalf("key: %v", err)
+	}
+	tmpl := &x509.Certificate{
+		SerialNumber: big.NewInt(time.Now().UnixNano()),
+		Subject:      pkix.Name{CommonName: "fake-etcd"},
+		NotBefore:    time.Now().Add(-time.Hour),
+		NotAfter:     time.Now().Add(time.Hour),
+		KeyUsage:     x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:  []x509.ExtKeyUsage{usage},
+		IPAddresses:  []net.IP{net.ParseIP("127.0.0.1")},
+		DNSNames:     []string{"localhost"},
+	}
+	der, err := x509.CreateCertificate(rand.Reader, tmpl, ca, &key.PublicKey, caKey)
+	if err != nil {
+		t.Fatalf("cert: %v", err)
+	}
+	return pair{tls: tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}, der: der, key: key}
+}
+
+func writeKeyPair(t *testing.T, dir, name string, p pair) {
+	t.Helper()
+	keyDER, err := x509.MarshalECPrivateKey(p.key)
+	if err != nil {
+		t.Fatalf("marshal key: %v", err)
+	}
+	writePEM(t, filepath.Join(dir, name+".crt"), "CERTIFICATE", p.der)
+	writePEM(t, filepath.Join(dir, name+".key"), "EC PRIVATE KEY", keyDER)
+}
+
+func writePEM(t *testing.T, path, blockType string, der []byte) {
+	t.Helper()
+	data := pem.EncodeToMemory(&pem.Block{Type: blockType, Bytes: der})
+	if err := os.WriteFile(path, data, 0o600); err != nil {
+		t.Fatalf("write %s: %v", path, err)
+	}
+}

scripts/README.md

@@ -130,6 +130,7 @@ Installed into the EKS node AMI and run at boot or on a timer. All host communic
 | `k3s-first-boot.sh` | Runs once after the server is healthy: reads the node-token and admin kubeconfig, rewrites the kubeconfig server address to the cluster NLB endpoint, and publishes both to the host. |
 | `konnectivity-server-prestart.sh` | Waits for the K3s CA and admin kubeconfig, mints the konnectivity serving cert, and writes the apiserver replica count for the caller to source. |
 | `mulga-eks-addon-sync.sh` | Pulls staged managed-addon descriptors through the gateway, renders each bundle into the K3s auto-deploy dir, and reports per-addon delivery status back. |
+| `eks-etcd-member` | Go helper built from `cmd/eks-etcd-member` and installed on control-plane (server) images. `remove --peer-ip <ip> [--self-ip <ip>]` evicts a terminated peer's etcd member over the local client endpoint, because control planes run no agent and k3s never prunes them. First boot calls it for a replacement control plane and treats an absent member as success. |
 | `mulga-eks-etcd-snapshot.sh` | Snapshots K3s embedded etcd to predastore. Installed into two crond periodic dirs and picks its tier — `frequent` or `daily` — from the dir it was invoked from. |
 | `mulga-eks-k3s-recovery.sh` | Applies a control-plane etcd-quorum recovery directive before k3s starts. Pull-based and a no-op unless the reconciler escalated a wedged control plane. |
 | `mulga-eks-state-report.sh` | Publishes server health (`healthz`, node count) to the cluster reconciler. The apiserver binds the VPC node-ip, which the host daemon cannot probe directly. |

scripts/images/eks-node/addons/argocd/3.0.23/argocd.yaml

@@ -24939,11 +24939,9 @@ spec:
       labels:
         app.kubernetes.io/name: argocd-applicationset-controller
     spec:
-      # Local delta over the vendored upstream manifest: a cluster with a
-      # GPU-only nodegroup has no untainted node — the CP carries
-      # CriticalAddonsOnly and every worker carries nvidia.com/gpu, so without
-      # these the pod stays Pending forever. Upstream never needs it because
-      # real EKS control planes are not schedulable nodes.
+      # Local delta over upstream: kept for clusters created before control-plane
+      # hosts stopped registering as Nodes (CriticalAddonsOnly), and for GPU-only
+      # nodegroups whose workers all carry nvidia.com/gpu.
       tolerations:
       - key: CriticalAddonsOnly
         operator: Exists
@@ -25191,11 +25189,9 @@ spec:
       labels:
         app.kubernetes.io/name: argocd-dex-server
     spec:
-      # Local delta over the vendored upstream manifest: a cluster with a
-      # GPU-only nodegroup has no untainted node — the CP carries
-      # CriticalAddonsOnly and every worker carries nvidia.com/gpu, so without
-      # these the pod stays Pending forever. Upstream never needs it because
-      # real EKS control planes are not schedulable nodes.
+      # Local delta over upstream: kept for clusters created before control-plane
+      # hosts stopped registering as Nodes (CriticalAddonsOnly), and for GPU-only
+      # nodegroups whose workers all carry nvidia.com/gpu.
       tolerations:
       - key: CriticalAddonsOnly
         operator: Exists
@@ -25326,11 +25322,9 @@ spec:
       labels:
         app.kubernetes.io/name: argocd-notifications-controller
     spec:
-      # Local delta over the vendored upstream manifest: a cluster with a
-      # GPU-only nodegroup has no untainted node — the CP carries
-      # CriticalAddonsOnly and every worker carries nvidia.com/gpu, so without
-      # these the pod stays Pending forever. Upstream never needs it because
-      # real EKS control planes are not schedulable nodes.
+      # Local delta over upstream: kept for clusters created before control-plane
+      # hosts stopped registering as Nodes (CriticalAddonsOnly), and for GPU-only
+      # nodegroups whose workers all carry nvidia.com/gpu.
       tolerations:
       - key: CriticalAddonsOnly
         operator: Exists
@@ -25436,11 +25430,9 @@ spec:
       labels:
         app.kubernetes.io/name: argocd-redis
     spec:
-      # Local delta over the vendored upstream manifest: a cluster with a
-      # GPU-only nodegroup has no untainted node — the CP carries
-      # CriticalAddonsOnly and every worker carries nvidia.com/gpu, so without
-      # these the pod stays Pending forever. Upstream never needs it because
-      # real EKS control planes are not schedulable nodes.
+      # Local delta over upstream: kept for clusters created before control-plane
+      # hosts stopped registering as Nodes (CriticalAddonsOnly), and for GPU-only
+      # nodegroups whose workers all carry nvidia.com/gpu.
       tolerations:
       - key: CriticalAddonsOnly
         operator: Exists
@@ -25530,11 +25522,9 @@ spec:
       labels:
         app.kubernetes.io/name: argocd-repo-server
     spec:
-      # Local delta over the vendored upstream manifest: a cluster with a
-      # GPU-only nodegroup has no untainted node — the CP carries
-      # CriticalAddonsOnly and every worker carries nvidia.com/gpu, so without
-      # these the pod stays Pending forever. Upstream never needs it because
-      # real EKS control planes are not schedulable nodes.
+      # Local delta over upstream: kept for clusters created before control-plane
+      # hosts stopped registering as Nodes (CriticalAddonsOnly), and for GPU-only
+      # nodegroups whose workers all carry nvidia.com/gpu.
       tolerations:
       - key: CriticalAddonsOnly
         operator: Exists
@@ -25895,11 +25885,9 @@ spec:
       labels:
         app.kubernetes.io/name: argocd-server
     spec:
-      # Local delta over the vendored upstream manifest: a cluster with a
-      # GPU-only nodegroup has no untainted node — the CP carries
-      # CriticalAddonsOnly and every worker carries nvidia.com/gpu, so without
-      # these the pod stays Pending forever. Upstream never needs it because
-      # real EKS control planes are not schedulable nodes.
+      # Local delta over upstream: kept for clusters created before control-plane
+      # hosts stopped registering as Nodes (CriticalAddonsOnly), and for GPU-only
+      # nodegroups whose workers all carry nvidia.com/gpu.
       tolerations:
       - key: CriticalAddonsOnly
         operator: Exists
@@ -26317,11 +26305,9 @@ spec:
       labels:
         app.kubernetes.io/name: argocd-application-controller
     spec:
-      # Local delta over the vendored upstream manifest: a cluster with a
-      # GPU-only nodegroup has no untainted node — the CP carries
-      # CriticalAddonsOnly and every worker carries nvidia.com/gpu, so without
-      # these the pod stays Pending forever. Upstream never needs it because
-      # real EKS control planes are not schedulable nodes.
+      # Local delta over upstream: kept for clusters created before control-plane
+      # hosts stopped registering as Nodes (CriticalAddonsOnly), and for GPU-only
+      # nodegroups whose workers all carry nvidia.com/gpu.
       tolerations:
       - key: CriticalAddonsOnly
         operator: Exists

scripts/images/eks-node/addons/aws-ebs-csi-driver/1.40.1/10-controller.yaml

@@ -44,12 +44,9 @@ spec:
         kubernetes.io/os: linux
       serviceAccountName: ebs-csi-controller-sa
       priorityClassName: system-cluster-critical
-      # A cluster with a GPU-only nodegroup has no untainted node: the CP carries
-      # CriticalAddonsOnly (EKS parity keeps user workloads off it) and every
-      # worker carries nvidia.com/gpu (nodegroup_userdata.go). Without these
-      # tolerations the controller has nowhere to schedule and stays Pending
-      # forever. Real EKS never needs this: its control plane isn't a schedulable
-      # node, so the stock upstream manifest carries no such toleration.
+      # Kept for clusters created before control-plane hosts stopped registering
+      # as Nodes (they carried CriticalAddonsOnly), and for GPU-only nodegroups
+      # whose workers all carry nvidia.com/gpu.
       tolerations:
       - key: CriticalAddonsOnly
         operator: Exists

scripts/images/eks-node/addons/aws-load-balancer-controller/2.11.0/10-aws-load-balancer-controller.yaml

@@ -348,12 +348,9 @@ spec:
           readOnly: true
 {{IRSA_VOLUME_MOUNT}}
       priorityClassName: system-cluster-critical
-      # A cluster with a GPU-only nodegroup has no untainted node: the CP carries
-      # CriticalAddonsOnly (EKS parity keeps user workloads off it) and every
-      # worker carries nvidia.com/gpu (nodegroup_userdata.go). Without these
-      # tolerations the controller has nowhere to schedule and stays Pending
-      # forever. Real EKS never needs this: its control plane isn't a schedulable
-      # node, so the stock upstream manifest carries no such toleration.
+      # Kept for clusters created before control-plane hosts stopped registering
+      # as Nodes (they carried CriticalAddonsOnly), and for GPU-only nodegroups
+      # whose workers all carry nvidia.com/gpu.
       tolerations:
       - key: CriticalAddonsOnly
         operator: Exists

scripts/images/eks-node/k3s-first-boot.sh

@@ -162,12 +162,9 @@ jq -n --arg k "${KUBECONFIG_REWRITTEN}" '{adminKubeconfig: $k}'      | publish_e
 jq -n --arg j "${JWKS}"                 '{jwks: $j}'                  | publish_envelope k3s-oidc-jwks
 jq -n --arg c "${CA_B64}"               '{certificateAuthority: $c}' | publish_envelope k3s-ca
 
-# 3.5 Prune a terminated control-plane peer this VM replaces. The spinifex
-# reconciler sets EKS_ETCD_PRUNE_PEER_IP to the dead member's node IP when this
-# VM is provisioned as its replacement (member-count reconcile). Deleting the
-# dead Node makes k3s embedded-etcd remove the stale member, so quorum width
-# returns to N rather than N+1-with-a-dead-peer. Best-effort: a failure leaves
-# the member for an operator and never blocks first boot.
+# 3.5 Prune the terminated CP peer this VM replaces: delete its Node if it still
+# has one, else remove its etcd member directly (CPs run no agent). Best-effort:
+# a failure is logged and never blocks first boot.
 if [ -n "${EKS_ETCD_PRUNE_PEER_IP:-}" ]; then
     log "pruning terminated CP peer at ${EKS_ETCD_PRUNE_PEER_IP} from etcd"
     dead_node=$(kubectl --kubeconfig "${KUBECONFIG_FILE}" get nodes \
@@ -181,7 +178,15 @@ if [ -n "${EKS_ETCD_PRUNE_PEER_IP:-}" ]; then
             log "WARN: delete of dead node ${dead_node} failed; etcd member left for operator"
         fi
     else
-        log "no node has internal IP ${EKS_ETCD_PRUNE_PEER_IP}; nothing to prune"
+        # The default-route source address is the address k3s advertises to etcd peers.
+        self_ip=$(ip -4 route get 1.1.1.1 2>/dev/null \
+            | awk '{for (i = 1; i < NF; i++) if ($i == "src") {print $(i + 1); exit}}') || self_ip=""
+        if member_out=$(eks-etcd-member remove --peer-ip "${EKS_ETCD_PRUNE_PEER_IP}" \
+            ${self_ip:+--self-ip "${self_ip}"} 2>&1); then
+            log "${member_out}"
+        else
+            log "WARN: etcd prune failed for peer ${EKS_ETCD_PRUNE_PEER_IP}: ${member_out}; member left for operator"
+        fi
     fi
 fi
 

scripts/images/eks-node/k3s-first-boot_test.sh

@@ -0,0 +1,123 @@
+#!/bin/sh
+# Test for the etcd peer-prune step (3.5) of k3s-first-boot.sh: kubectl and the helpers
+# are stubbed on PATH (jq required) and state paths rewritten into a temp dir.
+# Run: sh scripts/images/eks-node/k3s-first-boot_test.sh
+set -eu
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+SRC="${SCRIPT_DIR}/k3s-first-boot.sh"
+WORK=$(mktemp -d)
+trap 'rm -rf "${WORK}"' EXIT
+
+PASS=0
+FAIL=0
+pass() { PASS=$((PASS + 1)); echo "ok   - $1"; }
+fail() { FAIL=$((FAIL + 1)); echo "FAIL - $1"; }
+
+STUBBIN="${WORK}/bin"
+mkdir -p "${STUBBIN}"
+
+# kubectl stub: readiness and JWKS succeed; "get nodes" returns the fixture;
+# "delete node" is recorded so the test can see which Node the script removed.
+cat > "${STUBBIN}/kubectl" <<'EOF'
+#!/bin/sh
+case "$*" in
+    *"--raw=/readyz"*)         echo ok ;;
+    *"--raw=/openid/v1/jwks"*) echo '{"keys":[]}' ;;
+    *"get nodes"*)             printf '%s' "${NODES_BODY:-}" ;;
+    *"delete node"*)           echo "$*" >> "${DELETE_LOG}" ;;
+    *) : ;;
+esac
+EOF
+
+# eks-gateway-publish stub: swallow the envelope.
+cat > "${STUBBIN}/eks-gateway-publish" <<'EOF'
+#!/bin/sh
+cat > /dev/null
+EOF
+
+# eks-etcd-member stub: record argv; exit status is configurable so the
+# best-effort contract (a helper failure never blocks first boot) can be tested.
+cat > "${STUBBIN}/eks-etcd-member" <<'EOF'
+#!/bin/sh
+echo "$*" >> "${MEMBER_LOG}"
+exit "${MEMBER_RC:-0}"
+EOF
+
+chmod +x "${STUBBIN}"/*
+PATH="${STUBBIN}:${PATH}"
+export PATH
+
+command -v jq >/dev/null 2>&1 || { echo "SKIP - jq not installed"; exit 0; }
+
+# Rewrite the script's absolute state paths into the sandbox.
+mkdir -p "${WORK}/state" "${WORK}/etc" "${WORK}/k3s/server" "${WORK}/rancher"
+sed -e "s|/var/lib/spinifex-eks|${WORK}/state|g" \
+    -e "s|/etc/spinifex-eks|${WORK}/etc|g" \
+    -e "s|/var/lib/rancher/k3s/server|${WORK}/k3s/server|g" \
+    -e "s|/etc/rancher/k3s|${WORK}/rancher|g" \
+    "${SRC}" > "${WORK}/first-boot.sh"
+
+cat > "${WORK}/rancher/k3s.yaml" <<'EOF'
+apiVersion: v1
+clusters:
+- cluster:
+    certificate-authority-data: Q0FEQVRB
+    server: https://127.0.0.1:6443
+EOF
+echo node-token > "${WORK}/k3s/server/node-token"
+
+# run_case <prune-ip-or-empty>: fresh sentinel + env, then run the script copy.
+run_case() {
+    prune_ip=$1
+    : > "${WORK}/state/first-boot.pending"
+    {
+        echo "EKS_GATEWAY_URL=https://gw.invalid:9999"
+        echo "EKS_ACCOUNT_ID=000000000001"
+        echo "EKS_CLUSTER_NAME=demo"
+        echo "EKS_NLB_ENDPOINT=https://demo.invalid"
+        [ -z "${prune_ip}" ] || echo "EKS_ETCD_PRUNE_PEER_IP=${prune_ip}"
+    } > "${WORK}/etc/first-boot.env"
+    : > "${WORK}/delete.log"
+    : > "${WORK}/member.log"
+    DELETE_LOG="${WORK}/delete.log" MEMBER_LOG="${WORK}/member.log" \
+        EKS_NODE_INIT=none sh "${WORK}/first-boot.sh" > "${WORK}/out.log" 2>&1
+}
+
+# --- Case 1: dead peer has no Node (all-CPs-without-agent steady state) -> the
+# --- explicit member-remove helper is called with the peer IP.
+NODES_BODY="" run_case 10.0.0.12 && rc=0 || rc=$?
+[ "${rc}" -eq 0 ] && pass "no-node: first boot exits 0" || fail "no-node: exit ${rc}"
+[ ! -s "${WORK}/delete.log" ] && pass "no-node: no Node delete attempted" || fail "no-node: unexpected delete: $(cat "${WORK}/delete.log")"
+grep -q -- 'remove --peer-ip 10.0.0.12' "${WORK}/member.log" \
+    && pass "no-node: eks-etcd-member remove --peer-ip 10.0.0.12 called" \
+    || fail "no-node: member-remove helper not called with the peer IP (member.log: $(cat "${WORK}/member.log"))"
+
+# --- Case 2: dead peer still has a Node (mixed fleet) -> the Node is deleted so
+# --- k3s drops the member itself.
+NODES_BODY="ip-10-0-0-12 10.0.0.12" run_case 10.0.0.12 && rc=0 || rc=$?
+[ "${rc}" -eq 0 ] && pass "node-present: first boot exits 0" || fail "node-present: exit ${rc}"
+grep -q 'delete node ip-10-0-0-12' "${WORK}/delete.log" \
+    && pass "node-present: dead Node deleted" \
+    || fail "node-present: dead Node not deleted (delete.log: $(cat "${WORK}/delete.log"))"
+
+# --- Case 3: not a replacement (no prune IP) -> neither path runs.
+NODES_BODY="" run_case "" && rc=0 || rc=$?
+[ "${rc}" -eq 0 ] && pass "no-prune: first boot exits 0" || fail "no-prune: exit ${rc}"
+{ [ ! -s "${WORK}/delete.log" ] && [ ! -s "${WORK}/member.log" ]; } \
+    && pass "no-prune: neither delete nor member-remove ran" \
+    || fail "no-prune: unexpected prune activity"
+
+# --- Case 4: the helper fails -> best effort, first boot still completes (the
+# --- sentinel is consumed) and the helper was actually attempted.
+NODES_BODY="" MEMBER_RC=1 run_case 10.0.0.12 && rc=0 || rc=$?
+[ "${rc}" -eq 0 ] && pass "helper-fails: first boot still exits 0" || fail "helper-fails: exit ${rc}"
+[ ! -e "${WORK}/state/first-boot.pending" ] && pass "helper-fails: sentinel consumed" || fail "helper-fails: sentinel left behind"
+[ -s "${WORK}/member.log" ] && pass "helper-fails: helper was attempted" || fail "helper-fails: helper never attempted"
+grep -Eq 'prune.*fail.*10\.0\.0\.12' "${WORK}/out.log" \
+    && pass "helper-fails: prune failure logged with the peer IP" \
+    || fail "helper-fails: no prune-failure line naming 10.0.0.12 (out.log: $(cat "${WORK}/out.log"))"
+
+echo
+echo "passed=${PASS} failed=${FAIL}"
+[ "${FAIL}" -eq 0 ]

scripts/images/eks-node/konnectivity/konnectivity-agent.yaml

@@ -7,14 +7,9 @@
 # the K3s auto-deploy dir by k3s.initd, which fills the __KONN_HOST__ /
 # __KONN_AGENT_IMAGE__ tokens at boot.
 #
-# Also tolerates the control-plane CriticalAddonsOnly=true:NoExecute taint and
-# the GPU nvidia.com/gpu=present:NoSchedule taint, matching every other
-# CP-critical addon in this repo (aws-load-balancer-controller, ebs-csi-driver,
-# argocd): a zero-worker or GPU-only cluster has no untainted node, and other
-# addons already schedule onto the CP, so it needs its own tunnel too — without
-# an agent there, kubectl logs/exec against CP-hosted pods and apiserver
-# aggregated-API egress (metrics-server discovery) both fail. Real EKS never
-# needs this: its control plane isn't a schedulable node.
+# Also tolerates CriticalAddonsOnly, for clusters created before control-plane
+# hosts stopped registering as Nodes, and the GPU nvidia.com/gpu taint so a
+# GPU-only nodegroup still gets a tunnel agent.
 apiVersion: v1
 kind: ServiceAccount
 metadata:
@@ -44,11 +39,9 @@ spec:
     spec:
       serviceAccountName: konnectivity-agent
       priorityClassName: system-node-critical
-      # Without these, a zero-worker or GPU-only cluster (CP tainted
-      # CriticalAddonsOnly, workers tainted nvidia.com/gpu) shows DESIRED=0 for
-      # this DaemonSet: no tunnel exists, so kubectl logs/exec against any pod
-      # fails "No agent available" and apiserver aggregated-API egress
-      # (e.g. metrics-server discovery) fails, which stalls namespace GC.
+      # CriticalAddonsOnly is kept for clusters created before control-plane hosts
+      # stopped registering as Nodes. nvidia.com/gpu keeps the agent on GPU-only
+      # nodegroups, where logs/exec and aggregated-API egress need a tunnel.
       tolerations:
         - key: CriticalAddonsOnly
           operator: Exists

scripts/images/eks-node/manifest.conf

@@ -22,9 +22,9 @@ APK_PACKAGES="ca-certificates curl jq iptables ip6tables conntrack-tools cgroup-
 # at a pinned tag (its GitHub releases ship only container images, and its go.mod
 # carries replace directives that block `go install pkg@version`). The agent half
 # is the matching proxy-agent container image, pinned in k3s.initd.
-BUILD_COMMANDS="CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-token-webhook ./cmd/eks-token-webhook && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-gateway-publish ./cmd/eks-gateway-publish && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-gateway-fetch ./cmd/eks-gateway-fetch && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-webhook-cert ./cmd/eks-webhook-cert && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-konnectivity-cert ./cmd/eks-konnectivity-cert && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/ecr-credential-provider ./cmd/ecr-credential-provider && rm -rf /tmp/anp-build && git clone --depth 1 --branch v0.30.3 https://github.com/kubernetes-sigs/apiserver-network-proxy /tmp/anp-build && ( cd /tmp/anp-build && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o /tmp/anp-build/konnectivity-server ./cmd/server ) && cp /tmp/anp-build/konnectivity-server bin/konnectivity-server && rm -rf /tmp/anp-build"
+BUILD_COMMANDS="CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-token-webhook ./cmd/eks-token-webhook && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-gateway-publish ./cmd/eks-gateway-publish && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-gateway-fetch ./cmd/eks-gateway-fetch && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-webhook-cert ./cmd/eks-webhook-cert && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-konnectivity-cert ./cmd/eks-konnectivity-cert && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-etcd-member ./cmd/eks-etcd-member && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/ecr-credential-provider ./cmd/ecr-credential-provider && rm -rf /tmp/anp-build && git clone --depth 1 --branch v0.30.3 https://github.com/kubernetes-sigs/apiserver-network-proxy /tmp/anp-build && ( cd /tmp/anp-build && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o /tmp/anp-build/konnectivity-server ./cmd/server ) && cp /tmp/anp-build/konnectivity-server bin/konnectivity-server && rm -rf /tmp/anp-build"
 
-INSTALL_BINARIES="bin/eks-token-webhook:/usr/local/bin/eks-token-webhook bin/eks-gateway-publish:/usr/local/bin/eks-gateway-publish bin/eks-gateway-fetch:/usr/local/bin/eks-gateway-fetch bin/eks-webhook-cert:/usr/local/bin/eks-webhook-cert bin/eks-konnectivity-cert:/usr/local/bin/eks-konnectivity-cert bin/konnectivity-server:/usr/local/bin/konnectivity-server bin/ecr-credential-provider:/usr/local/bin/ecr-credential-provider"
+INSTALL_BINARIES="bin/eks-token-webhook:/usr/local/bin/eks-token-webhook bin/eks-gateway-publish:/usr/local/bin/eks-gateway-publish bin/eks-gateway-fetch:/usr/local/bin/eks-gateway-fetch bin/eks-webhook-cert:/usr/local/bin/eks-webhook-cert bin/eks-konnectivity-cert:/usr/local/bin/eks-konnectivity-cert bin/eks-etcd-member:/usr/local/bin/eks-etcd-member bin/konnectivity-server:/usr/local/bin/konnectivity-server bin/ecr-credential-provider:/usr/local/bin/ecr-credential-provider"
 
 # All role services ship; none but the selector are enabled at bake time.
 # eks-node-role picks server vs agent at first boot and enables the rest.

scripts/mkosi-build.sh

@@ -99,7 +99,8 @@ profile_binaries() {
 ./cmd/eks-gateway-publish:usr/local/bin/eks-gateway-publish \
 ./cmd/eks-gateway-fetch:usr/local/bin/eks-gateway-fetch \
 ./cmd/eks-webhook-cert:usr/local/bin/eks-webhook-cert \
-./cmd/eks-konnectivity-cert:usr/local/bin/eks-konnectivity-cert"
+./cmd/eks-konnectivity-cert:usr/local/bin/eks-konnectivity-cert \
+./cmd/eks-etcd-member:usr/local/bin/eks-etcd-member"
             ;;
         ecs) echo "./cmd/ecs-agent:usr/local/bin/ecs-agent" ;;
         *)   echo "" ;;

spinifex/agents/eks/etcdmember/etcdclient.go

@@ -0,0 +1,79 @@
+package etcdmember
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+	"time"
+
+	"go.etcd.io/etcd/client/pkg/v3/transport"
+	clientv3 "go.etcd.io/etcd/client/v3"
+)
+
+// DefaultEndpoint and DefaultCertDir match where K3s serves embedded etcd and
+// writes its client certificate.
+const (
+	DefaultEndpoint = "https://127.0.0.1:2379"
+	DefaultCertDir  = "/var/lib/rancher/k3s/server/tls/etcd"
+)
+
+// EtcdClient is a Client backed by a real etcd connection.
+type EtcdClient struct {
+	cli      *clientv3.Client
+	endpoint string
+}
+
+var _ Client = (*EtcdClient)(nil)
+
+// Dial connects to etcd with the K3s etcd client certificate in certDir.
+func Dial(endpoint, certDir string, timeout time.Duration) (*EtcdClient, error) {
+	tlsInfo := transport.TLSInfo{
+		CertFile:      filepath.Join(certDir, "client.crt"),
+		KeyFile:       filepath.Join(certDir, "client.key"),
+		TrustedCAFile: filepath.Join(certDir, "server-ca.crt"),
+	}
+	tlsCfg, err := tlsInfo.ClientConfig()
+	if err != nil {
+		return nil, fmt.Errorf("etcd client TLS: %w", err)
+	}
+	cli, err := clientv3.New(clientv3.Config{
+		Endpoints:   []string{endpoint},
+		DialTimeout: timeout,
+		TLS:         tlsCfg,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("dial etcd %s: %w", endpoint, err)
+	}
+	return &EtcdClient{cli: cli, endpoint: endpoint}, nil
+}
+
+// Close releases the connection.
+func (e *EtcdClient) Close() error { return e.cli.Close() }
+
+// MemberList implements Client.
+func (e *EtcdClient) MemberList(ctx context.Context) ([]Member, error) {
+	resp, err := e.cli.MemberList(ctx)
+	if err != nil {
+		return nil, err
+	}
+	members := make([]Member, 0, len(resp.Members))
+	for _, m := range resp.Members {
+		members = append(members, Member{ID: m.ID, Name: m.Name, PeerURLs: m.PeerURLs})
+	}
+	return members, nil
+}
+
+// MemberRemove implements Client.
+func (e *EtcdClient) MemberRemove(ctx context.Context, id uint64) error {
+	_, err := e.cli.MemberRemove(ctx, id)
+	return err
+}
+
+// SelfID implements Client using the local endpoint's own member ID.
+func (e *EtcdClient) SelfID(ctx context.Context) (uint64, error) {
+	resp, err := e.cli.Status(ctx, e.endpoint)
+	if err != nil {
+		return 0, err
+	}
+	return resp.Header.MemberId, nil
+}

spinifex/agents/eks/etcdmember/etcdclient_test.go

@@ -0,0 +1,99 @@
+package etcdmember_test
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/testkit/fakeetcd"
+	"github.com/mulgadc/spinifex/spinifex/agents/eks/etcdmember"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var members = []fakeetcd.Member{
+	{ID: 0xa1, Name: "cp-a", PeerURLs: []string{"https://10.0.0.10:2380"}},
+	{ID: 0xb2, Name: "cp-b", PeerURLs: []string{"https://10.0.0.12:2380"}},
+}
+
+func dial(t *testing.T, srv *fakeetcd.Server) *etcdmember.EtcdClient {
+	t.Helper()
+	cli, err := etcdmember.Dial(srv.Endpoint, srv.CertDir, 5*time.Second)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = cli.Close() })
+	return cli
+}
+
+func TestEtcdClientListsMembersOverMutualTLS(t *testing.T) {
+	t.Parallel()
+	srv := fakeetcd.Start(t, 0xa1, members)
+	cli := dial(t, srv)
+
+	got, err := cli.MemberList(t.Context())
+	require.NoError(t, err)
+	require.Len(t, got, 2)
+	assert.Equal(t, etcdmember.Member{ID: 0xb2, Name: "cp-b", PeerURLs: []string{"https://10.0.0.12:2380"}}, got[1])
+}
+
+func TestEtcdClientSelfIDIsTheAnsweringMember(t *testing.T) {
+	t.Parallel()
+	srv := fakeetcd.Start(t, 0xa1, members)
+
+	id, err := dial(t, srv).SelfID(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, uint64(0xa1), id)
+}
+
+func TestEtcdClientRemovesTheRequestedMember(t *testing.T) {
+	t.Parallel()
+	srv := fakeetcd.Start(t, 0xa1, members)
+
+	require.NoError(t, dial(t, srv).MemberRemove(t.Context(), 0xb2))
+	assert.Equal(t, []uint64{0xb2}, srv.Removed())
+}
+
+func TestEtcdClientSurfacesServerErrors(t *testing.T) {
+	t.Parallel()
+	srv := fakeetcd.Start(t, 0xa1, members)
+	cli := dial(t, srv)
+	srv.FailWith(errors.New("etcdserver: unhealthy cluster"))
+
+	_, err := cli.MemberList(t.Context())
+	assert.ErrorContains(t, err, "unhealthy cluster")
+	assert.ErrorContains(t, cli.MemberRemove(t.Context(), 0xb2), "unhealthy cluster")
+	_, err = cli.SelfID(t.Context())
+	assert.ErrorContains(t, err, "unhealthy cluster")
+	assert.Empty(t, srv.Removed())
+}
+
+func TestDialFailsWithoutClientCertificates(t *testing.T) {
+	t.Parallel()
+	_, err := etcdmember.Dial("https://127.0.0.1:2379", t.TempDir(), time.Second)
+	require.Error(t, err)
+	assert.ErrorContains(t, err, "etcd client TLS")
+}
+
+func TestRunRemovesTheDeadPeerThroughARealClient(t *testing.T) {
+	t.Parallel()
+	srv := fakeetcd.Start(t, 0xa1, members)
+	var out bytes.Buffer
+
+	code := etcdmember.Run(context.Background(), dial(t, srv),
+		etcdmember.Config{PeerIP: "10.0.0.12", SelfIP: "10.0.0.10"}, &out)
+
+	assert.Zero(t, code, out.String())
+	assert.Equal(t, []uint64{0xb2}, srv.Removed())
+}
+
+func TestRunRefusesToRemoveTheLocalMemberThroughARealClient(t *testing.T) {
+	t.Parallel()
+	srv := fakeetcd.Start(t, 0xa1, members)
+	var out bytes.Buffer
+
+	code := etcdmember.Run(context.Background(), dial(t, srv), etcdmember.Config{PeerIP: "10.0.0.10"}, &out)
+
+	assert.NotZero(t, code)
+	assert.Empty(t, srv.Removed())
+}

spinifex/agents/eks/etcdmember/etcdmember.go

@@ -0,0 +1,108 @@
+// Package etcdmember implements eks-etcd-member, which runs inside an EKS K3s
+// control-plane VM and removes a dead peer from the embedded etcd cluster.
+package etcdmember
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/url"
+)
+
+// Member is one etcd cluster member.
+type Member struct {
+	ID       uint64
+	Name     string
+	PeerURLs []string
+}
+
+// Client is the slice of the etcd cluster API the removal needs.
+type Client interface {
+	MemberList(ctx context.Context) ([]Member, error)
+	MemberRemove(ctx context.Context, id uint64) error
+	// SelfID is the ID of the member this client is connected to.
+	SelfID(ctx context.Context) (uint64, error)
+}
+
+// Config selects the member to remove.
+type Config struct {
+	PeerIP string
+	SelfIP string
+}
+
+var (
+	// ErrNoMatch means no member advertises the peer IP.
+	ErrNoMatch = errors.New("no etcd member matches the peer IP")
+	// ErrAmbiguous means more than one member advertises the peer IP.
+	ErrAmbiguous = errors.New("more than one etcd member matches the peer IP")
+	// ErrSelf means the matched member is the local one.
+	ErrSelf = errors.New("refusing to remove the local etcd member")
+)
+
+// Remove removes the single member whose peer URL host equals cfg.PeerIP.
+func Remove(ctx context.Context, c Client, cfg Config) error {
+	if cfg.PeerIP == "" {
+		return errors.New("peer IP is required")
+	}
+	if cfg.PeerIP == cfg.SelfIP {
+		return fmt.Errorf("%w: %s", ErrSelf, cfg.PeerIP)
+	}
+	members, err := c.MemberList(ctx)
+	if err != nil {
+		return fmt.Errorf("list etcd members: %w", err)
+	}
+	var matches []Member
+	for _, m := range members {
+		if advertises(m, cfg.PeerIP) {
+			matches = append(matches, m)
+		}
+	}
+	switch len(matches) {
+	case 0:
+		return fmt.Errorf("%w: %s", ErrNoMatch, cfg.PeerIP)
+	case 1:
+	default:
+		return fmt.Errorf("%w: %s (%d members)", ErrAmbiguous, cfg.PeerIP, len(matches))
+	}
+	selfID, err := c.SelfID(ctx)
+	if err != nil {
+		return fmt.Errorf("identify the local etcd member: %w", err)
+	}
+	if matches[0].ID == selfID {
+		return fmt.Errorf("%w: %s", ErrSelf, matches[0].Name)
+	}
+	if err := c.MemberRemove(ctx, matches[0].ID); err != nil {
+		return fmt.Errorf("remove etcd member %s: %w", matches[0].Name, err)
+	}
+	return nil
+}
+
+// advertises reports whether any peer URL of m has host ip. Matching the parsed
+// host keeps 10.0.0.12 from matching 10.0.0.123.
+func advertises(m Member, ip string) bool {
+	for _, raw := range m.PeerURLs {
+		u, err := url.Parse(raw)
+		if err == nil && u.Hostname() == ip {
+			return true
+		}
+	}
+	return false
+}
+
+// Run removes the peer and returns the process exit code. A member that is
+// already gone is success so a first-boot rerun stays idempotent.
+func Run(ctx context.Context, c Client, cfg Config, out io.Writer) int {
+	err := Remove(ctx, c, cfg)
+	switch {
+	case err == nil:
+		_, _ = fmt.Fprintf(out, "removed etcd member %s\n", cfg.PeerIP)
+		return 0
+	case errors.Is(err, ErrNoMatch):
+		_, _ = fmt.Fprintf(out, "member already absent: %s\n", cfg.PeerIP)
+		return 0
+	default:
+		_, _ = fmt.Fprintf(out, "etcd member remove failed: %v\n", err)
+		return 1
+	}
+}

spinifex/agents/eks/etcdmember/etcdmember_test.go

@@ -0,0 +1,150 @@
+package etcdmember
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type fakeClient struct {
+	members   []Member
+	listErr   error
+	removeErr error
+	removed   []uint64
+	selfID    uint64
+	selfErr   error
+}
+
+func (f *fakeClient) MemberList(context.Context) ([]Member, error) {
+	return f.members, f.listErr
+}
+
+func (f *fakeClient) MemberRemove(_ context.Context, id uint64) error {
+	f.removed = append(f.removed, id)
+	return f.removeErr
+}
+
+func (f *fakeClient) SelfID(context.Context) (uint64, error) {
+	return f.selfID, f.selfErr
+}
+
+var _ Client = (*fakeClient)(nil)
+
+func threeMembers() []Member {
+	return []Member{
+		{ID: 1, Name: "cp-1", PeerURLs: []string{"https://10.0.0.10:2380"}},
+		{ID: 2, Name: "cp-2", PeerURLs: []string{"https://10.0.0.12:2380"}},
+		{ID: 3, Name: "cp-3", PeerURLs: []string{"https://10.0.0.11:2380"}},
+	}
+}
+
+func TestRemove_RemovesOnlyTheMemberWithThatPeerIP(t *testing.T) {
+	c := &fakeClient{members: threeMembers()}
+	require.NoError(t, Remove(context.Background(), c, Config{PeerIP: "10.0.0.12", SelfIP: "10.0.0.10"}))
+	assert.Equal(t, []uint64{2}, c.removed)
+}
+
+func TestRemove_MatchesTheHostNotAPrefix(t *testing.T) {
+	c := &fakeClient{members: []Member{
+		{ID: 1, Name: "cp-1", PeerURLs: []string{"https://10.0.0.10:2380"}},
+		{ID: 2, Name: "cp-2", PeerURLs: []string{"https://10.0.0.123:2380"}},
+	}}
+	err := Remove(context.Background(), c, Config{PeerIP: "10.0.0.12", SelfIP: "10.0.0.10"})
+	require.ErrorIs(t, err, ErrNoMatch)
+	assert.Empty(t, c.removed)
+}
+
+func TestRemove_AnyAdvertisedPeerURLCounts(t *testing.T) {
+	c := &fakeClient{members: []Member{
+		{ID: 1, Name: "cp-1", PeerURLs: []string{"https://10.0.0.10:2380"}},
+		{ID: 2, Name: "cp-2", PeerURLs: []string{"https://127.0.0.1:2380", "https://10.0.0.12:2380"}},
+	}}
+	require.NoError(t, Remove(context.Background(), c, Config{PeerIP: "10.0.0.12", SelfIP: "10.0.0.10"}))
+	assert.Equal(t, []uint64{2}, c.removed)
+}
+
+func TestRemove_RefusesTheLocalMemberByIDWithoutASelfIP(t *testing.T) {
+	c := &fakeClient{members: threeMembers(), selfID: 2}
+	err := Remove(context.Background(), c, Config{PeerIP: "10.0.0.12"})
+	require.ErrorIs(t, err, ErrSelf)
+	assert.Empty(t, c.removed)
+}
+
+func TestRemove_RemovesNothingWhenTheLocalMemberCannotBeIdentified(t *testing.T) {
+	boom := errors.New("status unavailable")
+	c := &fakeClient{members: threeMembers(), selfErr: boom}
+	err := Remove(context.Background(), c, Config{PeerIP: "10.0.0.12"})
+	require.ErrorIs(t, err, boom)
+	assert.Empty(t, c.removed)
+}
+
+func TestRemove_RefusesWhenNothingMatches(t *testing.T) {
+	c := &fakeClient{members: threeMembers()}
+	err := Remove(context.Background(), c, Config{PeerIP: "10.0.0.99", SelfIP: "10.0.0.10"})
+	require.ErrorIs(t, err, ErrNoMatch)
+	assert.Empty(t, c.removed)
+}
+
+func TestRemove_RefusesWhenMoreThanOneMatches(t *testing.T) {
+	c := &fakeClient{members: []Member{
+		{ID: 1, Name: "cp-1", PeerURLs: []string{"https://10.0.0.10:2380"}},
+		{ID: 2, Name: "cp-2", PeerURLs: []string{"https://10.0.0.12:2380"}},
+		{ID: 3, Name: "cp-2-old", PeerURLs: []string{"https://10.0.0.12:2380"}},
+	}}
+	err := Remove(context.Background(), c, Config{PeerIP: "10.0.0.12", SelfIP: "10.0.0.10"})
+	require.ErrorIs(t, err, ErrAmbiguous)
+	assert.Empty(t, c.removed)
+}
+
+func TestRemove_RefusesToRemoveItself(t *testing.T) {
+	c := &fakeClient{members: threeMembers()}
+	err := Remove(context.Background(), c, Config{PeerIP: "10.0.0.10", SelfIP: "10.0.0.10"})
+	require.ErrorIs(t, err, ErrSelf)
+	assert.Empty(t, c.removed)
+}
+
+func TestRemove_ListFailureRemovesNothing(t *testing.T) {
+	boom := errors.New("etcd unreachable")
+	c := &fakeClient{members: threeMembers(), listErr: boom}
+	err := Remove(context.Background(), c, Config{PeerIP: "10.0.0.12", SelfIP: "10.0.0.10"})
+	require.ErrorIs(t, err, boom)
+	assert.Empty(t, c.removed)
+}
+
+func TestRemove_RemoveFailureIsReturned(t *testing.T) {
+	boom := errors.New("member not found")
+	c := &fakeClient{members: threeMembers(), removeErr: boom}
+	err := Remove(context.Background(), c, Config{PeerIP: "10.0.0.12", SelfIP: "10.0.0.10"})
+	require.ErrorIs(t, err, boom)
+	assert.Equal(t, []uint64{2}, c.removed)
+}
+
+func TestRun_ExitCodeMapping(t *testing.T) {
+	boom := errors.New("etcd unreachable")
+	cfg := Config{PeerIP: "10.0.0.12", SelfIP: "10.0.0.10"}
+	cases := []struct {
+		name     string
+		client   *fakeClient
+		cfg      Config
+		wantCode int
+		wantOut  string
+	}{
+		{"removed", &fakeClient{members: threeMembers()}, cfg, 0, "removed etcd member 10.0.0.12"},
+		{"already absent is success", &fakeClient{members: threeMembers()}, Config{PeerIP: "10.0.0.99", SelfIP: "10.0.0.10"}, 0, "member already absent: 10.0.0.99"},
+		{"self refused", &fakeClient{members: threeMembers()}, Config{PeerIP: "10.0.0.10", SelfIP: "10.0.0.10"}, 1, "refusing to remove the local etcd member"},
+		{"ambiguous refused", &fakeClient{members: append(threeMembers(), Member{ID: 4, Name: "dup", PeerURLs: []string{"https://10.0.0.12:2380"}})}, cfg, 1, "more than one etcd member"},
+		{"list error", &fakeClient{listErr: boom}, cfg, 1, "etcd unreachable"},
+		{"remove error", &fakeClient{members: threeMembers(), removeErr: boom}, cfg, 1, "etcd unreachable"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			var out bytes.Buffer
+			assert.Equal(t, tc.wantCode, Run(context.Background(), tc.client, tc.cfg, &out))
+			assert.Contains(t, out.String(), tc.wantOut)
+		})
+	}
+}

spinifex/domains/eks/cluster/cluster_reconciler_replace_test.go

@@ -298,6 +298,31 @@ func TestProvisionReplacementCP_LaunchesJoinOnFreeHost(t *testing.T) {
 		"replacement is told to prune the dead peer from etcd once joined")
 }
 
+func TestProvisionReplacementCP_RendersDisableAgent(t *testing.T) {
+	t.Parallel()
+	sched := &fakeHostScheduler{hosts: []string{"node-a", "node-b", "node-c"}}
+	inst := &seqK3sInst{}
+	svc := newPlacerService(sched, &fakePlacer{}, &seqK3sVPC{}, inst)
+	svc.deps.SystemAccessKey = "AKIATEST"
+	svc.deps.SystemSecretKey = "secret"
+	tmpl := validK3sInput()
+
+	_, err := svc.ProvisionReplacementCP(context.Background(), ReplacementCPRequest{
+		ClusterName:  "alpha",
+		Template:     &tmpl,
+		JoinURL:      k3sServerJoinURL("10.0.0.10"),
+		ExcludeHosts: []string{"node-a", "node-b"},
+		MemberCount:  haControlPlaneCount,
+		DeadPeerIP:   "10.0.0.12",
+	})
+
+	require.NoError(t, err)
+	assert.Contains(t, inst.userData["node-c"], "disable-agent: true",
+		"a replacement renders the current config, so it joins without an agent")
+	assert.Contains(t, inst.userData["node-c"], "EKS_ETCD_PRUNE_PEER_IP=10.0.0.12",
+		"the prune hint still reaches first boot")
+}
+
 func TestProvisionReplacementCP_NoFreeHostErrors(t *testing.T) {
 	t.Parallel()
 	sched := &fakeHostScheduler{hosts: []string{"node-a", "node-b"}}

spinifex/domains/eks/cluster/k3s_server_vm.go

@@ -598,11 +598,9 @@ func buildK3sUserData(in K3sServerInput) string {
 	// out and hold a tunnel to every apiserver replica, so apiserver→pod/kubelet
 	// egress is HA-correct (no single-VIP fan-out 502). EKS-parity datapath.
 	configLines = append(configLines, "egress-selector-mode: disabled")
-	configLines = append(configLines,
-		// Prevent user workloads on the CP (EKS parity). k3s packaged addons tolerate CriticalAddonsOnly.
-		"node-taint:",
-		"  - CriticalAddonsOnly=true:NoExecute",
-	)
+	// No agent: a control-plane host never registers as a Node (EKS parity), so
+	// no pod runs there and none needs a taint to stay away.
+	configLines = append(configLines, "disable-agent: true")
 	configLines = append(configLines,
 		"disable:",
 		"  - traefik",

spinifex/domains/eks/cluster/k3s_server_vm_test.go

@@ -461,6 +461,44 @@ func TestBuildK3sUserData_EgressSelectorDisabledWithKonnConfig(t *testing.T) {
 		"the EgressSelectorConfiguration file routes the cluster egress to the konn socket")
 }
 
+func TestBuildK3sUserData_DisablesAgentOnClusterInit(t *testing.T) {
+	t.Parallel()
+	in := validK3sInput()
+	in.ServerURL = ""
+
+	ud := buildK3sUserData(in)
+	assert.Contains(t, ud, "cluster-init: true")
+	assert.Contains(t, ud, "disable-agent: true",
+		"a control-plane host must not register as a Node (EKS parity)")
+}
+
+func TestBuildK3sUserData_DisablesAgentOnJoin(t *testing.T) {
+	t.Parallel()
+	in := validK3sInput()
+	in.ServerURL = k3sServerJoinURL("10.0.0.10")
+	in.JoinToken = "tok"
+
+	ud := buildK3sUserData(in)
+	assert.Contains(t, ud, "server: "+in.ServerURL)
+	assert.Contains(t, ud, "disable-agent: true",
+		"join servers run without an agent too")
+}
+
+func TestBuildK3sUserData_DisableAgentKeepsControlPlaneConfig(t *testing.T) {
+	t.Parallel()
+	in := validK3sInput()
+	in.PrivateEndpointIP = "10.20.0.5"
+
+	ud := buildK3sUserData(in)
+	require.Contains(t, ud, "disable-agent: true")
+	assert.Contains(t, ud, "egress-selector-mode: disabled")
+	assert.Contains(t, ud, "egress-selector-config-file="+k3sEgressSelectorConfigPath)
+	assert.Contains(t, ud, "advertise-address: 10.20.0.5")
+	for _, addon := range []string{"traefik", "servicelb", "local-storage"} {
+		assert.Contains(t, ud, "  - "+addon, "packaged addon %s stays disabled", addon)
+	}
+}
+
 func TestEgressSelectorConfigYAML_TunnelsClusterOnly(t *testing.T) {
 	t.Parallel()
 	cfg := egressSelectorConfigYAML()
@@ -587,10 +625,9 @@ func TestLaunchK3sServerVM_UserDataContainsAllArtifacts(t *testing.T) {
 	assert.Contains(t, udata, "FAKEPUB")
 
 	assert.Contains(t, udata, "path: "+k3sConfigPath)
-	// The control plane is tainted so user workloads never schedule on it (EKS
-	// parity, and it keeps image pulls off the etcd disk).
-	assert.Contains(t, udata, "node-taint:")
-	assert.Contains(t, udata, "  - CriticalAddonsOnly=true:NoExecute")
+	// The control plane runs no agent, so it is never a Node and needs no taint.
+	assert.Contains(t, udata, "disable-agent: true")
+	assert.NotContains(t, udata, "node-taint:")
 	// AWS parity: K3s' bundled traefik + servicelb are always disabled; Service
 	// type=LoadBalancer / Ingress are the AWS LB Controller's job. local-storage is
 	// disabled too so its local-path provisioner does not add a second default

tests/e2e/eks/cp_isolation_test.go

@@ -0,0 +1,215 @@
+//go:build e2e
+
+package eks
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/eks"
+	"github.com/mulgadc/spinifex/tests/e2e/harness"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const cpIsolationNodegroup = "cpiso-e2e-ng"
+
+// nodeGroupLabelRows prints one "<node> <nodegroup>" row per Node; the second
+// column is empty for a Node that is not a nodegroup worker.
+const nodeGroupLabelRows = `jsonpath={range .items[*]}{.metadata.name} {.metadata.labels.eks\.amazonaws\.com/nodegroup}{"\n"}{end}`
+
+// nodeReadyRows prints one "<node>=<Ready status>" row per Node.
+const nodeReadyRows = `jsonpath={range .items[*]}{.metadata.name}{"="}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}`
+
+// runCPIsolation proves control-plane hosts never register as Nodes, as on AWS:
+// only nodegroup workers are listed and one becomes Ready, a DaemonSet tolerating
+// every taint runs on workers only, metrics-server and its APIService become
+// available, and logs work for every kube-system pod on a Ready node.
+func runCPIsolation(t *testing.T, c *harness.AWSClient, env *harness.Env, artifacts string, fx *clusterFixture) {
+	nodeRoleArn, _ := ensureNodeRole(t, c, fx)
+	createWorkerNodegroup(t, c, fx, cpIsolationNodegroup, nodeRoleArn, nil)
+
+	kcPath := writeKubeconfig(t, artifacts, fx.Cluster)
+	kc := harness.NewKubectl(t, kcPath, getTokenEnv(t, env))
+
+	t.Run("WorkerNodeReady", func(t *testing.T) {
+		harness.EventuallyErr(t, func() error {
+			out, err := kc.Run(30*time.Second, "get", "nodes", "-o", nodeReadyRows)
+			if err != nil {
+				return fmt.Errorf("kubectl get nodes: %v\n%s", err, out)
+			}
+			if !strings.Contains(out, "=True") {
+				return fmt.Errorf("no Ready node yet:\n%s", out)
+			}
+			return nil
+		}, 6*time.Minute, 5*time.Second)
+	})
+
+	t.Run("OnlyWorkersAreNodes", func(t *testing.T) {
+		out, err := kc.Run(30*time.Second, "get", "nodes", "-o", nodeGroupLabelRows)
+		require.NoErrorf(t, err, "get nodes:\n%s", out)
+		rows := strings.Split(strings.TrimSpace(out), "\n")
+		require.NotEmpty(t, strings.TrimSpace(out), "the nodegroup worker must be listed")
+		for _, row := range rows {
+			f := strings.Fields(row)
+			assert.Lenf(t, f, 2, "Node %q must carry the eks.amazonaws.com/nodegroup label: a Node without it is not a nodegroup worker", row)
+		}
+	})
+
+	t.Run("TolerateAllDaemonSetRunsOnWorkersOnly", func(t *testing.T) {
+		const ds = "cpiso-tolerate-all"
+		manifest := filepath.Join(artifacts, "cpiso-ds.yaml")
+		require.NoError(t, os.WriteFile(manifest, []byte(tolerateAllDaemonSet(ds)), 0o600))
+		out, err := kc.Run(60*time.Second, "apply", "-f", manifest)
+		require.NoErrorf(t, err, "apply daemonset:\n%s", out)
+		t.Cleanup(func() {
+			_, _ = kc.Run(60*time.Second, "delete", "daemonset", ds, "-n", "default", "--ignore-not-found")
+		})
+
+		harness.EventuallyErr(t, func() error {
+			ready, err := kc.Run(30*time.Second, "get", "pods", "-l", "app="+ds,
+				"-o", `jsonpath={range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}`)
+			if err != nil {
+				return fmt.Errorf("get daemonset pods: %v\n%s", err, ready)
+			}
+			states := strings.Fields(ready)
+			if len(states) == 0 {
+				return fmt.Errorf("daemonset has no pods yet")
+			}
+			for _, s := range states {
+				if s != "True" {
+					return fmt.Errorf("daemonset pods not all Ready: %v", states)
+				}
+			}
+			return nil
+		}, 3*time.Minute, 5*time.Second)
+
+		nodes, err := kc.Run(30*time.Second, "get", "pods", "-l", "app="+ds,
+			"-o", `jsonpath={range .items[*]}{.spec.nodeName}{"\n"}{end}`)
+		require.NoErrorf(t, err, "get daemonset pod nodes:\n%s", nodes)
+		for _, node := range strings.Fields(nodes) {
+			label, err := kc.Run(30*time.Second, "get", "node", node, "-o",
+				`jsonpath={.metadata.labels.eks\.amazonaws\.com/nodegroup}`)
+			require.NoErrorf(t, err, "get node %s:\n%s", node, label)
+			assert.NotEmptyf(t, strings.TrimSpace(label), "daemonset pod ran on %s, which is not a nodegroup worker", node)
+		}
+	})
+
+	t.Run("MetricsServerReady", func(t *testing.T) {
+		harness.EventuallyErr(t, func() error {
+			out, err := kc.Run(30*time.Second, "get", "deployment", "metrics-server", "-n", "kube-system",
+				"-o", "jsonpath={.status.readyReplicas}")
+			if err != nil {
+				return fmt.Errorf("get metrics-server: %v\n%s", err, out)
+			}
+			if strings.TrimSpace(out) == "" || strings.TrimSpace(out) == "0" {
+				return fmt.Errorf("metrics-server has no ready replica (%q)", strings.TrimSpace(out))
+			}
+			return nil
+		}, 10*time.Minute, 10*time.Second)
+
+		// The aggregated metrics API must be Available: apiserver discovery dials
+		// it through the konnectivity tunnel, and while it is not, namespace GC
+		// never clears the kubernetes finalizer.
+		harness.EventuallyErr(t, func() error {
+			out, err := kc.Run(30*time.Second, "get", "apiservice", "v1beta1.metrics.k8s.io",
+				"-o", `jsonpath={.status.conditions[?(@.type=="Available")].status}`)
+			if err != nil {
+				return fmt.Errorf("get apiservice v1beta1.metrics.k8s.io: %v\n%s", err, out)
+			}
+			if strings.TrimSpace(out) != "True" {
+				return fmt.Errorf("v1beta1.metrics.k8s.io Available=%q", strings.TrimSpace(out))
+			}
+			return nil
+		}, 3*time.Minute, 10*time.Second)
+
+		harness.EventuallyErr(t, func() error {
+			out, err := kc.Run(60*time.Second, "top", "nodes")
+			if err != nil {
+				return fmt.Errorf("kubectl top nodes: %v\n%s", err, out)
+			}
+			return nil
+		}, 3*time.Minute, 10*time.Second)
+	})
+
+	t.Run("AddonDeletion", func(t *testing.T) {
+		// Namespace GC needs every aggregated API available, so addon deletion
+		// is checked once a worker exists.
+		if _, err := c.EKS.DescribeAddon(&eks.DescribeAddonInput{
+			ClusterName: aws.String(fx.ClusterName),
+			AddonName:   aws.String(noopAddon),
+		}); err != nil {
+			t.Skipf("addon %s was not created: %v", noopAddon, err)
+		}
+		_, err := c.EKS.DeleteAddon(&eks.DeleteAddonInput{
+			ClusterName: aws.String(fx.ClusterName),
+			AddonName:   aws.String(noopAddon),
+		})
+		require.NoError(t, err, "delete-addon")
+		harness.EventuallyErr(t, func() error {
+			out, runErr := kc.Run(30*time.Second, "get", "namespace", noopAddon,
+				"--ignore-not-found", "-o", `jsonpath={.metadata.name}`)
+			if runErr != nil {
+				return fmt.Errorf("kubectl get namespace: %v\n%s", runErr, out)
+			}
+			if strings.TrimSpace(out) != "" {
+				return fmt.Errorf("namespace still present")
+			}
+			return nil
+		}, 3*time.Minute, 10*time.Second)
+	})
+
+	t.Run("LogsWorkForEveryKubeSystemPod", func(t *testing.T) {
+		nodes, err := kc.Run(30*time.Second, "get", "nodes", "-o", nodeReadyRows)
+		require.NoErrorf(t, err, "get nodes:\n%s", nodes)
+		ready := map[string]bool{}
+		for _, row := range strings.Fields(nodes) {
+			if name, status, ok := strings.Cut(row, "="); ok && status == "True" {
+				ready[name] = true
+			}
+		}
+		out, err := kc.Run(30*time.Second, "get", "pods", "-n", "kube-system",
+			"-o", `jsonpath={range .items[*]}{.metadata.name} {.spec.nodeName}{"\n"}{end}`)
+		require.NoErrorf(t, err, "list kube-system pods:\n%s", out)
+		checked := 0
+		for _, row := range strings.Split(strings.TrimSpace(out), "\n") {
+			f := strings.Fields(row)
+			if len(f) != 2 || !ready[f[1]] {
+				continue
+			}
+			checked++
+			logs, err := kc.Run(60*time.Second, "logs", f[0], "-n", "kube-system", "--all-containers", "--tail=1")
+			assert.NoErrorf(t, err, "kubectl logs %s must work:\n%s", f[0], logs)
+		}
+		require.NotZero(t, checked, "kube-system must have pods on a Ready node")
+	})
+}
+
+func tolerateAllDaemonSet(name string) string {
+	return fmt.Sprintf(`apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: %[1]s
+  namespace: default
+spec:
+  selector:
+    matchLabels:
+      app: %[1]s
+  template:
+    metadata:
+      labels:
+        app: %[1]s
+    spec:
+      tolerations:
+      - operator: Exists
+      containers:
+      - name: pause
+        image: public.ecr.aws/docker/library/busybox:latest
+        command: ["sleep", "3600"]
+`, name)
+}

tests/e2e/eks/eks_test.go

@@ -45,6 +45,10 @@ const (
 // One cluster fixture is shared across the subtests (create once, delete in
 // Cleanup) — control-plane boot is the slowest step, so re-creating per subtest
 // would blow the suite timeout on dev nodes.
+// noopAddon is the delivery fixture (Namespace + ConfigMap), independent of the
+// CSI and load-balancer bundles.
+const noopAddon = "spinifex-noop"
+
 func TestEKS(t *testing.T) {
 	env := harness.LoadEnv(t)
 	harness.RequireDNSEnabled(t, env)
@@ -136,17 +140,15 @@ func TestEKS(t *testing.T) {
 		kcPath := writeKubeconfig(t, artifacts, fx.Cluster)
 		kc := harness.NewKubectl(t, kcPath, getTokenEnv(t, env))
 
-		// Poll until a Ready node appears. k3s may crash once during bootstrap
-		// (etcd fsync latency under I/O contention) and be respawned; the generous
-		// envelope allows the control plane to stabilise after the first ACTIVE.
+		// Control-plane hosts never register as Nodes, so a cluster without a
+		// nodegroup lists none. A Ready worker is asserted once a nodegroup exists.
 		harness.EventuallyErr(t, func() error {
-			out, err := kc.Run(30*time.Second, "get", "nodes",
-				"-o", `jsonpath={range .items[*]}{.metadata.name}{"="}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}`)
+			out, err := kc.Run(30*time.Second, "get", "nodes", "-o", "name")
 			if err != nil {
 				return fmt.Errorf("kubectl get nodes: %v\n%s", err, out)
 			}
-			if !strings.Contains(out, "=True") {
-				return fmt.Errorf("no Ready node yet:\n%s", out)
+			if strings.TrimSpace(out) != "" {
+				return fmt.Errorf("cluster without a nodegroup must list no Nodes:\n%s", out)
 			}
 			return nil
 		}, 6*time.Minute, 5*time.Second)
@@ -162,7 +164,7 @@ func TestEKS(t *testing.T) {
 		// gateway, renders the baked bundle into the K3s auto-deploy dir, and
 		// reports ready, which flips the record to ACTIVE. spinifex-noop is the
 		// fixture (Namespace + ConfigMap) — no dependency on the CSI/LB bundles.
-		const addon = "spinifex-noop"
+		const addon = noopAddon
 		_, err := c.EKS.CreateAddon(&eks.CreateAddonInput{
 			ClusterName: aws.String(fx.ClusterName),
 			AddonName:   aws.String(addon),
@@ -194,41 +196,15 @@ func TestEKS(t *testing.T) {
 		require.NoError(t, err, "addon ConfigMap must exist: %s", out)
 		assert.Contains(t, out, "delivered", "addon ConfigMap must carry the marker")
 
-		// The aggregated metrics API must be Available: apiserver discovery
-		// dials it through the konnectivity tunnel, and on a zero-worker or
-		// GPU-only cluster a konnectivity-agent gap makes it perpetually
-		// Unavailable — which is exactly what wedges namespace GC below, on
-		// every namespace, not just this one.
-		apiSvcOut, err := kc.Run(30*time.Second, "get", "apiservice", "v1beta1.metrics.k8s.io",
-			"-o", `jsonpath={.status.conditions[?(@.type=="Available")].status}`)
-		require.NoError(t, err, "get apiservice v1beta1.metrics.k8s.io: %s", apiSvcOut)
-		assert.Equal(t, "True", strings.TrimSpace(apiSvcOut),
-			"v1beta1.metrics.k8s.io must be Available or namespace GC (below) will never clear the kubernetes finalizer")
-
-		// DeleteAddon unstages the manifest; the agent GCs the rendered file and
-		// k3s' auto-deploy controller removes the objects.
-		_, err = c.EKS.DeleteAddon(&eks.DeleteAddonInput{
-			ClusterName: aws.String(fx.ClusterName),
-			AddonName:   aws.String(addon),
-		})
-		require.NoError(t, err, "delete-addon")
-
-		// Namespace GC enumerates every API group before it can clear the
-		// `kubernetes` finalizer, so a stuck aggregated API (checked above)
-		// leaves this stuck in Terminating forever rather than retrying it
-		// down — proving deletion actually completes, not just that it was
-		// attempted.
-		harness.EventuallyErr(t, func() error {
-			out, runErr := kc.Run(30*time.Second, "get", "namespace", "spinifex-noop",
-				"--ignore-not-found", "-o", `jsonpath={.metadata.name}`)
-			if runErr != nil {
-				return fmt.Errorf("kubectl get namespace: %v\n%s", runErr, out)
-			}
-			if strings.TrimSpace(out) != "" {
-				return fmt.Errorf("namespace still present")
-			}
-			return nil
-		}, 3*time.Minute, 10*time.Second)
+		// The bundled metrics-server objects are delivered with the cluster. They
+		// cannot become Available until a worker exists, so only presence is checked.
+		for _, obj := range [][]string{
+			{"deployment", "metrics-server", "-n", "kube-system"},
+			{"apiservice", "v1beta1.metrics.k8s.io"},
+		} {
+			got, gerr := kc.Run(30*time.Second, append([]string{"get"}, append(obj, "-o", "name")...)...)
+			require.NoErrorf(t, gerr, "bundled %s must be delivered: %s", obj[0], got)
+		}
 	})
 
 	t.Run("IRSAWebIdentity", func(t *testing.T) {
@@ -251,6 +227,11 @@ func TestEKS(t *testing.T) {
 		runWorkerIMDSLaunchTemplate(t, c, fx)
 	})
 
+	t.Run("CPIsolation", func(t *testing.T) {
+		requireClusterReady(t, fx)
+		runCPIsolation(t, c, env, artifacts, fx)
+	})
+
 	t.Run("EBSCSIVolume", func(t *testing.T) {
 		requireClusterReady(t, fx)
 		runEBSCSIVolume(t, c, env, artifacts, fx)