From 74b91b5cc565e501f39cb74e8b785f00a00b42bc Mon Sep 17 00:00:00 2001 From: Josh Siviour Date: Wed, 10 Jun 2026 12:25:47 +1000 Subject: [PATCH] =?UTF-8?q?feat(eks):=20HA=20control=20plane=20=E2=80=94?= =?UTF-8?q?=203-node=20etcd=20quorum=20spread=20across=20distinct=20hosts?= =?UTF-8?q?=20(#335)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(eks): node-targetable control-plane VM (HA spread transport prereq) Groundwork for HA control-plane placement (mulga-siv-231.7.1): make the k3s control-plane VM placeable on a specific Spinifex host. - Add a sys.medium instance type (2 vCPU / 4 GB) and boot the control-plane VM as it instead of t3.medium. A sys.* type keeps the CP VM out of customer DescribeInstanceTypes and makes the daemon register the node-targeted system.LaunchInstance.{type}.{nodeID} subject (only sys.* types get it). - Add K3sServerInput.TargetNodeID and a LaunchSystemInstanceOnNode method on the launcher: empty/local node launches in-process (unchanged single-CP path); any other node dispatches over the node-targeted subject, where the receiving daemon runs the launch and owns the VM. The spread orchestrator (reserve N hosts, launch per host, verify, rollback, <3-host fallback) lands separately; CreateCluster still launches locally (TargetNodeID empty), so behaviour is unchanged. * feat(eks): HA control-plane spread orchestrator Place N=3 k3s server VMs across distinct Spinifex hosts via a per-cluster spread placement group, with all-or-nothing rollback and single-CP fallback. - placeControlPlane: capacity fan-out (HostScheduler) -> >=3-host gate -> ensure spread group (system account) -> ReserveSpreadNodes(Min=Max=3) -> parallel per-node LaunchK3sServerVM(TargetNodeID) -> node-vms placement verify -> FinalizeSpreadInstances. Any launch/verify/finalize failure terminates launched VMs, releases the reservation, drops the group. - <3 schedulable hosts or a lost reservation race falls back to a single local control-plane VM (prior behaviour). - ClusterMeta gains ControlPlaneNodes[] + ControlPlaneSpreadGroup; [0] is mirrored into the scalar ControlPlane* fields for back-compat. DeleteCluster tears down every CP VM and the spread group. - Wire PlacementGroup + Scheduler into EKSServiceDeps + orchestration-deps gate. * fix(eks): size HA control-plane spread on raw host headroom SchedulableHosts matched the CP type by name in node.status's per-type capacity list, but that list omits sys.* types (system types are hidden from customers). The CP VM is sys.medium, so the match never hit and SchedulableHosts returned 0 on every node — placeControlPlane always fell back to a single CP, making the >=3-host spread unreachable in production. System VMs consume host vCPU/memory like any guest, so size the fit against each node's raw schedulable headroom (Total - Reserved - Alloc) and the type's footprint instead. Adds instancetypes.SpecForSystemType for the sys.* vCPU/memory spec and a pure nodeFitsSystemInstance helper. * feat(eks): k3s server-join role for HA control-plane quorum Servers 2..N now boot in join mode: server: https://:6443 + a shared cluster token, without cluster-init, so they join the first server's embedded-etcd quorum instead of each cluster-initing an independent datastore. - K3sServerInput gains JoinToken (shared cluster secret, set on every server) and ServerURL (set => join server, no cluster-init, server-join role). - buildK3sUserData branches cluster-init vs server:, emits token:, and selects SPINIFEX_K3S_ROLE server vs server-join. - launchControlPlaneSpread launches nodes[0] first (cluster-init), derives the join URL from its ENI IP, then launches the rest in parallel as join servers; a first-server failure skips the joins and rolls back. - CreateCluster mints a per-cluster crypto-random token (GenerateK3sClusterToken) threaded into the placement template. - eks-node-role.sh adds a server-join case: token-webhook + k3s + state-report, but not k3s-first-boot (the first server already publishes the bootstrap artifacts; a join re-publish only races the bootstrap bus). * fix: open etcd-peer/kubelet ingress on EKS control-plane SG for HA join Join servers (231.7.2) register over the supervisor port 6443 but their embedded etcd could not peer with the quorum: the control-plane SG only admitted 6443 from the VPC CIDR, omitting etcd client+peer 2379-2380 and kubelet 10250 between control-plane nodes. A single cluster-init server never exercises those ports, so the gap stayed hidden until multi-server HA — the join registered but its etcd never replicated the raft log, leaving the node perpetually NotReady. Add EnsureControlPlaneHAIngress with self-referencing (cpSG->cpSG) rules for etcd 2379-2380 and kubelet 10250, called from CreateCluster after EnsureControlPlaneIngress. Verified live on a 3-node cluster: join servers reach Ready and form a 3-member etcd quorum that survives loss of the original first server. * feat: register all control-plane ENI IPs as EKS cluster NLB targets Register every HA control-plane server ENI IP in the cluster NLB target group, not just the primary. The TG already health-checks :6443, so a dead member drains and kubectl/apiserver traffic fails over to surviving etcd-quorum members. Add RegisterClusterTargets batch helper; the single RegisterClusterTarget now delegates to it. --- scripts/images/eks-node/eks-node-role.sh | 18 ++++- scripts/images/eks-node/eks-node-role_test.sh | 14 ++++ spinifex/handlers/eks/k3s_ha_control_plane.go | 36 ++++++++-- .../handlers/eks/k3s_ha_control_plane_test.go | 58 +++++++++++++++- spinifex/handlers/eks/k3s_server_vm.go | 69 ++++++++++++++++--- spinifex/handlers/eks/k3s_server_vm_test.go | 64 +++++++++++++++++ spinifex/handlers/eks/nlb.go | 42 +++++++---- spinifex/handlers/eks/nlb_test.go | 46 +++++++++++++ spinifex/handlers/eks/security_groups.go | 58 ++++++++++++++++ spinifex/handlers/eks/security_groups_test.go | 59 ++++++++++++++++ spinifex/handlers/eks/service_impl.go | 31 +++++++-- 11 files changed, 459 insertions(+), 36 deletions(-) diff --git a/scripts/images/eks-node/eks-node-role.sh b/scripts/images/eks-node/eks-node-role.sh index 22948623f..f74c82c26 100755 --- a/scripts/images/eks-node/eks-node-role.sh +++ b/scripts/images/eks-node/eks-node-role.sh @@ -10,8 +10,13 @@ set -eu # ROLE_FILE so later boots and the etcd-snapshot cron can branch without # re-parsing, then removes itself from the runlevel so it runs exactly once. # -# server → eks-token-webhook, k3s (server), k3s-first-boot (bootstrap publish) -# agent → k3s-agent +# server → eks-token-webhook, k3s (server), k3s-first-boot (bootstrap +# publish), mulga-eks-state-report +# server-join → eks-token-webhook, k3s (server, joins the first server's etcd +# quorum), mulga-eks-state-report. NO k3s-first-boot: the first +# server already publishes the cluster-identical bootstrap +# artifacts; a join re-publish only races the bootstrap bus. +# agent → k3s-agent # # Paths are overridable via env so the selection logic is unit-testable (bats) # without root or a real /etc. @@ -55,6 +60,15 @@ case "${ROLE}" in rc-service k3s-first-boot start rc-service mulga-eks-state-report start ;; + server-join) + log "configuring server-join role" + rc-update add eks-token-webhook default + rc-update add k3s default + rc-update add mulga-eks-state-report default + rc-service eks-token-webhook start + rc-service k3s start + rc-service mulga-eks-state-report start + ;; agent) log "configuring agent role" rc-update add k3s-agent default diff --git a/scripts/images/eks-node/eks-node-role_test.sh b/scripts/images/eks-node/eks-node-role_test.sh index ac6d78f33..f9f3be2b2 100755 --- a/scripts/images/eks-node/eks-node-role_test.sh +++ b/scripts/images/eks-node/eks-node-role_test.sh @@ -66,6 +66,20 @@ check "server: starts k3s" "yes" "$(grep -q 'rc-service k3s start' "${CALLS}" && check "server: no k3s-agent" "no" "$(grep -q 'k3s-agent' "${CALLS}" && echo yes || echo no)" check "server: self-disables" "yes" "$(grep -q 'rc-update del eks-node-role default' "${CALLS}" && echo yes || echo no)" +# --- Case 1b: explicit server-join role --- +run_case server_join +printf 'SPINIFEX_K3S_ROLE=server-join\nEKS_CLUSTER_NAME=alpha\n' > "${EKS_NODE_ENVFILE}" +invoke +check "server-join: exit 0" 0 "${rc}" +check "server-join: role file" "server-join" "$(cat "${ROLE_FILE}")" +check "server-join: enables webhook" "yes" "$(grep -q 'rc-update add eks-token-webhook default' "${CALLS}" && echo yes || echo no)" +check "server-join: enables k3s" "yes" "$(grep -q 'rc-update add k3s default' "${CALLS}" && echo yes || echo no)" +check "server-join: enables state-report" "yes" "$(grep -q 'rc-update add mulga-eks-state-report default' "${CALLS}" && echo yes || echo no)" +# Join servers must NOT re-publish bootstrap; the first server already did. +check "server-join: no k3s-first-boot" "no" "$(grep -q 'k3s-first-boot' "${CALLS}" && echo yes || echo no)" +check "server-join: no k3s-agent" "no" "$(grep -q 'k3s-agent' "${CALLS}" && echo yes || echo no)" +check "server-join: self-disables" "yes" "$(grep -q 'rc-update del eks-node-role default' "${CALLS}" && echo yes || echo no)" + # --- Case 2: explicit agent role --- run_case agent_explicit printf 'SPINIFEX_K3S_ROLE=agent\n' > "${EKS_NODE_ENVFILE}" diff --git a/spinifex/handlers/eks/k3s_ha_control_plane.go b/spinifex/handlers/eks/k3s_ha_control_plane.go index 4cf1160a8..e838cde10 100644 --- a/spinifex/handlers/eks/k3s_ha_control_plane.go +++ b/spinifex/handlers/eks/k3s_ha_control_plane.go @@ -162,26 +162,48 @@ type cpLaunchResult struct { err error } -// launchControlPlaneSpread launches one server VM per reserved node in parallel -// (each remote launch blocks on a full-AMI boot round trip, so sequential would -// be N×). Results stay index-aligned with nodes. +// launchControlPlaneSpread launches the reserved nodes as one etcd quorum: +// nodes[0] is the first server (cluster-init), and its ENI IP is the join +// endpoint the remaining servers register against — so it must come up first. +// nodes[1..] then launch in parallel as join servers (each remote launch blocks +// on a full-AMI boot round trip, so sequential joins would be N×). Results stay +// index-aligned with nodes. If the first server fails, the joins are skipped +// (no endpoint to join) so the caller's all-or-nothing rollback fires. func (s *EKSServiceImpl) launchControlPlaneSpread(tmpl K3sServerInput, nodes []string) []cpLaunchResult { results := make([]cpLaunchResult, len(nodes)) + + first := tmpl + first.TargetNodeID = nodes[0] + out, err := LaunchK3sServerVM(s.deps.VPCK3s, s.deps.Instance, s.deps.Image, first) + results[0] = cpLaunchResult{node: nodes[0], out: out, err: err} + if err != nil { + for i := 1; i < len(nodes); i++ { + results[i] = cpLaunchResult{node: nodes[i], err: errFirstServerFailed} + } + return results + } + + joinURL := k3sServerJoinURL(out.ENIIP) var wg sync.WaitGroup - for i, node := range nodes { + for i := 1; i < len(nodes); i++ { wg.Add(1) go func(idx int, nodeID string) { defer wg.Done() in := tmpl in.TargetNodeID = nodeID - out, err := LaunchK3sServerVM(s.deps.VPCK3s, s.deps.Instance, s.deps.Image, in) - results[idx] = cpLaunchResult{node: nodeID, out: out, err: err} - }(i, node) + in.ServerURL = joinURL + o, e := LaunchK3sServerVM(s.deps.VPCK3s, s.deps.Instance, s.deps.Image, in) + results[idx] = cpLaunchResult{node: nodeID, out: o, err: e} + }(i, nodes[i]) } wg.Wait() return results } +// errFirstServerFailed marks join-server results skipped when the cluster-init +// server never came up — the joins have no endpoint to register against. +var errFirstServerFailed = errors.New("eks: first control-plane server failed; join servers skipped") + // verifyControlPlaneSpread confirms every launched VM sits on its reserved host // and that no two share a host. A VM not yet visible to the fan-out is tolerated // (listing lag — the node-targeted launch subject already guarantees the host). diff --git a/spinifex/handlers/eks/k3s_ha_control_plane_test.go b/spinifex/handlers/eks/k3s_ha_control_plane_test.go index 76e3475fc..1280e8f65 100644 --- a/spinifex/handlers/eks/k3s_ha_control_plane_test.go +++ b/spinifex/handlers/eks/k3s_ha_control_plane_test.go @@ -83,6 +83,7 @@ type seqK3sInst struct { nodes []string terminated []string failNodes map[string]bool + userData map[string]string // nodeID -> rendered cloud-init } var _ k3sInstanceLauncher = (*seqK3sInst)(nil) @@ -91,12 +92,16 @@ func (i *seqK3sInst) LaunchSystemInstance(in *sysinstance.SystemInstanceInput) ( return i.LaunchSystemInstanceOnNode("", in) } -func (i *seqK3sInst) LaunchSystemInstanceOnNode(nodeID string, _ *sysinstance.SystemInstanceInput) (*sysinstance.SystemInstanceOutput, error) { +func (i *seqK3sInst) LaunchSystemInstanceOnNode(nodeID string, in *sysinstance.SystemInstanceInput) (*sysinstance.SystemInstanceOutput, error) { i.mu.Lock() defer i.mu.Unlock() if i.failNodes[nodeID] { return nil, errors.New("InsufficientInstanceCapacity") } + if i.userData == nil { + i.userData = map[string]string{} + } + i.userData[nodeID] = in.UserData i.nodes = append(i.nodes, nodeID) id := "i-" + nodeID if nodeID == "" { @@ -258,6 +263,57 @@ func TestPlaceControlPlane_SpreadHappyPath(t *testing.T) { assert.Empty(t, inst.terminated) } +func TestPlaceControlPlane_SpreadFirstInitsRestJoin(t *testing.T) { + sched := &fakeHostScheduler{hosts: []string{"node-a", "node-b", "node-c"}} + placer := &fakePlacer{reserved: []string{"node-a", "node-b", "node-c"}} + vpc := &seqK3sVPC{} + inst := &seqK3sInst{} + svc := newPlacerService(sched, placer, vpc, inst) + + tmpl := validK3sInput() + tmpl.JoinToken = "sharedtok123" + _, _, err := svc.placeControlPlane(testHAAccountID, "alpha", tmpl) + require.NoError(t, err) + + // node-a is reserved[0] → the first server: cluster-inits, no join URL. + first := inst.userData["node-a"] + assert.Contains(t, first, "cluster-init: true") + assert.NotContains(t, first, "server: https://") + assert.Contains(t, first, "SPINIFEX_K3S_ROLE=server\n") + assert.Contains(t, first, "token: sharedtok123") + + // node-b/node-c join the first server's ENI IP (sequential first launch ⇒ + // the first ENI, 10.0.1.11) with the shared token, without cluster-init. + for _, n := range []string{"node-b", "node-c"} { + join := inst.userData[n] + assert.Contains(t, join, "server: https://10.0.1.11:6443", n) + assert.Contains(t, join, "token: sharedtok123", n) + assert.NotContains(t, join, "cluster-init: true", n) + assert.Contains(t, join, "SPINIFEX_K3S_ROLE=server-join", n) + } +} + +func TestPlaceControlPlane_FirstServerFailureRollsBackNoJoins(t *testing.T) { + sched := &fakeHostScheduler{hosts: []string{"node-a", "node-b", "node-c"}} + placer := &fakePlacer{reserved: []string{"node-a", "node-b", "node-c"}} + inst := &seqK3sInst{failNodes: map[string]bool{"node-a": true}} + svc := newPlacerService(sched, placer, &seqK3sVPC{}, inst) + + nodes, group, err := svc.placeControlPlane(testHAAccountID, "alpha", validK3sInput()) + require.Error(t, err) + assert.Nil(t, nodes) + assert.Equal(t, "", group) + + // The first server never came up, so no join servers were launched and there + // is nothing to terminate; the reservation + group are still cleaned up. + assert.Empty(t, inst.terminated) + assert.NotContains(t, inst.nodes, "node-b") + assert.NotContains(t, inst.nodes, "node-c") + require.Len(t, placer.releaseInputs, 1) + assert.Len(t, placer.deleteGroups, 1) + assert.Empty(t, placer.finalizeInputs) +} + func TestPlaceControlPlane_FallbackUnderThreeHosts(t *testing.T) { sched := &fakeHostScheduler{hosts: []string{"node-a", "node-b"}} placer := &fakePlacer{} diff --git a/spinifex/handlers/eks/k3s_server_vm.go b/spinifex/handlers/eks/k3s_server_vm.go index 3139a2641..c23dab4fc 100644 --- a/spinifex/handlers/eks/k3s_server_vm.go +++ b/spinifex/handlers/eks/k3s_server_vm.go @@ -1,6 +1,8 @@ package handlers_eks import ( + "crypto/rand" + "encoding/hex" "errors" "fmt" "log/slog" @@ -147,6 +149,16 @@ type K3sServerInput struct { // spread across distinct failure domains. Empty launches on the local // daemon node (the single-CP default). TargetNodeID string + // JoinToken is the shared k3s cluster token (server secret). Set on every + // control-plane server so servers 2..N authenticate into the first server's + // embedded-etcd quorum and workers continue to register. Empty preserves + // k3s' per-server auto-generated token (pre-HA single-CP behaviour). + JoinToken string + // ServerURL, when set, boots this VM as a JOIN server: it registers with the + // existing control plane at this https://:6443 endpoint and + // joins the etcd quorum WITHOUT cluster-init. Empty = the first server, which + // cluster-inits the datastore. A non-empty ServerURL requires JoinToken. + ServerURL string // BuiltinIngress keeps K3s' bundled traefik + servicelb enabled (interim // in-VPC app exposure). When false the config disables them for AWS parity, // where Service type=LoadBalancer / Ingress are satisfied by the AWS Load @@ -383,10 +395,32 @@ func validateK3sServerInput(in K3sServerInput) error { return errors.New("eks: K3sServerInput empty SecretKey") case strings.TrimSpace(in.GatewayCACert) == "": return errors.New("eks: K3sServerInput empty GatewayCACert") + case in.ServerURL != "" && in.JoinToken == "": + return errors.New("eks: K3sServerInput join server (ServerURL set) requires JoinToken") } return nil } +// GenerateK3sClusterToken mints the shared k3s cluster token (256 bits of +// crypto-random, hex-encoded) seeded into every control-plane server so servers +// 2..N join the first server's etcd quorum and workers register. Generated per +// CreateCluster; not persisted (the node-token derived from it is published on +// the bootstrap bus for workers). +func GenerateK3sClusterToken() (string, error) { + var b [32]byte + if _, err := rand.Read(b[:]); err != nil { + return "", fmt.Errorf("eks: generate k3s cluster token: %w", err) + } + return hex.EncodeToString(b[:]), nil +} + +// k3sServerJoinURL builds the registration endpoint a join server dials: the +// first server's ENI private IP on the k3s supervisor port. Token-based join is +// trust-on-first-use, so the IP need not be in the apiserver cert SANs. +func k3sServerJoinURL(ip string) string { + return "https://" + net.JoinHostPort(ip, "6443") +} + // boolFlag renders a bool as the "1"/"0" string the shell first-boot env reads. func boolFlag(b bool) string { if b { @@ -408,11 +442,18 @@ type userDataFile struct { func buildK3sUserData(in K3sServerInput) string { nlbEndpoint := "https://" + net.JoinHostPort(in.NLBDNS, strconv.FormatInt(clusterNLBListenPort, 10)) + // Role the eks-node-role first-boot selector reads to enable the + // control-plane services. The first server is "server" (cluster-init + + // bootstrap publisher); servers 2..N are "server-join" (register into the + // existing quorum, no bootstrap re-publish). Nodegroup workers seed "agent" + // through their own path. + role := "server" + if in.ServerURL != "" { + role = "server-join" + } + envBody := strings.Join([]string{ - // Role the eks-node-role first-boot selector reads to enable the - // control-plane services (k3s server + token webhook + bootstrap - // publisher). Nodegroup workers seed "agent" through their own path. - "SPINIFEX_K3S_ROLE=server", + "SPINIFEX_K3S_ROLE=" + role, "EKS_GATEWAY_URL=" + in.GatewayURL, "EKS_GATEWAY_CA=" + k3sGatewayCAPath, "EKS_ACCESS_KEY=" + in.AccessKey, @@ -428,8 +469,10 @@ func buildK3sUserData(in K3sServerInput) string { "EKS_DEFER_TRAEFIK=" + boolFlag(in.BuiltinIngress), }, "\n") - // cluster-init selects the embedded etcd datastore (required for multi-server - // HA). etcd-expose-metrics surfaces etcd's own wal_fsync_duration_seconds and + // cluster-init (first server) selects the embedded etcd datastore (required + // for multi-server HA); servers 2..N instead carry `server: ` + the + // shared token and join the quorum without cluster-init. + // etcd-expose-metrics surfaces etcd's own wal_fsync_duration_seconds and // backend_commit_duration_seconds on 127.0.0.1:2381/metrics so control-plane // commit latency is measurable directly, not inferred. // @@ -456,8 +499,16 @@ func buildK3sUserData(in K3sServerInput) string { // k3s never writes traefik.yaml, leaving no manifest to un-skip. servicelb // (klipper) is lazy — it acts only when a LoadBalancer Service exists — so it // carries no bootstrap cost and stays enabled whenever built-in ingress is on. - configLines := []string{ - "cluster-init: true", + var configLines []string + if in.ServerURL == "" { + configLines = append(configLines, "cluster-init: true") + } else { + configLines = append(configLines, "server: "+in.ServerURL) + } + if in.JoinToken != "" { + configLines = append(configLines, "token: "+in.JoinToken) + } + configLines = append(configLines, "etcd-expose-metrics: true", // Keep user workloads off the control plane (EKS parity — the CP is never // a worker). Critical here because the server node is schedulable by @@ -468,7 +519,7 @@ func buildK3sUserData(in K3sServerInput) string { // NoExecute also evicts anything already scheduled, not just future pods. "node-taint:", " - CriticalAddonsOnly=true:NoExecute", - } + ) if !in.BuiltinIngress { configLines = append(configLines, "disable:", diff --git a/spinifex/handlers/eks/k3s_server_vm_test.go b/spinifex/handlers/eks/k3s_server_vm_test.go index 7ed7af3de..ca167e950 100644 --- a/spinifex/handlers/eks/k3s_server_vm_test.go +++ b/spinifex/handlers/eks/k3s_server_vm_test.go @@ -128,6 +128,7 @@ func validK3sInput() K3sServerInput { AccessKey: "AKIAEXAMPLE", SecretKey: "s3cr3t-key", GatewayCACert: "-----BEGIN CERTIFICATE-----\nFAKECA\n-----END CERTIFICATE-----\n", + JoinToken: "clustertok-deadbeef", } } @@ -409,6 +410,69 @@ func TestLaunchK3sServerVM_UsesEmbeddedEtcd(t *testing.T) { assert.Contains(t, udata, "etcd-expose-metrics: true") } +func TestLaunchK3sServerVM_FirstServerTokenNoServerURL(t *testing.T) { + vpc, inst, ami := &fakeK3sVPC{}, &fakeK3sInst{}, &fakeK3sAMI{} + + in := validK3sInput() + in.JoinToken = "sharedtok123" + _, err := LaunchK3sServerVM(vpc, inst, ami, in) + require.NoError(t, err) + require.Len(t, inst.launchCalls, 1) + + udata := inst.launchCalls[0].UserData + // First server: cluster-inits the datastore, carries the shared token so + // servers 2..N and workers join, and boots the full server role. + assert.Contains(t, udata, "cluster-init: true") + assert.Contains(t, udata, "token: sharedtok123") + assert.NotContains(t, udata, "server: https://") + assert.Contains(t, udata, "SPINIFEX_K3S_ROLE=server\n") +} + +func TestLaunchK3sServerVM_JoinServerRendersServerAndJoinRole(t *testing.T) { + vpc, inst, ami := &fakeK3sVPC{}, &fakeK3sInst{}, &fakeK3sAMI{} + + in := validK3sInput() + in.JoinToken = "sharedtok123" + in.ServerURL = "https://10.0.1.7:6443" + _, err := LaunchK3sServerVM(vpc, inst, ami, in) + require.NoError(t, err) + require.Len(t, inst.launchCalls, 1) + + udata := inst.launchCalls[0].UserData + // Join server: registers at the first server's endpoint with the shared + // token, WITHOUT cluster-init, and boots the join role (no bootstrap publish). + assert.Contains(t, udata, "server: https://10.0.1.7:6443") + assert.Contains(t, udata, "token: sharedtok123") + assert.NotContains(t, udata, "cluster-init: true") + assert.Contains(t, udata, "SPINIFEX_K3S_ROLE=server-join") + assert.NotContains(t, udata, "SPINIFEX_K3S_ROLE=server\n") +} + +func TestLaunchK3sServerVM_JoinServerRequiresToken(t *testing.T) { + vpc, inst, ami := &fakeK3sVPC{}, &fakeK3sInst{}, &fakeK3sAMI{} + + in := validK3sInput() + in.JoinToken = "" + in.ServerURL = "https://10.0.1.7:6443" // ServerURL set, token cleared + _, err := LaunchK3sServerVM(vpc, inst, ami, in) + require.Error(t, err) + assert.Contains(t, err.Error(), "JoinToken") + assert.Empty(t, inst.launchCalls) +} + +func TestGenerateK3sClusterToken_UniqueHex(t *testing.T) { + a, err := GenerateK3sClusterToken() + require.NoError(t, err) + b, err := GenerateK3sClusterToken() + require.NoError(t, err) + assert.Len(t, a, 64) // 32 bytes hex + assert.NotEqual(t, a, b) +} + +func TestK3sServerJoinURL(t *testing.T) { + assert.Equal(t, "https://10.0.1.7:6443", k3sServerJoinURL("10.0.1.7")) +} + func TestLaunchK3sServerVM_RunInstancesEmptyReservationRollsBack(t *testing.T) { vpc, ami := &fakeK3sVPC{}, &fakeK3sAMI{} inst := &fakeK3sInst{launchOut: &sysinstance.SystemInstanceOutput{}} diff --git a/spinifex/handlers/eks/nlb.go b/spinifex/handlers/eks/nlb.go index 1aa6b04d2..e1fc0c5c1 100644 --- a/spinifex/handlers/eks/nlb.go +++ b/spinifex/handlers/eks/nlb.go @@ -174,25 +174,43 @@ func frontendIPFromLB(lb *elbv2.LoadBalancer, internetFacing bool) string { return "" } -// RegisterClusterTarget attaches the K3s server ENI IP to the cluster TG so -// the NLB starts forwarding :443 traffic to :6443 on the VM. The ELBv2 -// RegisterTargets impl already dedupes on (id, port), so re-invocation by -// the reconciler is idempotent. +// RegisterClusterTarget attaches one K3s server ENI IP to the cluster TG. Thin +// wrapper over RegisterClusterTargets for callers with a single IP. func RegisterClusterTarget(nlbp nlbProvisioner, accountID, tgArn, eniIP string) error { - if tgArn == "" { - return errors.New("eks: RegisterClusterTarget empty TG arn") - } if eniIP == "" { return errors.New("eks: RegisterClusterTarget empty ENI IP") } + return RegisterClusterTargets(nlbp, accountID, tgArn, []string{eniIP}) +} + +// RegisterClusterTargets attaches every control-plane server ENI IP to the +// cluster TG so the NLB forwards :443 traffic to :6443 on each CP VM. With all +// N HA servers registered, the TG health check (:6443) drains a member when its +// node dies and kubectl/apiserver traffic fails over to the survivors. Empty +// IPs are skipped; the ELBv2 RegisterTargets impl dedupes on (id, port), so +// re-invocation (e.g. by the reconciler) is idempotent. +func RegisterClusterTargets(nlbp nlbProvisioner, accountID, tgArn string, eniIPs []string) error { + if tgArn == "" { + return errors.New("eks: RegisterClusterTargets empty TG arn") + } + targets := make([]*elbv2.TargetDescription, 0, len(eniIPs)) + for _, ip := range eniIPs { + if ip == "" { + continue + } + targets = append(targets, &elbv2.TargetDescription{ + Id: aws.String(ip), + Port: aws.Int64(k3sAPIServerPort), + }) + } + if len(targets) == 0 { + return errors.New("eks: RegisterClusterTargets no non-empty ENI IPs") + } if _, err := nlbp.RegisterTargets(&elbv2.RegisterTargetsInput{ TargetGroupArn: aws.String(tgArn), - Targets: []*elbv2.TargetDescription{{ - Id: aws.String(eniIP), - Port: aws.Int64(k3sAPIServerPort), - }}, + Targets: targets, }, accountID); err != nil { - return fmt.Errorf("register target %s on TG %s: %w", eniIP, tgArn, err) + return fmt.Errorf("register %d targets on TG %s: %w", len(targets), tgArn, err) } return nil } diff --git a/spinifex/handlers/eks/nlb_test.go b/spinifex/handlers/eks/nlb_test.go index 632c4a4e2..263f2d320 100644 --- a/spinifex/handlers/eks/nlb_test.go +++ b/spinifex/handlers/eks/nlb_test.go @@ -440,6 +440,52 @@ func TestRegisterClusterTarget_EmptyInputsRejected(t *testing.T) { assert.Empty(t, nlbp.registerCalls) } +func TestRegisterClusterTargets_RegistersEveryENIIPInOneCall(t *testing.T) { + nlbp := newFakeNLBProvisioner() + + err := RegisterClusterTargets(nlbp, "111122223333", "arn:tg/alpha", + []string{"10.0.1.10", "10.0.2.11", "10.0.3.12"}) + require.NoError(t, err) + require.Len(t, nlbp.registerCalls, 1) + + in := nlbp.registerCalls[0] + assert.Equal(t, "arn:tg/alpha", aws.StringValue(in.TargetGroupArn)) + require.Len(t, in.Targets, 3) + for i, ip := range []string{"10.0.1.10", "10.0.2.11", "10.0.3.12"} { + assert.Equal(t, ip, aws.StringValue(in.Targets[i].Id)) + assert.Equal(t, k3sAPIServerPort, aws.Int64Value(in.Targets[i].Port)) + } +} + +func TestRegisterClusterTargets_SkipsEmptyIPs(t *testing.T) { + nlbp := newFakeNLBProvisioner() + + err := RegisterClusterTargets(nlbp, "111122223333", "arn:tg/alpha", + []string{"10.0.1.10", "", "10.0.3.12"}) + require.NoError(t, err) + require.Len(t, nlbp.registerCalls, 1) + require.Len(t, nlbp.registerCalls[0].Targets, 2) + assert.Equal(t, "10.0.1.10", aws.StringValue(nlbp.registerCalls[0].Targets[0].Id)) + assert.Equal(t, "10.0.3.12", aws.StringValue(nlbp.registerCalls[0].Targets[1].Id)) +} + +func TestRegisterClusterTargets_EmptyInputsRejected(t *testing.T) { + nlbp := newFakeNLBProvisioner() + require.Error(t, RegisterClusterTargets(nlbp, "111122223333", "", []string{"10.0.1.10"})) + require.Error(t, RegisterClusterTargets(nlbp, "111122223333", "arn:tg/alpha", nil)) + require.Error(t, RegisterClusterTargets(nlbp, "111122223333", "arn:tg/alpha", []string{"", ""})) + assert.Empty(t, nlbp.registerCalls) +} + +func TestRegisterClusterTargets_RegisterErrorSurfaced(t *testing.T) { + nlbp := newFakeNLBProvisioner() + nlbp.registerErr = errors.New("TargetGroupNotFound") + + err := RegisterClusterTargets(nlbp, "111122223333", "arn:tg/alpha", []string{"10.0.1.10"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "arn:tg/alpha") +} + func TestDeregisterClusterTarget_PostsENIIPAndAPIPort(t *testing.T) { nlbp := newFakeNLBProvisioner() diff --git a/spinifex/handlers/eks/security_groups.go b/spinifex/handlers/eks/security_groups.go index fd0f964a9..79b3567ef 100644 --- a/spinifex/handlers/eks/security_groups.go +++ b/spinifex/handlers/eks/security_groups.go @@ -242,6 +242,64 @@ func EnsureControlPlaneIngress(sgp sgProvisioner, accountID, cpSGID, vpcCIDR str return nil } +// EnsureControlPlaneHAIngress authorizes the server-to-server ingress an HA +// control plane needs so a join server's embedded etcd can peer with the rest +// of the quorum. The control-plane SG is self-referencing here: every rule's +// source and target are the same group, so the rules stay correct as CP VMs are +// launched, lost, and rebuilt. A single cluster-init server never exercises +// these ports, which is why they were absent until multi-server HA: the join +// server registers over the supervisor port (already admitted) but its etcd +// member can never replicate the raft log without the peer port, so the quorum +// silently never forms and the node never reports Ready. +// +// Rules (cpSG → cpSG): +// - etcd client+peer: 2379-2380/tcp — embedded-etcd quorum replication +// - kubelet: 10250/tcp — server-to-server kubelet (metrics, logs, exec) +// +// The supervisor/apiserver port (6443) is already admitted from the VPC CIDR by +// EnsureControlPlaneIngress, which covers the CP peers, so it is not repeated. +// Idempotent: the InvalidPermission.Duplicate error a re-run triggers for an +// already-authorized rule is treated as success. +func EnsureControlPlaneHAIngress(sgp sgProvisioner, accountID, cpSGID string) error { + if cpSGID == "" { + return errors.New("eks: EnsureControlPlaneHAIngress empty control-plane SG id") + } + + type rule struct { + proto string + from int64 + to int64 + desc string + } + rules := []rule{ + {"tcp", 2379, 2380, "EKS etcd client+peer control-plane to control-plane"}, + {"tcp", 10250, 10250, "EKS kubelet control-plane to control-plane"}, + } + + for _, r := range rules { + perm := &ec2.IpPermission{ + IpProtocol: aws.String(r.proto), + FromPort: aws.Int64(r.from), + ToPort: aws.Int64(r.to), + UserIdGroupPairs: []*ec2.UserIdGroupPair{{ + GroupId: aws.String(cpSGID), + Description: aws.String(r.desc), + }}, + } + _, err := sgp.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{ + GroupId: aws.String(cpSGID), + IpPermissions: []*ec2.IpPermission{perm}, + }, accountID) + if err != nil { + if awserrors.IsErrorCode(err, awserrors.ErrorInvalidPermissionDuplicate) { + continue + } + return fmt.Errorf("authorize %s ingress on %s: %w", r.desc, cpSGID, err) + } + } + return nil +} + func lookupSGByName(sgp sgProvisioner, accountID, vpcID, name string) (string, error) { out, err := sgp.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{ Filters: []*ec2.Filter{ diff --git a/spinifex/handlers/eks/security_groups_test.go b/spinifex/handlers/eks/security_groups_test.go index 41ff851f2..fc7e7fe80 100644 --- a/spinifex/handlers/eks/security_groups_test.go +++ b/spinifex/handlers/eks/security_groups_test.go @@ -234,6 +234,65 @@ func TestEnsureControlPlaneIngress_AuthorizeErrorSurfaced(t *testing.T) { assert.Contains(t, err.Error(), "sg-cp-001") } +func TestEnsureControlPlaneHAIngress_AuthorizesEtcdAndKubeletSelfReferenced(t *testing.T) { + sgp := newFakeSGProvisioner() + + err := EnsureControlPlaneHAIngress(sgp, "111122223333", "sg-cp-001") + require.NoError(t, err) + + require.Len(t, sgp.authorizeCalls, 2, "etcd peer/client + kubelet") + + // Collect the authorized (proto, from, to) tuples and assert each rule is + // self-referencing on the control-plane SG (source == target group). + type portRange struct { + proto string + from, to int64 + } + got := map[portRange]bool{} + for _, in := range sgp.authorizeCalls { + assert.Equal(t, "sg-cp-001", aws.StringValue(in.GroupId)) + require.Len(t, in.IpPermissions, 1) + perm := in.IpPermissions[0] + assert.Empty(t, perm.IpRanges, "HA rules are group-referenced, not CIDR") + require.Len(t, perm.UserIdGroupPairs, 1) + assert.Equal(t, "sg-cp-001", aws.StringValue(perm.UserIdGroupPairs[0].GroupId), + "control-plane SG must reference itself so the rule survives CP churn") + got[portRange{ + aws.StringValue(perm.IpProtocol), + aws.Int64Value(perm.FromPort), + aws.Int64Value(perm.ToPort), + }] = true + } + + // 2380 (etcd peer) is the port whose absence silently breaks the join quorum. + assert.True(t, got[portRange{"tcp", 2379, 2380}], "etcd client+peer 2379-2380 must be opened") + assert.True(t, got[portRange{"tcp", 10250, 10250}], "kubelet 10250 must be opened") +} + +func TestEnsureControlPlaneHAIngress_EmptyInputRejected(t *testing.T) { + sgp := newFakeSGProvisioner() + + require.Error(t, EnsureControlPlaneHAIngress(sgp, "111122223333", "")) + assert.Empty(t, sgp.authorizeCalls) +} + +func TestEnsureControlPlaneHAIngress_DuplicateTolerated(t *testing.T) { + sgp := newFakeSGProvisioner() + sgp.authorizeErr = errors.New(awserrors.ErrorInvalidPermissionDuplicate) + + err := EnsureControlPlaneHAIngress(sgp, "111122223333", "sg-cp-001") + require.NoError(t, err, "a duplicate rule on re-run must be treated as success") +} + +func TestEnsureControlPlaneHAIngress_AuthorizeErrorSurfaced(t *testing.T) { + sgp := newFakeSGProvisioner() + sgp.authorizeErr = errors.New("vpcd unavailable") + + err := EnsureControlPlaneHAIngress(sgp, "111122223333", "sg-cp-001") + require.Error(t, err) + assert.Contains(t, err.Error(), "sg-cp-001") +} + func assertSGCreateTagged(t *testing.T, in *ec2.CreateSecurityGroupInput, name, vpcID, clusterName, role string) { t.Helper() require.NotNil(t, in) diff --git a/spinifex/handlers/eks/service_impl.go b/spinifex/handlers/eks/service_impl.go index a9f153559..8aca862fe 100644 --- a/spinifex/handlers/eks/service_impl.go +++ b/spinifex/handlers/eks/service_impl.go @@ -402,6 +402,13 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID, s.markFailed(acctKV, name) return nil, logCreateErr(name, accountID, "ensure control-plane ingress", err) } + // HA control planes run servers 2..N as join servers whose embedded etcd must + // peer with the quorum; without these self-referencing CP-SG rules a join + // registers but its etcd never replicates, so the node never reports Ready. + if err := EnsureControlPlaneHAIngress(s.deps.VPCSG, accountID, cpSG); err != nil { + s.markFailed(acctKV, name) + return nil, logCreateErr(name, accountID, "ensure control-plane HA ingress", err) + } // Public access ⇒ internet-facing NLB (external-pool front-end IP, reachable // on the LAN/edge network); private-only ⇒ internal NLB (VPC-only). @@ -450,6 +457,15 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID, return nil, logCreateErr(name, accountID, "derive OIDC public key", err) } + // Shared cluster token seeded into every control-plane server so the HA + // spread's servers 2..N join the first server's etcd quorum (and workers + // register). Single-CP clusters carry it too — harmless, one code path. + joinToken, err := GenerateK3sClusterToken() + if err != nil { + s.markFailed(acctKV, name) + return nil, logCreateErr(name, accountID, "generate k3s cluster token", err) + } + cpNodes, spreadGroup, err := s.placeControlPlane(accountID, name, K3sServerInput{ AccountID: accountID, ClusterName: name, @@ -466,6 +482,7 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID, SecretKey: s.deps.SystemSecretKey, GatewayCACert: s.deps.GatewayCACert, BuiltinIngress: meta.BuiltinIngress, + JoinToken: joinToken, }) if err != nil { s.markFailed(acctKV, name) @@ -478,9 +495,9 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID, "cluster", name, "accountID", accountID, "err", err) return nil, fmt.Errorf("launch K3s VM: %w", err) } - // Mirror the primary ([0]) into the scalar fields the reconciler, NLB - // registration, egress wiring, and teardown read today. Until per-node NLB - // registration (231.7.3) only the primary is target-registered + egress-wired. + // Mirror the primary ([0]) into the scalar fields the reconciler, egress + // wiring, and teardown read today. All CP nodes are NLB target-registered + // (failover); egress is still primary-only (separate hidden-pool /32 wire). primary := cpNodes[0] meta.ControlPlaneNodes = cpNodes meta.ControlPlaneSpreadGroup = spreadGroup @@ -513,9 +530,13 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID, return nil, logCreateErr(name, accountID, "persist control-plane ids", err) } - if err := RegisterClusterTarget(s.deps.NLB, accountID, nlb.TargetGroupArn, primary.ENIIP); err != nil { + cpENIIPs := make([]string, 0, len(cpNodes)) + for _, n := range cpNodes { + cpENIIPs = append(cpENIIPs, n.ENIIP) + } + if err := RegisterClusterTargets(s.deps.NLB, accountID, nlb.TargetGroupArn, cpENIIPs); err != nil { s.markFailed(acctKV, name) - return nil, logCreateErr(name, accountID, "register NLB target", err) + return nil, logCreateErr(name, accountID, "register NLB targets", err) } if err := PutClusterMeta(acctKV, meta); err != nil {