From 852f61d9a321cbd1e34e4403deaaa90c8f73f953 Mon Sep 17 00:00:00 2001 From: joshsiv-mulga Date: Fri, 10 Jul 2026 15:48:59 +1000 Subject: [PATCH] fix(eks): write etcd-snapshot.env at CP launch so snapshots actually run --- spinifex/daemon/eks_deps.go | 81 ++++++++++++------- spinifex/handlers/eks/delete_cluster_test.go | 49 +++++------ spinifex/handlers/eks/k3s_ha_control_plane.go | 2 + spinifex/handlers/eks/k3s_server_vm.go | 27 +++++++ spinifex/handlers/eks/k3s_server_vm_test.go | 17 ++++ spinifex/handlers/eks/service_impl.go | 52 +++++++----- 6 files changed, 159 insertions(+), 69 deletions(-) diff --git a/spinifex/daemon/eks_deps.go b/spinifex/daemon/eks_deps.go index f6f7cd6d5..aca4b7362 100644 --- a/spinifex/daemon/eks_deps.go +++ b/spinifex/daemon/eks_deps.go @@ -8,6 +8,7 @@ import ( handlers_ec2_placementgroup "github.com/mulgadc/spinifex/spinifex/handlers/ec2/placementgroup" handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks" handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" + "github.com/mulgadc/spinifex/spinifex/objectstore" "log/slog" ) @@ -74,33 +75,37 @@ func (d *Daemon) buildEKSServiceDeps() handlers_eks.EKSServiceDeps { } deps := handlers_eks.EKSServiceDeps{ - Config: d.config, - NATSConn: d.natsConn, - MasterKey: masterKey, - GatewayBaseURL: d.resolveGatewayBaseURL(), - Region: d.config.Region, - HolderID: d.node, - ClusterSize: clusterSize, - InternalSuffix: internalSuffix, - SystemGatewayURL: d.resolveSystemGatewayBaseURL(), - SystemAccessKey: d.config.Predastore.AccessKey, - SystemSecretKey: d.config.Predastore.SecretKey, - GatewayCACert: gatewayCA, - VPCSG: d.vpcService, - VPCK3s: d.vpcService, - VPCSubnet: &daemonEKSSubnetResolver{d: d}, - NLB: d.elbv2Service, - Instance: d, - Image: d.imageService, - EIP: d.eipService, - IGW: d.igwService, - Worker: d, - VPCMgr: d.vpcService, - NATGW: d.natGatewayService, - RouteTable: d.routeTableService, - PlacementGroup: handlers_ec2_placementgroup.NewNATSPlacementGroupService(d.natsConn), - Scheduler: handlers_eks.NewNATSHostScheduler(d.natsConn), - CPControl: d.newEKSCPControl(), + Config: d.config, + NATSConn: d.natsConn, + MasterKey: masterKey, + GatewayBaseURL: d.resolveGatewayBaseURL(), + Region: d.config.Region, + HolderID: d.node, + ClusterSize: clusterSize, + InternalSuffix: internalSuffix, + SystemGatewayURL: d.resolveSystemGatewayBaseURL(), + SystemAccessKey: d.config.Predastore.AccessKey, + SystemSecretKey: d.config.Predastore.SecretKey, + GatewayCACert: gatewayCA, + SystemPredastoreURL: d.resolveSystemPredastoreURL(), + SnapshotStore: objectstore.NewS3ObjectStoreFromConfig( + d.config.Predastore.Host, d.config.Predastore.Region, + d.config.Predastore.AccessKey, d.config.Predastore.SecretKey), + VPCSG: d.vpcService, + VPCK3s: d.vpcService, + VPCSubnet: &daemonEKSSubnetResolver{d: d}, + NLB: d.elbv2Service, + Instance: d, + Image: d.imageService, + EIP: d.eipService, + IGW: d.igwService, + Worker: d, + VPCMgr: d.vpcService, + NATGW: d.natGatewayService, + RouteTable: d.routeTableService, + PlacementGroup: handlers_ec2_placementgroup.NewNATSPlacementGroupService(d.natsConn), + Scheduler: handlers_eks.NewNATSHostScheduler(d.natsConn), + CPControl: d.newEKSCPControl(), } // A KV-backed IAM service (sharing the gateway's buckets over NATS) lets EKS @@ -176,3 +181,25 @@ func (d *Daemon) resolveSystemGatewayBaseURL() string { } return "https://" + net.JoinHostPort(d.mgmtBridgeIP, gatewayPort) } + +// resolveSystemPredastoreURL derives the predastore URL a system instance (the +// K3s server VM) fetches/PUTs etcd snapshots to. Same mgmt-bridge-only +// reachability constraint as resolveSystemGatewayBaseURL: config.Predastore.Host +// is rewritten to 127.0.0.1 for the local daemon, which a guest cannot dial, so +// the bridge IP is substituted with the configured port. Falls back to the +// configured Predastore.Host (scheme-prefixed) when no mgmt bridge exists. +func (d *Daemon) resolveSystemPredastoreURL() string { + predastorePort := "8443" + if d.config.Predastore.Host != "" { + if _, port, splitErr := net.SplitHostPort(d.config.Predastore.Host); splitErr == nil && port != "" { + predastorePort = port + } + } + if d.mgmtBridgeIP == "" { + if d.config.Predastore.Host == "" { + return "" + } + return "https://" + d.config.Predastore.Host + } + return "https://" + net.JoinHostPort(d.mgmtBridgeIP, predastorePort) +} diff --git a/spinifex/handlers/eks/delete_cluster_test.go b/spinifex/handlers/eks/delete_cluster_test.go index 34332b57b..9ee70fc14 100644 --- a/spinifex/handlers/eks/delete_cluster_test.go +++ b/spinifex/handlers/eks/delete_cluster_test.go @@ -73,30 +73,31 @@ func newEKSServiceFixture(t *testing.T) *eksServiceFixture { rt := newFakeRouteTableProvisioner() svc, err := NewEKSServiceImpl(EKSServiceDeps{ - NATSConn: nc, - MasterKey: bootstrapTestMasterKey, - GatewayBaseURL: "https://gw.local:9999", - Region: "us-east-1", - HolderID: "node-1", - SystemGatewayURL: "https://gw.local:9999", - SystemAccessKey: "AKIAEXAMPLE", - SystemSecretKey: "s3cr3t-key", - GatewayCACert: "-----BEGIN CERTIFICATE-----\nFAKECA\n-----END CERTIFICATE-----\n", - VPCSG: sg, - VPCK3s: vpc, - VPCSubnet: fakeSubnetResolver{}, - IAM: newFakeEnsurer(), - NLB: nlb, - Instance: inst, - Image: ami, - EIP: eip, - Worker: worker, - VPCMgr: vpcMgr, - IGW: igw, - NATGW: ngw, - RouteTable: rt, - PlacementGroup: &fakePlacer{}, - Scheduler: &fakeHostScheduler{}, + NATSConn: nc, + MasterKey: bootstrapTestMasterKey, + GatewayBaseURL: "https://gw.local:9999", + Region: "us-east-1", + HolderID: "node-1", + SystemGatewayURL: "https://gw.local:9999", + SystemAccessKey: "AKIAEXAMPLE", + SystemSecretKey: "s3cr3t-key", + SystemPredastoreURL: "https://10.15.8.1:8443", + GatewayCACert: "-----BEGIN CERTIFICATE-----\nFAKECA\n-----END CERTIFICATE-----\n", + VPCSG: sg, + VPCK3s: vpc, + VPCSubnet: fakeSubnetResolver{}, + IAM: newFakeEnsurer(), + NLB: nlb, + Instance: inst, + Image: ami, + EIP: eip, + Worker: worker, + VPCMgr: vpcMgr, + IGW: igw, + NATGW: ngw, + RouteTable: rt, + PlacementGroup: &fakePlacer{}, + Scheduler: &fakeHostScheduler{}, }) require.NoError(t, err) // Tight Ready-gating knobs so nodegroup launches resolve in tests without the diff --git a/spinifex/handlers/eks/k3s_ha_control_plane.go b/spinifex/handlers/eks/k3s_ha_control_plane.go index 9edfb98c3..ab0ca3364 100644 --- a/spinifex/handlers/eks/k3s_ha_control_plane.go +++ b/spinifex/handlers/eks/k3s_ha_control_plane.go @@ -162,6 +162,8 @@ func (s *EKSServiceImpl) ProvisionReplacementCP(ctx context.Context, req Replace in.AccessKey = s.deps.SystemAccessKey in.SecretKey = s.deps.SystemSecretKey } + in.PredastoreAccessKey = s.deps.SystemAccessKey + in.PredastoreSecretKey = s.deps.SystemSecretKey instanceType := in.InstanceType if instanceType == "" { diff --git a/spinifex/handlers/eks/k3s_server_vm.go b/spinifex/handlers/eks/k3s_server_vm.go index fc6b2ace2..4e5baeb86 100644 --- a/spinifex/handlers/eks/k3s_server_vm.go +++ b/spinifex/handlers/eks/k3s_server_vm.go @@ -61,6 +61,10 @@ const ( // k3sFirstBootEnvPath is the env file k3s-first-boot.sh sources at boot. k3sFirstBootEnvPath = "/etc/spinifex-eks/first-boot.env" + // k3sSnapshotEnvPath is the env file mulga-eks-etcd-snapshot.sh (cron) and + // mulga-eks-k3s-recovery.sh (boot-time restore) source for predastore creds. + k3sSnapshotEnvPath = "/etc/spinifex-eks/etcd-snapshot.env" + // agentEnvPath is the env file the k3s-agent OpenRC service sources for // K3S_URL/K3S_TOKEN/K3S_NODE_NAME/K3S_NODE_LABEL. agentEnvPath = "/etc/spinifex-eks/agent.env" @@ -164,6 +168,13 @@ type K3sServerInput struct { // EKS_KONNECTIVITY_SERVER_COUNT so the konnectivity-server advertises // --server-count=N and every agent holds a tunnel to every replica (HA-correct). KonnServerCount int `json:"konnServerCount,omitempty"` + // PredastoreEndpoint/PredastoreAccessKey/PredastoreSecretKey are the + // mgmt-bridge-reachable predastore SigV4 creds baked into etcd-snapshot.env. + // The guest snapshot cron and boot-time recovery script have no IMDS-role + // support, so these are always static (unlike AccessKey/SecretKey above). + PredastoreEndpoint string `json:"predastoreEndpoint,omitempty"` + PredastoreAccessKey string `json:"predastoreAccessKey,omitempty"` + PredastoreSecretKey string `json:"predastoreSecretKey,omitempty"` } // K3sServerOutput carries identifiers to persist in ClusterMeta and register with the NLB. @@ -401,6 +412,12 @@ func validateK3sServerInput(in K3sServerInput) error { return errors.New("eks: K3sServerInput AccessKey and SecretKey must both be set or both empty") case strings.TrimSpace(in.GatewayCACert) == "": return errors.New("eks: K3sServerInput empty GatewayCACert") + case in.PredastoreEndpoint == "": + return errors.New("eks: K3sServerInput empty PredastoreEndpoint") + case in.PredastoreAccessKey == "": + return errors.New("eks: K3sServerInput empty PredastoreAccessKey") + case in.PredastoreSecretKey == "": + return errors.New("eks: K3sServerInput empty PredastoreSecretKey") case in.ServerURL != "" && in.JoinToken == "": return errors.New("eks: K3sServerInput join server (ServerURL set) requires JoinToken") } @@ -592,9 +609,19 @@ func buildK3sUserData(in K3sServerInput) string { ) k3sConfig := strings.Join(configLines, "\n") + snapshotEnvBody := strings.Join([]string{ + "EKS_ACCOUNT_ID=" + in.ClusterAccountID, + "EKS_CLUSTER_NAME=" + in.ClusterName, + "SPINIFEX_PREDASTORE_ENDPOINT=" + in.PredastoreEndpoint, + "SPINIFEX_PREDASTORE_AKID=" + in.PredastoreAccessKey, + "SPINIFEX_PREDASTORE_SECRET=" + in.PredastoreSecretKey, + }, "\n") + files := []userDataFile{ // 0600: may contain a system SigV4 secret key (static-cred mode). {Path: k3sFirstBootEnvPath, Perms: "0600", Body: envBody}, + // 0600: carries predastore SigV4 static creds for the snapshot cron + boot-time recovery. + {Path: k3sSnapshotEnvPath, Perms: "0600", Body: snapshotEnvBody}, {Path: k3sOIDCSigningKeyPath, Perms: "0600", Body: strings.TrimRight(in.OIDCPrivateKeyPEM, "\n")}, {Path: k3sOIDCPublicKeyPath, Perms: "0644", Body: strings.TrimRight(in.OIDCPublicKeyPEM, "\n")}, {Path: k3sConfigPath, Perms: "0644", Body: k3sConfig}, diff --git a/spinifex/handlers/eks/k3s_server_vm_test.go b/spinifex/handlers/eks/k3s_server_vm_test.go index 481851338..fc0d910e0 100644 --- a/spinifex/handlers/eks/k3s_server_vm_test.go +++ b/spinifex/handlers/eks/k3s_server_vm_test.go @@ -178,6 +178,10 @@ func validK3sInput() K3sServerInput { SecretKey: "s3cr3t-key", GatewayCACert: "-----BEGIN CERTIFICATE-----\nFAKECA\n-----END CERTIFICATE-----\n", JoinToken: "clustertok-deadbeef", + + PredastoreEndpoint: "https://10.15.8.1:8443", + PredastoreAccessKey: "AKIAPREDASTORE", + PredastoreSecretKey: "pred-s3cr3t-key", } } @@ -205,6 +209,9 @@ func TestLaunchK3sServerVM_EmptyInputsRejected(t *testing.T) { {"empty AccessKey", mk(func(in *K3sServerInput) { in.AccessKey = "" })}, {"empty SecretKey", mk(func(in *K3sServerInput) { in.SecretKey = "" })}, {"empty GatewayCACert", mk(func(in *K3sServerInput) { in.GatewayCACert = " \n" })}, + {"empty PredastoreEndpoint", mk(func(in *K3sServerInput) { in.PredastoreEndpoint = "" })}, + {"empty PredastoreAccessKey", mk(func(in *K3sServerInput) { in.PredastoreAccessKey = "" })}, + {"empty PredastoreSecretKey", mk(func(in *K3sServerInput) { in.PredastoreSecretKey = "" })}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -239,6 +246,16 @@ func TestBuildK3sUserData_StaticCredsBakeKeys(t *testing.T) { assert.Contains(t, ud, "EKS_SECRET_KEY=s3cr3t-key") } +func TestBuildK3sUserData_WritesEtcdSnapshotEnv(t *testing.T) { + ud := buildK3sUserData(validK3sInput()) + require.Contains(t, ud, "path: "+k3sSnapshotEnvPath) + assert.Contains(t, ud, "EKS_ACCOUNT_ID=111122223333") + assert.Contains(t, ud, "EKS_CLUSTER_NAME=alpha") + assert.Contains(t, ud, "SPINIFEX_PREDASTORE_ENDPOINT=https://10.15.8.1:8443") + assert.Contains(t, ud, "SPINIFEX_PREDASTORE_AKID=AKIAPREDASTORE") + assert.Contains(t, ud, "SPINIFEX_PREDASTORE_SECRET=pred-s3cr3t-key") +} + func TestBuildK3sUserData_ProfileModeOmitsKeys(t *testing.T) { in := validK3sInput() in.AccessKey = "" diff --git a/spinifex/handlers/eks/service_impl.go b/spinifex/handlers/eks/service_impl.go index d875fee33..24377221a 100644 --- a/spinifex/handlers/eks/service_impl.go +++ b/spinifex/handlers/eks/service_impl.go @@ -24,6 +24,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/config" handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" "github.com/mulgadc/spinifex/spinifex/handlers/sysinstance" + "github.com/mulgadc/spinifex/spinifex/objectstore" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/nats-io/nats.go" ) @@ -60,6 +61,16 @@ type EKSServiceDeps struct { SystemSecretKey string GatewayCACert string + // SystemPredastoreURL is the mgmt-bridge-reachable predastore endpoint baked + // into the CP VM's etcd-snapshot.env (SystemAccessKey/SystemSecretKey double + // as the predastore SigV4 creds — same system credential used for the gateway). + SystemPredastoreURL string + + // SnapshotStore lists/reads etcd snapshots from the eks-backups-system bucket + // for restore-snapshot's latest-snapshot resolution. Nil disables that lookup + // (callers must pass --snapshot explicitly). + SnapshotStore objectstore.ObjectStore + VPCSG sgProvisioner VPCK3s k3sVPCProvisioner VPCSubnet SubnetVPCResolver @@ -707,24 +718,27 @@ func (s *EKSServiceImpl) launchClusterInfra(ctx context.Context, lc clusterLaunc elbSubnets := dedupSubnetsByAZ(ctx, s.deps.VPCSubnet, accountID, lc.subnetIDs) serverIn := K3sServerInput{ - AccountID: sysAcct, - ClusterAccountID: accountID, - ClusterName: name, - Region: region, - SubnetID: cpRefs.PrivateSubnetIDs[0], - VpcID: meta.ResourcesVpcConfig.VpcId, - ELBSubnetIDs: elbSubnets, - ControlPlaneSGID: cpSG, - NLBDNS: nlb.DNSName, - EndpointIP: nlb.FrontendIP, - PrivateEndpointIP: meta.PrivateEndpointIP, - OIDCIssuer: oidcIssuer, - OIDCPrivateKeyPEM: privPEM, - OIDCPublicKeyPEM: pubPEM, - GatewayURL: s.deps.SystemGatewayURL, - AddonGatewayURL: s.deps.GatewayBaseURL, - GatewayCACert: s.deps.GatewayCACert, - JoinToken: joinToken, + AccountID: sysAcct, + ClusterAccountID: accountID, + ClusterName: name, + Region: region, + SubnetID: cpRefs.PrivateSubnetIDs[0], + VpcID: meta.ResourcesVpcConfig.VpcId, + ELBSubnetIDs: elbSubnets, + ControlPlaneSGID: cpSG, + NLBDNS: nlb.DNSName, + EndpointIP: nlb.FrontendIP, + PrivateEndpointIP: meta.PrivateEndpointIP, + OIDCIssuer: oidcIssuer, + OIDCPrivateKeyPEM: privPEM, + OIDCPublicKeyPEM: pubPEM, + GatewayURL: s.deps.SystemGatewayURL, + AddonGatewayURL: s.deps.GatewayBaseURL, + GatewayCACert: s.deps.GatewayCACert, + JoinToken: joinToken, + PredastoreEndpoint: s.deps.SystemPredastoreURL, + PredastoreAccessKey: s.deps.SystemAccessKey, + PredastoreSecretKey: s.deps.SystemSecretKey, } // Prefer IMDS instance-role creds: attach a system instance profile so the @@ -767,6 +781,8 @@ func (s *EKSServiceImpl) launchClusterInfra(ctx context.Context, lc clusterLaunc tmpl.AccessKey = "" tmpl.SecretKey = "" tmpl.IamInstanceProfileArn = "" + tmpl.PredastoreAccessKey = "" + tmpl.PredastoreSecretKey = "" meta.ControlPlaneTemplate = &tmpl // Persist the CP VM + ENI + spread-group refs now, before any further fallible