fix(eks): write etcd-snapshot.env at CP launch so snapshots actually run

spinifex/daemon/eks_deps.go

@@ -8,6 +8,7 @@ import (
 	handlers_ec2_placementgroup "github.com/mulgadc/spinifex/spinifex/handlers/ec2/placementgroup"
 	handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
 	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
+	"github.com/mulgadc/spinifex/spinifex/objectstore"
 	"log/slog"
 )
 
@@ -74,33 +75,37 @@ func (d *Daemon) buildEKSServiceDeps() handlers_eks.EKSServiceDeps {
 	}
 
 	deps := handlers_eks.EKSServiceDeps{
-		Config:           d.config,
+		Config:              d.config,
-		NATSConn:         d.natsConn,
+		NATSConn:            d.natsConn,
-		MasterKey:        masterKey,
+		MasterKey:           masterKey,
-		GatewayBaseURL:   d.resolveGatewayBaseURL(),
+		GatewayBaseURL:      d.resolveGatewayBaseURL(),
-		Region:           d.config.Region,
+		Region:              d.config.Region,
-		HolderID:         d.node,
+		HolderID:            d.node,
-		ClusterSize:      clusterSize,
+		ClusterSize:         clusterSize,
-		InternalSuffix:   internalSuffix,
+		InternalSuffix:      internalSuffix,
-		SystemGatewayURL: d.resolveSystemGatewayBaseURL(),
+		SystemGatewayURL:    d.resolveSystemGatewayBaseURL(),
-		SystemAccessKey:  d.config.Predastore.AccessKey,
+		SystemAccessKey:     d.config.Predastore.AccessKey,
-		SystemSecretKey:  d.config.Predastore.SecretKey,
+		SystemSecretKey:     d.config.Predastore.SecretKey,
-		GatewayCACert:    gatewayCA,
+		GatewayCACert:       gatewayCA,
-		VPCSG:            d.vpcService,
+		SystemPredastoreURL: d.resolveSystemPredastoreURL(),
-		VPCK3s:           d.vpcService,
+		SnapshotStore: objectstore.NewS3ObjectStoreFromConfig(
-		VPCSubnet:        &daemonEKSSubnetResolver{d: d},
+			d.config.Predastore.Host, d.config.Predastore.Region,
-		NLB:              d.elbv2Service,
+			d.config.Predastore.AccessKey, d.config.Predastore.SecretKey),
-		Instance:         d,
+		VPCSG:          d.vpcService,
-		Image:            d.imageService,
+		VPCK3s:         d.vpcService,
-		EIP:              d.eipService,
+		VPCSubnet:      &daemonEKSSubnetResolver{d: d},
-		IGW:              d.igwService,
+		NLB:            d.elbv2Service,
-		Worker:           d,
+		Instance:       d,
-		VPCMgr:           d.vpcService,
+		Image:          d.imageService,
-		NATGW:            d.natGatewayService,
+		EIP:            d.eipService,
-		RouteTable:       d.routeTableService,
+		IGW:            d.igwService,
-		PlacementGroup:   handlers_ec2_placementgroup.NewNATSPlacementGroupService(d.natsConn),
+		Worker:         d,
-		Scheduler:        handlers_eks.NewNATSHostScheduler(d.natsConn),
+		VPCMgr:         d.vpcService,
-		CPControl:        d.newEKSCPControl(),
+		NATGW:          d.natGatewayService,
+		RouteTable:     d.routeTableService,
+		PlacementGroup: handlers_ec2_placementgroup.NewNATSPlacementGroupService(d.natsConn),
+		Scheduler:      handlers_eks.NewNATSHostScheduler(d.natsConn),
+		CPControl:      d.newEKSCPControl(),
 	}
 
 	// A KV-backed IAM service (sharing the gateway's buckets over NATS) lets EKS
@@ -176,3 +181,25 @@ func (d *Daemon) resolveSystemGatewayBaseURL() string {
 	}
 	return "https://" + net.JoinHostPort(d.mgmtBridgeIP, gatewayPort)
 }
+
+// resolveSystemPredastoreURL derives the predastore URL a system instance (the
+// K3s server VM) fetches/PUTs etcd snapshots to. Same mgmt-bridge-only
+// reachability constraint as resolveSystemGatewayBaseURL: config.Predastore.Host
+// is rewritten to 127.0.0.1 for the local daemon, which a guest cannot dial, so
+// the bridge IP is substituted with the configured port. Falls back to the
+// configured Predastore.Host (scheme-prefixed) when no mgmt bridge exists.
+func (d *Daemon) resolveSystemPredastoreURL() string {
+	predastorePort := "8443"
+	if d.config.Predastore.Host != "" {
+		if _, port, splitErr := net.SplitHostPort(d.config.Predastore.Host); splitErr == nil && port != "" {
+			predastorePort = port
+		}
+	}
+	if d.mgmtBridgeIP == "" {
+		if d.config.Predastore.Host == "" {
+			return ""
+		}
+		return "https://" + d.config.Predastore.Host
+	}
+	return "https://" + net.JoinHostPort(d.mgmtBridgeIP, predastorePort)
+}

spinifex/handlers/eks/delete_cluster_test.go

@@ -73,30 +73,31 @@ func newEKSServiceFixture(t *testing.T) *eksServiceFixture {
 	rt := newFakeRouteTableProvisioner()
 
 	svc, err := NewEKSServiceImpl(EKSServiceDeps{
-		NATSConn:         nc,
+		NATSConn:            nc,
-		MasterKey:        bootstrapTestMasterKey,
+		MasterKey:           bootstrapTestMasterKey,
-		GatewayBaseURL:   "https://gw.local:9999",
+		GatewayBaseURL:      "https://gw.local:9999",
-		Region:           "us-east-1",
+		Region:              "us-east-1",
-		HolderID:         "node-1",
+		HolderID:            "node-1",
-		SystemGatewayURL: "https://gw.local:9999",
+		SystemGatewayURL:    "https://gw.local:9999",
-		SystemAccessKey:  "AKIAEXAMPLE",
+		SystemAccessKey:     "AKIAEXAMPLE",
-		SystemSecretKey:  "s3cr3t-key",
+		SystemSecretKey:     "s3cr3t-key",
-		GatewayCACert:    "-----BEGIN CERTIFICATE-----\nFAKECA\n-----END CERTIFICATE-----\n",
+		SystemPredastoreURL: "https://10.15.8.1:8443",
-		VPCSG:            sg,
+		GatewayCACert:       "-----BEGIN CERTIFICATE-----\nFAKECA\n-----END CERTIFICATE-----\n",
-		VPCK3s:           vpc,
+		VPCSG:               sg,
-		VPCSubnet:        fakeSubnetResolver{},
+		VPCK3s:              vpc,
-		IAM:              newFakeEnsurer(),
+		VPCSubnet:           fakeSubnetResolver{},
-		NLB:              nlb,
+		IAM:                 newFakeEnsurer(),
-		Instance:         inst,
+		NLB:                 nlb,
-		Image:            ami,
+		Instance:            inst,
-		EIP:              eip,
+		Image:               ami,
-		Worker:           worker,
+		EIP:                 eip,
-		VPCMgr:           vpcMgr,
+		Worker:              worker,
-		IGW:              igw,
+		VPCMgr:              vpcMgr,
-		NATGW:            ngw,
+		IGW:                 igw,
-		RouteTable:       rt,
+		NATGW:               ngw,
-		PlacementGroup:   &fakePlacer{},
+		RouteTable:          rt,
-		Scheduler:        &fakeHostScheduler{},
+		PlacementGroup:      &fakePlacer{},
+		Scheduler:           &fakeHostScheduler{},
 	})
 	require.NoError(t, err)
 	// Tight Ready-gating knobs so nodegroup launches resolve in tests without the

spinifex/handlers/eks/k3s_ha_control_plane.go

@@ -162,6 +162,8 @@ func (s *EKSServiceImpl) ProvisionReplacementCP(ctx context.Context, req Replace
 		in.AccessKey = s.deps.SystemAccessKey
 		in.SecretKey = s.deps.SystemSecretKey
 	}
+	in.PredastoreAccessKey = s.deps.SystemAccessKey
+	in.PredastoreSecretKey = s.deps.SystemSecretKey
 
 	instanceType := in.InstanceType
 	if instanceType == "" {

spinifex/handlers/eks/k3s_server_vm.go

@@ -61,6 +61,10 @@ const (
 	// k3sFirstBootEnvPath is the env file k3s-first-boot.sh sources at boot.
 	k3sFirstBootEnvPath = "/etc/spinifex-eks/first-boot.env"
 
+	// k3sSnapshotEnvPath is the env file mulga-eks-etcd-snapshot.sh (cron) and
+	// mulga-eks-k3s-recovery.sh (boot-time restore) source for predastore creds.
+	k3sSnapshotEnvPath = "/etc/spinifex-eks/etcd-snapshot.env"
+
 	// agentEnvPath is the env file the k3s-agent OpenRC service sources for
 	// K3S_URL/K3S_TOKEN/K3S_NODE_NAME/K3S_NODE_LABEL.
 	agentEnvPath = "/etc/spinifex-eks/agent.env"
@@ -164,6 +168,13 @@ type K3sServerInput struct {
 	// EKS_KONNECTIVITY_SERVER_COUNT so the konnectivity-server advertises
 	// --server-count=N and every agent holds a tunnel to every replica (HA-correct).
 	KonnServerCount int `json:"konnServerCount,omitempty"`
+	// PredastoreEndpoint/PredastoreAccessKey/PredastoreSecretKey are the
+	// mgmt-bridge-reachable predastore SigV4 creds baked into etcd-snapshot.env.
+	// The guest snapshot cron and boot-time recovery script have no IMDS-role
+	// support, so these are always static (unlike AccessKey/SecretKey above).
+	PredastoreEndpoint  string `json:"predastoreEndpoint,omitempty"`
+	PredastoreAccessKey string `json:"predastoreAccessKey,omitempty"`
+	PredastoreSecretKey string `json:"predastoreSecretKey,omitempty"`
 }
 
 // K3sServerOutput carries identifiers to persist in ClusterMeta and register with the NLB.
@@ -401,6 +412,12 @@ func validateK3sServerInput(in K3sServerInput) error {
 		return errors.New("eks: K3sServerInput AccessKey and SecretKey must both be set or both empty")
 	case strings.TrimSpace(in.GatewayCACert) == "":
 		return errors.New("eks: K3sServerInput empty GatewayCACert")
+	case in.PredastoreEndpoint == "":
+		return errors.New("eks: K3sServerInput empty PredastoreEndpoint")
+	case in.PredastoreAccessKey == "":
+		return errors.New("eks: K3sServerInput empty PredastoreAccessKey")
+	case in.PredastoreSecretKey == "":
+		return errors.New("eks: K3sServerInput empty PredastoreSecretKey")
 	case in.ServerURL != "" && in.JoinToken == "":
 		return errors.New("eks: K3sServerInput join server (ServerURL set) requires JoinToken")
 	}
@@ -592,9 +609,19 @@ func buildK3sUserData(in K3sServerInput) string {
 	)
 	k3sConfig := strings.Join(configLines, "\n")
 
+	snapshotEnvBody := strings.Join([]string{
+		"EKS_ACCOUNT_ID=" + in.ClusterAccountID,
+		"EKS_CLUSTER_NAME=" + in.ClusterName,
+		"SPINIFEX_PREDASTORE_ENDPOINT=" + in.PredastoreEndpoint,
+		"SPINIFEX_PREDASTORE_AKID=" + in.PredastoreAccessKey,
+		"SPINIFEX_PREDASTORE_SECRET=" + in.PredastoreSecretKey,
+	}, "\n")
+
 	files := []userDataFile{
 		// 0600: may contain a system SigV4 secret key (static-cred mode).
 		{Path: k3sFirstBootEnvPath, Perms: "0600", Body: envBody},
+		// 0600: carries predastore SigV4 static creds for the snapshot cron + boot-time recovery.
+		{Path: k3sSnapshotEnvPath, Perms: "0600", Body: snapshotEnvBody},
 		{Path: k3sOIDCSigningKeyPath, Perms: "0600", Body: strings.TrimRight(in.OIDCPrivateKeyPEM, "\n")},
 		{Path: k3sOIDCPublicKeyPath, Perms: "0644", Body: strings.TrimRight(in.OIDCPublicKeyPEM, "\n")},
 		{Path: k3sConfigPath, Perms: "0644", Body: k3sConfig},

spinifex/handlers/eks/k3s_server_vm_test.go

@@ -178,6 +178,10 @@ func validK3sInput() K3sServerInput {
 		SecretKey:         "s3cr3t-key",
 		GatewayCACert:     "-----BEGIN CERTIFICATE-----\nFAKECA\n-----END CERTIFICATE-----\n",
 		JoinToken:         "clustertok-deadbeef",
+
+		PredastoreEndpoint:  "https://10.15.8.1:8443",
+		PredastoreAccessKey: "AKIAPREDASTORE",
+		PredastoreSecretKey: "pred-s3cr3t-key",
 	}
 }
 
@@ -205,6 +209,9 @@ func TestLaunchK3sServerVM_EmptyInputsRejected(t *testing.T) {
 		{"empty AccessKey", mk(func(in *K3sServerInput) { in.AccessKey = "" })},
 		{"empty SecretKey", mk(func(in *K3sServerInput) { in.SecretKey = "" })},
 		{"empty GatewayCACert", mk(func(in *K3sServerInput) { in.GatewayCACert = "  \n" })},
+		{"empty PredastoreEndpoint", mk(func(in *K3sServerInput) { in.PredastoreEndpoint = "" })},
+		{"empty PredastoreAccessKey", mk(func(in *K3sServerInput) { in.PredastoreAccessKey = "" })},
+		{"empty PredastoreSecretKey", mk(func(in *K3sServerInput) { in.PredastoreSecretKey = "" })},
 	}
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
@@ -239,6 +246,16 @@ func TestBuildK3sUserData_StaticCredsBakeKeys(t *testing.T) {
 	assert.Contains(t, ud, "EKS_SECRET_KEY=s3cr3t-key")
 }
 
+func TestBuildK3sUserData_WritesEtcdSnapshotEnv(t *testing.T) {
+	ud := buildK3sUserData(validK3sInput())
+	require.Contains(t, ud, "path: "+k3sSnapshotEnvPath)
+	assert.Contains(t, ud, "EKS_ACCOUNT_ID=111122223333")
+	assert.Contains(t, ud, "EKS_CLUSTER_NAME=alpha")
+	assert.Contains(t, ud, "SPINIFEX_PREDASTORE_ENDPOINT=https://10.15.8.1:8443")
+	assert.Contains(t, ud, "SPINIFEX_PREDASTORE_AKID=AKIAPREDASTORE")
+	assert.Contains(t, ud, "SPINIFEX_PREDASTORE_SECRET=pred-s3cr3t-key")
+}
+
 func TestBuildK3sUserData_ProfileModeOmitsKeys(t *testing.T) {
 	in := validK3sInput()
 	in.AccessKey = ""

spinifex/handlers/eks/service_impl.go

@@ -24,6 +24,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/config"
 	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
 	"github.com/mulgadc/spinifex/spinifex/handlers/sysinstance"
+	"github.com/mulgadc/spinifex/spinifex/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/nats-io/nats.go"
 )
@@ -60,6 +61,16 @@ type EKSServiceDeps struct {
 	SystemSecretKey  string
 	GatewayCACert    string
 
+	// SystemPredastoreURL is the mgmt-bridge-reachable predastore endpoint baked
+	// into the CP VM's etcd-snapshot.env (SystemAccessKey/SystemSecretKey double
+	// as the predastore SigV4 creds — same system credential used for the gateway).
+	SystemPredastoreURL string
+
+	// SnapshotStore lists/reads etcd snapshots from the eks-backups-system bucket
+	// for restore-snapshot's latest-snapshot resolution. Nil disables that lookup
+	// (callers must pass --snapshot explicitly).
+	SnapshotStore objectstore.ObjectStore
+
 	VPCSG     sgProvisioner
 	VPCK3s    k3sVPCProvisioner
 	VPCSubnet SubnetVPCResolver
@@ -707,24 +718,27 @@ func (s *EKSServiceImpl) launchClusterInfra(ctx context.Context, lc clusterLaunc
 	elbSubnets := dedupSubnetsByAZ(ctx, s.deps.VPCSubnet, accountID, lc.subnetIDs)
 
 	serverIn := K3sServerInput{
-		AccountID:         sysAcct,
+		AccountID:           sysAcct,
-		ClusterAccountID:  accountID,
+		ClusterAccountID:    accountID,
-		ClusterName:       name,
+		ClusterName:         name,
-		Region:            region,
+		Region:              region,
-		SubnetID:          cpRefs.PrivateSubnetIDs[0],
+		SubnetID:            cpRefs.PrivateSubnetIDs[0],
-		VpcID:             meta.ResourcesVpcConfig.VpcId,
+		VpcID:               meta.ResourcesVpcConfig.VpcId,
-		ELBSubnetIDs:      elbSubnets,
+		ELBSubnetIDs:        elbSubnets,
-		ControlPlaneSGID:  cpSG,
+		ControlPlaneSGID:    cpSG,
-		NLBDNS:            nlb.DNSName,
+		NLBDNS:              nlb.DNSName,
-		EndpointIP:        nlb.FrontendIP,
+		EndpointIP:          nlb.FrontendIP,
-		PrivateEndpointIP: meta.PrivateEndpointIP,
+		PrivateEndpointIP:   meta.PrivateEndpointIP,
-		OIDCIssuer:        oidcIssuer,
+		OIDCIssuer:          oidcIssuer,
-		OIDCPrivateKeyPEM: privPEM,
+		OIDCPrivateKeyPEM:   privPEM,
-		OIDCPublicKeyPEM:  pubPEM,
+		OIDCPublicKeyPEM:    pubPEM,
-		GatewayURL:        s.deps.SystemGatewayURL,
+		GatewayURL:          s.deps.SystemGatewayURL,
-		AddonGatewayURL:   s.deps.GatewayBaseURL,
+		AddonGatewayURL:     s.deps.GatewayBaseURL,
-		GatewayCACert:     s.deps.GatewayCACert,
+		GatewayCACert:       s.deps.GatewayCACert,
-		JoinToken:         joinToken,
+		JoinToken:           joinToken,
+		PredastoreEndpoint:  s.deps.SystemPredastoreURL,
+		PredastoreAccessKey: s.deps.SystemAccessKey,
+		PredastoreSecretKey: s.deps.SystemSecretKey,
 	}
 
 	// Prefer IMDS instance-role creds: attach a system instance profile so the
@@ -767,6 +781,8 @@ func (s *EKSServiceImpl) launchClusterInfra(ctx context.Context, lc clusterLaunc
 	tmpl.AccessKey = ""
 	tmpl.SecretKey = ""
 	tmpl.IamInstanceProfileArn = ""
+	tmpl.PredastoreAccessKey = ""
+	tmpl.PredastoreSecretKey = ""
 	meta.ControlPlaneTemplate = &tmpl
 
 	// Persist the CP VM + ENI + spread-group refs now, before any further fallible