gsadmin/spinifex
1
0
Fork 0
mirror of https://github.com/mulgadc/spinifex.git synced 2026-10-15 17:11:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(eks): write etcd-snapshot.env at CP launch so snapshots actually run

Browse Source
This commit is contained in:
joshsiv-mulga
2026-07-10 15:48:59 +10:00
parent 8da043ce76
commit 852f61d9a3
6 changed files with 159 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
spinifex/daemon/eks_deps.go
+54 -27
View File
@@ -8,6 +8,7 @@ import (
handlers_ec2_placementgroup "github.com/mulgadc/spinifex/spinifex/handlers/ec2/placementgroup"
handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
"github.com/mulgadc/spinifex/spinifex/objectstore"
"log/slog"
)
@@ -74,33 +75,37 @@ func (d *Daemon) buildEKSServiceDeps() handlers_eks.EKSServiceDeps {
}
deps := handlers_eks.EKSServiceDeps{
Config: d.config,
NATSConn: d.natsConn,
MasterKey: masterKey,
GatewayBaseURL: d.resolveGatewayBaseURL(),
Region: d.config.Region,
HolderID: d.node,
ClusterSize: clusterSize,
InternalSuffix: internalSuffix,
SystemGatewayURL: d.resolveSystemGatewayBaseURL(),
SystemAccessKey: d.config.Predastore.AccessKey,
SystemSecretKey: d.config.Predastore.SecretKey,
GatewayCACert: gatewayCA,
VPCSG: d.vpcService,
VPCK3s: d.vpcService,
VPCSubnet: &daemonEKSSubnetResolver{d: d},
NLB: d.elbv2Service,
Instance: d,
Image: d.imageService,
EIP: d.eipService,
IGW: d.igwService,
Worker: d,
VPCMgr: d.vpcService,
NATGW: d.natGatewayService,
RouteTable: d.routeTableService,
PlacementGroup: handlers_ec2_placementgroup.NewNATSPlacementGroupService(d.natsConn),
Scheduler: handlers_eks.NewNATSHostScheduler(d.natsConn),
CPControl: d.newEKSCPControl(),
Config: d.config,
NATSConn: d.natsConn,
MasterKey: masterKey,
GatewayBaseURL: d.resolveGatewayBaseURL(),
Region: d.config.Region,
HolderID: d.node,
ClusterSize: clusterSize,
InternalSuffix: internalSuffix,
SystemGatewayURL: d.resolveSystemGatewayBaseURL(),
SystemAccessKey: d.config.Predastore.AccessKey,
SystemSecretKey: d.config.Predastore.SecretKey,
GatewayCACert: gatewayCA,
SystemPredastoreURL: d.resolveSystemPredastoreURL(),
SnapshotStore: objectstore.NewS3ObjectStoreFromConfig(
d.config.Predastore.Host, d.config.Predastore.Region,
d.config.Predastore.AccessKey, d.config.Predastore.SecretKey),
VPCSG: d.vpcService,
VPCK3s: d.vpcService,
VPCSubnet: &daemonEKSSubnetResolver{d: d},
NLB: d.elbv2Service,
Instance: d,
Image: d.imageService,
EIP: d.eipService,
IGW: d.igwService,
Worker: d,
VPCMgr: d.vpcService,
NATGW: d.natGatewayService,
RouteTable: d.routeTableService,
PlacementGroup: handlers_ec2_placementgroup.NewNATSPlacementGroupService(d.natsConn),
Scheduler: handlers_eks.NewNATSHostScheduler(d.natsConn),
CPControl: d.newEKSCPControl(),
}
// A KV-backed IAM service (sharing the gateway's buckets over NATS) lets EKS
@@ -176,3 +181,25 @@ func (d *Daemon) resolveSystemGatewayBaseURL() string {
}
return "https://" + net.JoinHostPort(d.mgmtBridgeIP, gatewayPort)
}
// resolveSystemPredastoreURL derives the predastore URL a system instance (the
// K3s server VM) fetches/PUTs etcd snapshots to. Same mgmt-bridge-only
// reachability constraint as resolveSystemGatewayBaseURL: config.Predastore.Host
// is rewritten to 127.0.0.1 for the local daemon, which a guest cannot dial, so
// the bridge IP is substituted with the configured port. Falls back to the
// configured Predastore.Host (scheme-prefixed) when no mgmt bridge exists.
func (d *Daemon) resolveSystemPredastoreURL() string {
predastorePort := "8443"
if d.config.Predastore.Host != "" {
if _, port, splitErr := net.SplitHostPort(d.config.Predastore.Host); splitErr == nil && port != "" {
predastorePort = port
}
}
if d.mgmtBridgeIP == "" {
if d.config.Predastore.Host == "" {
return ""
}
return "https://" + d.config.Predastore.Host
}
return "https://" + net.JoinHostPort(d.mgmtBridgeIP, predastorePort)
}
spinifex/handlers/eks/delete_cluster_test.go
+25 -24
View File
@@ -73,30 +73,31 @@ func newEKSServiceFixture(t *testing.T) *eksServiceFixture {
rt := newFakeRouteTableProvisioner()
svc, err := NewEKSServiceImpl(EKSServiceDeps{
NATSConn: nc,
MasterKey: bootstrapTestMasterKey,
GatewayBaseURL: "https://gw.local:9999",
Region: "us-east-1",
HolderID: "node-1",
SystemGatewayURL: "https://gw.local:9999",
SystemAccessKey: "AKIAEXAMPLE",
SystemSecretKey: "s3cr3t-key",
GatewayCACert: "-----BEGIN CERTIFICATE-----\nFAKECA\n-----END CERTIFICATE-----\n",
VPCSG: sg,
VPCK3s: vpc,
VPCSubnet: fakeSubnetResolver{},
IAM: newFakeEnsurer(),
NLB: nlb,
Instance: inst,
Image: ami,
EIP: eip,
Worker: worker,
VPCMgr: vpcMgr,
IGW: igw,
NATGW: ngw,
RouteTable: rt,
PlacementGroup: &fakePlacer{},
Scheduler: &fakeHostScheduler{},
NATSConn: nc,
MasterKey: bootstrapTestMasterKey,
GatewayBaseURL: "https://gw.local:9999",
Region: "us-east-1",
HolderID: "node-1",
SystemGatewayURL: "https://gw.local:9999",
SystemAccessKey: "AKIAEXAMPLE",
SystemSecretKey: "s3cr3t-key",
SystemPredastoreURL: "https://10.15.8.1:8443",
GatewayCACert: "-----BEGIN CERTIFICATE-----\nFAKECA\n-----END CERTIFICATE-----\n",
VPCSG: sg,
VPCK3s: vpc,
VPCSubnet: fakeSubnetResolver{},
IAM: newFakeEnsurer(),
NLB: nlb,
Instance: inst,
Image: ami,
EIP: eip,
Worker: worker,
VPCMgr: vpcMgr,
IGW: igw,
NATGW: ngw,
RouteTable: rt,
PlacementGroup: &fakePlacer{},
Scheduler: &fakeHostScheduler{},
})
require.NoError(t, err)
// Tight Ready-gating knobs so nodegroup launches resolve in tests without the
spinifex/handlers/eks/k3s_ha_control_plane.go
+2
View File
@@ -162,6 +162,8 @@ func (s *EKSServiceImpl) ProvisionReplacementCP(ctx context.Context, req Replace
in.AccessKey = s.deps.SystemAccessKey
in.SecretKey = s.deps.SystemSecretKey
}
in.PredastoreAccessKey = s.deps.SystemAccessKey
in.PredastoreSecretKey = s.deps.SystemSecretKey
instanceType := in.InstanceType
if instanceType == "" {
spinifex/handlers/eks/k3s_server_vm.go
+27
View File
@@ -61,6 +61,10 @@ const (
// k3sFirstBootEnvPath is the env file k3s-first-boot.sh sources at boot.
k3sFirstBootEnvPath = "/etc/spinifex-eks/first-boot.env"
// k3sSnapshotEnvPath is the env file mulga-eks-etcd-snapshot.sh (cron) and
// mulga-eks-k3s-recovery.sh (boot-time restore) source for predastore creds.
k3sSnapshotEnvPath = "/etc/spinifex-eks/etcd-snapshot.env"
// agentEnvPath is the env file the k3s-agent OpenRC service sources for
// K3S_URL/K3S_TOKEN/K3S_NODE_NAME/K3S_NODE_LABEL.
agentEnvPath = "/etc/spinifex-eks/agent.env"
@@ -164,6 +168,13 @@ type K3sServerInput struct {
// EKS_KONNECTIVITY_SERVER_COUNT so the konnectivity-server advertises
// --server-count=N and every agent holds a tunnel to every replica (HA-correct).
KonnServerCount int `json:"konnServerCount,omitempty"`
// PredastoreEndpoint/PredastoreAccessKey/PredastoreSecretKey are the
// mgmt-bridge-reachable predastore SigV4 creds baked into etcd-snapshot.env.
// The guest snapshot cron and boot-time recovery script have no IMDS-role
// support, so these are always static (unlike AccessKey/SecretKey above).
PredastoreEndpoint string `json:"predastoreEndpoint,omitempty"`
PredastoreAccessKey string `json:"predastoreAccessKey,omitempty"`
PredastoreSecretKey string `json:"predastoreSecretKey,omitempty"`
}
// K3sServerOutput carries identifiers to persist in ClusterMeta and register with the NLB.
@@ -401,6 +412,12 @@ func validateK3sServerInput(in K3sServerInput) error {
return errors.New("eks: K3sServerInput AccessKey and SecretKey must both be set or both empty")
case strings.TrimSpace(in.GatewayCACert) == "":
return errors.New("eks: K3sServerInput empty GatewayCACert")
case in.PredastoreEndpoint == "":
return errors.New("eks: K3sServerInput empty PredastoreEndpoint")
case in.PredastoreAccessKey == "":
return errors.New("eks: K3sServerInput empty PredastoreAccessKey")
case in.PredastoreSecretKey == "":
return errors.New("eks: K3sServerInput empty PredastoreSecretKey")
case in.ServerURL != "" && in.JoinToken == "":
return errors.New("eks: K3sServerInput join server (ServerURL set) requires JoinToken")
}
@@ -592,9 +609,19 @@ func buildK3sUserData(in K3sServerInput) string {
)
k3sConfig := strings.Join(configLines, "\n")
snapshotEnvBody := strings.Join([]string{
"EKS_ACCOUNT_ID=" + in.ClusterAccountID,
"EKS_CLUSTER_NAME=" + in.ClusterName,
"SPINIFEX_PREDASTORE_ENDPOINT=" + in.PredastoreEndpoint,
"SPINIFEX_PREDASTORE_AKID=" + in.PredastoreAccessKey,
"SPINIFEX_PREDASTORE_SECRET=" + in.PredastoreSecretKey,
}, "\n")
files := []userDataFile{
// 0600: may contain a system SigV4 secret key (static-cred mode).
{Path: k3sFirstBootEnvPath, Perms: "0600", Body: envBody},
// 0600: carries predastore SigV4 static creds for the snapshot cron + boot-time recovery.
{Path: k3sSnapshotEnvPath, Perms: "0600", Body: snapshotEnvBody},
{Path: k3sOIDCSigningKeyPath, Perms: "0600", Body: strings.TrimRight(in.OIDCPrivateKeyPEM, "\n")},
{Path: k3sOIDCPublicKeyPath, Perms: "0644", Body: strings.TrimRight(in.OIDCPublicKeyPEM, "\n")},
{Path: k3sConfigPath, Perms: "0644", Body: k3sConfig},
spinifex/handlers/eks/k3s_server_vm_test.go
+17
View File
@@ -178,6 +178,10 @@ func validK3sInput() K3sServerInput {
SecretKey: "s3cr3t-key",
GatewayCACert: "-----BEGIN CERTIFICATE-----\nFAKECA\n-----END CERTIFICATE-----\n",
JoinToken: "clustertok-deadbeef",
PredastoreEndpoint: "https://10.15.8.1:8443",
PredastoreAccessKey: "AKIAPREDASTORE",
PredastoreSecretKey: "pred-s3cr3t-key",
}
}
@@ -205,6 +209,9 @@ func TestLaunchK3sServerVM_EmptyInputsRejected(t *testing.T) {
{"empty AccessKey", mk(func(in *K3sServerInput) { in.AccessKey = "" })},
{"empty SecretKey", mk(func(in *K3sServerInput) { in.SecretKey = "" })},
{"empty GatewayCACert", mk(func(in *K3sServerInput) { in.GatewayCACert = " \n" })},
{"empty PredastoreEndpoint", mk(func(in *K3sServerInput) { in.PredastoreEndpoint = "" })},
{"empty PredastoreAccessKey", mk(func(in *K3sServerInput) { in.PredastoreAccessKey = "" })},
{"empty PredastoreSecretKey", mk(func(in *K3sServerInput) { in.PredastoreSecretKey = "" })},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
@@ -239,6 +246,16 @@ func TestBuildK3sUserData_StaticCredsBakeKeys(t *testing.T) {
assert.Contains(t, ud, "EKS_SECRET_KEY=s3cr3t-key")
}
func TestBuildK3sUserData_WritesEtcdSnapshotEnv(t *testing.T) {
ud := buildK3sUserData(validK3sInput())
require.Contains(t, ud, "path: "+k3sSnapshotEnvPath)
assert.Contains(t, ud, "EKS_ACCOUNT_ID=111122223333")
assert.Contains(t, ud, "EKS_CLUSTER_NAME=alpha")
assert.Contains(t, ud, "SPINIFEX_PREDASTORE_ENDPOINT=https://10.15.8.1:8443")
assert.Contains(t, ud, "SPINIFEX_PREDASTORE_AKID=AKIAPREDASTORE")
assert.Contains(t, ud, "SPINIFEX_PREDASTORE_SECRET=pred-s3cr3t-key")
}
func TestBuildK3sUserData_ProfileModeOmitsKeys(t *testing.T) {
in := validK3sInput()
in.AccessKey = ""
spinifex/handlers/eks/service_impl.go
+34 -18
View File
@@ -24,6 +24,7 @@ import (
"github.com/mulgadc/spinifex/spinifex/config"
handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
"github.com/mulgadc/spinifex/spinifex/handlers/sysinstance"
"github.com/mulgadc/spinifex/spinifex/objectstore"
"github.com/mulgadc/spinifex/spinifex/utils"
"github.com/nats-io/nats.go"
)
@@ -60,6 +61,16 @@ type EKSServiceDeps struct {
SystemSecretKey string
GatewayCACert string
// SystemPredastoreURL is the mgmt-bridge-reachable predastore endpoint baked
// into the CP VM's etcd-snapshot.env (SystemAccessKey/SystemSecretKey double
// as the predastore SigV4 creds — same system credential used for the gateway).
SystemPredastoreURL string
// SnapshotStore lists/reads etcd snapshots from the eks-backups-system bucket
// for restore-snapshot's latest-snapshot resolution. Nil disables that lookup
// (callers must pass --snapshot explicitly).
SnapshotStore objectstore.ObjectStore
VPCSG sgProvisioner
VPCK3s k3sVPCProvisioner
VPCSubnet SubnetVPCResolver
@@ -707,24 +718,27 @@ func (s *EKSServiceImpl) launchClusterInfra(ctx context.Context, lc clusterLaunc
elbSubnets := dedupSubnetsByAZ(ctx, s.deps.VPCSubnet, accountID, lc.subnetIDs)
serverIn := K3sServerInput{
AccountID: sysAcct,
ClusterAccountID: accountID,
ClusterName: name,
Region: region,
SubnetID: cpRefs.PrivateSubnetIDs[0],
VpcID: meta.ResourcesVpcConfig.VpcId,
ELBSubnetIDs: elbSubnets,
ControlPlaneSGID: cpSG,
NLBDNS: nlb.DNSName,
EndpointIP: nlb.FrontendIP,
PrivateEndpointIP: meta.PrivateEndpointIP,
OIDCIssuer: oidcIssuer,
OIDCPrivateKeyPEM: privPEM,
OIDCPublicKeyPEM: pubPEM,
GatewayURL: s.deps.SystemGatewayURL,
AddonGatewayURL: s.deps.GatewayBaseURL,
GatewayCACert: s.deps.GatewayCACert,
JoinToken: joinToken,
AccountID: sysAcct,
ClusterAccountID: accountID,
ClusterName: name,
Region: region,
SubnetID: cpRefs.PrivateSubnetIDs[0],
VpcID: meta.ResourcesVpcConfig.VpcId,
ELBSubnetIDs: elbSubnets,
ControlPlaneSGID: cpSG,
NLBDNS: nlb.DNSName,
EndpointIP: nlb.FrontendIP,
PrivateEndpointIP: meta.PrivateEndpointIP,
OIDCIssuer: oidcIssuer,
OIDCPrivateKeyPEM: privPEM,
OIDCPublicKeyPEM: pubPEM,
GatewayURL: s.deps.SystemGatewayURL,
AddonGatewayURL: s.deps.GatewayBaseURL,
GatewayCACert: s.deps.GatewayCACert,
JoinToken: joinToken,
PredastoreEndpoint: s.deps.SystemPredastoreURL,
PredastoreAccessKey: s.deps.SystemAccessKey,
PredastoreSecretKey: s.deps.SystemSecretKey,
}
// Prefer IMDS instance-role creds: attach a system instance profile so the
@@ -767,6 +781,8 @@ func (s *EKSServiceImpl) launchClusterInfra(ctx context.Context, lc clusterLaunc
tmpl.AccessKey = ""
tmpl.SecretKey = ""
tmpl.IamInstanceProfileArn = ""
tmpl.PredastoreAccessKey = ""
tmpl.PredastoreSecretKey = ""
meta.ControlPlaneTemplate = &tmpl
// Persist the CP VM + ENI + spread-group refs now, before any further fallible