gsadmin/spinifex
1
0
Fork 0
mirror of https://github.com/mulgadc/spinifex.git synced 2026-10-23 04:36:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(eks): control-plane disaster recovery — etcd snapshot + restore-snapshot (#503)

Browse Source 
* fix(eks): write etcd-snapshot.env at CP launch so snapshots actually run

* test(eks): cover resolveSystemPredastoreURL and buildEKSServiceDeps

Diff-coverage gate for 852f61d9 was unmet (48.8% < 60%) because the
mgmt-bridge branches and EKSServiceDeps assembly had no direct test.

* feat(eks): restore-snapshot DR path (fresh CP + cluster-reset restore + NLB re-point)

* fix(eks): make restore-snapshot fail-safe (validate snapshot, unwind on partial failure, honest status)

* fix(eks): sign etcd-snapshot uploads with deployment region (was 403ing)

* fix(eks): apply recovery directive on DR-seed first boot + single-node replacement host

* fix(daemon): log NATS handler failures at Error (were only on the trace span)
This commit is contained in:
Josh Siviour
2026-07-10 22:14:58 +10:00
committed by GitHub
parent 6879a3425a
commit d663760dec
28 changed files with 1358 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/eks-gateway-fetch/main.go
+13 -6
View File
@@ -156,9 +156,10 @@ func emitAddonsTSV(out io.Writer, body []byte) error {
// recoveryDirective mirrors the gateway's RecoveryDirective wire shape; duplicated
// here to keep this tiny VM binary free of the handler package's dependency graph.
type recoveryDirective struct {
Epoch int64 `json:"epoch"`
Action string `json:"action"`
Snapshot string `json:"snapshot"`
Epoch int64 `json:"epoch"`
Action string `json:"action"`
Snapshot string `json:"snapshot"`
SnapshotRequired bool `json:"snapshotRequired"`
}
type internalRecoveryResponse struct {
@@ -166,8 +167,10 @@ type internalRecoveryResponse struct {
}
// emitRecoveryTSV decodes the recovery directive and writes a single
// epoch\taction\tsnapshot line the on-VM k3s-recovery agent consumes without a
// JSON parser. An unset action defaults to "none" (steady state).
// epoch\taction\tsnapshot\tsnapshotRequired line the on-VM k3s-recovery agent
// consumes without a JSON parser. An unset action defaults to "none" (steady
// state); snapshotRequired is 1 when the snapshot MUST restore (fresh DR seed
// aborts boot on fetch failure) else 0.
func emitRecoveryTSV(out io.Writer, body []byte) error {
var resp internalRecoveryResponse
if err := json.Unmarshal(body, &resp); err != nil {
@@ -177,7 +180,11 @@ func emitRecoveryTSV(out io.Writer, body []byte) error {
if d.Action == "" {
d.Action = "none"
}
if _, err := fmt.Fprintf(out, "%d\t%s\t%s\n", d.Epoch, d.Action, d.Snapshot); err != nil {
required := "0"
if d.SnapshotRequired {
required = "1"
}
if _, err := fmt.Fprintf(out, "%d\t%s\t%s\t%s\n", d.Epoch, d.Action, d.Snapshot, required); err != nil {
return err
}
return nil
cmd/spinifex/cmd/admin_eks.go
+77
View File
@@ -0,0 +1,77 @@
package cmd
import (
"context"
"fmt"
"os"
"time"
"github.com/mulgadc/spinifex/spinifex/admin"
handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
"github.com/spf13/cobra"
)
var adminEksCmd = &cobra.Command{
Use: "eks",
Short: "Manage EKS control-plane disaster recovery",
}
var adminEksRestoreSnapshotCmd = &cobra.Command{
Use: "restore-snapshot",
Short: "Rebuild a single-CP cluster's control plane from an etcd snapshot",
Long: `restore-snapshot drives the single-CP total-loss DR path: it launches a fresh
control-plane VM as a cluster-init seed, directs its boot-time recovery agent to
run k3s server --cluster-reset (restoring the given etcd snapshot, or the latest
one found in predastore when --snapshot is omitted), and re-points the cluster
NLB target groups at the new CP.
Only single-CP clusters are supported. An HA cluster has a potentially
surviving etcd quorum and should be recovered via quorum reformation instead
of this destructive single-node rebuild.`,
Run: runAdminEksRestoreSnapshot,
}
func init() {
adminCmd.AddCommand(adminEksCmd)
adminEksCmd.AddCommand(adminEksRestoreSnapshotCmd)
adminEksRestoreSnapshotCmd.Flags().String("cluster", "", "Cluster name to restore (required)")
adminEksRestoreSnapshotCmd.Flags().String("snapshot", "", "Etcd snapshot key to restore; defaults to the latest snapshot found in predastore")
adminEksRestoreSnapshotCmd.Flags().String("account", "", "Account ID that owns the cluster; defaults to the bootstrap account")
_ = adminEksRestoreSnapshotCmd.MarkFlagRequired("cluster")
}
func runAdminEksRestoreSnapshot(cmd *cobra.Command, _ []string) {
clusterName, _ := cmd.Flags().GetString("cluster")
snapshot, _ := cmd.Flags().GetString("snapshot")
accountID, _ := cmd.Flags().GetString("account")
if accountID == "" {
accountID = admin.DefaultAccountID()
}
_, nc, err := loadConfigAndConnect()
if err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
os.Exit(1)
}
defer nc.Close()
svc := handlers_eks.NewNATSEKSService(nc)
// The server side launches a VM (30-60s) then fences the old CP with retries,
// so the RPC needs generous headroom over a bare launch.
ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second)
defer cancel()
fmt.Printf("Restoring control plane for cluster %q (account %s)...\n", clusterName, accountID)
out, err := svc.RestoreSnapshot(ctx, &handlers_eks.RestoreSnapshotInput{
ClusterName: clusterName,
Snapshot: snapshot,
}, accountID)
if err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
os.Exit(1)
}
fmt.Printf("New control plane %s launched (snapshot %s)\n", out.NewInstanceID, out.Snapshot)
fmt.Printf("Status: %s\n", out.Status)
}